1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #ifdef CONFIG_RFS_ACCEL
9 #include <linux/cpu_rmap.h>
10 #endif /* CONFIG_RFS_ACCEL */
11 #include <linux/ethtool.h>
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/numa.h>
15 #include <linux/pci.h>
16 #include <linux/utsname.h>
17 #include <linux/version.h>
18 #include <linux/vmalloc.h>
19 #include <net/ip.h>
20 
21 #include "ena_netdev.h"
22 #include "ena_pci_id_tbl.h"
23 #include "ena_xdp.h"
24 
25 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
26 MODULE_DESCRIPTION(DEVICE_NAME);
27 MODULE_LICENSE("GPL");
28 
29 /* Time in jiffies before concluding the transmitter is hung. */
30 #define TX_TIMEOUT  (5 * HZ)
31 
32 #define ENA_MAX_RINGS min_t(unsigned int, ENA_MAX_NUM_IO_QUEUES, num_possible_cpus())
33 
34 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \
35 		NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
36 
37 static struct ena_aenq_handlers aenq_handlers;
38 
39 static struct workqueue_struct *ena_wq;
40 
41 MODULE_DEVICE_TABLE(pci, ena_pci_tbl);
42 
43 static int ena_rss_init_default(struct ena_adapter *adapter);
44 static void check_for_admin_com_state(struct ena_adapter *adapter);
45 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful);
46 static int ena_restore_device(struct ena_adapter *adapter);
47 
ena_tx_timeout(struct net_device * dev,unsigned int txqueue)48 static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue)
49 {
50 	struct ena_adapter *adapter = netdev_priv(dev);
51 
52 	/* Change the state of the device to trigger reset
53 	 * Check that we are not in the middle or a trigger already
54 	 */
55 
56 	if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
57 		return;
58 
59 	ena_reset_device(adapter, ENA_REGS_RESET_OS_NETDEV_WD);
60 	ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp);
61 
62 	netif_err(adapter, tx_err, dev, "Transmit time out\n");
63 }
64 
update_rx_ring_mtu(struct ena_adapter * adapter,int mtu)65 static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
66 {
67 	int i;
68 
69 	for (i = 0; i < adapter->num_io_queues; i++)
70 		adapter->rx_ring[i].mtu = mtu;
71 }
72 
ena_change_mtu(struct net_device * dev,int new_mtu)73 static int ena_change_mtu(struct net_device *dev, int new_mtu)
74 {
75 	struct ena_adapter *adapter = netdev_priv(dev);
76 	int ret;
77 
78 	ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
79 	if (!ret) {
80 		netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu);
81 		update_rx_ring_mtu(adapter, new_mtu);
82 		dev->mtu = new_mtu;
83 	} else {
84 		netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",
85 			  new_mtu);
86 	}
87 
88 	return ret;
89 }
90 
ena_xmit_common(struct ena_adapter * adapter,struct ena_ring * ring,struct ena_tx_buffer * tx_info,struct ena_com_tx_ctx * ena_tx_ctx,u16 next_to_use,u32 bytes)91 int ena_xmit_common(struct ena_adapter *adapter,
92 		    struct ena_ring *ring,
93 		    struct ena_tx_buffer *tx_info,
94 		    struct ena_com_tx_ctx *ena_tx_ctx,
95 		    u16 next_to_use,
96 		    u32 bytes)
97 {
98 	int rc, nb_hw_desc;
99 
100 	if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq,
101 						ena_tx_ctx))) {
102 		netif_dbg(adapter, tx_queued, adapter->netdev,
103 			  "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
104 			  ring->qid);
105 		ena_ring_tx_doorbell(ring);
106 	}
107 
108 	/* prepare the packet's descriptors to dma engine */
109 	rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx,
110 				&nb_hw_desc);
111 
112 	/* In case there isn't enough space in the queue for the packet,
113 	 * we simply drop it. All other failure reasons of
114 	 * ena_com_prepare_tx() are fatal and therefore require a device reset.
115 	 */
116 	if (unlikely(rc)) {
117 		netif_err(adapter, tx_queued, adapter->netdev,
118 			  "Failed to prepare tx bufs\n");
119 		ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1, &ring->syncp);
120 		if (rc != -ENOMEM)
121 			ena_reset_device(adapter, ENA_REGS_RESET_DRIVER_INVALID_STATE);
122 		return rc;
123 	}
124 
125 	u64_stats_update_begin(&ring->syncp);
126 	ring->tx_stats.cnt++;
127 	ring->tx_stats.bytes += bytes;
128 	u64_stats_update_end(&ring->syncp);
129 
130 	tx_info->tx_descs = nb_hw_desc;
131 	tx_info->last_jiffies = jiffies;
132 	tx_info->print_once = 0;
133 
134 	ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
135 						 ring->ring_size);
136 	return 0;
137 }
138 
ena_init_rx_cpu_rmap(struct ena_adapter * adapter)139 static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
140 {
141 #ifdef CONFIG_RFS_ACCEL
142 	u32 i;
143 	int rc;
144 
145 	adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_io_queues);
146 	if (!adapter->netdev->rx_cpu_rmap)
147 		return -ENOMEM;
148 	for (i = 0; i < adapter->num_io_queues; i++) {
149 		int irq_idx = ENA_IO_IRQ_IDX(i);
150 
151 		rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
152 				      pci_irq_vector(adapter->pdev, irq_idx));
153 		if (rc) {
154 			free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
155 			adapter->netdev->rx_cpu_rmap = NULL;
156 			return rc;
157 		}
158 	}
159 #endif /* CONFIG_RFS_ACCEL */
160 	return 0;
161 }
162 
ena_init_io_rings_common(struct ena_adapter * adapter,struct ena_ring * ring,u16 qid)163 static void ena_init_io_rings_common(struct ena_adapter *adapter,
164 				     struct ena_ring *ring, u16 qid)
165 {
166 	ring->qid = qid;
167 	ring->pdev = adapter->pdev;
168 	ring->dev = &adapter->pdev->dev;
169 	ring->netdev = adapter->netdev;
170 	ring->napi = &adapter->ena_napi[qid].napi;
171 	ring->adapter = adapter;
172 	ring->ena_dev = adapter->ena_dev;
173 	ring->per_napi_packets = 0;
174 	ring->cpu = 0;
175 	ring->numa_node = 0;
176 	ring->no_interrupt_event_cnt = 0;
177 	u64_stats_init(&ring->syncp);
178 }
179 
ena_init_io_rings(struct ena_adapter * adapter,int first_index,int count)180 void ena_init_io_rings(struct ena_adapter *adapter,
181 		       int first_index, int count)
182 {
183 	struct ena_com_dev *ena_dev;
184 	struct ena_ring *txr, *rxr;
185 	int i;
186 
187 	ena_dev = adapter->ena_dev;
188 
189 	for (i = first_index; i < first_index + count; i++) {
190 		txr = &adapter->tx_ring[i];
191 		rxr = &adapter->rx_ring[i];
192 
193 		/* TX common ring state */
194 		ena_init_io_rings_common(adapter, txr, i);
195 
196 		/* TX specific ring state */
197 		txr->ring_size = adapter->requested_tx_ring_size;
198 		txr->tx_max_header_size = ena_dev->tx_max_header_size;
199 		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
200 		txr->sgl_size = adapter->max_tx_sgl_size;
201 		txr->smoothed_interval =
202 			ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
203 		txr->disable_meta_caching = adapter->disable_meta_caching;
204 		spin_lock_init(&txr->xdp_tx_lock);
205 
206 		/* Don't init RX queues for xdp queues */
207 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
208 			/* RX common ring state */
209 			ena_init_io_rings_common(adapter, rxr, i);
210 
211 			/* RX specific ring state */
212 			rxr->ring_size = adapter->requested_rx_ring_size;
213 			rxr->rx_copybreak = adapter->rx_copybreak;
214 			rxr->sgl_size = adapter->max_rx_sgl_size;
215 			rxr->smoothed_interval =
216 				ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
217 			rxr->empty_rx_queue = 0;
218 			rxr->rx_headroom = NET_SKB_PAD;
219 			adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
220 			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];
221 		}
222 	}
223 }
224 
225 /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)
226  * @adapter: network interface device structure
227  * @qid: queue index
228  *
229  * Return 0 on success, negative on failure
230  */
ena_setup_tx_resources(struct ena_adapter * adapter,int qid)231 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
232 {
233 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
234 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
235 	int size, i, node;
236 
237 	if (tx_ring->tx_buffer_info) {
238 		netif_err(adapter, ifup,
239 			  adapter->netdev, "tx_buffer_info info is not NULL");
240 		return -EEXIST;
241 	}
242 
243 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
244 	node = cpu_to_node(ena_irq->cpu);
245 
246 	tx_ring->tx_buffer_info = vzalloc_node(size, node);
247 	if (!tx_ring->tx_buffer_info) {
248 		tx_ring->tx_buffer_info = vzalloc(size);
249 		if (!tx_ring->tx_buffer_info)
250 			goto err_tx_buffer_info;
251 	}
252 
253 	size = sizeof(u16) * tx_ring->ring_size;
254 	tx_ring->free_ids = vzalloc_node(size, node);
255 	if (!tx_ring->free_ids) {
256 		tx_ring->free_ids = vzalloc(size);
257 		if (!tx_ring->free_ids)
258 			goto err_tx_free_ids;
259 	}
260 
261 	size = tx_ring->tx_max_header_size;
262 	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
263 	if (!tx_ring->push_buf_intermediate_buf) {
264 		tx_ring->push_buf_intermediate_buf = vzalloc(size);
265 		if (!tx_ring->push_buf_intermediate_buf)
266 			goto err_push_buf_intermediate_buf;
267 	}
268 
269 	/* Req id ring for TX out of order completions */
270 	for (i = 0; i < tx_ring->ring_size; i++)
271 		tx_ring->free_ids[i] = i;
272 
273 	/* Reset tx statistics */
274 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
275 
276 	tx_ring->next_to_use = 0;
277 	tx_ring->next_to_clean = 0;
278 	tx_ring->cpu = ena_irq->cpu;
279 	tx_ring->numa_node = node;
280 	return 0;
281 
282 err_push_buf_intermediate_buf:
283 	vfree(tx_ring->free_ids);
284 	tx_ring->free_ids = NULL;
285 err_tx_free_ids:
286 	vfree(tx_ring->tx_buffer_info);
287 	tx_ring->tx_buffer_info = NULL;
288 err_tx_buffer_info:
289 	return -ENOMEM;
290 }
291 
292 /* ena_free_tx_resources - Free I/O Tx Resources per Queue
293  * @adapter: network interface device structure
294  * @qid: queue index
295  *
296  * Free all transmit software resources
297  */
ena_free_tx_resources(struct ena_adapter * adapter,int qid)298 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
299 {
300 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
301 
302 	vfree(tx_ring->tx_buffer_info);
303 	tx_ring->tx_buffer_info = NULL;
304 
305 	vfree(tx_ring->free_ids);
306 	tx_ring->free_ids = NULL;
307 
308 	vfree(tx_ring->push_buf_intermediate_buf);
309 	tx_ring->push_buf_intermediate_buf = NULL;
310 }
311 
ena_setup_tx_resources_in_range(struct ena_adapter * adapter,int first_index,int count)312 int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
313 				    int first_index, int count)
314 {
315 	int i, rc = 0;
316 
317 	for (i = first_index; i < first_index + count; i++) {
318 		rc = ena_setup_tx_resources(adapter, i);
319 		if (rc)
320 			goto err_setup_tx;
321 	}
322 
323 	return 0;
324 
325 err_setup_tx:
326 
327 	netif_err(adapter, ifup, adapter->netdev,
328 		  "Tx queue %d: allocation failed\n", i);
329 
330 	/* rewind the index freeing the rings as we go */
331 	while (first_index < i--)
332 		ena_free_tx_resources(adapter, i);
333 	return rc;
334 }
335 
ena_free_all_io_tx_resources_in_range(struct ena_adapter * adapter,int first_index,int count)336 void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,
337 					   int first_index, int count)
338 {
339 	int i;
340 
341 	for (i = first_index; i < first_index + count; i++)
342 		ena_free_tx_resources(adapter, i);
343 }
344 
345 /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
346  * @adapter: board private structure
347  *
348  * Free all transmit software resources
349  */
ena_free_all_io_tx_resources(struct ena_adapter * adapter)350 void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
351 {
352 	ena_free_all_io_tx_resources_in_range(adapter,
353 					      0,
354 					      adapter->xdp_num_queues +
355 					      adapter->num_io_queues);
356 }
357 
358 /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
359  * @adapter: network interface device structure
360  * @qid: queue index
361  *
362  * Returns 0 on success, negative on failure
363  */
ena_setup_rx_resources(struct ena_adapter * adapter,u32 qid)364 static int ena_setup_rx_resources(struct ena_adapter *adapter,
365 				  u32 qid)
366 {
367 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
368 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
369 	int size, node, i;
370 
371 	if (rx_ring->rx_buffer_info) {
372 		netif_err(adapter, ifup, adapter->netdev,
373 			  "rx_buffer_info is not NULL");
374 		return -EEXIST;
375 	}
376 
377 	/* alloc extra element so in rx path
378 	 * we can always prefetch rx_info + 1
379 	 */
380 	size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);
381 	node = cpu_to_node(ena_irq->cpu);
382 
383 	rx_ring->rx_buffer_info = vzalloc_node(size, node);
384 	if (!rx_ring->rx_buffer_info) {
385 		rx_ring->rx_buffer_info = vzalloc(size);
386 		if (!rx_ring->rx_buffer_info)
387 			return -ENOMEM;
388 	}
389 
390 	size = sizeof(u16) * rx_ring->ring_size;
391 	rx_ring->free_ids = vzalloc_node(size, node);
392 	if (!rx_ring->free_ids) {
393 		rx_ring->free_ids = vzalloc(size);
394 		if (!rx_ring->free_ids) {
395 			vfree(rx_ring->rx_buffer_info);
396 			rx_ring->rx_buffer_info = NULL;
397 			return -ENOMEM;
398 		}
399 	}
400 
401 	/* Req id ring for receiving RX pkts out of order */
402 	for (i = 0; i < rx_ring->ring_size; i++)
403 		rx_ring->free_ids[i] = i;
404 
405 	/* Reset rx statistics */
406 	memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
407 
408 	rx_ring->next_to_clean = 0;
409 	rx_ring->next_to_use = 0;
410 	rx_ring->cpu = ena_irq->cpu;
411 	rx_ring->numa_node = node;
412 
413 	return 0;
414 }
415 
416 /* ena_free_rx_resources - Free I/O Rx Resources
417  * @adapter: network interface device structure
418  * @qid: queue index
419  *
420  * Free all receive software resources
421  */
ena_free_rx_resources(struct ena_adapter * adapter,u32 qid)422 static void ena_free_rx_resources(struct ena_adapter *adapter,
423 				  u32 qid)
424 {
425 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
426 
427 	vfree(rx_ring->rx_buffer_info);
428 	rx_ring->rx_buffer_info = NULL;
429 
430 	vfree(rx_ring->free_ids);
431 	rx_ring->free_ids = NULL;
432 }
433 
434 /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
435  * @adapter: board private structure
436  *
437  * Return 0 on success, negative on failure
438  */
ena_setup_all_rx_resources(struct ena_adapter * adapter)439 static int ena_setup_all_rx_resources(struct ena_adapter *adapter)
440 {
441 	int i, rc = 0;
442 
443 	for (i = 0; i < adapter->num_io_queues; i++) {
444 		rc = ena_setup_rx_resources(adapter, i);
445 		if (rc)
446 			goto err_setup_rx;
447 	}
448 
449 	return 0;
450 
451 err_setup_rx:
452 
453 	netif_err(adapter, ifup, adapter->netdev,
454 		  "Rx queue %d: allocation failed\n", i);
455 
456 	/* rewind the index freeing the rings as we go */
457 	while (i--)
458 		ena_free_rx_resources(adapter, i);
459 	return rc;
460 }
461 
462 /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues
463  * @adapter: board private structure
464  *
465  * Free all receive software resources
466  */
ena_free_all_io_rx_resources(struct ena_adapter * adapter)467 static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)
468 {
469 	int i;
470 
471 	for (i = 0; i < adapter->num_io_queues; i++)
472 		ena_free_rx_resources(adapter, i);
473 }
474 
ena_alloc_map_page(struct ena_ring * rx_ring,dma_addr_t * dma)475 static struct page *ena_alloc_map_page(struct ena_ring *rx_ring,
476 				       dma_addr_t *dma)
477 {
478 	struct page *page;
479 
480 	/* This would allocate the page on the same NUMA node the executing code
481 	 * is running on.
482 	 */
483 	page = dev_alloc_page();
484 	if (!page) {
485 		ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, &rx_ring->syncp);
486 		return ERR_PTR(-ENOSPC);
487 	}
488 
489 	/* To enable NIC-side port-mirroring, AKA SPAN port,
490 	 * we make the buffer readable from the nic as well
491 	 */
492 	*dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE,
493 			    DMA_BIDIRECTIONAL);
494 	if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) {
495 		ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1,
496 				  &rx_ring->syncp);
497 		__free_page(page);
498 		return ERR_PTR(-EIO);
499 	}
500 
501 	return page;
502 }
503 
ena_alloc_rx_buffer(struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)504 static int ena_alloc_rx_buffer(struct ena_ring *rx_ring,
505 			       struct ena_rx_buffer *rx_info)
506 {
507 	int headroom = rx_ring->rx_headroom;
508 	struct ena_com_buf *ena_buf;
509 	struct page *page;
510 	dma_addr_t dma;
511 	int tailroom;
512 
513 	/* restore page offset value in case it has been changed by device */
514 	rx_info->buf_offset = headroom;
515 
516 	/* if previous allocated page is not used */
517 	if (unlikely(rx_info->page))
518 		return 0;
519 
520 	/* We handle DMA here */
521 	page = ena_alloc_map_page(rx_ring, &dma);
522 	if (unlikely(IS_ERR(page)))
523 		return PTR_ERR(page);
524 
525 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
526 		  "Allocate page %p, rx_info %p\n", page, rx_info);
527 
528 	tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
529 
530 	rx_info->page = page;
531 	rx_info->dma_addr = dma;
532 	rx_info->page_offset = 0;
533 	ena_buf = &rx_info->ena_buf;
534 	ena_buf->paddr = dma + headroom;
535 	ena_buf->len = ENA_PAGE_SIZE - headroom - tailroom;
536 
537 	return 0;
538 }
539 
ena_unmap_rx_buff_attrs(struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info,unsigned long attrs)540 static void ena_unmap_rx_buff_attrs(struct ena_ring *rx_ring,
541 				    struct ena_rx_buffer *rx_info,
542 				    unsigned long attrs)
543 {
544 	dma_unmap_page_attrs(rx_ring->dev, rx_info->dma_addr, ENA_PAGE_SIZE, DMA_BIDIRECTIONAL,
545 			     attrs);
546 }
547 
ena_free_rx_page(struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)548 static void ena_free_rx_page(struct ena_ring *rx_ring,
549 			     struct ena_rx_buffer *rx_info)
550 {
551 	struct page *page = rx_info->page;
552 
553 	if (unlikely(!page)) {
554 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
555 			   "Trying to free unallocated buffer\n");
556 		return;
557 	}
558 
559 	ena_unmap_rx_buff_attrs(rx_ring, rx_info, 0);
560 
561 	__free_page(page);
562 	rx_info->page = NULL;
563 }
564 
ena_refill_rx_bufs(struct ena_ring * rx_ring,u32 num)565 static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
566 {
567 	u16 next_to_use, req_id;
568 	u32 i;
569 	int rc;
570 
571 	next_to_use = rx_ring->next_to_use;
572 
573 	for (i = 0; i < num; i++) {
574 		struct ena_rx_buffer *rx_info;
575 
576 		req_id = rx_ring->free_ids[next_to_use];
577 
578 		rx_info = &rx_ring->rx_buffer_info[req_id];
579 
580 		rc = ena_alloc_rx_buffer(rx_ring, rx_info);
581 		if (unlikely(rc < 0)) {
582 			netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
583 				   "Failed to allocate buffer for rx queue %d\n",
584 				   rx_ring->qid);
585 			break;
586 		}
587 		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
588 						&rx_info->ena_buf,
589 						req_id);
590 		if (unlikely(rc)) {
591 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
592 				   "Failed to add buffer for rx queue %d\n",
593 				   rx_ring->qid);
594 			break;
595 		}
596 		next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
597 						   rx_ring->ring_size);
598 	}
599 
600 	if (unlikely(i < num)) {
601 		ena_increase_stat(&rx_ring->rx_stats.refil_partial, 1,
602 				  &rx_ring->syncp);
603 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
604 			   "Refilled rx qid %d with only %d buffers (from %d)\n",
605 			   rx_ring->qid, i, num);
606 	}
607 
608 	/* ena_com_write_sq_doorbell issues a wmb() */
609 	if (likely(i))
610 		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
611 
612 	rx_ring->next_to_use = next_to_use;
613 
614 	return i;
615 }
616 
ena_free_rx_bufs(struct ena_adapter * adapter,u32 qid)617 static void ena_free_rx_bufs(struct ena_adapter *adapter,
618 			     u32 qid)
619 {
620 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
621 	u32 i;
622 
623 	for (i = 0; i < rx_ring->ring_size; i++) {
624 		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
625 
626 		if (rx_info->page)
627 			ena_free_rx_page(rx_ring, rx_info);
628 	}
629 }
630 
631 /* ena_refill_all_rx_bufs - allocate all queues Rx buffers
632  * @adapter: board private structure
633  */
ena_refill_all_rx_bufs(struct ena_adapter * adapter)634 static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
635 {
636 	struct ena_ring *rx_ring;
637 	int i, rc, bufs_num;
638 
639 	for (i = 0; i < adapter->num_io_queues; i++) {
640 		rx_ring = &adapter->rx_ring[i];
641 		bufs_num = rx_ring->ring_size - 1;
642 		rc = ena_refill_rx_bufs(rx_ring, bufs_num);
643 
644 		if (unlikely(rc != bufs_num))
645 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
646 				   "Refilling Queue %d failed. allocated %d buffers from: %d\n",
647 				   i, rc, bufs_num);
648 	}
649 }
650 
ena_free_all_rx_bufs(struct ena_adapter * adapter)651 static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
652 {
653 	int i;
654 
655 	for (i = 0; i < adapter->num_io_queues; i++)
656 		ena_free_rx_bufs(adapter, i);
657 }
658 
ena_unmap_tx_buff(struct ena_ring * tx_ring,struct ena_tx_buffer * tx_info)659 void ena_unmap_tx_buff(struct ena_ring *tx_ring,
660 		       struct ena_tx_buffer *tx_info)
661 {
662 	struct ena_com_buf *ena_buf;
663 	u32 cnt;
664 	int i;
665 
666 	ena_buf = tx_info->bufs;
667 	cnt = tx_info->num_of_bufs;
668 
669 	if (unlikely(!cnt))
670 		return;
671 
672 	if (tx_info->map_linear_data) {
673 		dma_unmap_single(tx_ring->dev,
674 				 dma_unmap_addr(ena_buf, paddr),
675 				 dma_unmap_len(ena_buf, len),
676 				 DMA_TO_DEVICE);
677 		ena_buf++;
678 		cnt--;
679 	}
680 
681 	/* unmap remaining mapped pages */
682 	for (i = 0; i < cnt; i++) {
683 		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
684 			       dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
685 		ena_buf++;
686 	}
687 }
688 
689 /* ena_free_tx_bufs - Free Tx Buffers per Queue
690  * @tx_ring: TX ring for which buffers be freed
691  */
ena_free_tx_bufs(struct ena_ring * tx_ring)692 static void ena_free_tx_bufs(struct ena_ring *tx_ring)
693 {
694 	bool print_once = true;
695 	bool is_xdp_ring;
696 	u32 i;
697 
698 	is_xdp_ring = ENA_IS_XDP_INDEX(tx_ring->adapter, tx_ring->qid);
699 
700 	for (i = 0; i < tx_ring->ring_size; i++) {
701 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
702 
703 		if (!tx_info->skb)
704 			continue;
705 
706 		if (print_once) {
707 			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,
708 				     "Free uncompleted tx skb qid %d idx 0x%x\n",
709 				     tx_ring->qid, i);
710 			print_once = false;
711 		} else {
712 			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,
713 				  "Free uncompleted tx skb qid %d idx 0x%x\n",
714 				  tx_ring->qid, i);
715 		}
716 
717 		ena_unmap_tx_buff(tx_ring, tx_info);
718 
719 		if (is_xdp_ring)
720 			xdp_return_frame(tx_info->xdpf);
721 		else
722 			dev_kfree_skb_any(tx_info->skb);
723 	}
724 
725 	if (!is_xdp_ring)
726 		netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
727 							  tx_ring->qid));
728 }
729 
ena_free_all_tx_bufs(struct ena_adapter * adapter)730 static void ena_free_all_tx_bufs(struct ena_adapter *adapter)
731 {
732 	struct ena_ring *tx_ring;
733 	int i;
734 
735 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
736 		tx_ring = &adapter->tx_ring[i];
737 		ena_free_tx_bufs(tx_ring);
738 	}
739 }
740 
ena_destroy_all_tx_queues(struct ena_adapter * adapter)741 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)
742 {
743 	u16 ena_qid;
744 	int i;
745 
746 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
747 		ena_qid = ENA_IO_TXQ_IDX(i);
748 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
749 	}
750 }
751 
ena_destroy_all_rx_queues(struct ena_adapter * adapter)752 static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
753 {
754 	u16 ena_qid;
755 	int i;
756 
757 	for (i = 0; i < adapter->num_io_queues; i++) {
758 		ena_qid = ENA_IO_RXQ_IDX(i);
759 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
760 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
761 	}
762 }
763 
ena_destroy_all_io_queues(struct ena_adapter * adapter)764 static void ena_destroy_all_io_queues(struct ena_adapter *adapter)
765 {
766 	ena_destroy_all_tx_queues(adapter);
767 	ena_destroy_all_rx_queues(adapter);
768 }
769 
handle_invalid_req_id(struct ena_ring * ring,u16 req_id,struct ena_tx_buffer * tx_info,bool is_xdp)770 int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,
771 			  struct ena_tx_buffer *tx_info, bool is_xdp)
772 {
773 	if (tx_info)
774 		netif_err(ring->adapter,
775 			  tx_done,
776 			  ring->netdev,
777 			  "tx_info doesn't have valid %s. qid %u req_id %u",
778 			   is_xdp ? "xdp frame" : "skb", ring->qid, req_id);
779 	else
780 		netif_err(ring->adapter,
781 			  tx_done,
782 			  ring->netdev,
783 			  "Invalid req_id %u in qid %u\n",
784 			  req_id, ring->qid);
785 
786 	ena_increase_stat(&ring->tx_stats.bad_req_id, 1, &ring->syncp);
787 	ena_reset_device(ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
788 
789 	return -EFAULT;
790 }
791 
validate_tx_req_id(struct ena_ring * tx_ring,u16 req_id)792 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
793 {
794 	struct ena_tx_buffer *tx_info;
795 
796 	tx_info = &tx_ring->tx_buffer_info[req_id];
797 	if (likely(tx_info->skb))
798 		return 0;
799 
800 	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
801 }
802 
ena_clean_tx_irq(struct ena_ring * tx_ring,u32 budget)803 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
804 {
805 	struct netdev_queue *txq;
806 	bool above_thresh;
807 	u32 tx_bytes = 0;
808 	u32 total_done = 0;
809 	u16 next_to_clean;
810 	u16 req_id;
811 	int tx_pkts = 0;
812 	int rc;
813 
814 	next_to_clean = tx_ring->next_to_clean;
815 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
816 
817 	while (tx_pkts < budget) {
818 		struct ena_tx_buffer *tx_info;
819 		struct sk_buff *skb;
820 
821 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
822 						&req_id);
823 		if (rc) {
824 			if (unlikely(rc == -EINVAL))
825 				handle_invalid_req_id(tx_ring, req_id, NULL, false);
826 			break;
827 		}
828 
829 		/* validate that the request id points to a valid skb */
830 		rc = validate_tx_req_id(tx_ring, req_id);
831 		if (rc)
832 			break;
833 
834 		tx_info = &tx_ring->tx_buffer_info[req_id];
835 		skb = tx_info->skb;
836 
837 		/* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */
838 		prefetch(&skb->end);
839 
840 		tx_info->skb = NULL;
841 		tx_info->last_jiffies = 0;
842 
843 		ena_unmap_tx_buff(tx_ring, tx_info);
844 
845 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
846 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,
847 			  skb);
848 
849 		tx_bytes += skb->len;
850 		dev_kfree_skb(skb);
851 		tx_pkts++;
852 		total_done += tx_info->tx_descs;
853 
854 		tx_ring->free_ids[next_to_clean] = req_id;
855 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
856 						     tx_ring->ring_size);
857 	}
858 
859 	tx_ring->next_to_clean = next_to_clean;
860 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
861 	ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
862 
863 	netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
864 
865 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
866 		  "tx_poll: q %d done. total pkts: %d\n",
867 		  tx_ring->qid, tx_pkts);
868 
869 	/* need to make the rings circular update visible to
870 	 * ena_start_xmit() before checking for netif_queue_stopped().
871 	 */
872 	smp_mb();
873 
874 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
875 						    ENA_TX_WAKEUP_THRESH);
876 	if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
877 		__netif_tx_lock(txq, smp_processor_id());
878 		above_thresh =
879 			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
880 						     ENA_TX_WAKEUP_THRESH);
881 		if (netif_tx_queue_stopped(txq) && above_thresh &&
882 		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
883 			netif_tx_wake_queue(txq);
884 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
885 					  &tx_ring->syncp);
886 		}
887 		__netif_tx_unlock(txq);
888 	}
889 
890 	return tx_pkts;
891 }
892 
ena_alloc_skb(struct ena_ring * rx_ring,void * first_frag,u16 len)893 static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag, u16 len)
894 {
895 	struct sk_buff *skb;
896 
897 	if (!first_frag)
898 		skb = napi_alloc_skb(rx_ring->napi, len);
899 	else
900 		skb = napi_build_skb(first_frag, len);
901 
902 	if (unlikely(!skb)) {
903 		ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1,
904 				  &rx_ring->syncp);
905 
906 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
907 			  "Failed to allocate skb. first_frag %s\n",
908 			  first_frag ? "provided" : "not provided");
909 	}
910 
911 	return skb;
912 }
913 
ena_try_rx_buf_page_reuse(struct ena_rx_buffer * rx_info,u16 buf_len,u16 len,int pkt_offset)914 static bool ena_try_rx_buf_page_reuse(struct ena_rx_buffer *rx_info, u16 buf_len,
915 				      u16 len, int pkt_offset)
916 {
917 	struct ena_com_buf *ena_buf = &rx_info->ena_buf;
918 
919 	/* More than ENA_MIN_RX_BUF_SIZE left in the reused buffer
920 	 * for data + headroom + tailroom.
921 	 */
922 	if (SKB_DATA_ALIGN(len + pkt_offset) + ENA_MIN_RX_BUF_SIZE <= ena_buf->len) {
923 		page_ref_inc(rx_info->page);
924 		rx_info->page_offset += buf_len;
925 		ena_buf->paddr += buf_len;
926 		ena_buf->len -= buf_len;
927 		return true;
928 	}
929 
930 	return false;
931 }
932 
ena_rx_skb(struct ena_ring * rx_ring,struct ena_com_rx_buf_info * ena_bufs,u32 descs,u16 * next_to_clean)933 static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
934 				  struct ena_com_rx_buf_info *ena_bufs,
935 				  u32 descs,
936 				  u16 *next_to_clean)
937 {
938 	int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
939 	bool is_xdp_loaded = ena_xdp_present_ring(rx_ring);
940 	struct ena_rx_buffer *rx_info;
941 	struct ena_adapter *adapter;
942 	int page_offset, pkt_offset;
943 	dma_addr_t pre_reuse_paddr;
944 	u16 len, req_id, buf = 0;
945 	bool reuse_rx_buf_page;
946 	struct sk_buff *skb;
947 	void *buf_addr;
948 	int buf_offset;
949 	u16 buf_len;
950 
951 	len = ena_bufs[buf].len;
952 	req_id = ena_bufs[buf].req_id;
953 
954 	rx_info = &rx_ring->rx_buffer_info[req_id];
955 
956 	if (unlikely(!rx_info->page)) {
957 		adapter = rx_ring->adapter;
958 		netif_err(adapter, rx_err, rx_ring->netdev,
959 			  "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id);
960 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp);
961 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
962 		return NULL;
963 	}
964 
965 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
966 		  "rx_info %p page %p\n",
967 		  rx_info, rx_info->page);
968 
969 	buf_offset = rx_info->buf_offset;
970 	pkt_offset = buf_offset - rx_ring->rx_headroom;
971 	page_offset = rx_info->page_offset;
972 	buf_addr = page_address(rx_info->page) + page_offset;
973 
974 	if (len <= rx_ring->rx_copybreak) {
975 		skb = ena_alloc_skb(rx_ring, NULL, len);
976 		if (unlikely(!skb))
977 			return NULL;
978 
979 		skb_copy_to_linear_data(skb, buf_addr + buf_offset, len);
980 		dma_sync_single_for_device(rx_ring->dev,
981 					   dma_unmap_addr(&rx_info->ena_buf, paddr) + pkt_offset,
982 					   len,
983 					   DMA_FROM_DEVICE);
984 
985 		skb_put(skb, len);
986 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
987 			  "RX allocated small packet. len %d.\n", skb->len);
988 		skb->protocol = eth_type_trans(skb, rx_ring->netdev);
989 		rx_ring->free_ids[*next_to_clean] = req_id;
990 		*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
991 						     rx_ring->ring_size);
992 		return skb;
993 	}
994 
995 	buf_len = SKB_DATA_ALIGN(len + buf_offset + tailroom);
996 
997 	/* If XDP isn't loaded try to reuse part of the RX buffer */
998 	reuse_rx_buf_page = !is_xdp_loaded &&
999 			    ena_try_rx_buf_page_reuse(rx_info, buf_len, len, pkt_offset);
1000 
1001 	if (!reuse_rx_buf_page)
1002 		ena_unmap_rx_buff_attrs(rx_ring, rx_info, DMA_ATTR_SKIP_CPU_SYNC);
1003 
1004 	skb = ena_alloc_skb(rx_ring, buf_addr, buf_len);
1005 	if (unlikely(!skb))
1006 		return NULL;
1007 
1008 	/* Populate skb's linear part */
1009 	skb_reserve(skb, buf_offset);
1010 	skb_put(skb, len);
1011 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1012 
1013 	do {
1014 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1015 			  "RX skb updated. len %d. data_len %d\n",
1016 			  skb->len, skb->data_len);
1017 
1018 		if (!reuse_rx_buf_page)
1019 			rx_info->page = NULL;
1020 
1021 		rx_ring->free_ids[*next_to_clean] = req_id;
1022 		*next_to_clean =
1023 			ENA_RX_RING_IDX_NEXT(*next_to_clean,
1024 					     rx_ring->ring_size);
1025 		if (likely(--descs == 0))
1026 			break;
1027 
1028 		buf++;
1029 		len = ena_bufs[buf].len;
1030 		req_id = ena_bufs[buf].req_id;
1031 
1032 		rx_info = &rx_ring->rx_buffer_info[req_id];
1033 
1034 		/* rx_info->buf_offset includes rx_ring->rx_headroom */
1035 		buf_offset = rx_info->buf_offset;
1036 		pkt_offset = buf_offset - rx_ring->rx_headroom;
1037 		buf_len = SKB_DATA_ALIGN(len + buf_offset + tailroom);
1038 		page_offset = rx_info->page_offset;
1039 
1040 		pre_reuse_paddr = dma_unmap_addr(&rx_info->ena_buf, paddr);
1041 
1042 		reuse_rx_buf_page = !is_xdp_loaded &&
1043 				    ena_try_rx_buf_page_reuse(rx_info, buf_len, len, pkt_offset);
1044 
1045 		dma_sync_single_for_cpu(rx_ring->dev,
1046 					pre_reuse_paddr + pkt_offset,
1047 					len,
1048 					DMA_FROM_DEVICE);
1049 
1050 		if (!reuse_rx_buf_page)
1051 			ena_unmap_rx_buff_attrs(rx_ring, rx_info, DMA_ATTR_SKIP_CPU_SYNC);
1052 
1053 		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
1054 				page_offset + buf_offset, len, buf_len);
1055 
1056 	} while (1);
1057 
1058 	return skb;
1059 }
1060 
1061 /* ena_rx_checksum - indicate in skb if hw indicated a good cksum
1062  * @adapter: structure containing adapter specific data
1063  * @ena_rx_ctx: received packet context/metadata
1064  * @skb: skb currently being received and modified
1065  */
ena_rx_checksum(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct sk_buff * skb)1066 static void ena_rx_checksum(struct ena_ring *rx_ring,
1067 				   struct ena_com_rx_ctx *ena_rx_ctx,
1068 				   struct sk_buff *skb)
1069 {
1070 	/* Rx csum disabled */
1071 	if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {
1072 		skb->ip_summed = CHECKSUM_NONE;
1073 		return;
1074 	}
1075 
1076 	/* For fragmented packets the checksum isn't valid */
1077 	if (ena_rx_ctx->frag) {
1078 		skb->ip_summed = CHECKSUM_NONE;
1079 		return;
1080 	}
1081 
1082 	/* if IP and error */
1083 	if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
1084 		     (ena_rx_ctx->l3_csum_err))) {
1085 		/* ipv4 checksum error */
1086 		skb->ip_summed = CHECKSUM_NONE;
1087 		ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1088 				  &rx_ring->syncp);
1089 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1090 			  "RX IPv4 header checksum error\n");
1091 		return;
1092 	}
1093 
1094 	/* if TCP/UDP */
1095 	if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1096 		   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {
1097 		if (unlikely(ena_rx_ctx->l4_csum_err)) {
1098 			/* TCP/UDP checksum error */
1099 			ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1100 					  &rx_ring->syncp);
1101 			netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1102 				  "RX L4 checksum error\n");
1103 			skb->ip_summed = CHECKSUM_NONE;
1104 			return;
1105 		}
1106 
1107 		if (likely(ena_rx_ctx->l4_csum_checked)) {
1108 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1109 			ena_increase_stat(&rx_ring->rx_stats.csum_good, 1,
1110 					  &rx_ring->syncp);
1111 		} else {
1112 			ena_increase_stat(&rx_ring->rx_stats.csum_unchecked, 1,
1113 					  &rx_ring->syncp);
1114 			skb->ip_summed = CHECKSUM_NONE;
1115 		}
1116 	} else {
1117 		skb->ip_summed = CHECKSUM_NONE;
1118 		return;
1119 	}
1120 
1121 }
1122 
ena_set_rx_hash(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct sk_buff * skb)1123 static void ena_set_rx_hash(struct ena_ring *rx_ring,
1124 			    struct ena_com_rx_ctx *ena_rx_ctx,
1125 			    struct sk_buff *skb)
1126 {
1127 	enum pkt_hash_types hash_type;
1128 
1129 	if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {
1130 		if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1131 			   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))
1132 
1133 			hash_type = PKT_HASH_TYPE_L4;
1134 		else
1135 			hash_type = PKT_HASH_TYPE_NONE;
1136 
1137 		/* Override hash type if the packet is fragmented */
1138 		if (ena_rx_ctx->frag)
1139 			hash_type = PKT_HASH_TYPE_NONE;
1140 
1141 		skb_set_hash(skb, ena_rx_ctx->hash, hash_type);
1142 	}
1143 }
1144 
ena_xdp_handle_buff(struct ena_ring * rx_ring,struct xdp_buff * xdp,u16 num_descs)1145 static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp, u16 num_descs)
1146 {
1147 	struct ena_rx_buffer *rx_info;
1148 	int ret;
1149 
1150 	/* XDP multi-buffer packets not supported */
1151 	if (unlikely(num_descs > 1)) {
1152 		netdev_err_once(rx_ring->adapter->netdev,
1153 				"xdp: dropped unsupported multi-buffer packets\n");
1154 		ena_increase_stat(&rx_ring->rx_stats.xdp_drop, 1, &rx_ring->syncp);
1155 		return ENA_XDP_DROP;
1156 	}
1157 
1158 	rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1159 	xdp_prepare_buff(xdp, page_address(rx_info->page),
1160 			 rx_info->buf_offset,
1161 			 rx_ring->ena_bufs[0].len, false);
1162 
1163 	ret = ena_xdp_execute(rx_ring, xdp);
1164 
1165 	/* The xdp program might expand the headers */
1166 	if (ret == ENA_XDP_PASS) {
1167 		rx_info->buf_offset = xdp->data - xdp->data_hard_start;
1168 		rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data;
1169 	}
1170 
1171 	return ret;
1172 }
1173 
1174 /* ena_clean_rx_irq - Cleanup RX irq
1175  * @rx_ring: RX ring to clean
1176  * @napi: napi handler
1177  * @budget: how many packets driver is allowed to clean
1178  *
1179  * Returns the number of cleaned buffers.
1180  */
ena_clean_rx_irq(struct ena_ring * rx_ring,struct napi_struct * napi,u32 budget)1181 static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
1182 			    u32 budget)
1183 {
1184 	u16 next_to_clean = rx_ring->next_to_clean;
1185 	struct ena_com_rx_ctx ena_rx_ctx;
1186 	struct ena_rx_buffer *rx_info;
1187 	struct ena_adapter *adapter;
1188 	u32 res_budget, work_done;
1189 	int rx_copybreak_pkt = 0;
1190 	int refill_threshold;
1191 	struct sk_buff *skb;
1192 	int refill_required;
1193 	struct xdp_buff xdp;
1194 	int xdp_flags = 0;
1195 	int total_len = 0;
1196 	int xdp_verdict;
1197 	u8 pkt_offset;
1198 	int rc = 0;
1199 	int i;
1200 
1201 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1202 		  "%s qid %d\n", __func__, rx_ring->qid);
1203 	res_budget = budget;
1204 	xdp_init_buff(&xdp, ENA_PAGE_SIZE, &rx_ring->xdp_rxq);
1205 
1206 	do {
1207 		xdp_verdict = ENA_XDP_PASS;
1208 		skb = NULL;
1209 		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1210 		ena_rx_ctx.max_bufs = rx_ring->sgl_size;
1211 		ena_rx_ctx.descs = 0;
1212 		ena_rx_ctx.pkt_offset = 0;
1213 		rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
1214 				    rx_ring->ena_com_io_sq,
1215 				    &ena_rx_ctx);
1216 		if (unlikely(rc))
1217 			goto error;
1218 
1219 		if (unlikely(ena_rx_ctx.descs == 0))
1220 			break;
1221 
1222 		/* First descriptor might have an offset set by the device */
1223 		rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1224 		pkt_offset = ena_rx_ctx.pkt_offset;
1225 		rx_info->buf_offset += pkt_offset;
1226 
1227 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1228 			  "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
1229 			  rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
1230 			  ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
1231 
1232 		dma_sync_single_for_cpu(rx_ring->dev,
1233 					dma_unmap_addr(&rx_info->ena_buf, paddr) + pkt_offset,
1234 					rx_ring->ena_bufs[0].len,
1235 					DMA_FROM_DEVICE);
1236 
1237 		if (ena_xdp_present_ring(rx_ring))
1238 			xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp, ena_rx_ctx.descs);
1239 
1240 		/* allocate skb and fill it */
1241 		if (xdp_verdict == ENA_XDP_PASS)
1242 			skb = ena_rx_skb(rx_ring,
1243 					 rx_ring->ena_bufs,
1244 					 ena_rx_ctx.descs,
1245 					 &next_to_clean);
1246 
1247 		if (unlikely(!skb)) {
1248 			for (i = 0; i < ena_rx_ctx.descs; i++) {
1249 				int req_id = rx_ring->ena_bufs[i].req_id;
1250 
1251 				rx_ring->free_ids[next_to_clean] = req_id;
1252 				next_to_clean =
1253 					ENA_RX_RING_IDX_NEXT(next_to_clean,
1254 							     rx_ring->ring_size);
1255 
1256 				/* Packets was passed for transmission, unmap it
1257 				 * from RX side.
1258 				 */
1259 				if (xdp_verdict & ENA_XDP_FORWARDED) {
1260 					ena_unmap_rx_buff_attrs(rx_ring,
1261 								&rx_ring->rx_buffer_info[req_id],
1262 								DMA_ATTR_SKIP_CPU_SYNC);
1263 					rx_ring->rx_buffer_info[req_id].page = NULL;
1264 				}
1265 			}
1266 			if (xdp_verdict != ENA_XDP_PASS) {
1267 				xdp_flags |= xdp_verdict;
1268 				total_len += ena_rx_ctx.ena_bufs[0].len;
1269 				res_budget--;
1270 				continue;
1271 			}
1272 			break;
1273 		}
1274 
1275 		ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);
1276 
1277 		ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);
1278 
1279 		skb_record_rx_queue(skb, rx_ring->qid);
1280 
1281 		if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak)
1282 			rx_copybreak_pkt++;
1283 
1284 		total_len += skb->len;
1285 
1286 		napi_gro_receive(napi, skb);
1287 
1288 		res_budget--;
1289 	} while (likely(res_budget));
1290 
1291 	work_done = budget - res_budget;
1292 	rx_ring->per_napi_packets += work_done;
1293 	u64_stats_update_begin(&rx_ring->syncp);
1294 	rx_ring->rx_stats.bytes += total_len;
1295 	rx_ring->rx_stats.cnt += work_done;
1296 	rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;
1297 	u64_stats_update_end(&rx_ring->syncp);
1298 
1299 	rx_ring->next_to_clean = next_to_clean;
1300 
1301 	refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
1302 	refill_threshold =
1303 		min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
1304 		      ENA_RX_REFILL_THRESH_PACKET);
1305 
1306 	/* Optimization, try to batch new rx buffers */
1307 	if (refill_required > refill_threshold) {
1308 		ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
1309 		ena_refill_rx_bufs(rx_ring, refill_required);
1310 	}
1311 
1312 	if (xdp_flags & ENA_XDP_REDIRECT)
1313 		xdp_do_flush_map();
1314 
1315 	return work_done;
1316 
1317 error:
1318 	if (xdp_flags & ENA_XDP_REDIRECT)
1319 		xdp_do_flush();
1320 
1321 	adapter = netdev_priv(rx_ring->netdev);
1322 
1323 	if (rc == -ENOSPC) {
1324 		ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1, &rx_ring->syncp);
1325 		ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS);
1326 	} else {
1327 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1,
1328 				  &rx_ring->syncp);
1329 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
1330 	}
1331 	return 0;
1332 }
1333 
ena_dim_work(struct work_struct * w)1334 static void ena_dim_work(struct work_struct *w)
1335 {
1336 	struct dim *dim = container_of(w, struct dim, work);
1337 	struct dim_cq_moder cur_moder =
1338 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1339 	struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);
1340 
1341 	ena_napi->rx_ring->smoothed_interval = cur_moder.usec;
1342 	dim->state = DIM_START_MEASURE;
1343 }
1344 
ena_adjust_adaptive_rx_intr_moderation(struct ena_napi * ena_napi)1345 static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)
1346 {
1347 	struct dim_sample dim_sample;
1348 	struct ena_ring *rx_ring = ena_napi->rx_ring;
1349 
1350 	if (!rx_ring->per_napi_packets)
1351 		return;
1352 
1353 	rx_ring->non_empty_napi_events++;
1354 
1355 	dim_update_sample(rx_ring->non_empty_napi_events,
1356 			  rx_ring->rx_stats.cnt,
1357 			  rx_ring->rx_stats.bytes,
1358 			  &dim_sample);
1359 
1360 	net_dim(&ena_napi->dim, dim_sample);
1361 
1362 	rx_ring->per_napi_packets = 0;
1363 }
1364 
ena_unmask_interrupt(struct ena_ring * tx_ring,struct ena_ring * rx_ring)1365 void ena_unmask_interrupt(struct ena_ring *tx_ring,
1366 			  struct ena_ring *rx_ring)
1367 {
1368 	u32 rx_interval = tx_ring->smoothed_interval;
1369 	struct ena_eth_io_intr_reg intr_reg;
1370 
1371 	/* Rx ring can be NULL when for XDP tx queues which don't have an
1372 	 * accompanying rx_ring pair.
1373 	 */
1374 	if (rx_ring)
1375 		rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?
1376 			rx_ring->smoothed_interval :
1377 			ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);
1378 
1379 	/* Update intr register: rx intr delay,
1380 	 * tx intr delay and interrupt unmask
1381 	 */
1382 	ena_com_update_intr_reg(&intr_reg,
1383 				rx_interval,
1384 				tx_ring->smoothed_interval,
1385 				true);
1386 
1387 	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,
1388 			  &tx_ring->syncp);
1389 
1390 	/* It is a shared MSI-X.
1391 	 * Tx and Rx CQ have pointer to it.
1392 	 * So we use one of them to reach the intr reg
1393 	 * The Tx ring is used because the rx_ring is NULL for XDP queues
1394 	 */
1395 	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);
1396 }
1397 
ena_update_ring_numa_node(struct ena_ring * tx_ring,struct ena_ring * rx_ring)1398 void ena_update_ring_numa_node(struct ena_ring *tx_ring,
1399 			       struct ena_ring *rx_ring)
1400 {
1401 	int cpu = get_cpu();
1402 	int numa_node;
1403 
1404 	/* Check only one ring since the 2 rings are running on the same cpu */
1405 	if (likely(tx_ring->cpu == cpu))
1406 		goto out;
1407 
1408 	tx_ring->cpu = cpu;
1409 	if (rx_ring)
1410 		rx_ring->cpu = cpu;
1411 
1412 	numa_node = cpu_to_node(cpu);
1413 
1414 	if (likely(tx_ring->numa_node == numa_node))
1415 		goto out;
1416 
1417 	put_cpu();
1418 
1419 	if (numa_node != NUMA_NO_NODE) {
1420 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
1421 		tx_ring->numa_node = numa_node;
1422 		if (rx_ring) {
1423 			rx_ring->numa_node = numa_node;
1424 			ena_com_update_numa_node(rx_ring->ena_com_io_cq,
1425 						 numa_node);
1426 		}
1427 	}
1428 
1429 	return;
1430 out:
1431 	put_cpu();
1432 }
1433 
ena_io_poll(struct napi_struct * napi,int budget)1434 static int ena_io_poll(struct napi_struct *napi, int budget)
1435 {
1436 	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
1437 	struct ena_ring *tx_ring, *rx_ring;
1438 	int tx_work_done;
1439 	int rx_work_done = 0;
1440 	int tx_budget;
1441 	int napi_comp_call = 0;
1442 	int ret;
1443 
1444 	tx_ring = ena_napi->tx_ring;
1445 	rx_ring = ena_napi->rx_ring;
1446 
1447 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
1448 
1449 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1450 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
1451 		napi_complete_done(napi, 0);
1452 		return 0;
1453 	}
1454 
1455 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
1456 	/* On netpoll the budget is zero and the handler should only clean the
1457 	 * tx completions.
1458 	 */
1459 	if (likely(budget))
1460 		rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
1461 
1462 	/* If the device is about to reset or down, avoid unmask
1463 	 * the interrupt and return 0 so NAPI won't reschedule
1464 	 */
1465 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1466 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
1467 		napi_complete_done(napi, 0);
1468 		ret = 0;
1469 
1470 	} else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
1471 		napi_comp_call = 1;
1472 
1473 		/* Update numa and unmask the interrupt only when schedule
1474 		 * from the interrupt context (vs from sk_busy_loop)
1475 		 */
1476 		if (napi_complete_done(napi, rx_work_done) &&
1477 		    READ_ONCE(ena_napi->interrupts_masked)) {
1478 			smp_rmb(); /* make sure interrupts_masked is read */
1479 			WRITE_ONCE(ena_napi->interrupts_masked, false);
1480 			/* We apply adaptive moderation on Rx path only.
1481 			 * Tx uses static interrupt moderation.
1482 			 */
1483 			if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
1484 				ena_adjust_adaptive_rx_intr_moderation(ena_napi);
1485 
1486 			ena_update_ring_numa_node(tx_ring, rx_ring);
1487 			ena_unmask_interrupt(tx_ring, rx_ring);
1488 		}
1489 
1490 		ret = rx_work_done;
1491 	} else {
1492 		ret = budget;
1493 	}
1494 
1495 	u64_stats_update_begin(&tx_ring->syncp);
1496 	tx_ring->tx_stats.napi_comp += napi_comp_call;
1497 	tx_ring->tx_stats.tx_poll++;
1498 	u64_stats_update_end(&tx_ring->syncp);
1499 
1500 	tx_ring->tx_stats.last_napi_jiffies = jiffies;
1501 
1502 	return ret;
1503 }
1504 
ena_intr_msix_mgmnt(int irq,void * data)1505 static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)
1506 {
1507 	struct ena_adapter *adapter = (struct ena_adapter *)data;
1508 
1509 	ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1510 
1511 	/* Don't call the aenq handler before probe is done */
1512 	if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))
1513 		ena_com_aenq_intr_handler(adapter->ena_dev, data);
1514 
1515 	return IRQ_HANDLED;
1516 }
1517 
1518 /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx
1519  * @irq: interrupt number
1520  * @data: pointer to a network interface private napi device structure
1521  */
ena_intr_msix_io(int irq,void * data)1522 static irqreturn_t ena_intr_msix_io(int irq, void *data)
1523 {
1524 	struct ena_napi *ena_napi = data;
1525 
1526 	/* Used to check HW health */
1527 	WRITE_ONCE(ena_napi->first_interrupt, true);
1528 
1529 	WRITE_ONCE(ena_napi->interrupts_masked, true);
1530 	smp_wmb(); /* write interrupts_masked before calling napi */
1531 
1532 	napi_schedule_irqoff(&ena_napi->napi);
1533 
1534 	return IRQ_HANDLED;
1535 }
1536 
1537 /* Reserve a single MSI-X vector for management (admin + aenq).
1538  * plus reserve one vector for each potential io queue.
1539  * the number of potential io queues is the minimum of what the device
1540  * supports and the number of vCPUs.
1541  */
ena_enable_msix(struct ena_adapter * adapter)1542 static int ena_enable_msix(struct ena_adapter *adapter)
1543 {
1544 	int msix_vecs, irq_cnt;
1545 
1546 	if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
1547 		netif_err(adapter, probe, adapter->netdev,
1548 			  "Error, MSI-X is already enabled\n");
1549 		return -EPERM;
1550 	}
1551 
1552 	/* Reserved the max msix vectors we might need */
1553 	msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
1554 	netif_dbg(adapter, probe, adapter->netdev,
1555 		  "Trying to enable MSI-X, vectors %d\n", msix_vecs);
1556 
1557 	irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
1558 					msix_vecs, PCI_IRQ_MSIX);
1559 
1560 	if (irq_cnt < 0) {
1561 		netif_err(adapter, probe, adapter->netdev,
1562 			  "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);
1563 		return -ENOSPC;
1564 	}
1565 
1566 	if (irq_cnt != msix_vecs) {
1567 		netif_notice(adapter, probe, adapter->netdev,
1568 			     "Enable only %d MSI-X (out of %d), reduce the number of queues\n",
1569 			     irq_cnt, msix_vecs);
1570 		adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
1571 	}
1572 
1573 	if (ena_init_rx_cpu_rmap(adapter))
1574 		netif_warn(adapter, probe, adapter->netdev,
1575 			   "Failed to map IRQs to CPUs\n");
1576 
1577 	adapter->msix_vecs = irq_cnt;
1578 	set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
1579 
1580 	return 0;
1581 }
1582 
ena_setup_mgmnt_intr(struct ena_adapter * adapter)1583 static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
1584 {
1585 	u32 cpu;
1586 
1587 	snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
1588 		 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
1589 		 pci_name(adapter->pdev));
1590 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =
1591 		ena_intr_msix_mgmnt;
1592 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
1593 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
1594 		pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
1595 	cpu = cpumask_first(cpu_online_mask);
1596 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
1597 	cpumask_set_cpu(cpu,
1598 			&adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);
1599 }
1600 
ena_setup_io_intr(struct ena_adapter * adapter)1601 static void ena_setup_io_intr(struct ena_adapter *adapter)
1602 {
1603 	struct net_device *netdev;
1604 	int irq_idx, i, cpu;
1605 	int io_queue_count;
1606 
1607 	netdev = adapter->netdev;
1608 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1609 
1610 	for (i = 0; i < io_queue_count; i++) {
1611 		irq_idx = ENA_IO_IRQ_IDX(i);
1612 		cpu = i % num_online_cpus();
1613 
1614 		snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
1615 			 "%s-Tx-Rx-%d", netdev->name, i);
1616 		adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
1617 		adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
1618 		adapter->irq_tbl[irq_idx].vector =
1619 			pci_irq_vector(adapter->pdev, irq_idx);
1620 		adapter->irq_tbl[irq_idx].cpu = cpu;
1621 
1622 		cpumask_set_cpu(cpu,
1623 				&adapter->irq_tbl[irq_idx].affinity_hint_mask);
1624 	}
1625 }
1626 
ena_request_mgmnt_irq(struct ena_adapter * adapter)1627 static int ena_request_mgmnt_irq(struct ena_adapter *adapter)
1628 {
1629 	unsigned long flags = 0;
1630 	struct ena_irq *irq;
1631 	int rc;
1632 
1633 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1634 	rc = request_irq(irq->vector, irq->handler, flags, irq->name,
1635 			 irq->data);
1636 	if (rc) {
1637 		netif_err(adapter, probe, adapter->netdev,
1638 			  "Failed to request admin irq\n");
1639 		return rc;
1640 	}
1641 
1642 	netif_dbg(adapter, probe, adapter->netdev,
1643 		  "Set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
1644 		  irq->affinity_hint_mask.bits[0], irq->vector);
1645 
1646 	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
1647 
1648 	return rc;
1649 }
1650 
ena_request_io_irq(struct ena_adapter * adapter)1651 static int ena_request_io_irq(struct ena_adapter *adapter)
1652 {
1653 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1654 	unsigned long flags = 0;
1655 	struct ena_irq *irq;
1656 	int rc = 0, i, k;
1657 
1658 	if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
1659 		netif_err(adapter, ifup, adapter->netdev,
1660 			  "Failed to request I/O IRQ: MSI-X is not enabled\n");
1661 		return -EINVAL;
1662 	}
1663 
1664 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
1665 		irq = &adapter->irq_tbl[i];
1666 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
1667 				 irq->data);
1668 		if (rc) {
1669 			netif_err(adapter, ifup, adapter->netdev,
1670 				  "Failed to request I/O IRQ. index %d rc %d\n",
1671 				   i, rc);
1672 			goto err;
1673 		}
1674 
1675 		netif_dbg(adapter, ifup, adapter->netdev,
1676 			  "Set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
1677 			  i, irq->affinity_hint_mask.bits[0], irq->vector);
1678 
1679 		irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
1680 	}
1681 
1682 	return rc;
1683 
1684 err:
1685 	for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {
1686 		irq = &adapter->irq_tbl[k];
1687 		free_irq(irq->vector, irq->data);
1688 	}
1689 
1690 	return rc;
1691 }
1692 
ena_free_mgmnt_irq(struct ena_adapter * adapter)1693 static void ena_free_mgmnt_irq(struct ena_adapter *adapter)
1694 {
1695 	struct ena_irq *irq;
1696 
1697 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1698 	synchronize_irq(irq->vector);
1699 	irq_set_affinity_hint(irq->vector, NULL);
1700 	free_irq(irq->vector, irq->data);
1701 }
1702 
ena_free_io_irq(struct ena_adapter * adapter)1703 static void ena_free_io_irq(struct ena_adapter *adapter)
1704 {
1705 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1706 	struct ena_irq *irq;
1707 	int i;
1708 
1709 #ifdef CONFIG_RFS_ACCEL
1710 	if (adapter->msix_vecs >= 1) {
1711 		free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
1712 		adapter->netdev->rx_cpu_rmap = NULL;
1713 	}
1714 #endif /* CONFIG_RFS_ACCEL */
1715 
1716 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
1717 		irq = &adapter->irq_tbl[i];
1718 		irq_set_affinity_hint(irq->vector, NULL);
1719 		free_irq(irq->vector, irq->data);
1720 	}
1721 }
1722 
ena_disable_msix(struct ena_adapter * adapter)1723 static void ena_disable_msix(struct ena_adapter *adapter)
1724 {
1725 	if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
1726 		pci_free_irq_vectors(adapter->pdev);
1727 }
1728 
ena_disable_io_intr_sync(struct ena_adapter * adapter)1729 static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
1730 {
1731 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1732 	int i;
1733 
1734 	if (!netif_running(adapter->netdev))
1735 		return;
1736 
1737 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++)
1738 		synchronize_irq(adapter->irq_tbl[i].vector);
1739 }
1740 
ena_del_napi_in_range(struct ena_adapter * adapter,int first_index,int count)1741 static void ena_del_napi_in_range(struct ena_adapter *adapter,
1742 				  int first_index,
1743 				  int count)
1744 {
1745 	int i;
1746 
1747 	for (i = first_index; i < first_index + count; i++) {
1748 		netif_napi_del(&adapter->ena_napi[i].napi);
1749 
1750 		WARN_ON(ENA_IS_XDP_INDEX(adapter, i) &&
1751 			adapter->ena_napi[i].rx_ring);
1752 	}
1753 }
1754 
ena_init_napi_in_range(struct ena_adapter * adapter,int first_index,int count)1755 static void ena_init_napi_in_range(struct ena_adapter *adapter,
1756 				   int first_index, int count)
1757 {
1758 	int i;
1759 
1760 	for (i = first_index; i < first_index + count; i++) {
1761 		struct ena_napi *napi = &adapter->ena_napi[i];
1762 
1763 		netif_napi_add(adapter->netdev, &napi->napi,
1764 			       ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll);
1765 
1766 		if (!ENA_IS_XDP_INDEX(adapter, i))
1767 			napi->rx_ring = &adapter->rx_ring[i];
1768 
1769 		napi->tx_ring = &adapter->tx_ring[i];
1770 		napi->qid = i;
1771 	}
1772 }
1773 
ena_napi_disable_in_range(struct ena_adapter * adapter,int first_index,int count)1774 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
1775 				      int first_index,
1776 				      int count)
1777 {
1778 	int i;
1779 
1780 	for (i = first_index; i < first_index + count; i++)
1781 		napi_disable(&adapter->ena_napi[i].napi);
1782 }
1783 
ena_napi_enable_in_range(struct ena_adapter * adapter,int first_index,int count)1784 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
1785 				     int first_index,
1786 				     int count)
1787 {
1788 	int i;
1789 
1790 	for (i = first_index; i < first_index + count; i++)
1791 		napi_enable(&adapter->ena_napi[i].napi);
1792 }
1793 
1794 /* Configure the Rx forwarding */
ena_rss_configure(struct ena_adapter * adapter)1795 static int ena_rss_configure(struct ena_adapter *adapter)
1796 {
1797 	struct ena_com_dev *ena_dev = adapter->ena_dev;
1798 	int rc;
1799 
1800 	/* In case the RSS table wasn't initialized by probe */
1801 	if (!ena_dev->rss.tbl_log_size) {
1802 		rc = ena_rss_init_default(adapter);
1803 		if (rc && (rc != -EOPNOTSUPP)) {
1804 			netif_err(adapter, ifup, adapter->netdev, "Failed to init RSS rc: %d\n", rc);
1805 			return rc;
1806 		}
1807 	}
1808 
1809 	/* Set indirect table */
1810 	rc = ena_com_indirect_table_set(ena_dev);
1811 	if (unlikely(rc && rc != -EOPNOTSUPP))
1812 		return rc;
1813 
1814 	/* Configure hash function (if supported) */
1815 	rc = ena_com_set_hash_function(ena_dev);
1816 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
1817 		return rc;
1818 
1819 	/* Configure hash inputs (if supported) */
1820 	rc = ena_com_set_hash_ctrl(ena_dev);
1821 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
1822 		return rc;
1823 
1824 	return 0;
1825 }
1826 
ena_up_complete(struct ena_adapter * adapter)1827 static int ena_up_complete(struct ena_adapter *adapter)
1828 {
1829 	int rc;
1830 
1831 	rc = ena_rss_configure(adapter);
1832 	if (rc)
1833 		return rc;
1834 
1835 	ena_change_mtu(adapter->netdev, adapter->netdev->mtu);
1836 
1837 	ena_refill_all_rx_bufs(adapter);
1838 
1839 	/* enable transmits */
1840 	netif_tx_start_all_queues(adapter->netdev);
1841 
1842 	ena_napi_enable_in_range(adapter,
1843 				 0,
1844 				 adapter->xdp_num_queues + adapter->num_io_queues);
1845 
1846 	return 0;
1847 }
1848 
ena_create_io_tx_queue(struct ena_adapter * adapter,int qid)1849 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
1850 {
1851 	struct ena_com_create_io_ctx ctx;
1852 	struct ena_com_dev *ena_dev;
1853 	struct ena_ring *tx_ring;
1854 	u32 msix_vector;
1855 	u16 ena_qid;
1856 	int rc;
1857 
1858 	ena_dev = adapter->ena_dev;
1859 
1860 	tx_ring = &adapter->tx_ring[qid];
1861 	msix_vector = ENA_IO_IRQ_IDX(qid);
1862 	ena_qid = ENA_IO_TXQ_IDX(qid);
1863 
1864 	memset(&ctx, 0x0, sizeof(ctx));
1865 
1866 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1867 	ctx.qid = ena_qid;
1868 	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1869 	ctx.msix_vector = msix_vector;
1870 	ctx.queue_size = tx_ring->ring_size;
1871 	ctx.numa_node = tx_ring->numa_node;
1872 
1873 	rc = ena_com_create_io_queue(ena_dev, &ctx);
1874 	if (rc) {
1875 		netif_err(adapter, ifup, adapter->netdev,
1876 			  "Failed to create I/O TX queue num %d rc: %d\n",
1877 			  qid, rc);
1878 		return rc;
1879 	}
1880 
1881 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1882 				     &tx_ring->ena_com_io_sq,
1883 				     &tx_ring->ena_com_io_cq);
1884 	if (rc) {
1885 		netif_err(adapter, ifup, adapter->netdev,
1886 			  "Failed to get TX queue handlers. TX queue num %d rc: %d\n",
1887 			  qid, rc);
1888 		ena_com_destroy_io_queue(ena_dev, ena_qid);
1889 		return rc;
1890 	}
1891 
1892 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
1893 	return rc;
1894 }
1895 
ena_create_io_tx_queues_in_range(struct ena_adapter * adapter,int first_index,int count)1896 int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
1897 				     int first_index, int count)
1898 {
1899 	struct ena_com_dev *ena_dev = adapter->ena_dev;
1900 	int rc, i;
1901 
1902 	for (i = first_index; i < first_index + count; i++) {
1903 		rc = ena_create_io_tx_queue(adapter, i);
1904 		if (rc)
1905 			goto create_err;
1906 	}
1907 
1908 	return 0;
1909 
1910 create_err:
1911 	while (i-- > first_index)
1912 		ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1913 
1914 	return rc;
1915 }
1916 
ena_create_io_rx_queue(struct ena_adapter * adapter,int qid)1917 static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
1918 {
1919 	struct ena_com_dev *ena_dev;
1920 	struct ena_com_create_io_ctx ctx;
1921 	struct ena_ring *rx_ring;
1922 	u32 msix_vector;
1923 	u16 ena_qid;
1924 	int rc;
1925 
1926 	ena_dev = adapter->ena_dev;
1927 
1928 	rx_ring = &adapter->rx_ring[qid];
1929 	msix_vector = ENA_IO_IRQ_IDX(qid);
1930 	ena_qid = ENA_IO_RXQ_IDX(qid);
1931 
1932 	memset(&ctx, 0x0, sizeof(ctx));
1933 
1934 	ctx.qid = ena_qid;
1935 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1936 	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1937 	ctx.msix_vector = msix_vector;
1938 	ctx.queue_size = rx_ring->ring_size;
1939 	ctx.numa_node = rx_ring->numa_node;
1940 
1941 	rc = ena_com_create_io_queue(ena_dev, &ctx);
1942 	if (rc) {
1943 		netif_err(adapter, ifup, adapter->netdev,
1944 			  "Failed to create I/O RX queue num %d rc: %d\n",
1945 			  qid, rc);
1946 		return rc;
1947 	}
1948 
1949 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1950 				     &rx_ring->ena_com_io_sq,
1951 				     &rx_ring->ena_com_io_cq);
1952 	if (rc) {
1953 		netif_err(adapter, ifup, adapter->netdev,
1954 			  "Failed to get RX queue handlers. RX queue num %d rc: %d\n",
1955 			  qid, rc);
1956 		goto err;
1957 	}
1958 
1959 	ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
1960 
1961 	return rc;
1962 err:
1963 	ena_com_destroy_io_queue(ena_dev, ena_qid);
1964 	return rc;
1965 }
1966 
ena_create_all_io_rx_queues(struct ena_adapter * adapter)1967 static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
1968 {
1969 	struct ena_com_dev *ena_dev = adapter->ena_dev;
1970 	int rc, i;
1971 
1972 	for (i = 0; i < adapter->num_io_queues; i++) {
1973 		rc = ena_create_io_rx_queue(adapter, i);
1974 		if (rc)
1975 			goto create_err;
1976 		INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);
1977 	}
1978 
1979 	return 0;
1980 
1981 create_err:
1982 	while (i--) {
1983 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
1984 		ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
1985 	}
1986 
1987 	return rc;
1988 }
1989 
set_io_rings_size(struct ena_adapter * adapter,int new_tx_size,int new_rx_size)1990 static void set_io_rings_size(struct ena_adapter *adapter,
1991 			      int new_tx_size,
1992 			      int new_rx_size)
1993 {
1994 	int i;
1995 
1996 	for (i = 0; i < adapter->num_io_queues; i++) {
1997 		adapter->tx_ring[i].ring_size = new_tx_size;
1998 		adapter->rx_ring[i].ring_size = new_rx_size;
1999 	}
2000 }
2001 
2002 /* This function allows queue allocation to backoff when the system is
2003  * low on memory. If there is not enough memory to allocate io queues
2004  * the driver will try to allocate smaller queues.
2005  *
2006  * The backoff algorithm is as follows:
2007  *  1. Try to allocate TX and RX and if successful.
2008  *  1.1. return success
2009  *
2010  *  2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same).
2011  *
2012  *  3. If TX or RX is smaller than 256
2013  *  3.1. return failure.
2014  *  4. else
2015  *  4.1. go back to 1.
2016  */
create_queues_with_size_backoff(struct ena_adapter * adapter)2017 static int create_queues_with_size_backoff(struct ena_adapter *adapter)
2018 {
2019 	int rc, cur_rx_ring_size, cur_tx_ring_size;
2020 	int new_rx_ring_size, new_tx_ring_size;
2021 
2022 	/* current queue sizes might be set to smaller than the requested
2023 	 * ones due to past queue allocation failures.
2024 	 */
2025 	set_io_rings_size(adapter, adapter->requested_tx_ring_size,
2026 			  adapter->requested_rx_ring_size);
2027 
2028 	while (1) {
2029 		if (ena_xdp_present(adapter)) {
2030 			rc = ena_setup_and_create_all_xdp_queues(adapter);
2031 
2032 			if (rc)
2033 				goto err_setup_tx;
2034 		}
2035 		rc = ena_setup_tx_resources_in_range(adapter,
2036 						     0,
2037 						     adapter->num_io_queues);
2038 		if (rc)
2039 			goto err_setup_tx;
2040 
2041 		rc = ena_create_io_tx_queues_in_range(adapter,
2042 						      0,
2043 						      adapter->num_io_queues);
2044 		if (rc)
2045 			goto err_create_tx_queues;
2046 
2047 		rc = ena_setup_all_rx_resources(adapter);
2048 		if (rc)
2049 			goto err_setup_rx;
2050 
2051 		rc = ena_create_all_io_rx_queues(adapter);
2052 		if (rc)
2053 			goto err_create_rx_queues;
2054 
2055 		return 0;
2056 
2057 err_create_rx_queues:
2058 		ena_free_all_io_rx_resources(adapter);
2059 err_setup_rx:
2060 		ena_destroy_all_tx_queues(adapter);
2061 err_create_tx_queues:
2062 		ena_free_all_io_tx_resources(adapter);
2063 err_setup_tx:
2064 		if (rc != -ENOMEM) {
2065 			netif_err(adapter, ifup, adapter->netdev,
2066 				  "Queue creation failed with error code %d\n",
2067 				  rc);
2068 			return rc;
2069 		}
2070 
2071 		cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2072 		cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2073 
2074 		netif_err(adapter, ifup, adapter->netdev,
2075 			  "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2076 			  cur_tx_ring_size, cur_rx_ring_size);
2077 
2078 		new_tx_ring_size = cur_tx_ring_size;
2079 		new_rx_ring_size = cur_rx_ring_size;
2080 
2081 		/* Decrease the size of the larger queue, or
2082 		 * decrease both if they are the same size.
2083 		 */
2084 		if (cur_rx_ring_size <= cur_tx_ring_size)
2085 			new_tx_ring_size = cur_tx_ring_size / 2;
2086 		if (cur_rx_ring_size >= cur_tx_ring_size)
2087 			new_rx_ring_size = cur_rx_ring_size / 2;
2088 
2089 		if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2090 		    new_rx_ring_size < ENA_MIN_RING_SIZE) {
2091 			netif_err(adapter, ifup, adapter->netdev,
2092 				  "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n",
2093 				  ENA_MIN_RING_SIZE);
2094 			return rc;
2095 		}
2096 
2097 		netif_err(adapter, ifup, adapter->netdev,
2098 			  "Retrying queue creation with sizes TX=%d, RX=%d\n",
2099 			  new_tx_ring_size,
2100 			  new_rx_ring_size);
2101 
2102 		set_io_rings_size(adapter, new_tx_ring_size,
2103 				  new_rx_ring_size);
2104 	}
2105 }
2106 
ena_up(struct ena_adapter * adapter)2107 int ena_up(struct ena_adapter *adapter)
2108 {
2109 	int io_queue_count, rc, i;
2110 
2111 	netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
2112 
2113 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2114 	ena_setup_io_intr(adapter);
2115 
2116 	/* napi poll functions should be initialized before running
2117 	 * request_irq(), to handle a rare condition where there is a pending
2118 	 * interrupt, causing the ISR to fire immediately while the poll
2119 	 * function wasn't set yet, causing a null dereference
2120 	 */
2121 	ena_init_napi_in_range(adapter, 0, io_queue_count);
2122 
2123 	rc = ena_request_io_irq(adapter);
2124 	if (rc)
2125 		goto err_req_irq;
2126 
2127 	rc = create_queues_with_size_backoff(adapter);
2128 	if (rc)
2129 		goto err_create_queues_with_backoff;
2130 
2131 	rc = ena_up_complete(adapter);
2132 	if (rc)
2133 		goto err_up;
2134 
2135 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
2136 		netif_carrier_on(adapter->netdev);
2137 
2138 	ena_increase_stat(&adapter->dev_stats.interface_up, 1,
2139 			  &adapter->syncp);
2140 
2141 	set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2142 
2143 	/* Enable completion queues interrupt */
2144 	for (i = 0; i < adapter->num_io_queues; i++)
2145 		ena_unmask_interrupt(&adapter->tx_ring[i],
2146 				     &adapter->rx_ring[i]);
2147 
2148 	/* schedule napi in case we had pending packets
2149 	 * from the last time we disable napi
2150 	 */
2151 	for (i = 0; i < io_queue_count; i++)
2152 		napi_schedule(&adapter->ena_napi[i].napi);
2153 
2154 	return rc;
2155 
2156 err_up:
2157 	ena_destroy_all_tx_queues(adapter);
2158 	ena_free_all_io_tx_resources(adapter);
2159 	ena_destroy_all_rx_queues(adapter);
2160 	ena_free_all_io_rx_resources(adapter);
2161 err_create_queues_with_backoff:
2162 	ena_free_io_irq(adapter);
2163 err_req_irq:
2164 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2165 
2166 	return rc;
2167 }
2168 
ena_down(struct ena_adapter * adapter)2169 void ena_down(struct ena_adapter *adapter)
2170 {
2171 	int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2172 
2173 	netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__);
2174 
2175 	clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2176 
2177 	ena_increase_stat(&adapter->dev_stats.interface_down, 1,
2178 			  &adapter->syncp);
2179 
2180 	netif_carrier_off(adapter->netdev);
2181 	netif_tx_disable(adapter->netdev);
2182 
2183 	/* After this point the napi handler won't enable the tx queue */
2184 	ena_napi_disable_in_range(adapter, 0, io_queue_count);
2185 
2186 	/* After destroy the queue there won't be any new interrupts */
2187 
2188 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
2189 		int rc;
2190 
2191 		rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
2192 		if (rc)
2193 			netif_err(adapter, ifdown, adapter->netdev,
2194 				  "Device reset failed\n");
2195 		/* stop submitting admin commands on a device that was reset */
2196 		ena_com_set_admin_running_state(adapter->ena_dev, false);
2197 	}
2198 
2199 	ena_destroy_all_io_queues(adapter);
2200 
2201 	ena_disable_io_intr_sync(adapter);
2202 	ena_free_io_irq(adapter);
2203 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2204 
2205 	ena_free_all_tx_bufs(adapter);
2206 	ena_free_all_rx_bufs(adapter);
2207 	ena_free_all_io_tx_resources(adapter);
2208 	ena_free_all_io_rx_resources(adapter);
2209 }
2210 
2211 /* ena_open - Called when a network interface is made active
2212  * @netdev: network interface device structure
2213  *
2214  * Returns 0 on success, negative value on failure
2215  *
2216  * The open entry point is called when a network interface is made
2217  * active by the system (IFF_UP).  At this point all resources needed
2218  * for transmit and receive operations are allocated, the interrupt
2219  * handler is registered with the OS, the watchdog timer is started,
2220  * and the stack is notified that the interface is ready.
2221  */
ena_open(struct net_device * netdev)2222 static int ena_open(struct net_device *netdev)
2223 {
2224 	struct ena_adapter *adapter = netdev_priv(netdev);
2225 	int rc;
2226 
2227 	/* Notify the stack of the actual queue counts. */
2228 	rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues);
2229 	if (rc) {
2230 		netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
2231 		return rc;
2232 	}
2233 
2234 	rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues);
2235 	if (rc) {
2236 		netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
2237 		return rc;
2238 	}
2239 
2240 	rc = ena_up(adapter);
2241 	if (rc)
2242 		return rc;
2243 
2244 	return rc;
2245 }
2246 
2247 /* ena_close - Disables a network interface
2248  * @netdev: network interface device structure
2249  *
2250  * Returns 0, this is not allowed to fail
2251  *
2252  * The close entry point is called when an interface is de-activated
2253  * by the OS.  The hardware is still under the drivers control, but
2254  * needs to be disabled.  A global MAC reset is issued to stop the
2255  * hardware, and all transmit and receive resources are freed.
2256  */
ena_close(struct net_device * netdev)2257 static int ena_close(struct net_device *netdev)
2258 {
2259 	struct ena_adapter *adapter = netdev_priv(netdev);
2260 
2261 	netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
2262 
2263 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
2264 		return 0;
2265 
2266 	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2267 		ena_down(adapter);
2268 
2269 	/* Check for device status and issue reset if needed*/
2270 	check_for_admin_com_state(adapter);
2271 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
2272 		netif_err(adapter, ifdown, adapter->netdev,
2273 			  "Destroy failure, restarting device\n");
2274 		ena_dump_stats_to_dmesg(adapter);
2275 		/* rtnl lock already obtained in dev_ioctl() layer */
2276 		ena_destroy_device(adapter, false);
2277 		ena_restore_device(adapter);
2278 	}
2279 
2280 	return 0;
2281 }
2282 
ena_update_queue_params(struct ena_adapter * adapter,u32 new_tx_size,u32 new_rx_size,u32 new_llq_header_len)2283 int ena_update_queue_params(struct ena_adapter *adapter,
2284 			    u32 new_tx_size,
2285 			    u32 new_rx_size,
2286 			    u32 new_llq_header_len)
2287 {
2288 	bool dev_was_up, large_llq_changed = false;
2289 	int rc = 0;
2290 
2291 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2292 	ena_close(adapter->netdev);
2293 	adapter->requested_tx_ring_size = new_tx_size;
2294 	adapter->requested_rx_ring_size = new_rx_size;
2295 	ena_init_io_rings(adapter,
2296 			  0,
2297 			  adapter->xdp_num_queues +
2298 			  adapter->num_io_queues);
2299 
2300 	large_llq_changed = adapter->ena_dev->tx_mem_queue_type ==
2301 			    ENA_ADMIN_PLACEMENT_POLICY_DEV;
2302 	large_llq_changed &=
2303 		new_llq_header_len != adapter->ena_dev->tx_max_header_size;
2304 
2305 	/* a check that the configuration is valid is done by caller */
2306 	if (large_llq_changed) {
2307 		adapter->large_llq_header_enabled = !adapter->large_llq_header_enabled;
2308 
2309 		ena_destroy_device(adapter, false);
2310 		rc = ena_restore_device(adapter);
2311 	}
2312 
2313 	return dev_was_up && !rc ? ena_up(adapter) : rc;
2314 }
2315 
ena_set_rx_copybreak(struct ena_adapter * adapter,u32 rx_copybreak)2316 int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak)
2317 {
2318 	struct ena_ring *rx_ring;
2319 	int i;
2320 
2321 	if (rx_copybreak > min_t(u16, adapter->netdev->mtu, ENA_PAGE_SIZE))
2322 		return -EINVAL;
2323 
2324 	adapter->rx_copybreak = rx_copybreak;
2325 
2326 	for (i = 0; i < adapter->num_io_queues; i++) {
2327 		rx_ring = &adapter->rx_ring[i];
2328 		rx_ring->rx_copybreak = rx_copybreak;
2329 	}
2330 
2331 	return 0;
2332 }
2333 
ena_update_queue_count(struct ena_adapter * adapter,u32 new_channel_count)2334 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)
2335 {
2336 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2337 	int prev_channel_count;
2338 	bool dev_was_up;
2339 
2340 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2341 	ena_close(adapter->netdev);
2342 	prev_channel_count = adapter->num_io_queues;
2343 	adapter->num_io_queues = new_channel_count;
2344 	if (ena_xdp_present(adapter) &&
2345 	    ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) {
2346 		adapter->xdp_first_ring = new_channel_count;
2347 		adapter->xdp_num_queues = new_channel_count;
2348 		if (prev_channel_count > new_channel_count)
2349 			ena_xdp_exchange_program_rx_in_range(adapter,
2350 							     NULL,
2351 							     new_channel_count,
2352 							     prev_channel_count);
2353 		else
2354 			ena_xdp_exchange_program_rx_in_range(adapter,
2355 							     adapter->xdp_bpf_prog,
2356 							     prev_channel_count,
2357 							     new_channel_count);
2358 	}
2359 
2360 	/* We need to destroy the rss table so that the indirection
2361 	 * table will be reinitialized by ena_up()
2362 	 */
2363 	ena_com_rss_destroy(ena_dev);
2364 	ena_init_io_rings(adapter,
2365 			  0,
2366 			  adapter->xdp_num_queues +
2367 			  adapter->num_io_queues);
2368 	return dev_was_up ? ena_open(adapter->netdev) : 0;
2369 }
2370 
ena_tx_csum(struct ena_com_tx_ctx * ena_tx_ctx,struct sk_buff * skb,bool disable_meta_caching)2371 static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,
2372 			struct sk_buff *skb,
2373 			bool disable_meta_caching)
2374 {
2375 	u32 mss = skb_shinfo(skb)->gso_size;
2376 	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
2377 	u8 l4_protocol = 0;
2378 
2379 	if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {
2380 		ena_tx_ctx->l4_csum_enable = 1;
2381 		if (mss) {
2382 			ena_tx_ctx->tso_enable = 1;
2383 			ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;
2384 			ena_tx_ctx->l4_csum_partial = 0;
2385 		} else {
2386 			ena_tx_ctx->tso_enable = 0;
2387 			ena_meta->l4_hdr_len = 0;
2388 			ena_tx_ctx->l4_csum_partial = 1;
2389 		}
2390 
2391 		switch (ip_hdr(skb)->version) {
2392 		case IPVERSION:
2393 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
2394 			if (ip_hdr(skb)->frag_off & htons(IP_DF))
2395 				ena_tx_ctx->df = 1;
2396 			if (mss)
2397 				ena_tx_ctx->l3_csum_enable = 1;
2398 			l4_protocol = ip_hdr(skb)->protocol;
2399 			break;
2400 		case 6:
2401 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
2402 			l4_protocol = ipv6_hdr(skb)->nexthdr;
2403 			break;
2404 		default:
2405 			break;
2406 		}
2407 
2408 		if (l4_protocol == IPPROTO_TCP)
2409 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
2410 		else
2411 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
2412 
2413 		ena_meta->mss = mss;
2414 		ena_meta->l3_hdr_len = skb_network_header_len(skb);
2415 		ena_meta->l3_hdr_offset = skb_network_offset(skb);
2416 		ena_tx_ctx->meta_valid = 1;
2417 	} else if (disable_meta_caching) {
2418 		memset(ena_meta, 0, sizeof(*ena_meta));
2419 		ena_tx_ctx->meta_valid = 1;
2420 	} else {
2421 		ena_tx_ctx->meta_valid = 0;
2422 	}
2423 }
2424 
ena_check_and_linearize_skb(struct ena_ring * tx_ring,struct sk_buff * skb)2425 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
2426 				       struct sk_buff *skb)
2427 {
2428 	int num_frags, header_len, rc;
2429 
2430 	num_frags = skb_shinfo(skb)->nr_frags;
2431 	header_len = skb_headlen(skb);
2432 
2433 	if (num_frags < tx_ring->sgl_size)
2434 		return 0;
2435 
2436 	if ((num_frags == tx_ring->sgl_size) &&
2437 	    (header_len < tx_ring->tx_max_header_size))
2438 		return 0;
2439 
2440 	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);
2441 
2442 	rc = skb_linearize(skb);
2443 	if (unlikely(rc)) {
2444 		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,
2445 				  &tx_ring->syncp);
2446 	}
2447 
2448 	return rc;
2449 }
2450 
ena_tx_map_skb(struct ena_ring * tx_ring,struct ena_tx_buffer * tx_info,struct sk_buff * skb,void ** push_hdr,u16 * header_len)2451 static int ena_tx_map_skb(struct ena_ring *tx_ring,
2452 			  struct ena_tx_buffer *tx_info,
2453 			  struct sk_buff *skb,
2454 			  void **push_hdr,
2455 			  u16 *header_len)
2456 {
2457 	struct ena_adapter *adapter = tx_ring->adapter;
2458 	struct ena_com_buf *ena_buf;
2459 	dma_addr_t dma;
2460 	u32 skb_head_len, frag_len, last_frag;
2461 	u16 push_len = 0;
2462 	u16 delta = 0;
2463 	int i = 0;
2464 
2465 	skb_head_len = skb_headlen(skb);
2466 	tx_info->skb = skb;
2467 	ena_buf = tx_info->bufs;
2468 
2469 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2470 		/* When the device is LLQ mode, the driver will copy
2471 		 * the header into the device memory space.
2472 		 * the ena_com layer assume the header is in a linear
2473 		 * memory space.
2474 		 * This assumption might be wrong since part of the header
2475 		 * can be in the fragmented buffers.
2476 		 * Use skb_header_pointer to make sure the header is in a
2477 		 * linear memory space.
2478 		 */
2479 
2480 		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
2481 		*push_hdr = skb_header_pointer(skb, 0, push_len,
2482 					       tx_ring->push_buf_intermediate_buf);
2483 		*header_len = push_len;
2484 		if (unlikely(skb->data != *push_hdr)) {
2485 			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,
2486 					  &tx_ring->syncp);
2487 
2488 			delta = push_len - skb_head_len;
2489 		}
2490 	} else {
2491 		*push_hdr = NULL;
2492 		*header_len = min_t(u32, skb_head_len,
2493 				    tx_ring->tx_max_header_size);
2494 	}
2495 
2496 	netif_dbg(adapter, tx_queued, adapter->netdev,
2497 		  "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
2498 		  *push_hdr, push_len);
2499 
2500 	if (skb_head_len > push_len) {
2501 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,
2502 				     skb_head_len - push_len, DMA_TO_DEVICE);
2503 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2504 			goto error_report_dma_error;
2505 
2506 		ena_buf->paddr = dma;
2507 		ena_buf->len = skb_head_len - push_len;
2508 
2509 		ena_buf++;
2510 		tx_info->num_of_bufs++;
2511 		tx_info->map_linear_data = 1;
2512 	} else {
2513 		tx_info->map_linear_data = 0;
2514 	}
2515 
2516 	last_frag = skb_shinfo(skb)->nr_frags;
2517 
2518 	for (i = 0; i < last_frag; i++) {
2519 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2520 
2521 		frag_len = skb_frag_size(frag);
2522 
2523 		if (unlikely(delta >= frag_len)) {
2524 			delta -= frag_len;
2525 			continue;
2526 		}
2527 
2528 		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
2529 				       frag_len - delta, DMA_TO_DEVICE);
2530 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2531 			goto error_report_dma_error;
2532 
2533 		ena_buf->paddr = dma;
2534 		ena_buf->len = frag_len - delta;
2535 		ena_buf++;
2536 		tx_info->num_of_bufs++;
2537 		delta = 0;
2538 	}
2539 
2540 	return 0;
2541 
2542 error_report_dma_error:
2543 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
2544 			  &tx_ring->syncp);
2545 	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n");
2546 
2547 	tx_info->skb = NULL;
2548 
2549 	tx_info->num_of_bufs += i;
2550 	ena_unmap_tx_buff(tx_ring, tx_info);
2551 
2552 	return -EINVAL;
2553 }
2554 
2555 /* Called with netif_tx_lock. */
ena_start_xmit(struct sk_buff * skb,struct net_device * dev)2556 static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
2557 {
2558 	struct ena_adapter *adapter = netdev_priv(dev);
2559 	struct ena_tx_buffer *tx_info;
2560 	struct ena_com_tx_ctx ena_tx_ctx;
2561 	struct ena_ring *tx_ring;
2562 	struct netdev_queue *txq;
2563 	void *push_hdr;
2564 	u16 next_to_use, req_id, header_len;
2565 	int qid, rc;
2566 
2567 	netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
2568 	/*  Determine which tx ring we will be placed on */
2569 	qid = skb_get_queue_mapping(skb);
2570 	tx_ring = &adapter->tx_ring[qid];
2571 	txq = netdev_get_tx_queue(dev, qid);
2572 
2573 	rc = ena_check_and_linearize_skb(tx_ring, skb);
2574 	if (unlikely(rc))
2575 		goto error_drop_packet;
2576 
2577 	skb_tx_timestamp(skb);
2578 
2579 	next_to_use = tx_ring->next_to_use;
2580 	req_id = tx_ring->free_ids[next_to_use];
2581 	tx_info = &tx_ring->tx_buffer_info[req_id];
2582 	tx_info->num_of_bufs = 0;
2583 
2584 	WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
2585 
2586 	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
2587 	if (unlikely(rc))
2588 		goto error_drop_packet;
2589 
2590 	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
2591 	ena_tx_ctx.ena_bufs = tx_info->bufs;
2592 	ena_tx_ctx.push_header = push_hdr;
2593 	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
2594 	ena_tx_ctx.req_id = req_id;
2595 	ena_tx_ctx.header_len = header_len;
2596 
2597 	/* set flags and meta data */
2598 	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
2599 
2600 	rc = ena_xmit_common(adapter,
2601 			     tx_ring,
2602 			     tx_info,
2603 			     &ena_tx_ctx,
2604 			     next_to_use,
2605 			     skb->len);
2606 	if (rc)
2607 		goto error_unmap_dma;
2608 
2609 	netdev_tx_sent_queue(txq, skb->len);
2610 
2611 	/* stop the queue when no more space available, the packet can have up
2612 	 * to sgl_size + 2. one for the meta descriptor and one for header
2613 	 * (if the header is larger than tx_max_header_size).
2614 	 */
2615 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
2616 						   tx_ring->sgl_size + 2))) {
2617 		netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
2618 			  __func__, qid);
2619 
2620 		netif_tx_stop_queue(txq);
2621 		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,
2622 				  &tx_ring->syncp);
2623 
2624 		/* There is a rare condition where this function decide to
2625 		 * stop the queue but meanwhile clean_tx_irq updates
2626 		 * next_to_completion and terminates.
2627 		 * The queue will remain stopped forever.
2628 		 * To solve this issue add a mb() to make sure that
2629 		 * netif_tx_stop_queue() write is vissible before checking if
2630 		 * there is additional space in the queue.
2631 		 */
2632 		smp_mb();
2633 
2634 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
2635 						 ENA_TX_WAKEUP_THRESH)) {
2636 			netif_tx_wake_queue(txq);
2637 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
2638 					  &tx_ring->syncp);
2639 		}
2640 	}
2641 
2642 	if (netif_xmit_stopped(txq) || !netdev_xmit_more())
2643 		/* trigger the dma engine. ena_ring_tx_doorbell()
2644 		 * calls a memory barrier inside it.
2645 		 */
2646 		ena_ring_tx_doorbell(tx_ring);
2647 
2648 	return NETDEV_TX_OK;
2649 
2650 error_unmap_dma:
2651 	ena_unmap_tx_buff(tx_ring, tx_info);
2652 	tx_info->skb = NULL;
2653 
2654 error_drop_packet:
2655 	dev_kfree_skb(skb);
2656 	return NETDEV_TX_OK;
2657 }
2658 
ena_config_host_info(struct ena_com_dev * ena_dev,struct pci_dev * pdev)2659 static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
2660 {
2661 	struct device *dev = &pdev->dev;
2662 	struct ena_admin_host_info *host_info;
2663 	int rc;
2664 
2665 	/* Allocate only the host info */
2666 	rc = ena_com_allocate_host_info(ena_dev);
2667 	if (rc) {
2668 		dev_err(dev, "Cannot allocate host info\n");
2669 		return;
2670 	}
2671 
2672 	host_info = ena_dev->host_attr.host_info;
2673 
2674 	host_info->bdf = pci_dev_id(pdev);
2675 	host_info->os_type = ENA_ADMIN_OS_LINUX;
2676 	host_info->kernel_ver = LINUX_VERSION_CODE;
2677 	strscpy(host_info->kernel_ver_str, utsname()->version,
2678 		sizeof(host_info->kernel_ver_str) - 1);
2679 	host_info->os_dist = 0;
2680 	strncpy(host_info->os_dist_str, utsname()->release,
2681 		sizeof(host_info->os_dist_str) - 1);
2682 	host_info->driver_version =
2683 		(DRV_MODULE_GEN_MAJOR) |
2684 		(DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
2685 		(DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) |
2686 		("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);
2687 	host_info->num_cpus = num_online_cpus();
2688 
2689 	host_info->driver_supported_features =
2690 		ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
2691 		ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK |
2692 		ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK |
2693 		ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK |
2694 		ENA_ADMIN_HOST_INFO_RX_PAGE_REUSE_MASK;
2695 
2696 	rc = ena_com_set_host_attributes(ena_dev);
2697 	if (rc) {
2698 		if (rc == -EOPNOTSUPP)
2699 			dev_warn(dev, "Cannot set host attributes\n");
2700 		else
2701 			dev_err(dev, "Cannot set host attributes\n");
2702 
2703 		goto err;
2704 	}
2705 
2706 	return;
2707 
2708 err:
2709 	ena_com_delete_host_info(ena_dev);
2710 }
2711 
ena_config_debug_area(struct ena_adapter * adapter)2712 static void ena_config_debug_area(struct ena_adapter *adapter)
2713 {
2714 	u32 debug_area_size;
2715 	int rc, ss_count;
2716 
2717 	ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);
2718 	if (ss_count <= 0) {
2719 		netif_err(adapter, drv, adapter->netdev,
2720 			  "SS count is negative\n");
2721 		return;
2722 	}
2723 
2724 	/* allocate 32 bytes for each string and 64bit for the value */
2725 	debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
2726 
2727 	rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
2728 	if (rc) {
2729 		netif_err(adapter, drv, adapter->netdev,
2730 			  "Cannot allocate debug area\n");
2731 		return;
2732 	}
2733 
2734 	rc = ena_com_set_host_attributes(adapter->ena_dev);
2735 	if (rc) {
2736 		if (rc == -EOPNOTSUPP)
2737 			netif_warn(adapter, drv, adapter->netdev, "Cannot set host attributes\n");
2738 		else
2739 			netif_err(adapter, drv, adapter->netdev,
2740 				  "Cannot set host attributes\n");
2741 		goto err;
2742 	}
2743 
2744 	return;
2745 err:
2746 	ena_com_delete_debug_area(adapter->ena_dev);
2747 }
2748 
ena_update_hw_stats(struct ena_adapter * adapter)2749 int ena_update_hw_stats(struct ena_adapter *adapter)
2750 {
2751 	int rc;
2752 
2753 	rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_stats);
2754 	if (rc) {
2755 		netdev_err(adapter->netdev, "Failed to get ENI stats\n");
2756 		return rc;
2757 	}
2758 
2759 	return 0;
2760 }
2761 
ena_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2762 static void ena_get_stats64(struct net_device *netdev,
2763 			    struct rtnl_link_stats64 *stats)
2764 {
2765 	struct ena_adapter *adapter = netdev_priv(netdev);
2766 	struct ena_ring *rx_ring, *tx_ring;
2767 	unsigned int start;
2768 	u64 rx_drops;
2769 	u64 tx_drops;
2770 	int i;
2771 
2772 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2773 		return;
2774 
2775 	for (i = 0; i < adapter->num_io_queues; i++) {
2776 		u64 bytes, packets;
2777 
2778 		tx_ring = &adapter->tx_ring[i];
2779 
2780 		do {
2781 			start = u64_stats_fetch_begin(&tx_ring->syncp);
2782 			packets = tx_ring->tx_stats.cnt;
2783 			bytes = tx_ring->tx_stats.bytes;
2784 		} while (u64_stats_fetch_retry(&tx_ring->syncp, start));
2785 
2786 		stats->tx_packets += packets;
2787 		stats->tx_bytes += bytes;
2788 
2789 		rx_ring = &adapter->rx_ring[i];
2790 
2791 		do {
2792 			start = u64_stats_fetch_begin(&rx_ring->syncp);
2793 			packets = rx_ring->rx_stats.cnt;
2794 			bytes = rx_ring->rx_stats.bytes;
2795 		} while (u64_stats_fetch_retry(&rx_ring->syncp, start));
2796 
2797 		stats->rx_packets += packets;
2798 		stats->rx_bytes += bytes;
2799 	}
2800 
2801 	do {
2802 		start = u64_stats_fetch_begin(&adapter->syncp);
2803 		rx_drops = adapter->dev_stats.rx_drops;
2804 		tx_drops = adapter->dev_stats.tx_drops;
2805 	} while (u64_stats_fetch_retry(&adapter->syncp, start));
2806 
2807 	stats->rx_dropped = rx_drops;
2808 	stats->tx_dropped = tx_drops;
2809 
2810 	stats->multicast = 0;
2811 	stats->collisions = 0;
2812 
2813 	stats->rx_length_errors = 0;
2814 	stats->rx_crc_errors = 0;
2815 	stats->rx_frame_errors = 0;
2816 	stats->rx_fifo_errors = 0;
2817 	stats->rx_missed_errors = 0;
2818 	stats->tx_window_errors = 0;
2819 
2820 	stats->rx_errors = 0;
2821 	stats->tx_errors = 0;
2822 }
2823 
2824 static const struct net_device_ops ena_netdev_ops = {
2825 	.ndo_open		= ena_open,
2826 	.ndo_stop		= ena_close,
2827 	.ndo_start_xmit		= ena_start_xmit,
2828 	.ndo_get_stats64	= ena_get_stats64,
2829 	.ndo_tx_timeout		= ena_tx_timeout,
2830 	.ndo_change_mtu		= ena_change_mtu,
2831 	.ndo_set_mac_address	= NULL,
2832 	.ndo_validate_addr	= eth_validate_addr,
2833 	.ndo_bpf		= ena_xdp,
2834 	.ndo_xdp_xmit		= ena_xdp_xmit,
2835 };
2836 
ena_calc_io_queue_size(struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * get_feat_ctx)2837 static void ena_calc_io_queue_size(struct ena_adapter *adapter,
2838 				   struct ena_com_dev_get_features_ctx *get_feat_ctx)
2839 {
2840 	struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq;
2841 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2842 	u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
2843 	u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
2844 	u32 max_tx_queue_size;
2845 	u32 max_rx_queue_size;
2846 
2847 	/* If this function is called after driver load, the ring sizes have already
2848 	 * been configured. Take it into account when recalculating ring size.
2849 	 */
2850 	if (adapter->tx_ring->ring_size)
2851 		tx_queue_size = adapter->tx_ring->ring_size;
2852 
2853 	if (adapter->rx_ring->ring_size)
2854 		rx_queue_size = adapter->rx_ring->ring_size;
2855 
2856 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2857 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2858 			&get_feat_ctx->max_queue_ext.max_queue_ext;
2859 		max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
2860 					  max_queue_ext->max_rx_sq_depth);
2861 		max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
2862 
2863 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
2864 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2865 						  llq->max_llq_depth);
2866 		else
2867 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2868 						  max_queue_ext->max_tx_sq_depth);
2869 
2870 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2871 						 max_queue_ext->max_per_packet_tx_descs);
2872 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2873 						 max_queue_ext->max_per_packet_rx_descs);
2874 	} else {
2875 		struct ena_admin_queue_feature_desc *max_queues =
2876 			&get_feat_ctx->max_queues;
2877 		max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
2878 					  max_queues->max_sq_depth);
2879 		max_tx_queue_size = max_queues->max_cq_depth;
2880 
2881 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
2882 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2883 						  llq->max_llq_depth);
2884 		else
2885 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2886 						  max_queues->max_sq_depth);
2887 
2888 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2889 						 max_queues->max_packet_tx_descs);
2890 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2891 						 max_queues->max_packet_rx_descs);
2892 	}
2893 
2894 	max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
2895 	max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
2896 
2897 	/* When forcing large headers, we multiply the entry size by 2, and therefore divide
2898 	 * the queue size by 2, leaving the amount of memory used by the queues unchanged.
2899 	 */
2900 	if (adapter->large_llq_header_enabled) {
2901 		if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&
2902 		    ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2903 			max_tx_queue_size /= 2;
2904 			dev_info(&adapter->pdev->dev,
2905 				 "Forcing large headers and decreasing maximum TX queue size to %d\n",
2906 				 max_tx_queue_size);
2907 		} else {
2908 			dev_err(&adapter->pdev->dev,
2909 				"Forcing large headers failed: LLQ is disabled or device does not support large headers\n");
2910 
2911 			adapter->large_llq_header_enabled = false;
2912 		}
2913 	}
2914 
2915 	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
2916 				  max_tx_queue_size);
2917 	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
2918 				  max_rx_queue_size);
2919 
2920 	tx_queue_size = rounddown_pow_of_two(tx_queue_size);
2921 	rx_queue_size = rounddown_pow_of_two(rx_queue_size);
2922 
2923 	adapter->max_tx_ring_size  = max_tx_queue_size;
2924 	adapter->max_rx_ring_size = max_rx_queue_size;
2925 	adapter->requested_tx_ring_size = tx_queue_size;
2926 	adapter->requested_rx_ring_size = rx_queue_size;
2927 }
2928 
ena_device_validate_params(struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * get_feat_ctx)2929 static int ena_device_validate_params(struct ena_adapter *adapter,
2930 				      struct ena_com_dev_get_features_ctx *get_feat_ctx)
2931 {
2932 	struct net_device *netdev = adapter->netdev;
2933 	int rc;
2934 
2935 	rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,
2936 			      adapter->mac_addr);
2937 	if (!rc) {
2938 		netif_err(adapter, drv, netdev,
2939 			  "Error, mac address are different\n");
2940 		return -EINVAL;
2941 	}
2942 
2943 	if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
2944 		netif_err(adapter, drv, netdev,
2945 			  "Error, device max mtu is smaller than netdev MTU\n");
2946 		return -EINVAL;
2947 	}
2948 
2949 	return 0;
2950 }
2951 
set_default_llq_configurations(struct ena_adapter * adapter,struct ena_llq_configurations * llq_config,struct ena_admin_feature_llq_desc * llq)2952 static void set_default_llq_configurations(struct ena_adapter *adapter,
2953 					   struct ena_llq_configurations *llq_config,
2954 					   struct ena_admin_feature_llq_desc *llq)
2955 {
2956 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2957 
2958 	llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
2959 	llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
2960 	llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
2961 
2962 	adapter->large_llq_header_supported =
2963 		!!(ena_dev->supported_features & BIT(ENA_ADMIN_LLQ));
2964 	adapter->large_llq_header_supported &=
2965 		!!(llq->entry_size_ctrl_supported &
2966 			ENA_ADMIN_LIST_ENTRY_SIZE_256B);
2967 
2968 	if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&
2969 	    adapter->large_llq_header_enabled) {
2970 		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
2971 		llq_config->llq_ring_entry_size_value = 256;
2972 	} else {
2973 		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
2974 		llq_config->llq_ring_entry_size_value = 128;
2975 	}
2976 }
2977 
ena_set_queues_placement_policy(struct pci_dev * pdev,struct ena_com_dev * ena_dev,struct ena_admin_feature_llq_desc * llq,struct ena_llq_configurations * llq_default_configurations)2978 static int ena_set_queues_placement_policy(struct pci_dev *pdev,
2979 					   struct ena_com_dev *ena_dev,
2980 					   struct ena_admin_feature_llq_desc *llq,
2981 					   struct ena_llq_configurations *llq_default_configurations)
2982 {
2983 	int rc;
2984 	u32 llq_feature_mask;
2985 
2986 	llq_feature_mask = 1 << ENA_ADMIN_LLQ;
2987 	if (!(ena_dev->supported_features & llq_feature_mask)) {
2988 		dev_warn(&pdev->dev,
2989 			"LLQ is not supported Fallback to host mode policy.\n");
2990 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2991 		return 0;
2992 	}
2993 
2994 	if (!ena_dev->mem_bar) {
2995 		netdev_err(ena_dev->net_device,
2996 			   "LLQ is advertised as supported but device doesn't expose mem bar\n");
2997 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2998 		return 0;
2999 	}
3000 
3001 	rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
3002 	if (unlikely(rc)) {
3003 		dev_err(&pdev->dev,
3004 			"Failed to configure the device mode.  Fallback to host mode policy.\n");
3005 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3006 	}
3007 
3008 	return 0;
3009 }
3010 
ena_map_llq_mem_bar(struct pci_dev * pdev,struct ena_com_dev * ena_dev,int bars)3011 static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
3012 			       int bars)
3013 {
3014 	bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));
3015 
3016 	if (!has_mem_bar)
3017 		return 0;
3018 
3019 	ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
3020 					   pci_resource_start(pdev, ENA_MEM_BAR),
3021 					   pci_resource_len(pdev, ENA_MEM_BAR));
3022 
3023 	if (!ena_dev->mem_bar)
3024 		return -EFAULT;
3025 
3026 	return 0;
3027 }
3028 
ena_device_init(struct ena_adapter * adapter,struct pci_dev * pdev,struct ena_com_dev_get_features_ctx * get_feat_ctx,bool * wd_state)3029 static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev,
3030 			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
3031 			   bool *wd_state)
3032 {
3033 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3034 	struct ena_llq_configurations llq_config;
3035 	struct device *dev = &pdev->dev;
3036 	bool readless_supported;
3037 	u32 aenq_groups;
3038 	int dma_width;
3039 	int rc;
3040 
3041 	rc = ena_com_mmio_reg_read_request_init(ena_dev);
3042 	if (rc) {
3043 		dev_err(dev, "Failed to init mmio read less\n");
3044 		return rc;
3045 	}
3046 
3047 	/* The PCIe configuration space revision id indicate if mmio reg
3048 	 * read is disabled
3049 	 */
3050 	readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
3051 	ena_com_set_mmio_read_mode(ena_dev, readless_supported);
3052 
3053 	rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
3054 	if (rc) {
3055 		dev_err(dev, "Can not reset device\n");
3056 		goto err_mmio_read_less;
3057 	}
3058 
3059 	rc = ena_com_validate_version(ena_dev);
3060 	if (rc) {
3061 		dev_err(dev, "Device version is too low\n");
3062 		goto err_mmio_read_less;
3063 	}
3064 
3065 	dma_width = ena_com_get_dma_width(ena_dev);
3066 	if (dma_width < 0) {
3067 		dev_err(dev, "Invalid dma width value %d", dma_width);
3068 		rc = dma_width;
3069 		goto err_mmio_read_less;
3070 	}
3071 
3072 	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(dma_width));
3073 	if (rc) {
3074 		dev_err(dev, "dma_set_mask_and_coherent failed %d\n", rc);
3075 		goto err_mmio_read_less;
3076 	}
3077 
3078 	/* ENA admin level init */
3079 	rc = ena_com_admin_init(ena_dev, &aenq_handlers);
3080 	if (rc) {
3081 		dev_err(dev,
3082 			"Can not initialize ena admin queue with device\n");
3083 		goto err_mmio_read_less;
3084 	}
3085 
3086 	/* To enable the msix interrupts the driver needs to know the number
3087 	 * of queues. So the driver uses polling mode to retrieve this
3088 	 * information
3089 	 */
3090 	ena_com_set_admin_polling_mode(ena_dev, true);
3091 
3092 	ena_config_host_info(ena_dev, pdev);
3093 
3094 	/* Get Device Attributes*/
3095 	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3096 	if (rc) {
3097 		dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);
3098 		goto err_admin_init;
3099 	}
3100 
3101 	/* Try to turn all the available aenq groups */
3102 	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3103 		BIT(ENA_ADMIN_FATAL_ERROR) |
3104 		BIT(ENA_ADMIN_WARNING) |
3105 		BIT(ENA_ADMIN_NOTIFICATION) |
3106 		BIT(ENA_ADMIN_KEEP_ALIVE);
3107 
3108 	aenq_groups &= get_feat_ctx->aenq.supported_groups;
3109 
3110 	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3111 	if (rc) {
3112 		dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);
3113 		goto err_admin_init;
3114 	}
3115 
3116 	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3117 
3118 	set_default_llq_configurations(adapter, &llq_config, &get_feat_ctx->llq);
3119 
3120 	rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
3121 					     &llq_config);
3122 	if (rc) {
3123 		dev_err(dev, "ENA device init failed\n");
3124 		goto err_admin_init;
3125 	}
3126 
3127 	ena_calc_io_queue_size(adapter, get_feat_ctx);
3128 
3129 	return 0;
3130 
3131 err_admin_init:
3132 	ena_com_delete_host_info(ena_dev);
3133 	ena_com_admin_destroy(ena_dev);
3134 err_mmio_read_less:
3135 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3136 
3137 	return rc;
3138 }
3139 
ena_enable_msix_and_set_admin_interrupts(struct ena_adapter * adapter)3140 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
3141 {
3142 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3143 	struct device *dev = &adapter->pdev->dev;
3144 	int rc;
3145 
3146 	rc = ena_enable_msix(adapter);
3147 	if (rc) {
3148 		dev_err(dev, "Can not reserve msix vectors\n");
3149 		return rc;
3150 	}
3151 
3152 	ena_setup_mgmnt_intr(adapter);
3153 
3154 	rc = ena_request_mgmnt_irq(adapter);
3155 	if (rc) {
3156 		dev_err(dev, "Can not setup management interrupts\n");
3157 		goto err_disable_msix;
3158 	}
3159 
3160 	ena_com_set_admin_polling_mode(ena_dev, false);
3161 
3162 	ena_com_admin_aenq_enable(ena_dev);
3163 
3164 	return 0;
3165 
3166 err_disable_msix:
3167 	ena_disable_msix(adapter);
3168 
3169 	return rc;
3170 }
3171 
ena_destroy_device(struct ena_adapter * adapter,bool graceful)3172 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3173 {
3174 	struct net_device *netdev = adapter->netdev;
3175 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3176 	bool dev_up;
3177 
3178 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
3179 		return;
3180 
3181 	netif_carrier_off(netdev);
3182 
3183 	del_timer_sync(&adapter->timer_service);
3184 
3185 	dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
3186 	adapter->dev_up_before_reset = dev_up;
3187 	if (!graceful)
3188 		ena_com_set_admin_running_state(ena_dev, false);
3189 
3190 	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3191 		ena_down(adapter);
3192 
3193 	/* Stop the device from sending AENQ events (in case reset flag is set
3194 	 *  and device is up, ena_down() already reset the device.
3195 	 */
3196 	if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up))
3197 		ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3198 
3199 	ena_free_mgmnt_irq(adapter);
3200 
3201 	ena_disable_msix(adapter);
3202 
3203 	ena_com_abort_admin_commands(ena_dev);
3204 
3205 	ena_com_wait_for_abort_completion(ena_dev);
3206 
3207 	ena_com_admin_destroy(ena_dev);
3208 
3209 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3210 
3211 	/* return reset reason to default value */
3212 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3213 
3214 	clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3215 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3216 }
3217 
ena_restore_device(struct ena_adapter * adapter)3218 static int ena_restore_device(struct ena_adapter *adapter)
3219 {
3220 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3221 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3222 	struct pci_dev *pdev = adapter->pdev;
3223 	struct ena_ring *txr;
3224 	int rc, count, i;
3225 	bool wd_state;
3226 
3227 	set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3228 	rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx, &wd_state);
3229 	if (rc) {
3230 		dev_err(&pdev->dev, "Can not initialize device\n");
3231 		goto err;
3232 	}
3233 	adapter->wd_state = wd_state;
3234 
3235 	count =  adapter->xdp_num_queues + adapter->num_io_queues;
3236 	for (i = 0 ; i < count; i++) {
3237 		txr = &adapter->tx_ring[i];
3238 		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
3239 		txr->tx_max_header_size = ena_dev->tx_max_header_size;
3240 	}
3241 
3242 	rc = ena_device_validate_params(adapter, &get_feat_ctx);
3243 	if (rc) {
3244 		dev_err(&pdev->dev, "Validation of device parameters failed\n");
3245 		goto err_device_destroy;
3246 	}
3247 
3248 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3249 	if (rc) {
3250 		dev_err(&pdev->dev, "Enable MSI-X failed\n");
3251 		goto err_device_destroy;
3252 	}
3253 	/* If the interface was up before the reset bring it up */
3254 	if (adapter->dev_up_before_reset) {
3255 		rc = ena_up(adapter);
3256 		if (rc) {
3257 			dev_err(&pdev->dev, "Failed to create I/O queues\n");
3258 			goto err_disable_msix;
3259 		}
3260 	}
3261 
3262 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3263 
3264 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3265 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
3266 		netif_carrier_on(adapter->netdev);
3267 
3268 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3269 	adapter->last_keep_alive_jiffies = jiffies;
3270 
3271 	return rc;
3272 err_disable_msix:
3273 	ena_free_mgmnt_irq(adapter);
3274 	ena_disable_msix(adapter);
3275 err_device_destroy:
3276 	ena_com_abort_admin_commands(ena_dev);
3277 	ena_com_wait_for_abort_completion(ena_dev);
3278 	ena_com_admin_destroy(ena_dev);
3279 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3280 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3281 err:
3282 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3283 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3284 	dev_err(&pdev->dev,
3285 		"Reset attempt failed. Can not reset the device\n");
3286 
3287 	return rc;
3288 }
3289 
ena_fw_reset_device(struct work_struct * work)3290 static void ena_fw_reset_device(struct work_struct *work)
3291 {
3292 	struct ena_adapter *adapter =
3293 		container_of(work, struct ena_adapter, reset_task);
3294 
3295 	rtnl_lock();
3296 
3297 	if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3298 		ena_destroy_device(adapter, false);
3299 		ena_restore_device(adapter);
3300 
3301 		dev_err(&adapter->pdev->dev, "Device reset completed successfully\n");
3302 	}
3303 
3304 	rtnl_unlock();
3305 }
3306 
check_for_rx_interrupt_queue(struct ena_adapter * adapter,struct ena_ring * rx_ring)3307 static int check_for_rx_interrupt_queue(struct ena_adapter *adapter,
3308 					struct ena_ring *rx_ring)
3309 {
3310 	struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi);
3311 
3312 	if (likely(READ_ONCE(ena_napi->first_interrupt)))
3313 		return 0;
3314 
3315 	if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
3316 		return 0;
3317 
3318 	rx_ring->no_interrupt_event_cnt++;
3319 
3320 	if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {
3321 		netif_err(adapter, rx_err, adapter->netdev,
3322 			  "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
3323 			  rx_ring->qid);
3324 
3325 		ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3326 		return -EIO;
3327 	}
3328 
3329 	return 0;
3330 }
3331 
check_missing_comp_in_tx_queue(struct ena_adapter * adapter,struct ena_ring * tx_ring)3332 static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
3333 					  struct ena_ring *tx_ring)
3334 {
3335 	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);
3336 	unsigned int time_since_last_napi;
3337 	unsigned int missing_tx_comp_to;
3338 	bool is_tx_comp_time_expired;
3339 	struct ena_tx_buffer *tx_buf;
3340 	unsigned long last_jiffies;
3341 	u32 missed_tx = 0;
3342 	int i, rc = 0;
3343 
3344 	for (i = 0; i < tx_ring->ring_size; i++) {
3345 		tx_buf = &tx_ring->tx_buffer_info[i];
3346 		last_jiffies = tx_buf->last_jiffies;
3347 
3348 		if (last_jiffies == 0)
3349 			/* no pending Tx at this location */
3350 			continue;
3351 
3352 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3353 			 2 * adapter->missing_tx_completion_to);
3354 
3355 		if (unlikely(!READ_ONCE(ena_napi->first_interrupt) && is_tx_comp_time_expired)) {
3356 			/* If after graceful period interrupt is still not
3357 			 * received, we schedule a reset
3358 			 */
3359 			netif_err(adapter, tx_err, adapter->netdev,
3360 				  "Potential MSIX issue on Tx side Queue = %d. Reset the device\n",
3361 				  tx_ring->qid);
3362 			ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3363 			return -EIO;
3364 		}
3365 
3366 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3367 			adapter->missing_tx_completion_to);
3368 
3369 		if (unlikely(is_tx_comp_time_expired)) {
3370 			if (!tx_buf->print_once) {
3371 				time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);
3372 				missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to);
3373 				netif_notice(adapter, tx_err, adapter->netdev,
3374 					     "Found a Tx that wasn't completed on time, qid %d, index %d. %u usecs have passed since last napi execution. Missing Tx timeout value %u msecs\n",
3375 					     tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to);
3376 			}
3377 
3378 			tx_buf->print_once = 1;
3379 			missed_tx++;
3380 		}
3381 	}
3382 
3383 	if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
3384 		netif_err(adapter, tx_err, adapter->netdev,
3385 			  "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n",
3386 			  missed_tx,
3387 			  adapter->missing_tx_completion_threshold);
3388 		ena_reset_device(adapter, ENA_REGS_RESET_MISS_TX_CMPL);
3389 		rc = -EIO;
3390 	}
3391 
3392 	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,
3393 			  &tx_ring->syncp);
3394 
3395 	return rc;
3396 }
3397 
check_for_missing_completions(struct ena_adapter * adapter)3398 static void check_for_missing_completions(struct ena_adapter *adapter)
3399 {
3400 	struct ena_ring *tx_ring;
3401 	struct ena_ring *rx_ring;
3402 	int qid, budget, rc;
3403 	int io_queue_count;
3404 
3405 	io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues;
3406 
3407 	/* Make sure the driver doesn't turn the device in other process */
3408 	smp_rmb();
3409 
3410 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3411 		return;
3412 
3413 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3414 		return;
3415 
3416 	if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)
3417 		return;
3418 
3419 	budget = min_t(u32, io_queue_count, ENA_MONITORED_TX_QUEUES);
3420 
3421 	qid = adapter->last_monitored_tx_qid;
3422 
3423 	while (budget) {
3424 		qid = (qid + 1) % io_queue_count;
3425 
3426 		tx_ring = &adapter->tx_ring[qid];
3427 		rx_ring = &adapter->rx_ring[qid];
3428 
3429 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
3430 		if (unlikely(rc))
3431 			return;
3432 
3433 		rc =  !ENA_IS_XDP_INDEX(adapter, qid) ?
3434 			check_for_rx_interrupt_queue(adapter, rx_ring) : 0;
3435 		if (unlikely(rc))
3436 			return;
3437 
3438 		budget--;
3439 	}
3440 
3441 	adapter->last_monitored_tx_qid = qid;
3442 }
3443 
3444 /* trigger napi schedule after 2 consecutive detections */
3445 #define EMPTY_RX_REFILL 2
3446 /* For the rare case where the device runs out of Rx descriptors and the
3447  * napi handler failed to refill new Rx descriptors (due to a lack of memory
3448  * for example).
3449  * This case will lead to a deadlock:
3450  * The device won't send interrupts since all the new Rx packets will be dropped
3451  * The napi handler won't allocate new Rx descriptors so the device will be
3452  * able to send new packets.
3453  *
3454  * This scenario can happen when the kernel's vm.min_free_kbytes is too small.
3455  * It is recommended to have at least 512MB, with a minimum of 128MB for
3456  * constrained environment).
3457  *
3458  * When such a situation is detected - Reschedule napi
3459  */
check_for_empty_rx_ring(struct ena_adapter * adapter)3460 static void check_for_empty_rx_ring(struct ena_adapter *adapter)
3461 {
3462 	struct ena_ring *rx_ring;
3463 	int i, refill_required;
3464 
3465 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3466 		return;
3467 
3468 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3469 		return;
3470 
3471 	for (i = 0; i < adapter->num_io_queues; i++) {
3472 		rx_ring = &adapter->rx_ring[i];
3473 
3474 		refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
3475 		if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3476 			rx_ring->empty_rx_queue++;
3477 
3478 			if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
3479 				ena_increase_stat(&rx_ring->rx_stats.empty_rx_ring, 1,
3480 						  &rx_ring->syncp);
3481 
3482 				netif_err(adapter, drv, adapter->netdev,
3483 					  "Trigger refill for ring %d\n", i);
3484 
3485 				napi_schedule(rx_ring->napi);
3486 				rx_ring->empty_rx_queue = 0;
3487 			}
3488 		} else {
3489 			rx_ring->empty_rx_queue = 0;
3490 		}
3491 	}
3492 }
3493 
3494 /* Check for keep alive expiration */
check_for_missing_keep_alive(struct ena_adapter * adapter)3495 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
3496 {
3497 	unsigned long keep_alive_expired;
3498 
3499 	if (!adapter->wd_state)
3500 		return;
3501 
3502 	if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3503 		return;
3504 
3505 	keep_alive_expired = adapter->last_keep_alive_jiffies +
3506 			     adapter->keep_alive_timeout;
3507 	if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
3508 		netif_err(adapter, drv, adapter->netdev,
3509 			  "Keep alive watchdog timeout.\n");
3510 		ena_increase_stat(&adapter->dev_stats.wd_expired, 1,
3511 				  &adapter->syncp);
3512 		ena_reset_device(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO);
3513 	}
3514 }
3515 
check_for_admin_com_state(struct ena_adapter * adapter)3516 static void check_for_admin_com_state(struct ena_adapter *adapter)
3517 {
3518 	if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
3519 		netif_err(adapter, drv, adapter->netdev,
3520 			  "ENA admin queue is not in running state!\n");
3521 		ena_increase_stat(&adapter->dev_stats.admin_q_pause, 1,
3522 				  &adapter->syncp);
3523 		ena_reset_device(adapter, ENA_REGS_RESET_ADMIN_TO);
3524 	}
3525 }
3526 
ena_update_hints(struct ena_adapter * adapter,struct ena_admin_ena_hw_hints * hints)3527 static void ena_update_hints(struct ena_adapter *adapter,
3528 			     struct ena_admin_ena_hw_hints *hints)
3529 {
3530 	struct net_device *netdev = adapter->netdev;
3531 
3532 	if (hints->admin_completion_tx_timeout)
3533 		adapter->ena_dev->admin_queue.completion_timeout =
3534 			hints->admin_completion_tx_timeout * 1000;
3535 
3536 	if (hints->mmio_read_timeout)
3537 		/* convert to usec */
3538 		adapter->ena_dev->mmio_read.reg_read_to =
3539 			hints->mmio_read_timeout * 1000;
3540 
3541 	if (hints->missed_tx_completion_count_threshold_to_reset)
3542 		adapter->missing_tx_completion_threshold =
3543 			hints->missed_tx_completion_count_threshold_to_reset;
3544 
3545 	if (hints->missing_tx_completion_timeout) {
3546 		if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3547 			adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT;
3548 		else
3549 			adapter->missing_tx_completion_to =
3550 				msecs_to_jiffies(hints->missing_tx_completion_timeout);
3551 	}
3552 
3553 	if (hints->netdev_wd_timeout)
3554 		netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout);
3555 
3556 	if (hints->driver_watchdog_timeout) {
3557 		if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3558 			adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3559 		else
3560 			adapter->keep_alive_timeout =
3561 				msecs_to_jiffies(hints->driver_watchdog_timeout);
3562 	}
3563 }
3564 
ena_update_host_info(struct ena_admin_host_info * host_info,struct net_device * netdev)3565 static void ena_update_host_info(struct ena_admin_host_info *host_info,
3566 				 struct net_device *netdev)
3567 {
3568 	host_info->supported_network_features[0] =
3569 		netdev->features & GENMASK_ULL(31, 0);
3570 	host_info->supported_network_features[1] =
3571 		(netdev->features & GENMASK_ULL(63, 32)) >> 32;
3572 }
3573 
ena_timer_service(struct timer_list * t)3574 static void ena_timer_service(struct timer_list *t)
3575 {
3576 	struct ena_adapter *adapter = from_timer(adapter, t, timer_service);
3577 	u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;
3578 	struct ena_admin_host_info *host_info =
3579 		adapter->ena_dev->host_attr.host_info;
3580 
3581 	check_for_missing_keep_alive(adapter);
3582 
3583 	check_for_admin_com_state(adapter);
3584 
3585 	check_for_missing_completions(adapter);
3586 
3587 	check_for_empty_rx_ring(adapter);
3588 
3589 	if (debug_area)
3590 		ena_dump_stats_to_buf(adapter, debug_area);
3591 
3592 	if (host_info)
3593 		ena_update_host_info(host_info, adapter->netdev);
3594 
3595 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3596 		netif_err(adapter, drv, adapter->netdev,
3597 			  "Trigger reset is on\n");
3598 		ena_dump_stats_to_dmesg(adapter);
3599 		queue_work(ena_wq, &adapter->reset_task);
3600 		return;
3601 	}
3602 
3603 	/* Reset the timer */
3604 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3605 }
3606 
ena_calc_max_io_queue_num(struct pci_dev * pdev,struct ena_com_dev * ena_dev,struct ena_com_dev_get_features_ctx * get_feat_ctx)3607 static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,
3608 				     struct ena_com_dev *ena_dev,
3609 				     struct ena_com_dev_get_features_ctx *get_feat_ctx)
3610 {
3611 	u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
3612 
3613 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
3614 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
3615 			&get_feat_ctx->max_queue_ext.max_queue_ext;
3616 		io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num,
3617 				  max_queue_ext->max_rx_cq_num);
3618 
3619 		io_tx_sq_num = max_queue_ext->max_tx_sq_num;
3620 		io_tx_cq_num = max_queue_ext->max_tx_cq_num;
3621 	} else {
3622 		struct ena_admin_queue_feature_desc *max_queues =
3623 			&get_feat_ctx->max_queues;
3624 		io_tx_sq_num = max_queues->max_sq_num;
3625 		io_tx_cq_num = max_queues->max_cq_num;
3626 		io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num);
3627 	}
3628 
3629 	/* In case of LLQ use the llq fields for the tx SQ/CQ */
3630 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3631 		io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
3632 
3633 	max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
3634 	max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num);
3635 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num);
3636 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);
3637 	/* 1 IRQ for mgmnt and 1 IRQs for each IO direction */
3638 	max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);
3639 
3640 	return max_num_io_queues;
3641 }
3642 
ena_set_dev_offloads(struct ena_com_dev_get_features_ctx * feat,struct net_device * netdev)3643 static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
3644 				 struct net_device *netdev)
3645 {
3646 	netdev_features_t dev_features = 0;
3647 
3648 	/* Set offload features */
3649 	if (feat->offload.tx &
3650 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
3651 		dev_features |= NETIF_F_IP_CSUM;
3652 
3653 	if (feat->offload.tx &
3654 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
3655 		dev_features |= NETIF_F_IPV6_CSUM;
3656 
3657 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
3658 		dev_features |= NETIF_F_TSO;
3659 
3660 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
3661 		dev_features |= NETIF_F_TSO6;
3662 
3663 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
3664 		dev_features |= NETIF_F_TSO_ECN;
3665 
3666 	if (feat->offload.rx_supported &
3667 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
3668 		dev_features |= NETIF_F_RXCSUM;
3669 
3670 	if (feat->offload.rx_supported &
3671 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
3672 		dev_features |= NETIF_F_RXCSUM;
3673 
3674 	netdev->features =
3675 		dev_features |
3676 		NETIF_F_SG |
3677 		NETIF_F_RXHASH |
3678 		NETIF_F_HIGHDMA;
3679 
3680 	netdev->hw_features |= netdev->features;
3681 	netdev->vlan_features |= netdev->features;
3682 }
3683 
ena_set_conf_feat_params(struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * feat)3684 static void ena_set_conf_feat_params(struct ena_adapter *adapter,
3685 				     struct ena_com_dev_get_features_ctx *feat)
3686 {
3687 	struct net_device *netdev = adapter->netdev;
3688 
3689 	/* Copy mac address */
3690 	if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {
3691 		eth_hw_addr_random(netdev);
3692 		ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
3693 	} else {
3694 		ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);
3695 		eth_hw_addr_set(netdev, adapter->mac_addr);
3696 	}
3697 
3698 	/* Set offload features */
3699 	ena_set_dev_offloads(feat, netdev);
3700 
3701 	adapter->max_mtu = feat->dev_attr.max_mtu;
3702 	netdev->max_mtu = adapter->max_mtu;
3703 	netdev->min_mtu = ENA_MIN_MTU;
3704 }
3705 
ena_rss_init_default(struct ena_adapter * adapter)3706 static int ena_rss_init_default(struct ena_adapter *adapter)
3707 {
3708 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3709 	struct device *dev = &adapter->pdev->dev;
3710 	int rc, i;
3711 	u32 val;
3712 
3713 	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
3714 	if (unlikely(rc)) {
3715 		dev_err(dev, "Cannot init indirect table\n");
3716 		goto err_rss_init;
3717 	}
3718 
3719 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
3720 		val = ethtool_rxfh_indir_default(i, adapter->num_io_queues);
3721 		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
3722 						       ENA_IO_RXQ_IDX(val));
3723 		if (unlikely(rc)) {
3724 			dev_err(dev, "Cannot fill indirect table\n");
3725 			goto err_fill_indir;
3726 		}
3727 	}
3728 
3729 	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL, ENA_HASH_KEY_SIZE,
3730 					0xFFFFFFFF);
3731 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
3732 		dev_err(dev, "Cannot fill hash function\n");
3733 		goto err_fill_indir;
3734 	}
3735 
3736 	rc = ena_com_set_default_hash_ctrl(ena_dev);
3737 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
3738 		dev_err(dev, "Cannot fill hash control\n");
3739 		goto err_fill_indir;
3740 	}
3741 
3742 	return 0;
3743 
3744 err_fill_indir:
3745 	ena_com_rss_destroy(ena_dev);
3746 err_rss_init:
3747 
3748 	return rc;
3749 }
3750 
ena_release_bars(struct ena_com_dev * ena_dev,struct pci_dev * pdev)3751 static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
3752 {
3753 	int release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
3754 
3755 	pci_release_selected_regions(pdev, release_bars);
3756 }
3757 
3758 /* ena_probe - Device Initialization Routine
3759  * @pdev: PCI device information struct
3760  * @ent: entry in ena_pci_tbl
3761  *
3762  * Returns 0 on success, negative on failure
3763  *
3764  * ena_probe initializes an adapter identified by a pci_dev structure.
3765  * The OS initialization, configuring of the adapter private structure,
3766  * and a hardware reset occur.
3767  */
ena_probe(struct pci_dev * pdev,const struct pci_device_id * ent)3768 static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3769 {
3770 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3771 	struct ena_com_dev *ena_dev = NULL;
3772 	struct ena_adapter *adapter;
3773 	struct net_device *netdev;
3774 	static int adapters_found;
3775 	u32 max_num_io_queues;
3776 	bool wd_state;
3777 	int bars, rc;
3778 
3779 	dev_dbg(&pdev->dev, "%s\n", __func__);
3780 
3781 	rc = pci_enable_device_mem(pdev);
3782 	if (rc) {
3783 		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
3784 		return rc;
3785 	}
3786 
3787 	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ENA_MAX_PHYS_ADDR_SIZE_BITS));
3788 	if (rc) {
3789 		dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", rc);
3790 		goto err_disable_device;
3791 	}
3792 
3793 	pci_set_master(pdev);
3794 
3795 	ena_dev = vzalloc(sizeof(*ena_dev));
3796 	if (!ena_dev) {
3797 		rc = -ENOMEM;
3798 		goto err_disable_device;
3799 	}
3800 
3801 	bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
3802 	rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
3803 	if (rc) {
3804 		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
3805 			rc);
3806 		goto err_free_ena_dev;
3807 	}
3808 
3809 	ena_dev->reg_bar = devm_ioremap(&pdev->dev,
3810 					pci_resource_start(pdev, ENA_REG_BAR),
3811 					pci_resource_len(pdev, ENA_REG_BAR));
3812 	if (!ena_dev->reg_bar) {
3813 		dev_err(&pdev->dev, "Failed to remap regs bar\n");
3814 		rc = -EFAULT;
3815 		goto err_free_region;
3816 	}
3817 
3818 	ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
3819 
3820 	ena_dev->dmadev = &pdev->dev;
3821 
3822 	netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), ENA_MAX_RINGS);
3823 	if (!netdev) {
3824 		dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
3825 		rc = -ENOMEM;
3826 		goto err_free_region;
3827 	}
3828 
3829 	SET_NETDEV_DEV(netdev, &pdev->dev);
3830 	adapter = netdev_priv(netdev);
3831 	adapter->ena_dev = ena_dev;
3832 	adapter->netdev = netdev;
3833 	adapter->pdev = pdev;
3834 	adapter->msg_enable = DEFAULT_MSG_ENABLE;
3835 
3836 	ena_dev->net_device = netdev;
3837 
3838 	pci_set_drvdata(pdev, adapter);
3839 
3840 	rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
3841 	if (rc) {
3842 		dev_err(&pdev->dev, "ENA LLQ bar mapping failed\n");
3843 		goto err_netdev_destroy;
3844 	}
3845 
3846 	rc = ena_device_init(adapter, pdev, &get_feat_ctx, &wd_state);
3847 	if (rc) {
3848 		dev_err(&pdev->dev, "ENA device init failed\n");
3849 		if (rc == -ETIME)
3850 			rc = -EPROBE_DEFER;
3851 		goto err_netdev_destroy;
3852 	}
3853 
3854 	/* Initial TX and RX interrupt delay. Assumes 1 usec granularity.
3855 	 * Updated during device initialization with the real granularity
3856 	 */
3857 	ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
3858 	ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
3859 	ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
3860 	max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);
3861 	if (unlikely(!max_num_io_queues)) {
3862 		rc = -EFAULT;
3863 		goto err_device_destroy;
3864 	}
3865 
3866 	ena_set_conf_feat_params(adapter, &get_feat_ctx);
3867 
3868 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3869 
3870 	adapter->num_io_queues = max_num_io_queues;
3871 	adapter->max_num_io_queues = max_num_io_queues;
3872 	adapter->last_monitored_tx_qid = 0;
3873 
3874 	adapter->xdp_first_ring = 0;
3875 	adapter->xdp_num_queues = 0;
3876 
3877 	adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
3878 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3879 		adapter->disable_meta_caching =
3880 			!!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
3881 			   BIT(ENA_ADMIN_DISABLE_META_CACHING));
3882 
3883 	adapter->wd_state = wd_state;
3884 
3885 	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
3886 
3887 	rc = ena_com_init_interrupt_moderation(adapter->ena_dev);
3888 	if (rc) {
3889 		dev_err(&pdev->dev,
3890 			"Failed to query interrupt moderation feature\n");
3891 		goto err_device_destroy;
3892 	}
3893 
3894 	ena_init_io_rings(adapter,
3895 			  0,
3896 			  adapter->xdp_num_queues +
3897 			  adapter->num_io_queues);
3898 
3899 	netdev->netdev_ops = &ena_netdev_ops;
3900 	netdev->watchdog_timeo = TX_TIMEOUT;
3901 	ena_set_ethtool_ops(netdev);
3902 
3903 	netdev->priv_flags |= IFF_UNICAST_FLT;
3904 
3905 	u64_stats_init(&adapter->syncp);
3906 
3907 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3908 	if (rc) {
3909 		dev_err(&pdev->dev,
3910 			"Failed to enable and set the admin interrupts\n");
3911 		goto err_worker_destroy;
3912 	}
3913 	rc = ena_rss_init_default(adapter);
3914 	if (rc && (rc != -EOPNOTSUPP)) {
3915 		dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
3916 		goto err_free_msix;
3917 	}
3918 
3919 	ena_config_debug_area(adapter);
3920 
3921 	if (ena_xdp_legal_queue_count(adapter, adapter->num_io_queues))
3922 		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
3923 				       NETDEV_XDP_ACT_REDIRECT;
3924 
3925 	memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
3926 
3927 	netif_carrier_off(netdev);
3928 
3929 	rc = register_netdev(netdev);
3930 	if (rc) {
3931 		dev_err(&pdev->dev, "Cannot register net device\n");
3932 		goto err_rss;
3933 	}
3934 
3935 	INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
3936 
3937 	adapter->last_keep_alive_jiffies = jiffies;
3938 	adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
3939 	adapter->missing_tx_completion_to = TX_TIMEOUT;
3940 	adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS;
3941 
3942 	ena_update_hints(adapter, &get_feat_ctx.hw_hints);
3943 
3944 	timer_setup(&adapter->timer_service, ena_timer_service, 0);
3945 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3946 
3947 	dev_info(&pdev->dev,
3948 		 "%s found at mem %lx, mac addr %pM\n",
3949 		 DEVICE_NAME, (long)pci_resource_start(pdev, 0),
3950 		 netdev->dev_addr);
3951 
3952 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3953 
3954 	adapters_found++;
3955 
3956 	return 0;
3957 
3958 err_rss:
3959 	ena_com_delete_debug_area(ena_dev);
3960 	ena_com_rss_destroy(ena_dev);
3961 err_free_msix:
3962 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
3963 	/* stop submitting admin commands on a device that was reset */
3964 	ena_com_set_admin_running_state(ena_dev, false);
3965 	ena_free_mgmnt_irq(adapter);
3966 	ena_disable_msix(adapter);
3967 err_worker_destroy:
3968 	del_timer(&adapter->timer_service);
3969 err_device_destroy:
3970 	ena_com_delete_host_info(ena_dev);
3971 	ena_com_admin_destroy(ena_dev);
3972 err_netdev_destroy:
3973 	free_netdev(netdev);
3974 err_free_region:
3975 	ena_release_bars(ena_dev, pdev);
3976 err_free_ena_dev:
3977 	vfree(ena_dev);
3978 err_disable_device:
3979 	pci_disable_device(pdev);
3980 	return rc;
3981 }
3982 
3983 /*****************************************************************************/
3984 
3985 /* __ena_shutoff - Helper used in both PCI remove/shutdown routines
3986  * @pdev: PCI device information struct
3987  * @shutdown: Is it a shutdown operation? If false, means it is a removal
3988  *
3989  * __ena_shutoff is a helper routine that does the real work on shutdown and
3990  * removal paths; the difference between those paths is with regards to whether
3991  * dettach or unregister the netdevice.
3992  */
__ena_shutoff(struct pci_dev * pdev,bool shutdown)3993 static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)
3994 {
3995 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
3996 	struct ena_com_dev *ena_dev;
3997 	struct net_device *netdev;
3998 
3999 	ena_dev = adapter->ena_dev;
4000 	netdev = adapter->netdev;
4001 
4002 #ifdef CONFIG_RFS_ACCEL
4003 	if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) {
4004 		free_irq_cpu_rmap(netdev->rx_cpu_rmap);
4005 		netdev->rx_cpu_rmap = NULL;
4006 	}
4007 #endif /* CONFIG_RFS_ACCEL */
4008 
4009 	/* Make sure timer and reset routine won't be called after
4010 	 * freeing device resources.
4011 	 */
4012 	del_timer_sync(&adapter->timer_service);
4013 	cancel_work_sync(&adapter->reset_task);
4014 
4015 	rtnl_lock(); /* lock released inside the below if-else block */
4016 	adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;
4017 	ena_destroy_device(adapter, true);
4018 
4019 	if (shutdown) {
4020 		netif_device_detach(netdev);
4021 		dev_close(netdev);
4022 		rtnl_unlock();
4023 	} else {
4024 		rtnl_unlock();
4025 		unregister_netdev(netdev);
4026 		free_netdev(netdev);
4027 	}
4028 
4029 	ena_com_rss_destroy(ena_dev);
4030 
4031 	ena_com_delete_debug_area(ena_dev);
4032 
4033 	ena_com_delete_host_info(ena_dev);
4034 
4035 	ena_release_bars(ena_dev, pdev);
4036 
4037 	pci_disable_device(pdev);
4038 
4039 	vfree(ena_dev);
4040 }
4041 
4042 /* ena_remove - Device Removal Routine
4043  * @pdev: PCI device information struct
4044  *
4045  * ena_remove is called by the PCI subsystem to alert the driver
4046  * that it should release a PCI device.
4047  */
4048 
ena_remove(struct pci_dev * pdev)4049 static void ena_remove(struct pci_dev *pdev)
4050 {
4051 	__ena_shutoff(pdev, false);
4052 }
4053 
4054 /* ena_shutdown - Device Shutdown Routine
4055  * @pdev: PCI device information struct
4056  *
4057  * ena_shutdown is called by the PCI subsystem to alert the driver that
4058  * a shutdown/reboot (or kexec) is happening and device must be disabled.
4059  */
4060 
ena_shutdown(struct pci_dev * pdev)4061 static void ena_shutdown(struct pci_dev *pdev)
4062 {
4063 	__ena_shutoff(pdev, true);
4064 }
4065 
4066 /* ena_suspend - PM suspend callback
4067  * @dev_d: Device information struct
4068  */
ena_suspend(struct device * dev_d)4069 static int __maybe_unused ena_suspend(struct device *dev_d)
4070 {
4071 	struct pci_dev *pdev = to_pci_dev(dev_d);
4072 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
4073 
4074 	ena_increase_stat(&adapter->dev_stats.suspend, 1, &adapter->syncp);
4075 
4076 	rtnl_lock();
4077 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
4078 		dev_err(&pdev->dev,
4079 			"Ignoring device reset request as the device is being suspended\n");
4080 		clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
4081 	}
4082 	ena_destroy_device(adapter, true);
4083 	rtnl_unlock();
4084 	return 0;
4085 }
4086 
4087 /* ena_resume - PM resume callback
4088  * @dev_d: Device information struct
4089  */
ena_resume(struct device * dev_d)4090 static int __maybe_unused ena_resume(struct device *dev_d)
4091 {
4092 	struct ena_adapter *adapter = dev_get_drvdata(dev_d);
4093 	int rc;
4094 
4095 	ena_increase_stat(&adapter->dev_stats.resume, 1, &adapter->syncp);
4096 
4097 	rtnl_lock();
4098 	rc = ena_restore_device(adapter);
4099 	rtnl_unlock();
4100 	return rc;
4101 }
4102 
4103 static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume);
4104 
4105 static struct pci_driver ena_pci_driver = {
4106 	.name		= DRV_MODULE_NAME,
4107 	.id_table	= ena_pci_tbl,
4108 	.probe		= ena_probe,
4109 	.remove		= ena_remove,
4110 	.shutdown	= ena_shutdown,
4111 	.driver.pm	= &ena_pm_ops,
4112 	.sriov_configure = pci_sriov_configure_simple,
4113 };
4114 
ena_init(void)4115 static int __init ena_init(void)
4116 {
4117 	int ret;
4118 
4119 	ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
4120 	if (!ena_wq) {
4121 		pr_err("Failed to create workqueue\n");
4122 		return -ENOMEM;
4123 	}
4124 
4125 	ret = pci_register_driver(&ena_pci_driver);
4126 	if (ret)
4127 		destroy_workqueue(ena_wq);
4128 
4129 	return ret;
4130 }
4131 
ena_cleanup(void)4132 static void __exit ena_cleanup(void)
4133 {
4134 	pci_unregister_driver(&ena_pci_driver);
4135 
4136 	if (ena_wq) {
4137 		destroy_workqueue(ena_wq);
4138 		ena_wq = NULL;
4139 	}
4140 }
4141 
4142 /******************************************************************************
4143  ******************************** AENQ Handlers *******************************
4144  *****************************************************************************/
4145 /* ena_update_on_link_change:
4146  * Notify the network interface about the change in link status
4147  */
ena_update_on_link_change(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4148 static void ena_update_on_link_change(void *adapter_data,
4149 				      struct ena_admin_aenq_entry *aenq_e)
4150 {
4151 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4152 	struct ena_admin_aenq_link_change_desc *aenq_desc =
4153 		(struct ena_admin_aenq_link_change_desc *)aenq_e;
4154 	int status = aenq_desc->flags &
4155 		ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
4156 
4157 	if (status) {
4158 		netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
4159 		set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4160 		if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags))
4161 			netif_carrier_on(adapter->netdev);
4162 	} else {
4163 		clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4164 		netif_carrier_off(adapter->netdev);
4165 	}
4166 }
4167 
ena_keep_alive_wd(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4168 static void ena_keep_alive_wd(void *adapter_data,
4169 			      struct ena_admin_aenq_entry *aenq_e)
4170 {
4171 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4172 	struct ena_admin_aenq_keep_alive_desc *desc;
4173 	u64 rx_drops;
4174 	u64 tx_drops;
4175 
4176 	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
4177 	adapter->last_keep_alive_jiffies = jiffies;
4178 
4179 	rx_drops = ((u64)desc->rx_drops_high << 32) | desc->rx_drops_low;
4180 	tx_drops = ((u64)desc->tx_drops_high << 32) | desc->tx_drops_low;
4181 
4182 	u64_stats_update_begin(&adapter->syncp);
4183 	/* These stats are accumulated by the device, so the counters indicate
4184 	 * all drops since last reset.
4185 	 */
4186 	adapter->dev_stats.rx_drops = rx_drops;
4187 	adapter->dev_stats.tx_drops = tx_drops;
4188 	u64_stats_update_end(&adapter->syncp);
4189 }
4190 
ena_notification(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4191 static void ena_notification(void *adapter_data,
4192 			     struct ena_admin_aenq_entry *aenq_e)
4193 {
4194 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4195 	struct ena_admin_ena_hw_hints *hints;
4196 
4197 	WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
4198 	     "Invalid group(%x) expected %x\n",
4199 	     aenq_e->aenq_common_desc.group,
4200 	     ENA_ADMIN_NOTIFICATION);
4201 
4202 	switch (aenq_e->aenq_common_desc.syndrome) {
4203 	case ENA_ADMIN_UPDATE_HINTS:
4204 		hints = (struct ena_admin_ena_hw_hints *)
4205 			(&aenq_e->inline_data_w4);
4206 		ena_update_hints(adapter, hints);
4207 		break;
4208 	default:
4209 		netif_err(adapter, drv, adapter->netdev,
4210 			  "Invalid aenq notification link state %d\n",
4211 			  aenq_e->aenq_common_desc.syndrome);
4212 	}
4213 }
4214 
4215 /* This handler will called for unknown event group or unimplemented handlers*/
unimplemented_aenq_handler(void * data,struct ena_admin_aenq_entry * aenq_e)4216 static void unimplemented_aenq_handler(void *data,
4217 				       struct ena_admin_aenq_entry *aenq_e)
4218 {
4219 	struct ena_adapter *adapter = (struct ena_adapter *)data;
4220 
4221 	netif_err(adapter, drv, adapter->netdev,
4222 		  "Unknown event was received or event with unimplemented handler\n");
4223 }
4224 
4225 static struct ena_aenq_handlers aenq_handlers = {
4226 	.handlers = {
4227 		[ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
4228 		[ENA_ADMIN_NOTIFICATION] = ena_notification,
4229 		[ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
4230 	},
4231 	.unimplemented_handler = unimplemented_aenq_handler
4232 };
4233 
4234 module_init(ena_init);
4235 module_exit(ena_cleanup);
4236