1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #ifdef CONFIG_RFS_ACCEL 9 #include <linux/cpu_rmap.h> 10 #endif /* CONFIG_RFS_ACCEL */ 11 #include <linux/ethtool.h> 12 #include <linux/kernel.h> 13 #include <linux/module.h> 14 #include <linux/numa.h> 15 #include <linux/pci.h> 16 #include <linux/utsname.h> 17 #include <linux/version.h> 18 #include <linux/vmalloc.h> 19 #include <net/ip.h> 20 21 #include "ena_netdev.h" 22 #include <linux/bpf_trace.h> 23 #include "ena_pci_id_tbl.h" 24 25 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates"); 26 MODULE_DESCRIPTION(DEVICE_NAME); 27 MODULE_LICENSE("GPL"); 28 29 /* Time in jiffies before concluding the transmitter is hung. */ 30 #define TX_TIMEOUT (5 * HZ) 31 32 #define ENA_MAX_RINGS min_t(unsigned int, ENA_MAX_NUM_IO_QUEUES, num_possible_cpus()) 33 34 #define ENA_NAPI_BUDGET 64 35 36 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \ 37 NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR) 38 39 static struct ena_aenq_handlers aenq_handlers; 40 41 static struct workqueue_struct *ena_wq; 42 43 MODULE_DEVICE_TABLE(pci, ena_pci_tbl); 44 45 static int ena_rss_init_default(struct ena_adapter *adapter); 46 static void check_for_admin_com_state(struct ena_adapter *adapter); 47 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful); 48 static int ena_restore_device(struct ena_adapter *adapter); 49 50 static void ena_init_io_rings(struct ena_adapter *adapter, 51 int first_index, int count); 52 static void ena_init_napi_in_range(struct ena_adapter *adapter, int first_index, 53 int count); 54 static void ena_del_napi_in_range(struct ena_adapter *adapter, int first_index, 55 int count); 56 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid); 57 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter, 58 int first_index, 59 int count); 60 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid); 61 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid); 62 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget); 63 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter); 64 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter); 65 static void ena_napi_disable_in_range(struct ena_adapter *adapter, 66 int first_index, int count); 67 static void ena_napi_enable_in_range(struct ena_adapter *adapter, 68 int first_index, int count); 69 static int ena_up(struct ena_adapter *adapter); 70 static void ena_down(struct ena_adapter *adapter); 71 static void ena_unmask_interrupt(struct ena_ring *tx_ring, 72 struct ena_ring *rx_ring); 73 static void ena_update_ring_numa_node(struct ena_ring *tx_ring, 74 struct ena_ring *rx_ring); 75 static void ena_unmap_tx_buff(struct ena_ring *tx_ring, 76 struct ena_tx_buffer *tx_info); 77 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter, 78 int first_index, int count); 79 80 /* Increase a stat by cnt while holding syncp seqlock on 32bit machines */ 81 static void ena_increase_stat(u64 *statp, u64 cnt, 82 struct u64_stats_sync *syncp) 83 { 84 u64_stats_update_begin(syncp); 85 (*statp) += cnt; 86 u64_stats_update_end(syncp); 87 } 88 89 static void ena_ring_tx_doorbell(struct ena_ring *tx_ring) 90 { 91 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 92 ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, &tx_ring->syncp); 93 } 94 95 static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue) 96 { 97 struct ena_adapter *adapter = netdev_priv(dev); 98 99 /* Change the state of the device to trigger reset 100 * Check that we are not in the middle or a trigger already 101 */ 102 103 if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) 104 return; 105 106 ena_reset_device(adapter, ENA_REGS_RESET_OS_NETDEV_WD); 107 ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp); 108 109 netif_err(adapter, tx_err, dev, "Transmit time out\n"); 110 } 111 112 static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu) 113 { 114 int i; 115 116 for (i = 0; i < adapter->num_io_queues; i++) 117 adapter->rx_ring[i].mtu = mtu; 118 } 119 120 static int ena_change_mtu(struct net_device *dev, int new_mtu) 121 { 122 struct ena_adapter *adapter = netdev_priv(dev); 123 int ret; 124 125 ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); 126 if (!ret) { 127 netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu); 128 update_rx_ring_mtu(adapter, new_mtu); 129 dev->mtu = new_mtu; 130 } else { 131 netif_err(adapter, drv, dev, "Failed to set MTU to %d\n", 132 new_mtu); 133 } 134 135 return ret; 136 } 137 138 static int ena_xmit_common(struct net_device *dev, 139 struct ena_ring *ring, 140 struct ena_tx_buffer *tx_info, 141 struct ena_com_tx_ctx *ena_tx_ctx, 142 u16 next_to_use, 143 u32 bytes) 144 { 145 struct ena_adapter *adapter = netdev_priv(dev); 146 int rc, nb_hw_desc; 147 148 if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq, 149 ena_tx_ctx))) { 150 netif_dbg(adapter, tx_queued, dev, 151 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n", 152 ring->qid); 153 ena_ring_tx_doorbell(ring); 154 } 155 156 /* prepare the packet's descriptors to dma engine */ 157 rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx, 158 &nb_hw_desc); 159 160 /* In case there isn't enough space in the queue for the packet, 161 * we simply drop it. All other failure reasons of 162 * ena_com_prepare_tx() are fatal and therefore require a device reset. 163 */ 164 if (unlikely(rc)) { 165 netif_err(adapter, tx_queued, dev, 166 "Failed to prepare tx bufs\n"); 167 ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1, 168 &ring->syncp); 169 if (rc != -ENOMEM) 170 ena_reset_device(adapter, 171 ENA_REGS_RESET_DRIVER_INVALID_STATE); 172 return rc; 173 } 174 175 u64_stats_update_begin(&ring->syncp); 176 ring->tx_stats.cnt++; 177 ring->tx_stats.bytes += bytes; 178 u64_stats_update_end(&ring->syncp); 179 180 tx_info->tx_descs = nb_hw_desc; 181 tx_info->last_jiffies = jiffies; 182 tx_info->print_once = 0; 183 184 ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, 185 ring->ring_size); 186 return 0; 187 } 188 189 /* This is the XDP napi callback. XDP queues use a separate napi callback 190 * than Rx/Tx queues. 191 */ 192 static int ena_xdp_io_poll(struct napi_struct *napi, int budget) 193 { 194 struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi); 195 u32 xdp_work_done, xdp_budget; 196 struct ena_ring *xdp_ring; 197 int napi_comp_call = 0; 198 int ret; 199 200 xdp_ring = ena_napi->xdp_ring; 201 202 xdp_budget = budget; 203 204 if (!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags) || 205 test_bit(ENA_FLAG_TRIGGER_RESET, &xdp_ring->adapter->flags)) { 206 napi_complete_done(napi, 0); 207 return 0; 208 } 209 210 xdp_work_done = ena_clean_xdp_irq(xdp_ring, xdp_budget); 211 212 /* If the device is about to reset or down, avoid unmask 213 * the interrupt and return 0 so NAPI won't reschedule 214 */ 215 if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags))) { 216 napi_complete_done(napi, 0); 217 ret = 0; 218 } else if (xdp_budget > xdp_work_done) { 219 napi_comp_call = 1; 220 if (napi_complete_done(napi, xdp_work_done)) 221 ena_unmask_interrupt(xdp_ring, NULL); 222 ena_update_ring_numa_node(xdp_ring, NULL); 223 ret = xdp_work_done; 224 } else { 225 ret = xdp_budget; 226 } 227 228 u64_stats_update_begin(&xdp_ring->syncp); 229 xdp_ring->tx_stats.napi_comp += napi_comp_call; 230 xdp_ring->tx_stats.tx_poll++; 231 u64_stats_update_end(&xdp_ring->syncp); 232 xdp_ring->tx_stats.last_napi_jiffies = jiffies; 233 234 return ret; 235 } 236 237 static int ena_xdp_tx_map_frame(struct ena_ring *xdp_ring, 238 struct ena_tx_buffer *tx_info, 239 struct xdp_frame *xdpf, 240 struct ena_com_tx_ctx *ena_tx_ctx) 241 { 242 struct ena_adapter *adapter = xdp_ring->adapter; 243 struct ena_com_buf *ena_buf; 244 int push_len = 0; 245 dma_addr_t dma; 246 void *data; 247 u32 size; 248 249 tx_info->xdpf = xdpf; 250 data = tx_info->xdpf->data; 251 size = tx_info->xdpf->len; 252 253 if (xdp_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 254 /* Designate part of the packet for LLQ */ 255 push_len = min_t(u32, size, xdp_ring->tx_max_header_size); 256 257 ena_tx_ctx->push_header = data; 258 259 size -= push_len; 260 data += push_len; 261 } 262 263 ena_tx_ctx->header_len = push_len; 264 265 if (size > 0) { 266 dma = dma_map_single(xdp_ring->dev, 267 data, 268 size, 269 DMA_TO_DEVICE); 270 if (unlikely(dma_mapping_error(xdp_ring->dev, dma))) 271 goto error_report_dma_error; 272 273 tx_info->map_linear_data = 0; 274 275 ena_buf = tx_info->bufs; 276 ena_buf->paddr = dma; 277 ena_buf->len = size; 278 279 ena_tx_ctx->ena_bufs = ena_buf; 280 ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1; 281 } 282 283 return 0; 284 285 error_report_dma_error: 286 ena_increase_stat(&xdp_ring->tx_stats.dma_mapping_err, 1, 287 &xdp_ring->syncp); 288 netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n"); 289 290 return -EINVAL; 291 } 292 293 static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring, 294 struct net_device *dev, 295 struct xdp_frame *xdpf, 296 int flags) 297 { 298 struct ena_com_tx_ctx ena_tx_ctx = {}; 299 struct ena_tx_buffer *tx_info; 300 u16 next_to_use, req_id; 301 int rc; 302 303 next_to_use = xdp_ring->next_to_use; 304 req_id = xdp_ring->free_ids[next_to_use]; 305 tx_info = &xdp_ring->tx_buffer_info[req_id]; 306 tx_info->num_of_bufs = 0; 307 308 rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &ena_tx_ctx); 309 if (unlikely(rc)) 310 return rc; 311 312 ena_tx_ctx.req_id = req_id; 313 314 rc = ena_xmit_common(dev, 315 xdp_ring, 316 tx_info, 317 &ena_tx_ctx, 318 next_to_use, 319 xdpf->len); 320 if (rc) 321 goto error_unmap_dma; 322 323 /* trigger the dma engine. ena_ring_tx_doorbell() 324 * calls a memory barrier inside it. 325 */ 326 if (flags & XDP_XMIT_FLUSH) 327 ena_ring_tx_doorbell(xdp_ring); 328 329 return rc; 330 331 error_unmap_dma: 332 ena_unmap_tx_buff(xdp_ring, tx_info); 333 tx_info->xdpf = NULL; 334 return rc; 335 } 336 337 static int ena_xdp_xmit(struct net_device *dev, int n, 338 struct xdp_frame **frames, u32 flags) 339 { 340 struct ena_adapter *adapter = netdev_priv(dev); 341 struct ena_ring *xdp_ring; 342 int qid, i, nxmit = 0; 343 344 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 345 return -EINVAL; 346 347 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) 348 return -ENETDOWN; 349 350 /* We assume that all rings have the same XDP program */ 351 if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog)) 352 return -ENXIO; 353 354 qid = smp_processor_id() % adapter->xdp_num_queues; 355 qid += adapter->xdp_first_ring; 356 xdp_ring = &adapter->tx_ring[qid]; 357 358 /* Other CPU ids might try to send thorugh this queue */ 359 spin_lock(&xdp_ring->xdp_tx_lock); 360 361 for (i = 0; i < n; i++) { 362 if (ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0)) 363 break; 364 nxmit++; 365 } 366 367 /* Ring doorbell to make device aware of the packets */ 368 if (flags & XDP_XMIT_FLUSH) 369 ena_ring_tx_doorbell(xdp_ring); 370 371 spin_unlock(&xdp_ring->xdp_tx_lock); 372 373 /* Return number of packets sent */ 374 return nxmit; 375 } 376 377 static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) 378 { 379 struct bpf_prog *xdp_prog; 380 struct ena_ring *xdp_ring; 381 u32 verdict = XDP_PASS; 382 struct xdp_frame *xdpf; 383 u64 *xdp_stat; 384 385 xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog); 386 387 if (!xdp_prog) 388 goto out; 389 390 verdict = bpf_prog_run_xdp(xdp_prog, xdp); 391 392 switch (verdict) { 393 case XDP_TX: 394 xdpf = xdp_convert_buff_to_frame(xdp); 395 if (unlikely(!xdpf)) { 396 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); 397 xdp_stat = &rx_ring->rx_stats.xdp_aborted; 398 verdict = XDP_ABORTED; 399 break; 400 } 401 402 /* Find xmit queue */ 403 xdp_ring = rx_ring->xdp_ring; 404 405 /* The XDP queues are shared between XDP_TX and XDP_REDIRECT */ 406 spin_lock(&xdp_ring->xdp_tx_lock); 407 408 if (ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf, 409 XDP_XMIT_FLUSH)) 410 xdp_return_frame(xdpf); 411 412 spin_unlock(&xdp_ring->xdp_tx_lock); 413 xdp_stat = &rx_ring->rx_stats.xdp_tx; 414 break; 415 case XDP_REDIRECT: 416 if (likely(!xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))) { 417 xdp_stat = &rx_ring->rx_stats.xdp_redirect; 418 break; 419 } 420 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); 421 xdp_stat = &rx_ring->rx_stats.xdp_aborted; 422 verdict = XDP_ABORTED; 423 break; 424 case XDP_ABORTED: 425 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); 426 xdp_stat = &rx_ring->rx_stats.xdp_aborted; 427 break; 428 case XDP_DROP: 429 xdp_stat = &rx_ring->rx_stats.xdp_drop; 430 break; 431 case XDP_PASS: 432 xdp_stat = &rx_ring->rx_stats.xdp_pass; 433 break; 434 default: 435 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict); 436 xdp_stat = &rx_ring->rx_stats.xdp_invalid; 437 } 438 439 ena_increase_stat(xdp_stat, 1, &rx_ring->syncp); 440 out: 441 return verdict; 442 } 443 444 static void ena_init_all_xdp_queues(struct ena_adapter *adapter) 445 { 446 adapter->xdp_first_ring = adapter->num_io_queues; 447 adapter->xdp_num_queues = adapter->num_io_queues; 448 449 ena_init_io_rings(adapter, 450 adapter->xdp_first_ring, 451 adapter->xdp_num_queues); 452 } 453 454 static int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter) 455 { 456 int rc = 0; 457 458 rc = ena_setup_tx_resources_in_range(adapter, adapter->xdp_first_ring, 459 adapter->xdp_num_queues); 460 if (rc) 461 goto setup_err; 462 463 rc = ena_create_io_tx_queues_in_range(adapter, 464 adapter->xdp_first_ring, 465 adapter->xdp_num_queues); 466 if (rc) 467 goto create_err; 468 469 return 0; 470 471 create_err: 472 ena_free_all_io_tx_resources(adapter); 473 setup_err: 474 return rc; 475 } 476 477 /* Provides a way for both kernel and bpf-prog to know 478 * more about the RX-queue a given XDP frame arrived on. 479 */ 480 static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring) 481 { 482 int rc; 483 484 rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0); 485 486 if (rc) { 487 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, 488 "Failed to register xdp rx queue info. RX queue num %d rc: %d\n", 489 rx_ring->qid, rc); 490 goto err; 491 } 492 493 rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, 494 NULL); 495 496 if (rc) { 497 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, 498 "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n", 499 rx_ring->qid, rc); 500 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 501 } 502 503 err: 504 return rc; 505 } 506 507 static void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring) 508 { 509 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq); 510 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 511 } 512 513 static void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter, 514 struct bpf_prog *prog, 515 int first, int count) 516 { 517 struct ena_ring *rx_ring; 518 int i = 0; 519 520 for (i = first; i < count; i++) { 521 rx_ring = &adapter->rx_ring[i]; 522 xchg(&rx_ring->xdp_bpf_prog, prog); 523 if (prog) { 524 ena_xdp_register_rxq_info(rx_ring); 525 rx_ring->rx_headroom = XDP_PACKET_HEADROOM; 526 } else { 527 ena_xdp_unregister_rxq_info(rx_ring); 528 rx_ring->rx_headroom = NET_SKB_PAD; 529 } 530 } 531 } 532 533 static void ena_xdp_exchange_program(struct ena_adapter *adapter, 534 struct bpf_prog *prog) 535 { 536 struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog); 537 538 ena_xdp_exchange_program_rx_in_range(adapter, 539 prog, 540 0, 541 adapter->num_io_queues); 542 543 if (old_bpf_prog) 544 bpf_prog_put(old_bpf_prog); 545 } 546 547 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter) 548 { 549 bool was_up; 550 int rc; 551 552 was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); 553 554 if (was_up) 555 ena_down(adapter); 556 557 adapter->xdp_first_ring = 0; 558 adapter->xdp_num_queues = 0; 559 ena_xdp_exchange_program(adapter, NULL); 560 if (was_up) { 561 rc = ena_up(adapter); 562 if (rc) 563 return rc; 564 } 565 return 0; 566 } 567 568 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf) 569 { 570 struct ena_adapter *adapter = netdev_priv(netdev); 571 struct bpf_prog *prog = bpf->prog; 572 struct bpf_prog *old_bpf_prog; 573 int rc, prev_mtu; 574 bool is_up; 575 576 is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); 577 rc = ena_xdp_allowed(adapter); 578 if (rc == ENA_XDP_ALLOWED) { 579 old_bpf_prog = adapter->xdp_bpf_prog; 580 if (prog) { 581 if (!is_up) { 582 ena_init_all_xdp_queues(adapter); 583 } else if (!old_bpf_prog) { 584 ena_down(adapter); 585 ena_init_all_xdp_queues(adapter); 586 } 587 ena_xdp_exchange_program(adapter, prog); 588 589 if (is_up && !old_bpf_prog) { 590 rc = ena_up(adapter); 591 if (rc) 592 return rc; 593 } 594 } else if (old_bpf_prog) { 595 rc = ena_destroy_and_free_all_xdp_queues(adapter); 596 if (rc) 597 return rc; 598 } 599 600 prev_mtu = netdev->max_mtu; 601 netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu; 602 603 if (!old_bpf_prog) 604 netif_info(adapter, drv, adapter->netdev, 605 "XDP program is set, changing the max_mtu from %d to %d", 606 prev_mtu, netdev->max_mtu); 607 608 } else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) { 609 netif_err(adapter, drv, adapter->netdev, 610 "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on", 611 netdev->mtu, ENA_XDP_MAX_MTU); 612 NL_SET_ERR_MSG_MOD(bpf->extack, 613 "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info"); 614 return -EINVAL; 615 } else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) { 616 netif_err(adapter, drv, adapter->netdev, 617 "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n", 618 adapter->num_io_queues, adapter->max_num_io_queues); 619 NL_SET_ERR_MSG_MOD(bpf->extack, 620 "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info"); 621 return -EINVAL; 622 } 623 624 return 0; 625 } 626 627 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp 628 * program as well as to query the current xdp program id. 629 */ 630 static int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf) 631 { 632 switch (bpf->command) { 633 case XDP_SETUP_PROG: 634 return ena_xdp_set(netdev, bpf); 635 default: 636 return -EINVAL; 637 } 638 return 0; 639 } 640 641 static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter) 642 { 643 #ifdef CONFIG_RFS_ACCEL 644 u32 i; 645 int rc; 646 647 adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_io_queues); 648 if (!adapter->netdev->rx_cpu_rmap) 649 return -ENOMEM; 650 for (i = 0; i < adapter->num_io_queues; i++) { 651 int irq_idx = ENA_IO_IRQ_IDX(i); 652 653 rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap, 654 pci_irq_vector(adapter->pdev, irq_idx)); 655 if (rc) { 656 free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap); 657 adapter->netdev->rx_cpu_rmap = NULL; 658 return rc; 659 } 660 } 661 #endif /* CONFIG_RFS_ACCEL */ 662 return 0; 663 } 664 665 static void ena_init_io_rings_common(struct ena_adapter *adapter, 666 struct ena_ring *ring, u16 qid) 667 { 668 ring->qid = qid; 669 ring->pdev = adapter->pdev; 670 ring->dev = &adapter->pdev->dev; 671 ring->netdev = adapter->netdev; 672 ring->napi = &adapter->ena_napi[qid].napi; 673 ring->adapter = adapter; 674 ring->ena_dev = adapter->ena_dev; 675 ring->per_napi_packets = 0; 676 ring->cpu = 0; 677 ring->no_interrupt_event_cnt = 0; 678 u64_stats_init(&ring->syncp); 679 } 680 681 static void ena_init_io_rings(struct ena_adapter *adapter, 682 int first_index, int count) 683 { 684 struct ena_com_dev *ena_dev; 685 struct ena_ring *txr, *rxr; 686 int i; 687 688 ena_dev = adapter->ena_dev; 689 690 for (i = first_index; i < first_index + count; i++) { 691 txr = &adapter->tx_ring[i]; 692 rxr = &adapter->rx_ring[i]; 693 694 /* TX common ring state */ 695 ena_init_io_rings_common(adapter, txr, i); 696 697 /* TX specific ring state */ 698 txr->ring_size = adapter->requested_tx_ring_size; 699 txr->tx_max_header_size = ena_dev->tx_max_header_size; 700 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; 701 txr->sgl_size = adapter->max_tx_sgl_size; 702 txr->smoothed_interval = 703 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev); 704 txr->disable_meta_caching = adapter->disable_meta_caching; 705 spin_lock_init(&txr->xdp_tx_lock); 706 707 /* Don't init RX queues for xdp queues */ 708 if (!ENA_IS_XDP_INDEX(adapter, i)) { 709 /* RX common ring state */ 710 ena_init_io_rings_common(adapter, rxr, i); 711 712 /* RX specific ring state */ 713 rxr->ring_size = adapter->requested_rx_ring_size; 714 rxr->rx_copybreak = adapter->rx_copybreak; 715 rxr->sgl_size = adapter->max_rx_sgl_size; 716 rxr->smoothed_interval = 717 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev); 718 rxr->empty_rx_queue = 0; 719 rxr->rx_headroom = NET_SKB_PAD; 720 adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 721 rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues]; 722 } 723 } 724 } 725 726 /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors) 727 * @adapter: network interface device structure 728 * @qid: queue index 729 * 730 * Return 0 on success, negative on failure 731 */ 732 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid) 733 { 734 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 735 struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)]; 736 int size, i, node; 737 738 if (tx_ring->tx_buffer_info) { 739 netif_err(adapter, ifup, 740 adapter->netdev, "tx_buffer_info info is not NULL"); 741 return -EEXIST; 742 } 743 744 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; 745 node = cpu_to_node(ena_irq->cpu); 746 747 tx_ring->tx_buffer_info = vzalloc_node(size, node); 748 if (!tx_ring->tx_buffer_info) { 749 tx_ring->tx_buffer_info = vzalloc(size); 750 if (!tx_ring->tx_buffer_info) 751 goto err_tx_buffer_info; 752 } 753 754 size = sizeof(u16) * tx_ring->ring_size; 755 tx_ring->free_ids = vzalloc_node(size, node); 756 if (!tx_ring->free_ids) { 757 tx_ring->free_ids = vzalloc(size); 758 if (!tx_ring->free_ids) 759 goto err_tx_free_ids; 760 } 761 762 size = tx_ring->tx_max_header_size; 763 tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node); 764 if (!tx_ring->push_buf_intermediate_buf) { 765 tx_ring->push_buf_intermediate_buf = vzalloc(size); 766 if (!tx_ring->push_buf_intermediate_buf) 767 goto err_push_buf_intermediate_buf; 768 } 769 770 /* Req id ring for TX out of order completions */ 771 for (i = 0; i < tx_ring->ring_size; i++) 772 tx_ring->free_ids[i] = i; 773 774 /* Reset tx statistics */ 775 memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats)); 776 777 tx_ring->next_to_use = 0; 778 tx_ring->next_to_clean = 0; 779 tx_ring->cpu = ena_irq->cpu; 780 return 0; 781 782 err_push_buf_intermediate_buf: 783 vfree(tx_ring->free_ids); 784 tx_ring->free_ids = NULL; 785 err_tx_free_ids: 786 vfree(tx_ring->tx_buffer_info); 787 tx_ring->tx_buffer_info = NULL; 788 err_tx_buffer_info: 789 return -ENOMEM; 790 } 791 792 /* ena_free_tx_resources - Free I/O Tx Resources per Queue 793 * @adapter: network interface device structure 794 * @qid: queue index 795 * 796 * Free all transmit software resources 797 */ 798 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid) 799 { 800 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 801 802 vfree(tx_ring->tx_buffer_info); 803 tx_ring->tx_buffer_info = NULL; 804 805 vfree(tx_ring->free_ids); 806 tx_ring->free_ids = NULL; 807 808 vfree(tx_ring->push_buf_intermediate_buf); 809 tx_ring->push_buf_intermediate_buf = NULL; 810 } 811 812 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter, 813 int first_index, 814 int count) 815 { 816 int i, rc = 0; 817 818 for (i = first_index; i < first_index + count; i++) { 819 rc = ena_setup_tx_resources(adapter, i); 820 if (rc) 821 goto err_setup_tx; 822 } 823 824 return 0; 825 826 err_setup_tx: 827 828 netif_err(adapter, ifup, adapter->netdev, 829 "Tx queue %d: allocation failed\n", i); 830 831 /* rewind the index freeing the rings as we go */ 832 while (first_index < i--) 833 ena_free_tx_resources(adapter, i); 834 return rc; 835 } 836 837 static void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter, 838 int first_index, int count) 839 { 840 int i; 841 842 for (i = first_index; i < first_index + count; i++) 843 ena_free_tx_resources(adapter, i); 844 } 845 846 /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues 847 * @adapter: board private structure 848 * 849 * Free all transmit software resources 850 */ 851 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter) 852 { 853 ena_free_all_io_tx_resources_in_range(adapter, 854 0, 855 adapter->xdp_num_queues + 856 adapter->num_io_queues); 857 } 858 859 /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors) 860 * @adapter: network interface device structure 861 * @qid: queue index 862 * 863 * Returns 0 on success, negative on failure 864 */ 865 static int ena_setup_rx_resources(struct ena_adapter *adapter, 866 u32 qid) 867 { 868 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 869 struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)]; 870 int size, node, i; 871 872 if (rx_ring->rx_buffer_info) { 873 netif_err(adapter, ifup, adapter->netdev, 874 "rx_buffer_info is not NULL"); 875 return -EEXIST; 876 } 877 878 /* alloc extra element so in rx path 879 * we can always prefetch rx_info + 1 880 */ 881 size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1); 882 node = cpu_to_node(ena_irq->cpu); 883 884 rx_ring->rx_buffer_info = vzalloc_node(size, node); 885 if (!rx_ring->rx_buffer_info) { 886 rx_ring->rx_buffer_info = vzalloc(size); 887 if (!rx_ring->rx_buffer_info) 888 return -ENOMEM; 889 } 890 891 size = sizeof(u16) * rx_ring->ring_size; 892 rx_ring->free_ids = vzalloc_node(size, node); 893 if (!rx_ring->free_ids) { 894 rx_ring->free_ids = vzalloc(size); 895 if (!rx_ring->free_ids) { 896 vfree(rx_ring->rx_buffer_info); 897 rx_ring->rx_buffer_info = NULL; 898 return -ENOMEM; 899 } 900 } 901 902 /* Req id ring for receiving RX pkts out of order */ 903 for (i = 0; i < rx_ring->ring_size; i++) 904 rx_ring->free_ids[i] = i; 905 906 /* Reset rx statistics */ 907 memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats)); 908 909 rx_ring->next_to_clean = 0; 910 rx_ring->next_to_use = 0; 911 rx_ring->cpu = ena_irq->cpu; 912 913 return 0; 914 } 915 916 /* ena_free_rx_resources - Free I/O Rx Resources 917 * @adapter: network interface device structure 918 * @qid: queue index 919 * 920 * Free all receive software resources 921 */ 922 static void ena_free_rx_resources(struct ena_adapter *adapter, 923 u32 qid) 924 { 925 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 926 927 vfree(rx_ring->rx_buffer_info); 928 rx_ring->rx_buffer_info = NULL; 929 930 vfree(rx_ring->free_ids); 931 rx_ring->free_ids = NULL; 932 } 933 934 /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues 935 * @adapter: board private structure 936 * 937 * Return 0 on success, negative on failure 938 */ 939 static int ena_setup_all_rx_resources(struct ena_adapter *adapter) 940 { 941 int i, rc = 0; 942 943 for (i = 0; i < adapter->num_io_queues; i++) { 944 rc = ena_setup_rx_resources(adapter, i); 945 if (rc) 946 goto err_setup_rx; 947 } 948 949 return 0; 950 951 err_setup_rx: 952 953 netif_err(adapter, ifup, adapter->netdev, 954 "Rx queue %d: allocation failed\n", i); 955 956 /* rewind the index freeing the rings as we go */ 957 while (i--) 958 ena_free_rx_resources(adapter, i); 959 return rc; 960 } 961 962 /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues 963 * @adapter: board private structure 964 * 965 * Free all receive software resources 966 */ 967 static void ena_free_all_io_rx_resources(struct ena_adapter *adapter) 968 { 969 int i; 970 971 for (i = 0; i < adapter->num_io_queues; i++) 972 ena_free_rx_resources(adapter, i); 973 } 974 975 static struct page *ena_alloc_map_page(struct ena_ring *rx_ring, 976 dma_addr_t *dma) 977 { 978 struct page *page; 979 980 /* This would allocate the page on the same NUMA node the executing code 981 * is running on. 982 */ 983 page = dev_alloc_page(); 984 if (!page) { 985 ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, 986 &rx_ring->syncp); 987 return ERR_PTR(-ENOSPC); 988 } 989 990 /* To enable NIC-side port-mirroring, AKA SPAN port, 991 * we make the buffer readable from the nic as well 992 */ 993 *dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE, 994 DMA_BIDIRECTIONAL); 995 if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) { 996 ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1, 997 &rx_ring->syncp); 998 __free_page(page); 999 return ERR_PTR(-EIO); 1000 } 1001 1002 return page; 1003 } 1004 1005 static int ena_alloc_rx_buffer(struct ena_ring *rx_ring, 1006 struct ena_rx_buffer *rx_info) 1007 { 1008 int headroom = rx_ring->rx_headroom; 1009 struct ena_com_buf *ena_buf; 1010 struct page *page; 1011 dma_addr_t dma; 1012 int tailroom; 1013 1014 /* restore page offset value in case it has been changed by device */ 1015 rx_info->page_offset = headroom; 1016 1017 /* if previous allocated page is not used */ 1018 if (unlikely(rx_info->page)) 1019 return 0; 1020 1021 /* We handle DMA here */ 1022 page = ena_alloc_map_page(rx_ring, &dma); 1023 if (unlikely(IS_ERR(page))) 1024 return PTR_ERR(page); 1025 1026 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, 1027 "Allocate page %p, rx_info %p\n", page, rx_info); 1028 1029 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1030 1031 rx_info->page = page; 1032 ena_buf = &rx_info->ena_buf; 1033 ena_buf->paddr = dma + headroom; 1034 ena_buf->len = ENA_PAGE_SIZE - headroom - tailroom; 1035 1036 return 0; 1037 } 1038 1039 static void ena_unmap_rx_buff(struct ena_ring *rx_ring, 1040 struct ena_rx_buffer *rx_info) 1041 { 1042 struct ena_com_buf *ena_buf = &rx_info->ena_buf; 1043 1044 dma_unmap_page(rx_ring->dev, ena_buf->paddr - rx_ring->rx_headroom, 1045 ENA_PAGE_SIZE, 1046 DMA_BIDIRECTIONAL); 1047 } 1048 1049 static void ena_free_rx_page(struct ena_ring *rx_ring, 1050 struct ena_rx_buffer *rx_info) 1051 { 1052 struct page *page = rx_info->page; 1053 1054 if (unlikely(!page)) { 1055 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, 1056 "Trying to free unallocated buffer\n"); 1057 return; 1058 } 1059 1060 ena_unmap_rx_buff(rx_ring, rx_info); 1061 1062 __free_page(page); 1063 rx_info->page = NULL; 1064 } 1065 1066 static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num) 1067 { 1068 u16 next_to_use, req_id; 1069 u32 i; 1070 int rc; 1071 1072 next_to_use = rx_ring->next_to_use; 1073 1074 for (i = 0; i < num; i++) { 1075 struct ena_rx_buffer *rx_info; 1076 1077 req_id = rx_ring->free_ids[next_to_use]; 1078 1079 rx_info = &rx_ring->rx_buffer_info[req_id]; 1080 1081 rc = ena_alloc_rx_buffer(rx_ring, rx_info); 1082 if (unlikely(rc < 0)) { 1083 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, 1084 "Failed to allocate buffer for rx queue %d\n", 1085 rx_ring->qid); 1086 break; 1087 } 1088 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, 1089 &rx_info->ena_buf, 1090 req_id); 1091 if (unlikely(rc)) { 1092 netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev, 1093 "Failed to add buffer for rx queue %d\n", 1094 rx_ring->qid); 1095 break; 1096 } 1097 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, 1098 rx_ring->ring_size); 1099 } 1100 1101 if (unlikely(i < num)) { 1102 ena_increase_stat(&rx_ring->rx_stats.refil_partial, 1, 1103 &rx_ring->syncp); 1104 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, 1105 "Refilled rx qid %d with only %d buffers (from %d)\n", 1106 rx_ring->qid, i, num); 1107 } 1108 1109 /* ena_com_write_sq_doorbell issues a wmb() */ 1110 if (likely(i)) 1111 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); 1112 1113 rx_ring->next_to_use = next_to_use; 1114 1115 return i; 1116 } 1117 1118 static void ena_free_rx_bufs(struct ena_adapter *adapter, 1119 u32 qid) 1120 { 1121 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 1122 u32 i; 1123 1124 for (i = 0; i < rx_ring->ring_size; i++) { 1125 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; 1126 1127 if (rx_info->page) 1128 ena_free_rx_page(rx_ring, rx_info); 1129 } 1130 } 1131 1132 /* ena_refill_all_rx_bufs - allocate all queues Rx buffers 1133 * @adapter: board private structure 1134 */ 1135 static void ena_refill_all_rx_bufs(struct ena_adapter *adapter) 1136 { 1137 struct ena_ring *rx_ring; 1138 int i, rc, bufs_num; 1139 1140 for (i = 0; i < adapter->num_io_queues; i++) { 1141 rx_ring = &adapter->rx_ring[i]; 1142 bufs_num = rx_ring->ring_size - 1; 1143 rc = ena_refill_rx_bufs(rx_ring, bufs_num); 1144 1145 if (unlikely(rc != bufs_num)) 1146 netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev, 1147 "Refilling Queue %d failed. allocated %d buffers from: %d\n", 1148 i, rc, bufs_num); 1149 } 1150 } 1151 1152 static void ena_free_all_rx_bufs(struct ena_adapter *adapter) 1153 { 1154 int i; 1155 1156 for (i = 0; i < adapter->num_io_queues; i++) 1157 ena_free_rx_bufs(adapter, i); 1158 } 1159 1160 static void ena_unmap_tx_buff(struct ena_ring *tx_ring, 1161 struct ena_tx_buffer *tx_info) 1162 { 1163 struct ena_com_buf *ena_buf; 1164 u32 cnt; 1165 int i; 1166 1167 ena_buf = tx_info->bufs; 1168 cnt = tx_info->num_of_bufs; 1169 1170 if (unlikely(!cnt)) 1171 return; 1172 1173 if (tx_info->map_linear_data) { 1174 dma_unmap_single(tx_ring->dev, 1175 dma_unmap_addr(ena_buf, paddr), 1176 dma_unmap_len(ena_buf, len), 1177 DMA_TO_DEVICE); 1178 ena_buf++; 1179 cnt--; 1180 } 1181 1182 /* unmap remaining mapped pages */ 1183 for (i = 0; i < cnt; i++) { 1184 dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr), 1185 dma_unmap_len(ena_buf, len), DMA_TO_DEVICE); 1186 ena_buf++; 1187 } 1188 } 1189 1190 /* ena_free_tx_bufs - Free Tx Buffers per Queue 1191 * @tx_ring: TX ring for which buffers be freed 1192 */ 1193 static void ena_free_tx_bufs(struct ena_ring *tx_ring) 1194 { 1195 bool print_once = true; 1196 u32 i; 1197 1198 for (i = 0; i < tx_ring->ring_size; i++) { 1199 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; 1200 1201 if (!tx_info->skb) 1202 continue; 1203 1204 if (print_once) { 1205 netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev, 1206 "Free uncompleted tx skb qid %d idx 0x%x\n", 1207 tx_ring->qid, i); 1208 print_once = false; 1209 } else { 1210 netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev, 1211 "Free uncompleted tx skb qid %d idx 0x%x\n", 1212 tx_ring->qid, i); 1213 } 1214 1215 ena_unmap_tx_buff(tx_ring, tx_info); 1216 1217 dev_kfree_skb_any(tx_info->skb); 1218 } 1219 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev, 1220 tx_ring->qid)); 1221 } 1222 1223 static void ena_free_all_tx_bufs(struct ena_adapter *adapter) 1224 { 1225 struct ena_ring *tx_ring; 1226 int i; 1227 1228 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { 1229 tx_ring = &adapter->tx_ring[i]; 1230 ena_free_tx_bufs(tx_ring); 1231 } 1232 } 1233 1234 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter) 1235 { 1236 u16 ena_qid; 1237 int i; 1238 1239 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { 1240 ena_qid = ENA_IO_TXQ_IDX(i); 1241 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1242 } 1243 } 1244 1245 static void ena_destroy_all_rx_queues(struct ena_adapter *adapter) 1246 { 1247 u16 ena_qid; 1248 int i; 1249 1250 for (i = 0; i < adapter->num_io_queues; i++) { 1251 ena_qid = ENA_IO_RXQ_IDX(i); 1252 cancel_work_sync(&adapter->ena_napi[i].dim.work); 1253 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1254 } 1255 } 1256 1257 static void ena_destroy_all_io_queues(struct ena_adapter *adapter) 1258 { 1259 ena_destroy_all_tx_queues(adapter); 1260 ena_destroy_all_rx_queues(adapter); 1261 } 1262 1263 static int handle_invalid_req_id(struct ena_ring *ring, u16 req_id, 1264 struct ena_tx_buffer *tx_info, bool is_xdp) 1265 { 1266 if (tx_info) 1267 netif_err(ring->adapter, 1268 tx_done, 1269 ring->netdev, 1270 "tx_info doesn't have valid %s. qid %u req_id %u", 1271 is_xdp ? "xdp frame" : "skb", ring->qid, req_id); 1272 else 1273 netif_err(ring->adapter, 1274 tx_done, 1275 ring->netdev, 1276 "Invalid req_id %u in qid %u\n", 1277 req_id, ring->qid); 1278 1279 ena_increase_stat(&ring->tx_stats.bad_req_id, 1, &ring->syncp); 1280 ena_reset_device(ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID); 1281 1282 return -EFAULT; 1283 } 1284 1285 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) 1286 { 1287 struct ena_tx_buffer *tx_info; 1288 1289 tx_info = &tx_ring->tx_buffer_info[req_id]; 1290 if (likely(tx_info->skb)) 1291 return 0; 1292 1293 return handle_invalid_req_id(tx_ring, req_id, tx_info, false); 1294 } 1295 1296 static int validate_xdp_req_id(struct ena_ring *xdp_ring, u16 req_id) 1297 { 1298 struct ena_tx_buffer *tx_info; 1299 1300 tx_info = &xdp_ring->tx_buffer_info[req_id]; 1301 if (likely(tx_info->xdpf)) 1302 return 0; 1303 1304 return handle_invalid_req_id(xdp_ring, req_id, tx_info, true); 1305 } 1306 1307 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget) 1308 { 1309 struct netdev_queue *txq; 1310 bool above_thresh; 1311 u32 tx_bytes = 0; 1312 u32 total_done = 0; 1313 u16 next_to_clean; 1314 u16 req_id; 1315 int tx_pkts = 0; 1316 int rc; 1317 1318 next_to_clean = tx_ring->next_to_clean; 1319 txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid); 1320 1321 while (tx_pkts < budget) { 1322 struct ena_tx_buffer *tx_info; 1323 struct sk_buff *skb; 1324 1325 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, 1326 &req_id); 1327 if (rc) { 1328 if (unlikely(rc == -EINVAL)) 1329 handle_invalid_req_id(tx_ring, req_id, NULL, 1330 false); 1331 break; 1332 } 1333 1334 /* validate that the request id points to a valid skb */ 1335 rc = validate_tx_req_id(tx_ring, req_id); 1336 if (rc) 1337 break; 1338 1339 tx_info = &tx_ring->tx_buffer_info[req_id]; 1340 skb = tx_info->skb; 1341 1342 /* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */ 1343 prefetch(&skb->end); 1344 1345 tx_info->skb = NULL; 1346 tx_info->last_jiffies = 0; 1347 1348 ena_unmap_tx_buff(tx_ring, tx_info); 1349 1350 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, 1351 "tx_poll: q %d skb %p completed\n", tx_ring->qid, 1352 skb); 1353 1354 tx_bytes += skb->len; 1355 dev_kfree_skb(skb); 1356 tx_pkts++; 1357 total_done += tx_info->tx_descs; 1358 1359 tx_ring->free_ids[next_to_clean] = req_id; 1360 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, 1361 tx_ring->ring_size); 1362 } 1363 1364 tx_ring->next_to_clean = next_to_clean; 1365 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done); 1366 ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); 1367 1368 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes); 1369 1370 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, 1371 "tx_poll: q %d done. total pkts: %d\n", 1372 tx_ring->qid, tx_pkts); 1373 1374 /* need to make the rings circular update visible to 1375 * ena_start_xmit() before checking for netif_queue_stopped(). 1376 */ 1377 smp_mb(); 1378 1379 above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1380 ENA_TX_WAKEUP_THRESH); 1381 if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) { 1382 __netif_tx_lock(txq, smp_processor_id()); 1383 above_thresh = 1384 ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1385 ENA_TX_WAKEUP_THRESH); 1386 if (netif_tx_queue_stopped(txq) && above_thresh && 1387 test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) { 1388 netif_tx_wake_queue(txq); 1389 ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1, 1390 &tx_ring->syncp); 1391 } 1392 __netif_tx_unlock(txq); 1393 } 1394 1395 return tx_pkts; 1396 } 1397 1398 static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag) 1399 { 1400 struct sk_buff *skb; 1401 1402 if (!first_frag) 1403 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 1404 rx_ring->rx_copybreak); 1405 else 1406 skb = build_skb(first_frag, ENA_PAGE_SIZE); 1407 1408 if (unlikely(!skb)) { 1409 ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1, 1410 &rx_ring->syncp); 1411 1412 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, 1413 "Failed to allocate skb. first_frag %s\n", 1414 first_frag ? "provided" : "not provided"); 1415 return NULL; 1416 } 1417 1418 return skb; 1419 } 1420 1421 static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, 1422 struct ena_com_rx_buf_info *ena_bufs, 1423 u32 descs, 1424 u16 *next_to_clean) 1425 { 1426 struct ena_rx_buffer *rx_info; 1427 struct ena_adapter *adapter; 1428 u16 len, req_id, buf = 0; 1429 struct sk_buff *skb; 1430 void *page_addr; 1431 u32 page_offset; 1432 void *data_addr; 1433 1434 len = ena_bufs[buf].len; 1435 req_id = ena_bufs[buf].req_id; 1436 1437 rx_info = &rx_ring->rx_buffer_info[req_id]; 1438 1439 if (unlikely(!rx_info->page)) { 1440 adapter = rx_ring->adapter; 1441 netif_err(adapter, rx_err, rx_ring->netdev, 1442 "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id); 1443 ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp); 1444 ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); 1445 return NULL; 1446 } 1447 1448 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, 1449 "rx_info %p page %p\n", 1450 rx_info, rx_info->page); 1451 1452 /* save virt address of first buffer */ 1453 page_addr = page_address(rx_info->page); 1454 page_offset = rx_info->page_offset; 1455 data_addr = page_addr + page_offset; 1456 1457 prefetch(data_addr); 1458 1459 if (len <= rx_ring->rx_copybreak) { 1460 skb = ena_alloc_skb(rx_ring, NULL); 1461 if (unlikely(!skb)) 1462 return NULL; 1463 1464 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, 1465 "RX allocated small packet. len %d. data_len %d\n", 1466 skb->len, skb->data_len); 1467 1468 /* sync this buffer for CPU use */ 1469 dma_sync_single_for_cpu(rx_ring->dev, 1470 dma_unmap_addr(&rx_info->ena_buf, paddr), 1471 len, 1472 DMA_FROM_DEVICE); 1473 skb_copy_to_linear_data(skb, data_addr, len); 1474 dma_sync_single_for_device(rx_ring->dev, 1475 dma_unmap_addr(&rx_info->ena_buf, paddr), 1476 len, 1477 DMA_FROM_DEVICE); 1478 1479 skb_put(skb, len); 1480 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1481 rx_ring->free_ids[*next_to_clean] = req_id; 1482 *next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs, 1483 rx_ring->ring_size); 1484 return skb; 1485 } 1486 1487 ena_unmap_rx_buff(rx_ring, rx_info); 1488 1489 skb = ena_alloc_skb(rx_ring, page_addr); 1490 if (unlikely(!skb)) 1491 return NULL; 1492 1493 /* Populate skb's linear part */ 1494 skb_reserve(skb, page_offset); 1495 skb_put(skb, len); 1496 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1497 1498 do { 1499 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, 1500 "RX skb updated. len %d. data_len %d\n", 1501 skb->len, skb->data_len); 1502 1503 rx_info->page = NULL; 1504 1505 rx_ring->free_ids[*next_to_clean] = req_id; 1506 *next_to_clean = 1507 ENA_RX_RING_IDX_NEXT(*next_to_clean, 1508 rx_ring->ring_size); 1509 if (likely(--descs == 0)) 1510 break; 1511 1512 buf++; 1513 len = ena_bufs[buf].len; 1514 req_id = ena_bufs[buf].req_id; 1515 1516 rx_info = &rx_ring->rx_buffer_info[req_id]; 1517 1518 ena_unmap_rx_buff(rx_ring, rx_info); 1519 1520 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page, 1521 rx_info->page_offset, len, ENA_PAGE_SIZE); 1522 1523 } while (1); 1524 1525 return skb; 1526 } 1527 1528 /* ena_rx_checksum - indicate in skb if hw indicated a good cksum 1529 * @adapter: structure containing adapter specific data 1530 * @ena_rx_ctx: received packet context/metadata 1531 * @skb: skb currently being received and modified 1532 */ 1533 static void ena_rx_checksum(struct ena_ring *rx_ring, 1534 struct ena_com_rx_ctx *ena_rx_ctx, 1535 struct sk_buff *skb) 1536 { 1537 /* Rx csum disabled */ 1538 if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) { 1539 skb->ip_summed = CHECKSUM_NONE; 1540 return; 1541 } 1542 1543 /* For fragmented packets the checksum isn't valid */ 1544 if (ena_rx_ctx->frag) { 1545 skb->ip_summed = CHECKSUM_NONE; 1546 return; 1547 } 1548 1549 /* if IP and error */ 1550 if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) && 1551 (ena_rx_ctx->l3_csum_err))) { 1552 /* ipv4 checksum error */ 1553 skb->ip_summed = CHECKSUM_NONE; 1554 ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1, 1555 &rx_ring->syncp); 1556 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, 1557 "RX IPv4 header checksum error\n"); 1558 return; 1559 } 1560 1561 /* if TCP/UDP */ 1562 if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || 1563 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) { 1564 if (unlikely(ena_rx_ctx->l4_csum_err)) { 1565 /* TCP/UDP checksum error */ 1566 ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1, 1567 &rx_ring->syncp); 1568 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, 1569 "RX L4 checksum error\n"); 1570 skb->ip_summed = CHECKSUM_NONE; 1571 return; 1572 } 1573 1574 if (likely(ena_rx_ctx->l4_csum_checked)) { 1575 skb->ip_summed = CHECKSUM_UNNECESSARY; 1576 ena_increase_stat(&rx_ring->rx_stats.csum_good, 1, 1577 &rx_ring->syncp); 1578 } else { 1579 ena_increase_stat(&rx_ring->rx_stats.csum_unchecked, 1, 1580 &rx_ring->syncp); 1581 skb->ip_summed = CHECKSUM_NONE; 1582 } 1583 } else { 1584 skb->ip_summed = CHECKSUM_NONE; 1585 return; 1586 } 1587 1588 } 1589 1590 static void ena_set_rx_hash(struct ena_ring *rx_ring, 1591 struct ena_com_rx_ctx *ena_rx_ctx, 1592 struct sk_buff *skb) 1593 { 1594 enum pkt_hash_types hash_type; 1595 1596 if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) { 1597 if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || 1598 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) 1599 1600 hash_type = PKT_HASH_TYPE_L4; 1601 else 1602 hash_type = PKT_HASH_TYPE_NONE; 1603 1604 /* Override hash type if the packet is fragmented */ 1605 if (ena_rx_ctx->frag) 1606 hash_type = PKT_HASH_TYPE_NONE; 1607 1608 skb_set_hash(skb, ena_rx_ctx->hash, hash_type); 1609 } 1610 } 1611 1612 static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp) 1613 { 1614 struct ena_rx_buffer *rx_info; 1615 int ret; 1616 1617 rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id]; 1618 xdp_prepare_buff(xdp, page_address(rx_info->page), 1619 rx_info->page_offset, 1620 rx_ring->ena_bufs[0].len, false); 1621 /* If for some reason we received a bigger packet than 1622 * we expect, then we simply drop it 1623 */ 1624 if (unlikely(rx_ring->ena_bufs[0].len > ENA_XDP_MAX_MTU)) 1625 return XDP_DROP; 1626 1627 ret = ena_xdp_execute(rx_ring, xdp); 1628 1629 /* The xdp program might expand the headers */ 1630 if (ret == XDP_PASS) { 1631 rx_info->page_offset = xdp->data - xdp->data_hard_start; 1632 rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data; 1633 } 1634 1635 return ret; 1636 } 1637 /* ena_clean_rx_irq - Cleanup RX irq 1638 * @rx_ring: RX ring to clean 1639 * @napi: napi handler 1640 * @budget: how many packets driver is allowed to clean 1641 * 1642 * Returns the number of cleaned buffers. 1643 */ 1644 static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, 1645 u32 budget) 1646 { 1647 u16 next_to_clean = rx_ring->next_to_clean; 1648 struct ena_com_rx_ctx ena_rx_ctx; 1649 struct ena_rx_buffer *rx_info; 1650 struct ena_adapter *adapter; 1651 u32 res_budget, work_done; 1652 int rx_copybreak_pkt = 0; 1653 int refill_threshold; 1654 struct sk_buff *skb; 1655 int refill_required; 1656 struct xdp_buff xdp; 1657 int xdp_flags = 0; 1658 int total_len = 0; 1659 int xdp_verdict; 1660 int rc = 0; 1661 int i; 1662 1663 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, 1664 "%s qid %d\n", __func__, rx_ring->qid); 1665 res_budget = budget; 1666 xdp_init_buff(&xdp, ENA_PAGE_SIZE, &rx_ring->xdp_rxq); 1667 1668 do { 1669 xdp_verdict = XDP_PASS; 1670 skb = NULL; 1671 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1672 ena_rx_ctx.max_bufs = rx_ring->sgl_size; 1673 ena_rx_ctx.descs = 0; 1674 ena_rx_ctx.pkt_offset = 0; 1675 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1676 rx_ring->ena_com_io_sq, 1677 &ena_rx_ctx); 1678 if (unlikely(rc)) 1679 goto error; 1680 1681 if (unlikely(ena_rx_ctx.descs == 0)) 1682 break; 1683 1684 /* First descriptor might have an offset set by the device */ 1685 rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id]; 1686 rx_info->page_offset += ena_rx_ctx.pkt_offset; 1687 1688 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, 1689 "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n", 1690 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, 1691 ena_rx_ctx.l4_proto, ena_rx_ctx.hash); 1692 1693 if (ena_xdp_present_ring(rx_ring)) 1694 xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp); 1695 1696 /* allocate skb and fill it */ 1697 if (xdp_verdict == XDP_PASS) 1698 skb = ena_rx_skb(rx_ring, 1699 rx_ring->ena_bufs, 1700 ena_rx_ctx.descs, 1701 &next_to_clean); 1702 1703 if (unlikely(!skb)) { 1704 for (i = 0; i < ena_rx_ctx.descs; i++) { 1705 int req_id = rx_ring->ena_bufs[i].req_id; 1706 1707 rx_ring->free_ids[next_to_clean] = req_id; 1708 next_to_clean = 1709 ENA_RX_RING_IDX_NEXT(next_to_clean, 1710 rx_ring->ring_size); 1711 1712 /* Packets was passed for transmission, unmap it 1713 * from RX side. 1714 */ 1715 if (xdp_verdict == XDP_TX || xdp_verdict == XDP_REDIRECT) { 1716 ena_unmap_rx_buff(rx_ring, 1717 &rx_ring->rx_buffer_info[req_id]); 1718 rx_ring->rx_buffer_info[req_id].page = NULL; 1719 } 1720 } 1721 if (xdp_verdict != XDP_PASS) { 1722 xdp_flags |= xdp_verdict; 1723 res_budget--; 1724 continue; 1725 } 1726 break; 1727 } 1728 1729 ena_rx_checksum(rx_ring, &ena_rx_ctx, skb); 1730 1731 ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb); 1732 1733 skb_record_rx_queue(skb, rx_ring->qid); 1734 1735 if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak) 1736 rx_copybreak_pkt++; 1737 1738 total_len += skb->len; 1739 1740 napi_gro_receive(napi, skb); 1741 1742 res_budget--; 1743 } while (likely(res_budget)); 1744 1745 work_done = budget - res_budget; 1746 rx_ring->per_napi_packets += work_done; 1747 u64_stats_update_begin(&rx_ring->syncp); 1748 rx_ring->rx_stats.bytes += total_len; 1749 rx_ring->rx_stats.cnt += work_done; 1750 rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt; 1751 u64_stats_update_end(&rx_ring->syncp); 1752 1753 rx_ring->next_to_clean = next_to_clean; 1754 1755 refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq); 1756 refill_threshold = 1757 min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER, 1758 ENA_RX_REFILL_THRESH_PACKET); 1759 1760 /* Optimization, try to batch new rx buffers */ 1761 if (refill_required > refill_threshold) { 1762 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); 1763 ena_refill_rx_bufs(rx_ring, refill_required); 1764 } 1765 1766 if (xdp_flags & XDP_REDIRECT) 1767 xdp_do_flush_map(); 1768 1769 return work_done; 1770 1771 error: 1772 adapter = netdev_priv(rx_ring->netdev); 1773 1774 if (rc == -ENOSPC) { 1775 ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1, 1776 &rx_ring->syncp); 1777 ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS); 1778 } else { 1779 ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, 1780 &rx_ring->syncp); 1781 ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); 1782 } 1783 return 0; 1784 } 1785 1786 static void ena_dim_work(struct work_struct *w) 1787 { 1788 struct dim *dim = container_of(w, struct dim, work); 1789 struct dim_cq_moder cur_moder = 1790 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 1791 struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim); 1792 1793 ena_napi->rx_ring->smoothed_interval = cur_moder.usec; 1794 dim->state = DIM_START_MEASURE; 1795 } 1796 1797 static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi) 1798 { 1799 struct dim_sample dim_sample; 1800 struct ena_ring *rx_ring = ena_napi->rx_ring; 1801 1802 if (!rx_ring->per_napi_packets) 1803 return; 1804 1805 rx_ring->non_empty_napi_events++; 1806 1807 dim_update_sample(rx_ring->non_empty_napi_events, 1808 rx_ring->rx_stats.cnt, 1809 rx_ring->rx_stats.bytes, 1810 &dim_sample); 1811 1812 net_dim(&ena_napi->dim, dim_sample); 1813 1814 rx_ring->per_napi_packets = 0; 1815 } 1816 1817 static void ena_unmask_interrupt(struct ena_ring *tx_ring, 1818 struct ena_ring *rx_ring) 1819 { 1820 struct ena_eth_io_intr_reg intr_reg; 1821 u32 rx_interval = 0; 1822 /* Rx ring can be NULL when for XDP tx queues which don't have an 1823 * accompanying rx_ring pair. 1824 */ 1825 if (rx_ring) 1826 rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ? 1827 rx_ring->smoothed_interval : 1828 ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev); 1829 1830 /* Update intr register: rx intr delay, 1831 * tx intr delay and interrupt unmask 1832 */ 1833 ena_com_update_intr_reg(&intr_reg, 1834 rx_interval, 1835 tx_ring->smoothed_interval, 1836 true); 1837 1838 ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1, 1839 &tx_ring->syncp); 1840 1841 /* It is a shared MSI-X. 1842 * Tx and Rx CQ have pointer to it. 1843 * So we use one of them to reach the intr reg 1844 * The Tx ring is used because the rx_ring is NULL for XDP queues 1845 */ 1846 ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg); 1847 } 1848 1849 static void ena_update_ring_numa_node(struct ena_ring *tx_ring, 1850 struct ena_ring *rx_ring) 1851 { 1852 int cpu = get_cpu(); 1853 int numa_node; 1854 1855 /* Check only one ring since the 2 rings are running on the same cpu */ 1856 if (likely(tx_ring->cpu == cpu)) 1857 goto out; 1858 1859 numa_node = cpu_to_node(cpu); 1860 put_cpu(); 1861 1862 if (numa_node != NUMA_NO_NODE) { 1863 ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node); 1864 if (rx_ring) 1865 ena_com_update_numa_node(rx_ring->ena_com_io_cq, 1866 numa_node); 1867 } 1868 1869 tx_ring->cpu = cpu; 1870 if (rx_ring) 1871 rx_ring->cpu = cpu; 1872 1873 return; 1874 out: 1875 put_cpu(); 1876 } 1877 1878 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget) 1879 { 1880 u32 total_done = 0; 1881 u16 next_to_clean; 1882 u32 tx_bytes = 0; 1883 int tx_pkts = 0; 1884 u16 req_id; 1885 int rc; 1886 1887 if (unlikely(!xdp_ring)) 1888 return 0; 1889 next_to_clean = xdp_ring->next_to_clean; 1890 1891 while (tx_pkts < budget) { 1892 struct ena_tx_buffer *tx_info; 1893 struct xdp_frame *xdpf; 1894 1895 rc = ena_com_tx_comp_req_id_get(xdp_ring->ena_com_io_cq, 1896 &req_id); 1897 if (rc) { 1898 if (unlikely(rc == -EINVAL)) 1899 handle_invalid_req_id(xdp_ring, req_id, NULL, 1900 true); 1901 break; 1902 } 1903 1904 /* validate that the request id points to a valid xdp_frame */ 1905 rc = validate_xdp_req_id(xdp_ring, req_id); 1906 if (rc) 1907 break; 1908 1909 tx_info = &xdp_ring->tx_buffer_info[req_id]; 1910 xdpf = tx_info->xdpf; 1911 1912 tx_info->xdpf = NULL; 1913 tx_info->last_jiffies = 0; 1914 ena_unmap_tx_buff(xdp_ring, tx_info); 1915 1916 netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev, 1917 "tx_poll: q %d skb %p completed\n", xdp_ring->qid, 1918 xdpf); 1919 1920 tx_bytes += xdpf->len; 1921 tx_pkts++; 1922 total_done += tx_info->tx_descs; 1923 1924 xdp_return_frame(xdpf); 1925 xdp_ring->free_ids[next_to_clean] = req_id; 1926 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, 1927 xdp_ring->ring_size); 1928 } 1929 1930 xdp_ring->next_to_clean = next_to_clean; 1931 ena_com_comp_ack(xdp_ring->ena_com_io_sq, total_done); 1932 ena_com_update_dev_comp_head(xdp_ring->ena_com_io_cq); 1933 1934 netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev, 1935 "tx_poll: q %d done. total pkts: %d\n", 1936 xdp_ring->qid, tx_pkts); 1937 1938 return tx_pkts; 1939 } 1940 1941 static int ena_io_poll(struct napi_struct *napi, int budget) 1942 { 1943 struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi); 1944 struct ena_ring *tx_ring, *rx_ring; 1945 int tx_work_done; 1946 int rx_work_done = 0; 1947 int tx_budget; 1948 int napi_comp_call = 0; 1949 int ret; 1950 1951 tx_ring = ena_napi->tx_ring; 1952 rx_ring = ena_napi->rx_ring; 1953 1954 tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER; 1955 1956 if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) || 1957 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) { 1958 napi_complete_done(napi, 0); 1959 return 0; 1960 } 1961 1962 tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget); 1963 /* On netpoll the budget is zero and the handler should only clean the 1964 * tx completions. 1965 */ 1966 if (likely(budget)) 1967 rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget); 1968 1969 /* If the device is about to reset or down, avoid unmask 1970 * the interrupt and return 0 so NAPI won't reschedule 1971 */ 1972 if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) || 1973 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) { 1974 napi_complete_done(napi, 0); 1975 ret = 0; 1976 1977 } else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) { 1978 napi_comp_call = 1; 1979 1980 /* Update numa and unmask the interrupt only when schedule 1981 * from the interrupt context (vs from sk_busy_loop) 1982 */ 1983 if (napi_complete_done(napi, rx_work_done) && 1984 READ_ONCE(ena_napi->interrupts_masked)) { 1985 smp_rmb(); /* make sure interrupts_masked is read */ 1986 WRITE_ONCE(ena_napi->interrupts_masked, false); 1987 /* We apply adaptive moderation on Rx path only. 1988 * Tx uses static interrupt moderation. 1989 */ 1990 if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev)) 1991 ena_adjust_adaptive_rx_intr_moderation(ena_napi); 1992 1993 ena_unmask_interrupt(tx_ring, rx_ring); 1994 } 1995 1996 ena_update_ring_numa_node(tx_ring, rx_ring); 1997 1998 ret = rx_work_done; 1999 } else { 2000 ret = budget; 2001 } 2002 2003 u64_stats_update_begin(&tx_ring->syncp); 2004 tx_ring->tx_stats.napi_comp += napi_comp_call; 2005 tx_ring->tx_stats.tx_poll++; 2006 u64_stats_update_end(&tx_ring->syncp); 2007 2008 tx_ring->tx_stats.last_napi_jiffies = jiffies; 2009 2010 return ret; 2011 } 2012 2013 static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data) 2014 { 2015 struct ena_adapter *adapter = (struct ena_adapter *)data; 2016 2017 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); 2018 2019 /* Don't call the aenq handler before probe is done */ 2020 if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))) 2021 ena_com_aenq_intr_handler(adapter->ena_dev, data); 2022 2023 return IRQ_HANDLED; 2024 } 2025 2026 /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx 2027 * @irq: interrupt number 2028 * @data: pointer to a network interface private napi device structure 2029 */ 2030 static irqreturn_t ena_intr_msix_io(int irq, void *data) 2031 { 2032 struct ena_napi *ena_napi = data; 2033 2034 /* Used to check HW health */ 2035 WRITE_ONCE(ena_napi->first_interrupt, true); 2036 2037 WRITE_ONCE(ena_napi->interrupts_masked, true); 2038 smp_wmb(); /* write interrupts_masked before calling napi */ 2039 2040 napi_schedule_irqoff(&ena_napi->napi); 2041 2042 return IRQ_HANDLED; 2043 } 2044 2045 /* Reserve a single MSI-X vector for management (admin + aenq). 2046 * plus reserve one vector for each potential io queue. 2047 * the number of potential io queues is the minimum of what the device 2048 * supports and the number of vCPUs. 2049 */ 2050 static int ena_enable_msix(struct ena_adapter *adapter) 2051 { 2052 int msix_vecs, irq_cnt; 2053 2054 if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) { 2055 netif_err(adapter, probe, adapter->netdev, 2056 "Error, MSI-X is already enabled\n"); 2057 return -EPERM; 2058 } 2059 2060 /* Reserved the max msix vectors we might need */ 2061 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); 2062 netif_dbg(adapter, probe, adapter->netdev, 2063 "Trying to enable MSI-X, vectors %d\n", msix_vecs); 2064 2065 irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC, 2066 msix_vecs, PCI_IRQ_MSIX); 2067 2068 if (irq_cnt < 0) { 2069 netif_err(adapter, probe, adapter->netdev, 2070 "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt); 2071 return -ENOSPC; 2072 } 2073 2074 if (irq_cnt != msix_vecs) { 2075 netif_notice(adapter, probe, adapter->netdev, 2076 "Enable only %d MSI-X (out of %d), reduce the number of queues\n", 2077 irq_cnt, msix_vecs); 2078 adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC; 2079 } 2080 2081 if (ena_init_rx_cpu_rmap(adapter)) 2082 netif_warn(adapter, probe, adapter->netdev, 2083 "Failed to map IRQs to CPUs\n"); 2084 2085 adapter->msix_vecs = irq_cnt; 2086 set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags); 2087 2088 return 0; 2089 } 2090 2091 static void ena_setup_mgmnt_intr(struct ena_adapter *adapter) 2092 { 2093 u32 cpu; 2094 2095 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, 2096 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s", 2097 pci_name(adapter->pdev)); 2098 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = 2099 ena_intr_msix_mgmnt; 2100 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; 2101 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = 2102 pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX); 2103 cpu = cpumask_first(cpu_online_mask); 2104 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu; 2105 cpumask_set_cpu(cpu, 2106 &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask); 2107 } 2108 2109 static void ena_setup_io_intr(struct ena_adapter *adapter) 2110 { 2111 struct net_device *netdev; 2112 int irq_idx, i, cpu; 2113 int io_queue_count; 2114 2115 netdev = adapter->netdev; 2116 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; 2117 2118 for (i = 0; i < io_queue_count; i++) { 2119 irq_idx = ENA_IO_IRQ_IDX(i); 2120 cpu = i % num_online_cpus(); 2121 2122 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, 2123 "%s-Tx-Rx-%d", netdev->name, i); 2124 adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io; 2125 adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i]; 2126 adapter->irq_tbl[irq_idx].vector = 2127 pci_irq_vector(adapter->pdev, irq_idx); 2128 adapter->irq_tbl[irq_idx].cpu = cpu; 2129 2130 cpumask_set_cpu(cpu, 2131 &adapter->irq_tbl[irq_idx].affinity_hint_mask); 2132 } 2133 } 2134 2135 static int ena_request_mgmnt_irq(struct ena_adapter *adapter) 2136 { 2137 unsigned long flags = 0; 2138 struct ena_irq *irq; 2139 int rc; 2140 2141 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 2142 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 2143 irq->data); 2144 if (rc) { 2145 netif_err(adapter, probe, adapter->netdev, 2146 "Failed to request admin irq\n"); 2147 return rc; 2148 } 2149 2150 netif_dbg(adapter, probe, adapter->netdev, 2151 "Set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n", 2152 irq->affinity_hint_mask.bits[0], irq->vector); 2153 2154 irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask); 2155 2156 return rc; 2157 } 2158 2159 static int ena_request_io_irq(struct ena_adapter *adapter) 2160 { 2161 u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; 2162 unsigned long flags = 0; 2163 struct ena_irq *irq; 2164 int rc = 0, i, k; 2165 2166 if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) { 2167 netif_err(adapter, ifup, adapter->netdev, 2168 "Failed to request I/O IRQ: MSI-X is not enabled\n"); 2169 return -EINVAL; 2170 } 2171 2172 for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) { 2173 irq = &adapter->irq_tbl[i]; 2174 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 2175 irq->data); 2176 if (rc) { 2177 netif_err(adapter, ifup, adapter->netdev, 2178 "Failed to request I/O IRQ. index %d rc %d\n", 2179 i, rc); 2180 goto err; 2181 } 2182 2183 netif_dbg(adapter, ifup, adapter->netdev, 2184 "Set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n", 2185 i, irq->affinity_hint_mask.bits[0], irq->vector); 2186 2187 irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask); 2188 } 2189 2190 return rc; 2191 2192 err: 2193 for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) { 2194 irq = &adapter->irq_tbl[k]; 2195 free_irq(irq->vector, irq->data); 2196 } 2197 2198 return rc; 2199 } 2200 2201 static void ena_free_mgmnt_irq(struct ena_adapter *adapter) 2202 { 2203 struct ena_irq *irq; 2204 2205 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 2206 synchronize_irq(irq->vector); 2207 irq_set_affinity_hint(irq->vector, NULL); 2208 free_irq(irq->vector, irq->data); 2209 } 2210 2211 static void ena_free_io_irq(struct ena_adapter *adapter) 2212 { 2213 u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; 2214 struct ena_irq *irq; 2215 int i; 2216 2217 #ifdef CONFIG_RFS_ACCEL 2218 if (adapter->msix_vecs >= 1) { 2219 free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap); 2220 adapter->netdev->rx_cpu_rmap = NULL; 2221 } 2222 #endif /* CONFIG_RFS_ACCEL */ 2223 2224 for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) { 2225 irq = &adapter->irq_tbl[i]; 2226 irq_set_affinity_hint(irq->vector, NULL); 2227 free_irq(irq->vector, irq->data); 2228 } 2229 } 2230 2231 static void ena_disable_msix(struct ena_adapter *adapter) 2232 { 2233 if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) 2234 pci_free_irq_vectors(adapter->pdev); 2235 } 2236 2237 static void ena_disable_io_intr_sync(struct ena_adapter *adapter) 2238 { 2239 u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; 2240 int i; 2241 2242 if (!netif_running(adapter->netdev)) 2243 return; 2244 2245 for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) 2246 synchronize_irq(adapter->irq_tbl[i].vector); 2247 } 2248 2249 static void ena_del_napi_in_range(struct ena_adapter *adapter, 2250 int first_index, 2251 int count) 2252 { 2253 int i; 2254 2255 for (i = first_index; i < first_index + count; i++) { 2256 netif_napi_del(&adapter->ena_napi[i].napi); 2257 2258 WARN_ON(!ENA_IS_XDP_INDEX(adapter, i) && 2259 adapter->ena_napi[i].xdp_ring); 2260 } 2261 } 2262 2263 static void ena_init_napi_in_range(struct ena_adapter *adapter, 2264 int first_index, int count) 2265 { 2266 int i; 2267 2268 for (i = first_index; i < first_index + count; i++) { 2269 struct ena_napi *napi = &adapter->ena_napi[i]; 2270 2271 netif_napi_add(adapter->netdev, 2272 &napi->napi, 2273 ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll, 2274 ENA_NAPI_BUDGET); 2275 2276 if (!ENA_IS_XDP_INDEX(adapter, i)) { 2277 napi->rx_ring = &adapter->rx_ring[i]; 2278 napi->tx_ring = &adapter->tx_ring[i]; 2279 } else { 2280 napi->xdp_ring = &adapter->tx_ring[i]; 2281 } 2282 napi->qid = i; 2283 } 2284 } 2285 2286 static void ena_napi_disable_in_range(struct ena_adapter *adapter, 2287 int first_index, 2288 int count) 2289 { 2290 int i; 2291 2292 for (i = first_index; i < first_index + count; i++) 2293 napi_disable(&adapter->ena_napi[i].napi); 2294 } 2295 2296 static void ena_napi_enable_in_range(struct ena_adapter *adapter, 2297 int first_index, 2298 int count) 2299 { 2300 int i; 2301 2302 for (i = first_index; i < first_index + count; i++) 2303 napi_enable(&adapter->ena_napi[i].napi); 2304 } 2305 2306 /* Configure the Rx forwarding */ 2307 static int ena_rss_configure(struct ena_adapter *adapter) 2308 { 2309 struct ena_com_dev *ena_dev = adapter->ena_dev; 2310 int rc; 2311 2312 /* In case the RSS table wasn't initialized by probe */ 2313 if (!ena_dev->rss.tbl_log_size) { 2314 rc = ena_rss_init_default(adapter); 2315 if (rc && (rc != -EOPNOTSUPP)) { 2316 netif_err(adapter, ifup, adapter->netdev, 2317 "Failed to init RSS rc: %d\n", rc); 2318 return rc; 2319 } 2320 } 2321 2322 /* Set indirect table */ 2323 rc = ena_com_indirect_table_set(ena_dev); 2324 if (unlikely(rc && rc != -EOPNOTSUPP)) 2325 return rc; 2326 2327 /* Configure hash function (if supported) */ 2328 rc = ena_com_set_hash_function(ena_dev); 2329 if (unlikely(rc && (rc != -EOPNOTSUPP))) 2330 return rc; 2331 2332 /* Configure hash inputs (if supported) */ 2333 rc = ena_com_set_hash_ctrl(ena_dev); 2334 if (unlikely(rc && (rc != -EOPNOTSUPP))) 2335 return rc; 2336 2337 return 0; 2338 } 2339 2340 static int ena_up_complete(struct ena_adapter *adapter) 2341 { 2342 int rc; 2343 2344 rc = ena_rss_configure(adapter); 2345 if (rc) 2346 return rc; 2347 2348 ena_change_mtu(adapter->netdev, adapter->netdev->mtu); 2349 2350 ena_refill_all_rx_bufs(adapter); 2351 2352 /* enable transmits */ 2353 netif_tx_start_all_queues(adapter->netdev); 2354 2355 ena_napi_enable_in_range(adapter, 2356 0, 2357 adapter->xdp_num_queues + adapter->num_io_queues); 2358 2359 return 0; 2360 } 2361 2362 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid) 2363 { 2364 struct ena_com_create_io_ctx ctx; 2365 struct ena_com_dev *ena_dev; 2366 struct ena_ring *tx_ring; 2367 u32 msix_vector; 2368 u16 ena_qid; 2369 int rc; 2370 2371 ena_dev = adapter->ena_dev; 2372 2373 tx_ring = &adapter->tx_ring[qid]; 2374 msix_vector = ENA_IO_IRQ_IDX(qid); 2375 ena_qid = ENA_IO_TXQ_IDX(qid); 2376 2377 memset(&ctx, 0x0, sizeof(ctx)); 2378 2379 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 2380 ctx.qid = ena_qid; 2381 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 2382 ctx.msix_vector = msix_vector; 2383 ctx.queue_size = tx_ring->ring_size; 2384 ctx.numa_node = cpu_to_node(tx_ring->cpu); 2385 2386 rc = ena_com_create_io_queue(ena_dev, &ctx); 2387 if (rc) { 2388 netif_err(adapter, ifup, adapter->netdev, 2389 "Failed to create I/O TX queue num %d rc: %d\n", 2390 qid, rc); 2391 return rc; 2392 } 2393 2394 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 2395 &tx_ring->ena_com_io_sq, 2396 &tx_ring->ena_com_io_cq); 2397 if (rc) { 2398 netif_err(adapter, ifup, adapter->netdev, 2399 "Failed to get TX queue handlers. TX queue num %d rc: %d\n", 2400 qid, rc); 2401 ena_com_destroy_io_queue(ena_dev, ena_qid); 2402 return rc; 2403 } 2404 2405 ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node); 2406 return rc; 2407 } 2408 2409 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter, 2410 int first_index, int count) 2411 { 2412 struct ena_com_dev *ena_dev = adapter->ena_dev; 2413 int rc, i; 2414 2415 for (i = first_index; i < first_index + count; i++) { 2416 rc = ena_create_io_tx_queue(adapter, i); 2417 if (rc) 2418 goto create_err; 2419 } 2420 2421 return 0; 2422 2423 create_err: 2424 while (i-- > first_index) 2425 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i)); 2426 2427 return rc; 2428 } 2429 2430 static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid) 2431 { 2432 struct ena_com_dev *ena_dev; 2433 struct ena_com_create_io_ctx ctx; 2434 struct ena_ring *rx_ring; 2435 u32 msix_vector; 2436 u16 ena_qid; 2437 int rc; 2438 2439 ena_dev = adapter->ena_dev; 2440 2441 rx_ring = &adapter->rx_ring[qid]; 2442 msix_vector = ENA_IO_IRQ_IDX(qid); 2443 ena_qid = ENA_IO_RXQ_IDX(qid); 2444 2445 memset(&ctx, 0x0, sizeof(ctx)); 2446 2447 ctx.qid = ena_qid; 2448 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 2449 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2450 ctx.msix_vector = msix_vector; 2451 ctx.queue_size = rx_ring->ring_size; 2452 ctx.numa_node = cpu_to_node(rx_ring->cpu); 2453 2454 rc = ena_com_create_io_queue(ena_dev, &ctx); 2455 if (rc) { 2456 netif_err(adapter, ifup, adapter->netdev, 2457 "Failed to create I/O RX queue num %d rc: %d\n", 2458 qid, rc); 2459 return rc; 2460 } 2461 2462 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 2463 &rx_ring->ena_com_io_sq, 2464 &rx_ring->ena_com_io_cq); 2465 if (rc) { 2466 netif_err(adapter, ifup, adapter->netdev, 2467 "Failed to get RX queue handlers. RX queue num %d rc: %d\n", 2468 qid, rc); 2469 goto err; 2470 } 2471 2472 ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node); 2473 2474 return rc; 2475 err: 2476 ena_com_destroy_io_queue(ena_dev, ena_qid); 2477 return rc; 2478 } 2479 2480 static int ena_create_all_io_rx_queues(struct ena_adapter *adapter) 2481 { 2482 struct ena_com_dev *ena_dev = adapter->ena_dev; 2483 int rc, i; 2484 2485 for (i = 0; i < adapter->num_io_queues; i++) { 2486 rc = ena_create_io_rx_queue(adapter, i); 2487 if (rc) 2488 goto create_err; 2489 INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work); 2490 } 2491 2492 return 0; 2493 2494 create_err: 2495 while (i--) { 2496 cancel_work_sync(&adapter->ena_napi[i].dim.work); 2497 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i)); 2498 } 2499 2500 return rc; 2501 } 2502 2503 static void set_io_rings_size(struct ena_adapter *adapter, 2504 int new_tx_size, 2505 int new_rx_size) 2506 { 2507 int i; 2508 2509 for (i = 0; i < adapter->num_io_queues; i++) { 2510 adapter->tx_ring[i].ring_size = new_tx_size; 2511 adapter->rx_ring[i].ring_size = new_rx_size; 2512 } 2513 } 2514 2515 /* This function allows queue allocation to backoff when the system is 2516 * low on memory. If there is not enough memory to allocate io queues 2517 * the driver will try to allocate smaller queues. 2518 * 2519 * The backoff algorithm is as follows: 2520 * 1. Try to allocate TX and RX and if successful. 2521 * 1.1. return success 2522 * 2523 * 2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same). 2524 * 2525 * 3. If TX or RX is smaller than 256 2526 * 3.1. return failure. 2527 * 4. else 2528 * 4.1. go back to 1. 2529 */ 2530 static int create_queues_with_size_backoff(struct ena_adapter *adapter) 2531 { 2532 int rc, cur_rx_ring_size, cur_tx_ring_size; 2533 int new_rx_ring_size, new_tx_ring_size; 2534 2535 /* current queue sizes might be set to smaller than the requested 2536 * ones due to past queue allocation failures. 2537 */ 2538 set_io_rings_size(adapter, adapter->requested_tx_ring_size, 2539 adapter->requested_rx_ring_size); 2540 2541 while (1) { 2542 if (ena_xdp_present(adapter)) { 2543 rc = ena_setup_and_create_all_xdp_queues(adapter); 2544 2545 if (rc) 2546 goto err_setup_tx; 2547 } 2548 rc = ena_setup_tx_resources_in_range(adapter, 2549 0, 2550 adapter->num_io_queues); 2551 if (rc) 2552 goto err_setup_tx; 2553 2554 rc = ena_create_io_tx_queues_in_range(adapter, 2555 0, 2556 adapter->num_io_queues); 2557 if (rc) 2558 goto err_create_tx_queues; 2559 2560 rc = ena_setup_all_rx_resources(adapter); 2561 if (rc) 2562 goto err_setup_rx; 2563 2564 rc = ena_create_all_io_rx_queues(adapter); 2565 if (rc) 2566 goto err_create_rx_queues; 2567 2568 return 0; 2569 2570 err_create_rx_queues: 2571 ena_free_all_io_rx_resources(adapter); 2572 err_setup_rx: 2573 ena_destroy_all_tx_queues(adapter); 2574 err_create_tx_queues: 2575 ena_free_all_io_tx_resources(adapter); 2576 err_setup_tx: 2577 if (rc != -ENOMEM) { 2578 netif_err(adapter, ifup, adapter->netdev, 2579 "Queue creation failed with error code %d\n", 2580 rc); 2581 return rc; 2582 } 2583 2584 cur_tx_ring_size = adapter->tx_ring[0].ring_size; 2585 cur_rx_ring_size = adapter->rx_ring[0].ring_size; 2586 2587 netif_err(adapter, ifup, adapter->netdev, 2588 "Not enough memory to create queues with sizes TX=%d, RX=%d\n", 2589 cur_tx_ring_size, cur_rx_ring_size); 2590 2591 new_tx_ring_size = cur_tx_ring_size; 2592 new_rx_ring_size = cur_rx_ring_size; 2593 2594 /* Decrease the size of the larger queue, or 2595 * decrease both if they are the same size. 2596 */ 2597 if (cur_rx_ring_size <= cur_tx_ring_size) 2598 new_tx_ring_size = cur_tx_ring_size / 2; 2599 if (cur_rx_ring_size >= cur_tx_ring_size) 2600 new_rx_ring_size = cur_rx_ring_size / 2; 2601 2602 if (new_tx_ring_size < ENA_MIN_RING_SIZE || 2603 new_rx_ring_size < ENA_MIN_RING_SIZE) { 2604 netif_err(adapter, ifup, adapter->netdev, 2605 "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n", 2606 ENA_MIN_RING_SIZE); 2607 return rc; 2608 } 2609 2610 netif_err(adapter, ifup, adapter->netdev, 2611 "Retrying queue creation with sizes TX=%d, RX=%d\n", 2612 new_tx_ring_size, 2613 new_rx_ring_size); 2614 2615 set_io_rings_size(adapter, new_tx_ring_size, 2616 new_rx_ring_size); 2617 } 2618 } 2619 2620 static int ena_up(struct ena_adapter *adapter) 2621 { 2622 int io_queue_count, rc, i; 2623 2624 netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__); 2625 2626 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; 2627 ena_setup_io_intr(adapter); 2628 2629 /* napi poll functions should be initialized before running 2630 * request_irq(), to handle a rare condition where there is a pending 2631 * interrupt, causing the ISR to fire immediately while the poll 2632 * function wasn't set yet, causing a null dereference 2633 */ 2634 ena_init_napi_in_range(adapter, 0, io_queue_count); 2635 2636 rc = ena_request_io_irq(adapter); 2637 if (rc) 2638 goto err_req_irq; 2639 2640 rc = create_queues_with_size_backoff(adapter); 2641 if (rc) 2642 goto err_create_queues_with_backoff; 2643 2644 rc = ena_up_complete(adapter); 2645 if (rc) 2646 goto err_up; 2647 2648 if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags)) 2649 netif_carrier_on(adapter->netdev); 2650 2651 ena_increase_stat(&adapter->dev_stats.interface_up, 1, 2652 &adapter->syncp); 2653 2654 set_bit(ENA_FLAG_DEV_UP, &adapter->flags); 2655 2656 /* Enable completion queues interrupt */ 2657 for (i = 0; i < adapter->num_io_queues; i++) 2658 ena_unmask_interrupt(&adapter->tx_ring[i], 2659 &adapter->rx_ring[i]); 2660 2661 /* schedule napi in case we had pending packets 2662 * from the last time we disable napi 2663 */ 2664 for (i = 0; i < io_queue_count; i++) 2665 napi_schedule(&adapter->ena_napi[i].napi); 2666 2667 return rc; 2668 2669 err_up: 2670 ena_destroy_all_tx_queues(adapter); 2671 ena_free_all_io_tx_resources(adapter); 2672 ena_destroy_all_rx_queues(adapter); 2673 ena_free_all_io_rx_resources(adapter); 2674 err_create_queues_with_backoff: 2675 ena_free_io_irq(adapter); 2676 err_req_irq: 2677 ena_del_napi_in_range(adapter, 0, io_queue_count); 2678 2679 return rc; 2680 } 2681 2682 static void ena_down(struct ena_adapter *adapter) 2683 { 2684 int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; 2685 2686 netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__); 2687 2688 clear_bit(ENA_FLAG_DEV_UP, &adapter->flags); 2689 2690 ena_increase_stat(&adapter->dev_stats.interface_down, 1, 2691 &adapter->syncp); 2692 2693 netif_carrier_off(adapter->netdev); 2694 netif_tx_disable(adapter->netdev); 2695 2696 /* After this point the napi handler won't enable the tx queue */ 2697 ena_napi_disable_in_range(adapter, 0, io_queue_count); 2698 2699 /* After destroy the queue there won't be any new interrupts */ 2700 2701 if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) { 2702 int rc; 2703 2704 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); 2705 if (rc) 2706 netif_err(adapter, ifdown, adapter->netdev, 2707 "Device reset failed\n"); 2708 /* stop submitting admin commands on a device that was reset */ 2709 ena_com_set_admin_running_state(adapter->ena_dev, false); 2710 } 2711 2712 ena_destroy_all_io_queues(adapter); 2713 2714 ena_disable_io_intr_sync(adapter); 2715 ena_free_io_irq(adapter); 2716 ena_del_napi_in_range(adapter, 0, io_queue_count); 2717 2718 ena_free_all_tx_bufs(adapter); 2719 ena_free_all_rx_bufs(adapter); 2720 ena_free_all_io_tx_resources(adapter); 2721 ena_free_all_io_rx_resources(adapter); 2722 } 2723 2724 /* ena_open - Called when a network interface is made active 2725 * @netdev: network interface device structure 2726 * 2727 * Returns 0 on success, negative value on failure 2728 * 2729 * The open entry point is called when a network interface is made 2730 * active by the system (IFF_UP). At this point all resources needed 2731 * for transmit and receive operations are allocated, the interrupt 2732 * handler is registered with the OS, the watchdog timer is started, 2733 * and the stack is notified that the interface is ready. 2734 */ 2735 static int ena_open(struct net_device *netdev) 2736 { 2737 struct ena_adapter *adapter = netdev_priv(netdev); 2738 int rc; 2739 2740 /* Notify the stack of the actual queue counts. */ 2741 rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues); 2742 if (rc) { 2743 netif_err(adapter, ifup, netdev, "Can't set num tx queues\n"); 2744 return rc; 2745 } 2746 2747 rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues); 2748 if (rc) { 2749 netif_err(adapter, ifup, netdev, "Can't set num rx queues\n"); 2750 return rc; 2751 } 2752 2753 rc = ena_up(adapter); 2754 if (rc) 2755 return rc; 2756 2757 return rc; 2758 } 2759 2760 /* ena_close - Disables a network interface 2761 * @netdev: network interface device structure 2762 * 2763 * Returns 0, this is not allowed to fail 2764 * 2765 * The close entry point is called when an interface is de-activated 2766 * by the OS. The hardware is still under the drivers control, but 2767 * needs to be disabled. A global MAC reset is issued to stop the 2768 * hardware, and all transmit and receive resources are freed. 2769 */ 2770 static int ena_close(struct net_device *netdev) 2771 { 2772 struct ena_adapter *adapter = netdev_priv(netdev); 2773 2774 netif_dbg(adapter, ifdown, netdev, "%s\n", __func__); 2775 2776 if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)) 2777 return 0; 2778 2779 if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) 2780 ena_down(adapter); 2781 2782 /* Check for device status and issue reset if needed*/ 2783 check_for_admin_com_state(adapter); 2784 if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { 2785 netif_err(adapter, ifdown, adapter->netdev, 2786 "Destroy failure, restarting device\n"); 2787 ena_dump_stats_to_dmesg(adapter); 2788 /* rtnl lock already obtained in dev_ioctl() layer */ 2789 ena_destroy_device(adapter, false); 2790 ena_restore_device(adapter); 2791 } 2792 2793 return 0; 2794 } 2795 2796 int ena_update_queue_sizes(struct ena_adapter *adapter, 2797 u32 new_tx_size, 2798 u32 new_rx_size) 2799 { 2800 bool dev_was_up; 2801 2802 dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); 2803 ena_close(adapter->netdev); 2804 adapter->requested_tx_ring_size = new_tx_size; 2805 adapter->requested_rx_ring_size = new_rx_size; 2806 ena_init_io_rings(adapter, 2807 0, 2808 adapter->xdp_num_queues + 2809 adapter->num_io_queues); 2810 return dev_was_up ? ena_up(adapter) : 0; 2811 } 2812 2813 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count) 2814 { 2815 struct ena_com_dev *ena_dev = adapter->ena_dev; 2816 int prev_channel_count; 2817 bool dev_was_up; 2818 2819 dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); 2820 ena_close(adapter->netdev); 2821 prev_channel_count = adapter->num_io_queues; 2822 adapter->num_io_queues = new_channel_count; 2823 if (ena_xdp_present(adapter) && 2824 ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) { 2825 adapter->xdp_first_ring = new_channel_count; 2826 adapter->xdp_num_queues = new_channel_count; 2827 if (prev_channel_count > new_channel_count) 2828 ena_xdp_exchange_program_rx_in_range(adapter, 2829 NULL, 2830 new_channel_count, 2831 prev_channel_count); 2832 else 2833 ena_xdp_exchange_program_rx_in_range(adapter, 2834 adapter->xdp_bpf_prog, 2835 prev_channel_count, 2836 new_channel_count); 2837 } 2838 2839 /* We need to destroy the rss table so that the indirection 2840 * table will be reinitialized by ena_up() 2841 */ 2842 ena_com_rss_destroy(ena_dev); 2843 ena_init_io_rings(adapter, 2844 0, 2845 adapter->xdp_num_queues + 2846 adapter->num_io_queues); 2847 return dev_was_up ? ena_open(adapter->netdev) : 0; 2848 } 2849 2850 static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, 2851 struct sk_buff *skb, 2852 bool disable_meta_caching) 2853 { 2854 u32 mss = skb_shinfo(skb)->gso_size; 2855 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 2856 u8 l4_protocol = 0; 2857 2858 if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) { 2859 ena_tx_ctx->l4_csum_enable = 1; 2860 if (mss) { 2861 ena_tx_ctx->tso_enable = 1; 2862 ena_meta->l4_hdr_len = tcp_hdr(skb)->doff; 2863 ena_tx_ctx->l4_csum_partial = 0; 2864 } else { 2865 ena_tx_ctx->tso_enable = 0; 2866 ena_meta->l4_hdr_len = 0; 2867 ena_tx_ctx->l4_csum_partial = 1; 2868 } 2869 2870 switch (ip_hdr(skb)->version) { 2871 case IPVERSION: 2872 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 2873 if (ip_hdr(skb)->frag_off & htons(IP_DF)) 2874 ena_tx_ctx->df = 1; 2875 if (mss) 2876 ena_tx_ctx->l3_csum_enable = 1; 2877 l4_protocol = ip_hdr(skb)->protocol; 2878 break; 2879 case 6: 2880 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 2881 l4_protocol = ipv6_hdr(skb)->nexthdr; 2882 break; 2883 default: 2884 break; 2885 } 2886 2887 if (l4_protocol == IPPROTO_TCP) 2888 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 2889 else 2890 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 2891 2892 ena_meta->mss = mss; 2893 ena_meta->l3_hdr_len = skb_network_header_len(skb); 2894 ena_meta->l3_hdr_offset = skb_network_offset(skb); 2895 ena_tx_ctx->meta_valid = 1; 2896 } else if (disable_meta_caching) { 2897 memset(ena_meta, 0, sizeof(*ena_meta)); 2898 ena_tx_ctx->meta_valid = 1; 2899 } else { 2900 ena_tx_ctx->meta_valid = 0; 2901 } 2902 } 2903 2904 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring, 2905 struct sk_buff *skb) 2906 { 2907 int num_frags, header_len, rc; 2908 2909 num_frags = skb_shinfo(skb)->nr_frags; 2910 header_len = skb_headlen(skb); 2911 2912 if (num_frags < tx_ring->sgl_size) 2913 return 0; 2914 2915 if ((num_frags == tx_ring->sgl_size) && 2916 (header_len < tx_ring->tx_max_header_size)) 2917 return 0; 2918 2919 ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp); 2920 2921 rc = skb_linearize(skb); 2922 if (unlikely(rc)) { 2923 ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1, 2924 &tx_ring->syncp); 2925 } 2926 2927 return rc; 2928 } 2929 2930 static int ena_tx_map_skb(struct ena_ring *tx_ring, 2931 struct ena_tx_buffer *tx_info, 2932 struct sk_buff *skb, 2933 void **push_hdr, 2934 u16 *header_len) 2935 { 2936 struct ena_adapter *adapter = tx_ring->adapter; 2937 struct ena_com_buf *ena_buf; 2938 dma_addr_t dma; 2939 u32 skb_head_len, frag_len, last_frag; 2940 u16 push_len = 0; 2941 u16 delta = 0; 2942 int i = 0; 2943 2944 skb_head_len = skb_headlen(skb); 2945 tx_info->skb = skb; 2946 ena_buf = tx_info->bufs; 2947 2948 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2949 /* When the device is LLQ mode, the driver will copy 2950 * the header into the device memory space. 2951 * the ena_com layer assume the header is in a linear 2952 * memory space. 2953 * This assumption might be wrong since part of the header 2954 * can be in the fragmented buffers. 2955 * Use skb_header_pointer to make sure the header is in a 2956 * linear memory space. 2957 */ 2958 2959 push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size); 2960 *push_hdr = skb_header_pointer(skb, 0, push_len, 2961 tx_ring->push_buf_intermediate_buf); 2962 *header_len = push_len; 2963 if (unlikely(skb->data != *push_hdr)) { 2964 ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1, 2965 &tx_ring->syncp); 2966 2967 delta = push_len - skb_head_len; 2968 } 2969 } else { 2970 *push_hdr = NULL; 2971 *header_len = min_t(u32, skb_head_len, 2972 tx_ring->tx_max_header_size); 2973 } 2974 2975 netif_dbg(adapter, tx_queued, adapter->netdev, 2976 "skb: %p header_buf->vaddr: %p push_len: %d\n", skb, 2977 *push_hdr, push_len); 2978 2979 if (skb_head_len > push_len) { 2980 dma = dma_map_single(tx_ring->dev, skb->data + push_len, 2981 skb_head_len - push_len, DMA_TO_DEVICE); 2982 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 2983 goto error_report_dma_error; 2984 2985 ena_buf->paddr = dma; 2986 ena_buf->len = skb_head_len - push_len; 2987 2988 ena_buf++; 2989 tx_info->num_of_bufs++; 2990 tx_info->map_linear_data = 1; 2991 } else { 2992 tx_info->map_linear_data = 0; 2993 } 2994 2995 last_frag = skb_shinfo(skb)->nr_frags; 2996 2997 for (i = 0; i < last_frag; i++) { 2998 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2999 3000 frag_len = skb_frag_size(frag); 3001 3002 if (unlikely(delta >= frag_len)) { 3003 delta -= frag_len; 3004 continue; 3005 } 3006 3007 dma = skb_frag_dma_map(tx_ring->dev, frag, delta, 3008 frag_len - delta, DMA_TO_DEVICE); 3009 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 3010 goto error_report_dma_error; 3011 3012 ena_buf->paddr = dma; 3013 ena_buf->len = frag_len - delta; 3014 ena_buf++; 3015 tx_info->num_of_bufs++; 3016 delta = 0; 3017 } 3018 3019 return 0; 3020 3021 error_report_dma_error: 3022 ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1, 3023 &tx_ring->syncp); 3024 netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n"); 3025 3026 tx_info->skb = NULL; 3027 3028 tx_info->num_of_bufs += i; 3029 ena_unmap_tx_buff(tx_ring, tx_info); 3030 3031 return -EINVAL; 3032 } 3033 3034 /* Called with netif_tx_lock. */ 3035 static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev) 3036 { 3037 struct ena_adapter *adapter = netdev_priv(dev); 3038 struct ena_tx_buffer *tx_info; 3039 struct ena_com_tx_ctx ena_tx_ctx; 3040 struct ena_ring *tx_ring; 3041 struct netdev_queue *txq; 3042 void *push_hdr; 3043 u16 next_to_use, req_id, header_len; 3044 int qid, rc; 3045 3046 netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb); 3047 /* Determine which tx ring we will be placed on */ 3048 qid = skb_get_queue_mapping(skb); 3049 tx_ring = &adapter->tx_ring[qid]; 3050 txq = netdev_get_tx_queue(dev, qid); 3051 3052 rc = ena_check_and_linearize_skb(tx_ring, skb); 3053 if (unlikely(rc)) 3054 goto error_drop_packet; 3055 3056 skb_tx_timestamp(skb); 3057 3058 next_to_use = tx_ring->next_to_use; 3059 req_id = tx_ring->free_ids[next_to_use]; 3060 tx_info = &tx_ring->tx_buffer_info[req_id]; 3061 tx_info->num_of_bufs = 0; 3062 3063 WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id); 3064 3065 rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len); 3066 if (unlikely(rc)) 3067 goto error_drop_packet; 3068 3069 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 3070 ena_tx_ctx.ena_bufs = tx_info->bufs; 3071 ena_tx_ctx.push_header = push_hdr; 3072 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 3073 ena_tx_ctx.req_id = req_id; 3074 ena_tx_ctx.header_len = header_len; 3075 3076 /* set flags and meta data */ 3077 ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching); 3078 3079 rc = ena_xmit_common(dev, 3080 tx_ring, 3081 tx_info, 3082 &ena_tx_ctx, 3083 next_to_use, 3084 skb->len); 3085 if (rc) 3086 goto error_unmap_dma; 3087 3088 netdev_tx_sent_queue(txq, skb->len); 3089 3090 /* stop the queue when no more space available, the packet can have up 3091 * to sgl_size + 2. one for the meta descriptor and one for header 3092 * (if the header is larger than tx_max_header_size). 3093 */ 3094 if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 3095 tx_ring->sgl_size + 2))) { 3096 netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n", 3097 __func__, qid); 3098 3099 netif_tx_stop_queue(txq); 3100 ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1, 3101 &tx_ring->syncp); 3102 3103 /* There is a rare condition where this function decide to 3104 * stop the queue but meanwhile clean_tx_irq updates 3105 * next_to_completion and terminates. 3106 * The queue will remain stopped forever. 3107 * To solve this issue add a mb() to make sure that 3108 * netif_tx_stop_queue() write is vissible before checking if 3109 * there is additional space in the queue. 3110 */ 3111 smp_mb(); 3112 3113 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 3114 ENA_TX_WAKEUP_THRESH)) { 3115 netif_tx_wake_queue(txq); 3116 ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1, 3117 &tx_ring->syncp); 3118 } 3119 } 3120 3121 if (netif_xmit_stopped(txq) || !netdev_xmit_more()) 3122 /* trigger the dma engine. ena_ring_tx_doorbell() 3123 * calls a memory barrier inside it. 3124 */ 3125 ena_ring_tx_doorbell(tx_ring); 3126 3127 return NETDEV_TX_OK; 3128 3129 error_unmap_dma: 3130 ena_unmap_tx_buff(tx_ring, tx_info); 3131 tx_info->skb = NULL; 3132 3133 error_drop_packet: 3134 dev_kfree_skb(skb); 3135 return NETDEV_TX_OK; 3136 } 3137 3138 static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb, 3139 struct net_device *sb_dev) 3140 { 3141 u16 qid; 3142 /* we suspect that this is good for in--kernel network services that 3143 * want to loop incoming skb rx to tx in normal user generated traffic, 3144 * most probably we will not get to this 3145 */ 3146 if (skb_rx_queue_recorded(skb)) 3147 qid = skb_get_rx_queue(skb); 3148 else 3149 qid = netdev_pick_tx(dev, skb, NULL); 3150 3151 return qid; 3152 } 3153 3154 static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev) 3155 { 3156 struct device *dev = &pdev->dev; 3157 struct ena_admin_host_info *host_info; 3158 int rc; 3159 3160 /* Allocate only the host info */ 3161 rc = ena_com_allocate_host_info(ena_dev); 3162 if (rc) { 3163 dev_err(dev, "Cannot allocate host info\n"); 3164 return; 3165 } 3166 3167 host_info = ena_dev->host_attr.host_info; 3168 3169 host_info->bdf = (pdev->bus->number << 8) | pdev->devfn; 3170 host_info->os_type = ENA_ADMIN_OS_LINUX; 3171 host_info->kernel_ver = LINUX_VERSION_CODE; 3172 strlcpy(host_info->kernel_ver_str, utsname()->version, 3173 sizeof(host_info->kernel_ver_str) - 1); 3174 host_info->os_dist = 0; 3175 strncpy(host_info->os_dist_str, utsname()->release, 3176 sizeof(host_info->os_dist_str) - 1); 3177 host_info->driver_version = 3178 (DRV_MODULE_GEN_MAJOR) | 3179 (DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 3180 (DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) | 3181 ("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT); 3182 host_info->num_cpus = num_online_cpus(); 3183 3184 host_info->driver_supported_features = 3185 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK | 3186 ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK | 3187 ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK | 3188 ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK; 3189 3190 rc = ena_com_set_host_attributes(ena_dev); 3191 if (rc) { 3192 if (rc == -EOPNOTSUPP) 3193 dev_warn(dev, "Cannot set host attributes\n"); 3194 else 3195 dev_err(dev, "Cannot set host attributes\n"); 3196 3197 goto err; 3198 } 3199 3200 return; 3201 3202 err: 3203 ena_com_delete_host_info(ena_dev); 3204 } 3205 3206 static void ena_config_debug_area(struct ena_adapter *adapter) 3207 { 3208 u32 debug_area_size; 3209 int rc, ss_count; 3210 3211 ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS); 3212 if (ss_count <= 0) { 3213 netif_err(adapter, drv, adapter->netdev, 3214 "SS count is negative\n"); 3215 return; 3216 } 3217 3218 /* allocate 32 bytes for each string and 64bit for the value */ 3219 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 3220 3221 rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size); 3222 if (rc) { 3223 netif_err(adapter, drv, adapter->netdev, 3224 "Cannot allocate debug area\n"); 3225 return; 3226 } 3227 3228 rc = ena_com_set_host_attributes(adapter->ena_dev); 3229 if (rc) { 3230 if (rc == -EOPNOTSUPP) 3231 netif_warn(adapter, drv, adapter->netdev, 3232 "Cannot set host attributes\n"); 3233 else 3234 netif_err(adapter, drv, adapter->netdev, 3235 "Cannot set host attributes\n"); 3236 goto err; 3237 } 3238 3239 return; 3240 err: 3241 ena_com_delete_debug_area(adapter->ena_dev); 3242 } 3243 3244 int ena_update_hw_stats(struct ena_adapter *adapter) 3245 { 3246 int rc; 3247 3248 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_stats); 3249 if (rc) { 3250 netdev_err(adapter->netdev, "Failed to get ENI stats\n"); 3251 return rc; 3252 } 3253 3254 return 0; 3255 } 3256 3257 static void ena_get_stats64(struct net_device *netdev, 3258 struct rtnl_link_stats64 *stats) 3259 { 3260 struct ena_adapter *adapter = netdev_priv(netdev); 3261 struct ena_ring *rx_ring, *tx_ring; 3262 unsigned int start; 3263 u64 rx_drops; 3264 u64 tx_drops; 3265 int i; 3266 3267 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) 3268 return; 3269 3270 for (i = 0; i < adapter->num_io_queues; i++) { 3271 u64 bytes, packets; 3272 3273 tx_ring = &adapter->tx_ring[i]; 3274 3275 do { 3276 start = u64_stats_fetch_begin_irq(&tx_ring->syncp); 3277 packets = tx_ring->tx_stats.cnt; 3278 bytes = tx_ring->tx_stats.bytes; 3279 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); 3280 3281 stats->tx_packets += packets; 3282 stats->tx_bytes += bytes; 3283 3284 rx_ring = &adapter->rx_ring[i]; 3285 3286 do { 3287 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 3288 packets = rx_ring->rx_stats.cnt; 3289 bytes = rx_ring->rx_stats.bytes; 3290 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 3291 3292 stats->rx_packets += packets; 3293 stats->rx_bytes += bytes; 3294 } 3295 3296 do { 3297 start = u64_stats_fetch_begin_irq(&adapter->syncp); 3298 rx_drops = adapter->dev_stats.rx_drops; 3299 tx_drops = adapter->dev_stats.tx_drops; 3300 } while (u64_stats_fetch_retry_irq(&adapter->syncp, start)); 3301 3302 stats->rx_dropped = rx_drops; 3303 stats->tx_dropped = tx_drops; 3304 3305 stats->multicast = 0; 3306 stats->collisions = 0; 3307 3308 stats->rx_length_errors = 0; 3309 stats->rx_crc_errors = 0; 3310 stats->rx_frame_errors = 0; 3311 stats->rx_fifo_errors = 0; 3312 stats->rx_missed_errors = 0; 3313 stats->tx_window_errors = 0; 3314 3315 stats->rx_errors = 0; 3316 stats->tx_errors = 0; 3317 } 3318 3319 static const struct net_device_ops ena_netdev_ops = { 3320 .ndo_open = ena_open, 3321 .ndo_stop = ena_close, 3322 .ndo_start_xmit = ena_start_xmit, 3323 .ndo_select_queue = ena_select_queue, 3324 .ndo_get_stats64 = ena_get_stats64, 3325 .ndo_tx_timeout = ena_tx_timeout, 3326 .ndo_change_mtu = ena_change_mtu, 3327 .ndo_set_mac_address = NULL, 3328 .ndo_validate_addr = eth_validate_addr, 3329 .ndo_bpf = ena_xdp, 3330 .ndo_xdp_xmit = ena_xdp_xmit, 3331 }; 3332 3333 static int ena_device_validate_params(struct ena_adapter *adapter, 3334 struct ena_com_dev_get_features_ctx *get_feat_ctx) 3335 { 3336 struct net_device *netdev = adapter->netdev; 3337 int rc; 3338 3339 rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr, 3340 adapter->mac_addr); 3341 if (!rc) { 3342 netif_err(adapter, drv, netdev, 3343 "Error, mac address are different\n"); 3344 return -EINVAL; 3345 } 3346 3347 if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) { 3348 netif_err(adapter, drv, netdev, 3349 "Error, device max mtu is smaller than netdev MTU\n"); 3350 return -EINVAL; 3351 } 3352 3353 return 0; 3354 } 3355 3356 static void set_default_llq_configurations(struct ena_llq_configurations *llq_config) 3357 { 3358 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 3359 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 3360 llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 3361 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B; 3362 llq_config->llq_ring_entry_size_value = 128; 3363 } 3364 3365 static int ena_set_queues_placement_policy(struct pci_dev *pdev, 3366 struct ena_com_dev *ena_dev, 3367 struct ena_admin_feature_llq_desc *llq, 3368 struct ena_llq_configurations *llq_default_configurations) 3369 { 3370 int rc; 3371 u32 llq_feature_mask; 3372 3373 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 3374 if (!(ena_dev->supported_features & llq_feature_mask)) { 3375 dev_warn(&pdev->dev, 3376 "LLQ is not supported Fallback to host mode policy.\n"); 3377 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 3378 return 0; 3379 } 3380 3381 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 3382 if (unlikely(rc)) { 3383 dev_err(&pdev->dev, 3384 "Failed to configure the device mode. Fallback to host mode policy.\n"); 3385 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 3386 } 3387 3388 return 0; 3389 } 3390 3391 static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev, 3392 int bars) 3393 { 3394 bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR)); 3395 3396 if (!has_mem_bar) { 3397 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 3398 dev_err(&pdev->dev, 3399 "ENA device does not expose LLQ bar. Fallback to host mode policy.\n"); 3400 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 3401 } 3402 3403 return 0; 3404 } 3405 3406 ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev, 3407 pci_resource_start(pdev, ENA_MEM_BAR), 3408 pci_resource_len(pdev, ENA_MEM_BAR)); 3409 3410 if (!ena_dev->mem_bar) 3411 return -EFAULT; 3412 3413 return 0; 3414 } 3415 3416 static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev, 3417 struct ena_com_dev_get_features_ctx *get_feat_ctx, 3418 bool *wd_state) 3419 { 3420 struct ena_llq_configurations llq_config; 3421 struct device *dev = &pdev->dev; 3422 bool readless_supported; 3423 u32 aenq_groups; 3424 int dma_width; 3425 int rc; 3426 3427 rc = ena_com_mmio_reg_read_request_init(ena_dev); 3428 if (rc) { 3429 dev_err(dev, "Failed to init mmio read less\n"); 3430 return rc; 3431 } 3432 3433 /* The PCIe configuration space revision id indicate if mmio reg 3434 * read is disabled 3435 */ 3436 readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ); 3437 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 3438 3439 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 3440 if (rc) { 3441 dev_err(dev, "Can not reset device\n"); 3442 goto err_mmio_read_less; 3443 } 3444 3445 rc = ena_com_validate_version(ena_dev); 3446 if (rc) { 3447 dev_err(dev, "Device version is too low\n"); 3448 goto err_mmio_read_less; 3449 } 3450 3451 dma_width = ena_com_get_dma_width(ena_dev); 3452 if (dma_width < 0) { 3453 dev_err(dev, "Invalid dma width value %d", dma_width); 3454 rc = dma_width; 3455 goto err_mmio_read_less; 3456 } 3457 3458 rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(dma_width)); 3459 if (rc) { 3460 dev_err(dev, "dma_set_mask_and_coherent failed %d\n", rc); 3461 goto err_mmio_read_less; 3462 } 3463 3464 /* ENA admin level init */ 3465 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 3466 if (rc) { 3467 dev_err(dev, 3468 "Can not initialize ena admin queue with device\n"); 3469 goto err_mmio_read_less; 3470 } 3471 3472 /* To enable the msix interrupts the driver needs to know the number 3473 * of queues. So the driver uses polling mode to retrieve this 3474 * information 3475 */ 3476 ena_com_set_admin_polling_mode(ena_dev, true); 3477 3478 ena_config_host_info(ena_dev, pdev); 3479 3480 /* Get Device Attributes*/ 3481 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 3482 if (rc) { 3483 dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc); 3484 goto err_admin_init; 3485 } 3486 3487 /* Try to turn all the available aenq groups */ 3488 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 3489 BIT(ENA_ADMIN_FATAL_ERROR) | 3490 BIT(ENA_ADMIN_WARNING) | 3491 BIT(ENA_ADMIN_NOTIFICATION) | 3492 BIT(ENA_ADMIN_KEEP_ALIVE); 3493 3494 aenq_groups &= get_feat_ctx->aenq.supported_groups; 3495 3496 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 3497 if (rc) { 3498 dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc); 3499 goto err_admin_init; 3500 } 3501 3502 *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 3503 3504 set_default_llq_configurations(&llq_config); 3505 3506 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq, 3507 &llq_config); 3508 if (rc) { 3509 dev_err(dev, "ENA device init failed\n"); 3510 goto err_admin_init; 3511 } 3512 3513 return 0; 3514 3515 err_admin_init: 3516 ena_com_delete_host_info(ena_dev); 3517 ena_com_admin_destroy(ena_dev); 3518 err_mmio_read_less: 3519 ena_com_mmio_reg_read_request_destroy(ena_dev); 3520 3521 return rc; 3522 } 3523 3524 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) 3525 { 3526 struct ena_com_dev *ena_dev = adapter->ena_dev; 3527 struct device *dev = &adapter->pdev->dev; 3528 int rc; 3529 3530 rc = ena_enable_msix(adapter); 3531 if (rc) { 3532 dev_err(dev, "Can not reserve msix vectors\n"); 3533 return rc; 3534 } 3535 3536 ena_setup_mgmnt_intr(adapter); 3537 3538 rc = ena_request_mgmnt_irq(adapter); 3539 if (rc) { 3540 dev_err(dev, "Can not setup management interrupts\n"); 3541 goto err_disable_msix; 3542 } 3543 3544 ena_com_set_admin_polling_mode(ena_dev, false); 3545 3546 ena_com_admin_aenq_enable(ena_dev); 3547 3548 return 0; 3549 3550 err_disable_msix: 3551 ena_disable_msix(adapter); 3552 3553 return rc; 3554 } 3555 3556 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful) 3557 { 3558 struct net_device *netdev = adapter->netdev; 3559 struct ena_com_dev *ena_dev = adapter->ena_dev; 3560 bool dev_up; 3561 3562 if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)) 3563 return; 3564 3565 netif_carrier_off(netdev); 3566 3567 del_timer_sync(&adapter->timer_service); 3568 3569 dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); 3570 adapter->dev_up_before_reset = dev_up; 3571 if (!graceful) 3572 ena_com_set_admin_running_state(ena_dev, false); 3573 3574 if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) 3575 ena_down(adapter); 3576 3577 /* Stop the device from sending AENQ events (in case reset flag is set 3578 * and device is up, ena_down() already reset the device. 3579 */ 3580 if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up)) 3581 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); 3582 3583 ena_free_mgmnt_irq(adapter); 3584 3585 ena_disable_msix(adapter); 3586 3587 ena_com_abort_admin_commands(ena_dev); 3588 3589 ena_com_wait_for_abort_completion(ena_dev); 3590 3591 ena_com_admin_destroy(ena_dev); 3592 3593 ena_com_mmio_reg_read_request_destroy(ena_dev); 3594 3595 /* return reset reason to default value */ 3596 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3597 3598 clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags); 3599 clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); 3600 } 3601 3602 static int ena_restore_device(struct ena_adapter *adapter) 3603 { 3604 struct ena_com_dev_get_features_ctx get_feat_ctx; 3605 struct ena_com_dev *ena_dev = adapter->ena_dev; 3606 struct pci_dev *pdev = adapter->pdev; 3607 bool wd_state; 3608 int rc; 3609 3610 set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags); 3611 rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state); 3612 if (rc) { 3613 dev_err(&pdev->dev, "Can not initialize device\n"); 3614 goto err; 3615 } 3616 adapter->wd_state = wd_state; 3617 3618 rc = ena_device_validate_params(adapter, &get_feat_ctx); 3619 if (rc) { 3620 dev_err(&pdev->dev, "Validation of device parameters failed\n"); 3621 goto err_device_destroy; 3622 } 3623 3624 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 3625 if (rc) { 3626 dev_err(&pdev->dev, "Enable MSI-X failed\n"); 3627 goto err_device_destroy; 3628 } 3629 /* If the interface was up before the reset bring it up */ 3630 if (adapter->dev_up_before_reset) { 3631 rc = ena_up(adapter); 3632 if (rc) { 3633 dev_err(&pdev->dev, "Failed to create I/O queues\n"); 3634 goto err_disable_msix; 3635 } 3636 } 3637 3638 set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); 3639 3640 clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags); 3641 if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags)) 3642 netif_carrier_on(adapter->netdev); 3643 3644 mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ)); 3645 adapter->last_keep_alive_jiffies = jiffies; 3646 3647 return rc; 3648 err_disable_msix: 3649 ena_free_mgmnt_irq(adapter); 3650 ena_disable_msix(adapter); 3651 err_device_destroy: 3652 ena_com_abort_admin_commands(ena_dev); 3653 ena_com_wait_for_abort_completion(ena_dev); 3654 ena_com_admin_destroy(ena_dev); 3655 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE); 3656 ena_com_mmio_reg_read_request_destroy(ena_dev); 3657 err: 3658 clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); 3659 clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags); 3660 dev_err(&pdev->dev, 3661 "Reset attempt failed. Can not reset the device\n"); 3662 3663 return rc; 3664 } 3665 3666 static void ena_fw_reset_device(struct work_struct *work) 3667 { 3668 struct ena_adapter *adapter = 3669 container_of(work, struct ena_adapter, reset_task); 3670 3671 rtnl_lock(); 3672 3673 if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { 3674 ena_destroy_device(adapter, false); 3675 ena_restore_device(adapter); 3676 3677 dev_err(&adapter->pdev->dev, "Device reset completed successfully\n"); 3678 } 3679 3680 rtnl_unlock(); 3681 } 3682 3683 static int check_for_rx_interrupt_queue(struct ena_adapter *adapter, 3684 struct ena_ring *rx_ring) 3685 { 3686 struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi); 3687 3688 if (likely(READ_ONCE(ena_napi->first_interrupt))) 3689 return 0; 3690 3691 if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) 3692 return 0; 3693 3694 rx_ring->no_interrupt_event_cnt++; 3695 3696 if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) { 3697 netif_err(adapter, rx_err, adapter->netdev, 3698 "Potential MSIX issue on Rx side Queue = %d. Reset the device\n", 3699 rx_ring->qid); 3700 3701 ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT); 3702 return -EIO; 3703 } 3704 3705 return 0; 3706 } 3707 3708 static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, 3709 struct ena_ring *tx_ring) 3710 { 3711 struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi); 3712 unsigned int time_since_last_napi; 3713 unsigned int missing_tx_comp_to; 3714 bool is_tx_comp_time_expired; 3715 struct ena_tx_buffer *tx_buf; 3716 unsigned long last_jiffies; 3717 u32 missed_tx = 0; 3718 int i, rc = 0; 3719 3720 for (i = 0; i < tx_ring->ring_size; i++) { 3721 tx_buf = &tx_ring->tx_buffer_info[i]; 3722 last_jiffies = tx_buf->last_jiffies; 3723 3724 if (last_jiffies == 0) 3725 /* no pending Tx at this location */ 3726 continue; 3727 3728 is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies + 3729 2 * adapter->missing_tx_completion_to); 3730 3731 if (unlikely(!READ_ONCE(ena_napi->first_interrupt) && is_tx_comp_time_expired)) { 3732 /* If after graceful period interrupt is still not 3733 * received, we schedule a reset 3734 */ 3735 netif_err(adapter, tx_err, adapter->netdev, 3736 "Potential MSIX issue on Tx side Queue = %d. Reset the device\n", 3737 tx_ring->qid); 3738 ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT); 3739 return -EIO; 3740 } 3741 3742 is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies + 3743 adapter->missing_tx_completion_to); 3744 3745 if (unlikely(is_tx_comp_time_expired)) { 3746 if (!tx_buf->print_once) { 3747 time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies); 3748 missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to); 3749 netif_notice(adapter, tx_err, adapter->netdev, 3750 "Found a Tx that wasn't completed on time, qid %d, index %d. %u usecs have passed since last napi execution. Missing Tx timeout value %u msecs\n", 3751 tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to); 3752 } 3753 3754 tx_buf->print_once = 1; 3755 missed_tx++; 3756 } 3757 } 3758 3759 if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) { 3760 netif_err(adapter, tx_err, adapter->netdev, 3761 "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n", 3762 missed_tx, 3763 adapter->missing_tx_completion_threshold); 3764 ena_reset_device(adapter, ENA_REGS_RESET_MISS_TX_CMPL); 3765 rc = -EIO; 3766 } 3767 3768 ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx, 3769 &tx_ring->syncp); 3770 3771 return rc; 3772 } 3773 3774 static void check_for_missing_completions(struct ena_adapter *adapter) 3775 { 3776 struct ena_ring *tx_ring; 3777 struct ena_ring *rx_ring; 3778 int i, budget, rc; 3779 int io_queue_count; 3780 3781 io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues; 3782 /* Make sure the driver doesn't turn the device in other process */ 3783 smp_rmb(); 3784 3785 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) 3786 return; 3787 3788 if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) 3789 return; 3790 3791 if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT) 3792 return; 3793 3794 budget = ENA_MONITORED_TX_QUEUES; 3795 3796 for (i = adapter->last_monitored_tx_qid; i < io_queue_count; i++) { 3797 tx_ring = &adapter->tx_ring[i]; 3798 rx_ring = &adapter->rx_ring[i]; 3799 3800 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); 3801 if (unlikely(rc)) 3802 return; 3803 3804 rc = !ENA_IS_XDP_INDEX(adapter, i) ? 3805 check_for_rx_interrupt_queue(adapter, rx_ring) : 0; 3806 if (unlikely(rc)) 3807 return; 3808 3809 budget--; 3810 if (!budget) 3811 break; 3812 } 3813 3814 adapter->last_monitored_tx_qid = i % io_queue_count; 3815 } 3816 3817 /* trigger napi schedule after 2 consecutive detections */ 3818 #define EMPTY_RX_REFILL 2 3819 /* For the rare case where the device runs out of Rx descriptors and the 3820 * napi handler failed to refill new Rx descriptors (due to a lack of memory 3821 * for example). 3822 * This case will lead to a deadlock: 3823 * The device won't send interrupts since all the new Rx packets will be dropped 3824 * The napi handler won't allocate new Rx descriptors so the device will be 3825 * able to send new packets. 3826 * 3827 * This scenario can happen when the kernel's vm.min_free_kbytes is too small. 3828 * It is recommended to have at least 512MB, with a minimum of 128MB for 3829 * constrained environment). 3830 * 3831 * When such a situation is detected - Reschedule napi 3832 */ 3833 static void check_for_empty_rx_ring(struct ena_adapter *adapter) 3834 { 3835 struct ena_ring *rx_ring; 3836 int i, refill_required; 3837 3838 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) 3839 return; 3840 3841 if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) 3842 return; 3843 3844 for (i = 0; i < adapter->num_io_queues; i++) { 3845 rx_ring = &adapter->rx_ring[i]; 3846 3847 refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq); 3848 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { 3849 rx_ring->empty_rx_queue++; 3850 3851 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { 3852 ena_increase_stat(&rx_ring->rx_stats.empty_rx_ring, 1, 3853 &rx_ring->syncp); 3854 3855 netif_err(adapter, drv, adapter->netdev, 3856 "Trigger refill for ring %d\n", i); 3857 3858 napi_schedule(rx_ring->napi); 3859 rx_ring->empty_rx_queue = 0; 3860 } 3861 } else { 3862 rx_ring->empty_rx_queue = 0; 3863 } 3864 } 3865 } 3866 3867 /* Check for keep alive expiration */ 3868 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 3869 { 3870 unsigned long keep_alive_expired; 3871 3872 if (!adapter->wd_state) 3873 return; 3874 3875 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3876 return; 3877 3878 keep_alive_expired = adapter->last_keep_alive_jiffies + 3879 adapter->keep_alive_timeout; 3880 if (unlikely(time_is_before_jiffies(keep_alive_expired))) { 3881 netif_err(adapter, drv, adapter->netdev, 3882 "Keep alive watchdog timeout.\n"); 3883 ena_increase_stat(&adapter->dev_stats.wd_expired, 1, 3884 &adapter->syncp); 3885 ena_reset_device(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO); 3886 } 3887 } 3888 3889 static void check_for_admin_com_state(struct ena_adapter *adapter) 3890 { 3891 if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) { 3892 netif_err(adapter, drv, adapter->netdev, 3893 "ENA admin queue is not in running state!\n"); 3894 ena_increase_stat(&adapter->dev_stats.admin_q_pause, 1, 3895 &adapter->syncp); 3896 ena_reset_device(adapter, ENA_REGS_RESET_ADMIN_TO); 3897 } 3898 } 3899 3900 static void ena_update_hints(struct ena_adapter *adapter, 3901 struct ena_admin_ena_hw_hints *hints) 3902 { 3903 struct net_device *netdev = adapter->netdev; 3904 3905 if (hints->admin_completion_tx_timeout) 3906 adapter->ena_dev->admin_queue.completion_timeout = 3907 hints->admin_completion_tx_timeout * 1000; 3908 3909 if (hints->mmio_read_timeout) 3910 /* convert to usec */ 3911 adapter->ena_dev->mmio_read.reg_read_to = 3912 hints->mmio_read_timeout * 1000; 3913 3914 if (hints->missed_tx_completion_count_threshold_to_reset) 3915 adapter->missing_tx_completion_threshold = 3916 hints->missed_tx_completion_count_threshold_to_reset; 3917 3918 if (hints->missing_tx_completion_timeout) { 3919 if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3920 adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT; 3921 else 3922 adapter->missing_tx_completion_to = 3923 msecs_to_jiffies(hints->missing_tx_completion_timeout); 3924 } 3925 3926 if (hints->netdev_wd_timeout) 3927 netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout); 3928 3929 if (hints->driver_watchdog_timeout) { 3930 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3931 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 3932 else 3933 adapter->keep_alive_timeout = 3934 msecs_to_jiffies(hints->driver_watchdog_timeout); 3935 } 3936 } 3937 3938 static void ena_update_host_info(struct ena_admin_host_info *host_info, 3939 struct net_device *netdev) 3940 { 3941 host_info->supported_network_features[0] = 3942 netdev->features & GENMASK_ULL(31, 0); 3943 host_info->supported_network_features[1] = 3944 (netdev->features & GENMASK_ULL(63, 32)) >> 32; 3945 } 3946 3947 static void ena_timer_service(struct timer_list *t) 3948 { 3949 struct ena_adapter *adapter = from_timer(adapter, t, timer_service); 3950 u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr; 3951 struct ena_admin_host_info *host_info = 3952 adapter->ena_dev->host_attr.host_info; 3953 3954 check_for_missing_keep_alive(adapter); 3955 3956 check_for_admin_com_state(adapter); 3957 3958 check_for_missing_completions(adapter); 3959 3960 check_for_empty_rx_ring(adapter); 3961 3962 if (debug_area) 3963 ena_dump_stats_to_buf(adapter, debug_area); 3964 3965 if (host_info) 3966 ena_update_host_info(host_info, adapter->netdev); 3967 3968 if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { 3969 netif_err(adapter, drv, adapter->netdev, 3970 "Trigger reset is on\n"); 3971 ena_dump_stats_to_dmesg(adapter); 3972 queue_work(ena_wq, &adapter->reset_task); 3973 return; 3974 } 3975 3976 /* Reset the timer */ 3977 mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ)); 3978 } 3979 3980 static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev, 3981 struct ena_com_dev *ena_dev, 3982 struct ena_com_dev_get_features_ctx *get_feat_ctx) 3983 { 3984 u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues; 3985 3986 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 3987 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 3988 &get_feat_ctx->max_queue_ext.max_queue_ext; 3989 io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num, 3990 max_queue_ext->max_rx_cq_num); 3991 3992 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 3993 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 3994 } else { 3995 struct ena_admin_queue_feature_desc *max_queues = 3996 &get_feat_ctx->max_queues; 3997 io_tx_sq_num = max_queues->max_sq_num; 3998 io_tx_cq_num = max_queues->max_cq_num; 3999 io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num); 4000 } 4001 4002 /* In case of LLQ use the llq fields for the tx SQ/CQ */ 4003 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 4004 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 4005 4006 max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES); 4007 max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num); 4008 max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num); 4009 max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num); 4010 /* 1 IRQ for mgmnt and 1 IRQs for each IO direction */ 4011 max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1); 4012 4013 return max_num_io_queues; 4014 } 4015 4016 static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat, 4017 struct net_device *netdev) 4018 { 4019 netdev_features_t dev_features = 0; 4020 4021 /* Set offload features */ 4022 if (feat->offload.tx & 4023 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 4024 dev_features |= NETIF_F_IP_CSUM; 4025 4026 if (feat->offload.tx & 4027 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK) 4028 dev_features |= NETIF_F_IPV6_CSUM; 4029 4030 if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 4031 dev_features |= NETIF_F_TSO; 4032 4033 if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) 4034 dev_features |= NETIF_F_TSO6; 4035 4036 if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK) 4037 dev_features |= NETIF_F_TSO_ECN; 4038 4039 if (feat->offload.rx_supported & 4040 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 4041 dev_features |= NETIF_F_RXCSUM; 4042 4043 if (feat->offload.rx_supported & 4044 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) 4045 dev_features |= NETIF_F_RXCSUM; 4046 4047 netdev->features = 4048 dev_features | 4049 NETIF_F_SG | 4050 NETIF_F_RXHASH | 4051 NETIF_F_HIGHDMA; 4052 4053 netdev->hw_features |= netdev->features; 4054 netdev->vlan_features |= netdev->features; 4055 } 4056 4057 static void ena_set_conf_feat_params(struct ena_adapter *adapter, 4058 struct ena_com_dev_get_features_ctx *feat) 4059 { 4060 struct net_device *netdev = adapter->netdev; 4061 4062 /* Copy mac address */ 4063 if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) { 4064 eth_hw_addr_random(netdev); 4065 ether_addr_copy(adapter->mac_addr, netdev->dev_addr); 4066 } else { 4067 ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr); 4068 eth_hw_addr_set(netdev, adapter->mac_addr); 4069 } 4070 4071 /* Set offload features */ 4072 ena_set_dev_offloads(feat, netdev); 4073 4074 adapter->max_mtu = feat->dev_attr.max_mtu; 4075 netdev->max_mtu = adapter->max_mtu; 4076 netdev->min_mtu = ENA_MIN_MTU; 4077 } 4078 4079 static int ena_rss_init_default(struct ena_adapter *adapter) 4080 { 4081 struct ena_com_dev *ena_dev = adapter->ena_dev; 4082 struct device *dev = &adapter->pdev->dev; 4083 int rc, i; 4084 u32 val; 4085 4086 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 4087 if (unlikely(rc)) { 4088 dev_err(dev, "Cannot init indirect table\n"); 4089 goto err_rss_init; 4090 } 4091 4092 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 4093 val = ethtool_rxfh_indir_default(i, adapter->num_io_queues); 4094 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 4095 ENA_IO_RXQ_IDX(val)); 4096 if (unlikely(rc)) { 4097 dev_err(dev, "Cannot fill indirect table\n"); 4098 goto err_fill_indir; 4099 } 4100 } 4101 4102 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL, 4103 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 4104 if (unlikely(rc && (rc != -EOPNOTSUPP))) { 4105 dev_err(dev, "Cannot fill hash function\n"); 4106 goto err_fill_indir; 4107 } 4108 4109 rc = ena_com_set_default_hash_ctrl(ena_dev); 4110 if (unlikely(rc && (rc != -EOPNOTSUPP))) { 4111 dev_err(dev, "Cannot fill hash control\n"); 4112 goto err_fill_indir; 4113 } 4114 4115 return 0; 4116 4117 err_fill_indir: 4118 ena_com_rss_destroy(ena_dev); 4119 err_rss_init: 4120 4121 return rc; 4122 } 4123 4124 static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev) 4125 { 4126 int release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK; 4127 4128 pci_release_selected_regions(pdev, release_bars); 4129 } 4130 4131 4132 static void ena_calc_io_queue_size(struct ena_adapter *adapter, 4133 struct ena_com_dev_get_features_ctx *get_feat_ctx) 4134 { 4135 struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq; 4136 struct ena_com_dev *ena_dev = adapter->ena_dev; 4137 u32 tx_queue_size = ENA_DEFAULT_RING_SIZE; 4138 u32 rx_queue_size = ENA_DEFAULT_RING_SIZE; 4139 u32 max_tx_queue_size; 4140 u32 max_rx_queue_size; 4141 4142 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 4143 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 4144 &get_feat_ctx->max_queue_ext.max_queue_ext; 4145 max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth, 4146 max_queue_ext->max_rx_sq_depth); 4147 max_tx_queue_size = max_queue_ext->max_tx_cq_depth; 4148 4149 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 4150 max_tx_queue_size = min_t(u32, max_tx_queue_size, 4151 llq->max_llq_depth); 4152 else 4153 max_tx_queue_size = min_t(u32, max_tx_queue_size, 4154 max_queue_ext->max_tx_sq_depth); 4155 4156 adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, 4157 max_queue_ext->max_per_packet_tx_descs); 4158 adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, 4159 max_queue_ext->max_per_packet_rx_descs); 4160 } else { 4161 struct ena_admin_queue_feature_desc *max_queues = 4162 &get_feat_ctx->max_queues; 4163 max_rx_queue_size = min_t(u32, max_queues->max_cq_depth, 4164 max_queues->max_sq_depth); 4165 max_tx_queue_size = max_queues->max_cq_depth; 4166 4167 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 4168 max_tx_queue_size = min_t(u32, max_tx_queue_size, 4169 llq->max_llq_depth); 4170 else 4171 max_tx_queue_size = min_t(u32, max_tx_queue_size, 4172 max_queues->max_sq_depth); 4173 4174 adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, 4175 max_queues->max_packet_tx_descs); 4176 adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, 4177 max_queues->max_packet_rx_descs); 4178 } 4179 4180 max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size); 4181 max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size); 4182 4183 tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE, 4184 max_tx_queue_size); 4185 rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE, 4186 max_rx_queue_size); 4187 4188 tx_queue_size = rounddown_pow_of_two(tx_queue_size); 4189 rx_queue_size = rounddown_pow_of_two(rx_queue_size); 4190 4191 adapter->max_tx_ring_size = max_tx_queue_size; 4192 adapter->max_rx_ring_size = max_rx_queue_size; 4193 adapter->requested_tx_ring_size = tx_queue_size; 4194 adapter->requested_rx_ring_size = rx_queue_size; 4195 } 4196 4197 /* ena_probe - Device Initialization Routine 4198 * @pdev: PCI device information struct 4199 * @ent: entry in ena_pci_tbl 4200 * 4201 * Returns 0 on success, negative on failure 4202 * 4203 * ena_probe initializes an adapter identified by a pci_dev structure. 4204 * The OS initialization, configuring of the adapter private structure, 4205 * and a hardware reset occur. 4206 */ 4207 static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 4208 { 4209 struct ena_com_dev_get_features_ctx get_feat_ctx; 4210 struct ena_com_dev *ena_dev = NULL; 4211 struct ena_adapter *adapter; 4212 struct net_device *netdev; 4213 static int adapters_found; 4214 u32 max_num_io_queues; 4215 bool wd_state; 4216 int bars, rc; 4217 4218 dev_dbg(&pdev->dev, "%s\n", __func__); 4219 4220 rc = pci_enable_device_mem(pdev); 4221 if (rc) { 4222 dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n"); 4223 return rc; 4224 } 4225 4226 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ENA_MAX_PHYS_ADDR_SIZE_BITS)); 4227 if (rc) { 4228 dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", rc); 4229 goto err_disable_device; 4230 } 4231 4232 pci_set_master(pdev); 4233 4234 ena_dev = vzalloc(sizeof(*ena_dev)); 4235 if (!ena_dev) { 4236 rc = -ENOMEM; 4237 goto err_disable_device; 4238 } 4239 4240 bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK; 4241 rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME); 4242 if (rc) { 4243 dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n", 4244 rc); 4245 goto err_free_ena_dev; 4246 } 4247 4248 ena_dev->reg_bar = devm_ioremap(&pdev->dev, 4249 pci_resource_start(pdev, ENA_REG_BAR), 4250 pci_resource_len(pdev, ENA_REG_BAR)); 4251 if (!ena_dev->reg_bar) { 4252 dev_err(&pdev->dev, "Failed to remap regs bar\n"); 4253 rc = -EFAULT; 4254 goto err_free_region; 4255 } 4256 4257 ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US; 4258 4259 ena_dev->dmadev = &pdev->dev; 4260 4261 netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), ENA_MAX_RINGS); 4262 if (!netdev) { 4263 dev_err(&pdev->dev, "alloc_etherdev_mq failed\n"); 4264 rc = -ENOMEM; 4265 goto err_free_region; 4266 } 4267 4268 SET_NETDEV_DEV(netdev, &pdev->dev); 4269 adapter = netdev_priv(netdev); 4270 adapter->ena_dev = ena_dev; 4271 adapter->netdev = netdev; 4272 adapter->pdev = pdev; 4273 adapter->msg_enable = DEFAULT_MSG_ENABLE; 4274 4275 ena_dev->net_device = netdev; 4276 4277 pci_set_drvdata(pdev, adapter); 4278 4279 rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state); 4280 if (rc) { 4281 dev_err(&pdev->dev, "ENA device init failed\n"); 4282 if (rc == -ETIME) 4283 rc = -EPROBE_DEFER; 4284 goto err_netdev_destroy; 4285 } 4286 4287 rc = ena_map_llq_mem_bar(pdev, ena_dev, bars); 4288 if (rc) { 4289 dev_err(&pdev->dev, "ENA llq bar mapping failed\n"); 4290 goto err_device_destroy; 4291 } 4292 4293 /* Initial TX and RX interrupt delay. Assumes 1 usec granularity. 4294 * Updated during device initialization with the real granularity 4295 */ 4296 ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS; 4297 ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS; 4298 ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION; 4299 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx); 4300 ena_calc_io_queue_size(adapter, &get_feat_ctx); 4301 if (unlikely(!max_num_io_queues)) { 4302 rc = -EFAULT; 4303 goto err_device_destroy; 4304 } 4305 4306 ena_set_conf_feat_params(adapter, &get_feat_ctx); 4307 4308 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 4309 4310 adapter->num_io_queues = max_num_io_queues; 4311 adapter->max_num_io_queues = max_num_io_queues; 4312 adapter->last_monitored_tx_qid = 0; 4313 4314 adapter->xdp_first_ring = 0; 4315 adapter->xdp_num_queues = 0; 4316 4317 adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK; 4318 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 4319 adapter->disable_meta_caching = 4320 !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags & 4321 BIT(ENA_ADMIN_DISABLE_META_CACHING)); 4322 4323 adapter->wd_state = wd_state; 4324 4325 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found); 4326 4327 rc = ena_com_init_interrupt_moderation(adapter->ena_dev); 4328 if (rc) { 4329 dev_err(&pdev->dev, 4330 "Failed to query interrupt moderation feature\n"); 4331 goto err_device_destroy; 4332 } 4333 ena_init_io_rings(adapter, 4334 0, 4335 adapter->xdp_num_queues + 4336 adapter->num_io_queues); 4337 4338 netdev->netdev_ops = &ena_netdev_ops; 4339 netdev->watchdog_timeo = TX_TIMEOUT; 4340 ena_set_ethtool_ops(netdev); 4341 4342 netdev->priv_flags |= IFF_UNICAST_FLT; 4343 4344 u64_stats_init(&adapter->syncp); 4345 4346 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 4347 if (rc) { 4348 dev_err(&pdev->dev, 4349 "Failed to enable and set the admin interrupts\n"); 4350 goto err_worker_destroy; 4351 } 4352 rc = ena_rss_init_default(adapter); 4353 if (rc && (rc != -EOPNOTSUPP)) { 4354 dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc); 4355 goto err_free_msix; 4356 } 4357 4358 ena_config_debug_area(adapter); 4359 4360 memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len); 4361 4362 netif_carrier_off(netdev); 4363 4364 rc = register_netdev(netdev); 4365 if (rc) { 4366 dev_err(&pdev->dev, "Cannot register net device\n"); 4367 goto err_rss; 4368 } 4369 4370 INIT_WORK(&adapter->reset_task, ena_fw_reset_device); 4371 4372 adapter->last_keep_alive_jiffies = jiffies; 4373 adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; 4374 adapter->missing_tx_completion_to = TX_TIMEOUT; 4375 adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS; 4376 4377 ena_update_hints(adapter, &get_feat_ctx.hw_hints); 4378 4379 timer_setup(&adapter->timer_service, ena_timer_service, 0); 4380 mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ)); 4381 4382 dev_info(&pdev->dev, 4383 "%s found at mem %lx, mac addr %pM\n", 4384 DEVICE_NAME, (long)pci_resource_start(pdev, 0), 4385 netdev->dev_addr); 4386 4387 set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); 4388 4389 adapters_found++; 4390 4391 return 0; 4392 4393 err_rss: 4394 ena_com_delete_debug_area(ena_dev); 4395 ena_com_rss_destroy(ena_dev); 4396 err_free_msix: 4397 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR); 4398 /* stop submitting admin commands on a device that was reset */ 4399 ena_com_set_admin_running_state(ena_dev, false); 4400 ena_free_mgmnt_irq(adapter); 4401 ena_disable_msix(adapter); 4402 err_worker_destroy: 4403 del_timer(&adapter->timer_service); 4404 err_device_destroy: 4405 ena_com_delete_host_info(ena_dev); 4406 ena_com_admin_destroy(ena_dev); 4407 err_netdev_destroy: 4408 free_netdev(netdev); 4409 err_free_region: 4410 ena_release_bars(ena_dev, pdev); 4411 err_free_ena_dev: 4412 vfree(ena_dev); 4413 err_disable_device: 4414 pci_disable_device(pdev); 4415 return rc; 4416 } 4417 4418 /*****************************************************************************/ 4419 4420 /* __ena_shutoff - Helper used in both PCI remove/shutdown routines 4421 * @pdev: PCI device information struct 4422 * @shutdown: Is it a shutdown operation? If false, means it is a removal 4423 * 4424 * __ena_shutoff is a helper routine that does the real work on shutdown and 4425 * removal paths; the difference between those paths is with regards to whether 4426 * dettach or unregister the netdevice. 4427 */ 4428 static void __ena_shutoff(struct pci_dev *pdev, bool shutdown) 4429 { 4430 struct ena_adapter *adapter = pci_get_drvdata(pdev); 4431 struct ena_com_dev *ena_dev; 4432 struct net_device *netdev; 4433 4434 ena_dev = adapter->ena_dev; 4435 netdev = adapter->netdev; 4436 4437 #ifdef CONFIG_RFS_ACCEL 4438 if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) { 4439 free_irq_cpu_rmap(netdev->rx_cpu_rmap); 4440 netdev->rx_cpu_rmap = NULL; 4441 } 4442 #endif /* CONFIG_RFS_ACCEL */ 4443 4444 /* Make sure timer and reset routine won't be called after 4445 * freeing device resources. 4446 */ 4447 del_timer_sync(&adapter->timer_service); 4448 cancel_work_sync(&adapter->reset_task); 4449 4450 rtnl_lock(); /* lock released inside the below if-else block */ 4451 adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN; 4452 ena_destroy_device(adapter, true); 4453 if (shutdown) { 4454 netif_device_detach(netdev); 4455 dev_close(netdev); 4456 rtnl_unlock(); 4457 } else { 4458 rtnl_unlock(); 4459 unregister_netdev(netdev); 4460 free_netdev(netdev); 4461 } 4462 4463 ena_com_rss_destroy(ena_dev); 4464 4465 ena_com_delete_debug_area(ena_dev); 4466 4467 ena_com_delete_host_info(ena_dev); 4468 4469 ena_release_bars(ena_dev, pdev); 4470 4471 pci_disable_device(pdev); 4472 4473 vfree(ena_dev); 4474 } 4475 4476 /* ena_remove - Device Removal Routine 4477 * @pdev: PCI device information struct 4478 * 4479 * ena_remove is called by the PCI subsystem to alert the driver 4480 * that it should release a PCI device. 4481 */ 4482 4483 static void ena_remove(struct pci_dev *pdev) 4484 { 4485 __ena_shutoff(pdev, false); 4486 } 4487 4488 /* ena_shutdown - Device Shutdown Routine 4489 * @pdev: PCI device information struct 4490 * 4491 * ena_shutdown is called by the PCI subsystem to alert the driver that 4492 * a shutdown/reboot (or kexec) is happening and device must be disabled. 4493 */ 4494 4495 static void ena_shutdown(struct pci_dev *pdev) 4496 { 4497 __ena_shutoff(pdev, true); 4498 } 4499 4500 /* ena_suspend - PM suspend callback 4501 * @dev_d: Device information struct 4502 */ 4503 static int __maybe_unused ena_suspend(struct device *dev_d) 4504 { 4505 struct pci_dev *pdev = to_pci_dev(dev_d); 4506 struct ena_adapter *adapter = pci_get_drvdata(pdev); 4507 4508 ena_increase_stat(&adapter->dev_stats.suspend, 1, &adapter->syncp); 4509 4510 rtnl_lock(); 4511 if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { 4512 dev_err(&pdev->dev, 4513 "Ignoring device reset request as the device is being suspended\n"); 4514 clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags); 4515 } 4516 ena_destroy_device(adapter, true); 4517 rtnl_unlock(); 4518 return 0; 4519 } 4520 4521 /* ena_resume - PM resume callback 4522 * @dev_d: Device information struct 4523 */ 4524 static int __maybe_unused ena_resume(struct device *dev_d) 4525 { 4526 struct ena_adapter *adapter = dev_get_drvdata(dev_d); 4527 int rc; 4528 4529 ena_increase_stat(&adapter->dev_stats.resume, 1, &adapter->syncp); 4530 4531 rtnl_lock(); 4532 rc = ena_restore_device(adapter); 4533 rtnl_unlock(); 4534 return rc; 4535 } 4536 4537 static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume); 4538 4539 static struct pci_driver ena_pci_driver = { 4540 .name = DRV_MODULE_NAME, 4541 .id_table = ena_pci_tbl, 4542 .probe = ena_probe, 4543 .remove = ena_remove, 4544 .shutdown = ena_shutdown, 4545 .driver.pm = &ena_pm_ops, 4546 .sriov_configure = pci_sriov_configure_simple, 4547 }; 4548 4549 static int __init ena_init(void) 4550 { 4551 ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME); 4552 if (!ena_wq) { 4553 pr_err("Failed to create workqueue\n"); 4554 return -ENOMEM; 4555 } 4556 4557 return pci_register_driver(&ena_pci_driver); 4558 } 4559 4560 static void __exit ena_cleanup(void) 4561 { 4562 pci_unregister_driver(&ena_pci_driver); 4563 4564 if (ena_wq) { 4565 destroy_workqueue(ena_wq); 4566 ena_wq = NULL; 4567 } 4568 } 4569 4570 /****************************************************************************** 4571 ******************************** AENQ Handlers ******************************* 4572 *****************************************************************************/ 4573 /* ena_update_on_link_change: 4574 * Notify the network interface about the change in link status 4575 */ 4576 static void ena_update_on_link_change(void *adapter_data, 4577 struct ena_admin_aenq_entry *aenq_e) 4578 { 4579 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 4580 struct ena_admin_aenq_link_change_desc *aenq_desc = 4581 (struct ena_admin_aenq_link_change_desc *)aenq_e; 4582 int status = aenq_desc->flags & 4583 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; 4584 4585 if (status) { 4586 netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__); 4587 set_bit(ENA_FLAG_LINK_UP, &adapter->flags); 4588 if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags)) 4589 netif_carrier_on(adapter->netdev); 4590 } else { 4591 clear_bit(ENA_FLAG_LINK_UP, &adapter->flags); 4592 netif_carrier_off(adapter->netdev); 4593 } 4594 } 4595 4596 static void ena_keep_alive_wd(void *adapter_data, 4597 struct ena_admin_aenq_entry *aenq_e) 4598 { 4599 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 4600 struct ena_admin_aenq_keep_alive_desc *desc; 4601 u64 rx_drops; 4602 u64 tx_drops; 4603 4604 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 4605 adapter->last_keep_alive_jiffies = jiffies; 4606 4607 rx_drops = ((u64)desc->rx_drops_high << 32) | desc->rx_drops_low; 4608 tx_drops = ((u64)desc->tx_drops_high << 32) | desc->tx_drops_low; 4609 4610 u64_stats_update_begin(&adapter->syncp); 4611 /* These stats are accumulated by the device, so the counters indicate 4612 * all drops since last reset. 4613 */ 4614 adapter->dev_stats.rx_drops = rx_drops; 4615 adapter->dev_stats.tx_drops = tx_drops; 4616 u64_stats_update_end(&adapter->syncp); 4617 } 4618 4619 static void ena_notification(void *adapter_data, 4620 struct ena_admin_aenq_entry *aenq_e) 4621 { 4622 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 4623 struct ena_admin_ena_hw_hints *hints; 4624 4625 WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION, 4626 "Invalid group(%x) expected %x\n", 4627 aenq_e->aenq_common_desc.group, 4628 ENA_ADMIN_NOTIFICATION); 4629 4630 switch (aenq_e->aenq_common_desc.syndrome) { 4631 case ENA_ADMIN_UPDATE_HINTS: 4632 hints = (struct ena_admin_ena_hw_hints *) 4633 (&aenq_e->inline_data_w4); 4634 ena_update_hints(adapter, hints); 4635 break; 4636 default: 4637 netif_err(adapter, drv, adapter->netdev, 4638 "Invalid aenq notification link state %d\n", 4639 aenq_e->aenq_common_desc.syndrome); 4640 } 4641 } 4642 4643 /* This handler will called for unknown event group or unimplemented handlers*/ 4644 static void unimplemented_aenq_handler(void *data, 4645 struct ena_admin_aenq_entry *aenq_e) 4646 { 4647 struct ena_adapter *adapter = (struct ena_adapter *)data; 4648 4649 netif_err(adapter, drv, adapter->netdev, 4650 "Unknown event was received or event with unimplemented handler\n"); 4651 } 4652 4653 static struct ena_aenq_handlers aenq_handlers = { 4654 .handlers = { 4655 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 4656 [ENA_ADMIN_NOTIFICATION] = ena_notification, 4657 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd, 4658 }, 4659 .unimplemented_handler = unimplemented_aenq_handler 4660 }; 4661 4662 module_init(ena_init); 4663 module_exit(ena_cleanup); 4664