1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #ifndef ENA_COM 7 #define ENA_COM 8 9 #include <linux/compiler.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/gfp.h> 13 #include <linux/io.h> 14 #include <linux/prefetch.h> 15 #include <linux/sched.h> 16 #include <linux/sizes.h> 17 #include <linux/spinlock.h> 18 #include <linux/types.h> 19 #include <linux/wait.h> 20 #include <linux/netdevice.h> 21 22 #include "ena_common_defs.h" 23 #include "ena_admin_defs.h" 24 #include "ena_eth_io_defs.h" 25 #include "ena_regs_defs.h" 26 27 #undef pr_fmt 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #define ENA_MAX_NUM_IO_QUEUES 128U 31 /* We need to queues for each IO (on for Tx and one for Rx) */ 32 #define ENA_TOTAL_NUM_QUEUES (2 * (ENA_MAX_NUM_IO_QUEUES)) 33 34 #define ENA_MAX_HANDLERS 256 35 36 #define ENA_MAX_PHYS_ADDR_SIZE_BITS 48 37 38 /* Unit in usec */ 39 #define ENA_REG_READ_TIMEOUT 200000 40 41 #define ADMIN_SQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aq_entry)) 42 #define ADMIN_CQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_acq_entry)) 43 #define ADMIN_AENQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aenq_entry)) 44 45 /*****************************************************************************/ 46 /*****************************************************************************/ 47 /* ENA adaptive interrupt moderation settings */ 48 49 #define ENA_INTR_INITIAL_TX_INTERVAL_USECS 64 50 #define ENA_INTR_INITIAL_RX_INTERVAL_USECS 0 51 #define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1 52 53 #define ENA_HASH_KEY_SIZE 40 54 55 #define ENA_HW_HINTS_NO_TIMEOUT 0xFFFF 56 57 #define ENA_FEATURE_MAX_QUEUE_EXT_VER 1 58 59 struct ena_llq_configurations { 60 enum ena_admin_llq_header_location llq_header_location; 61 enum ena_admin_llq_ring_entry_size llq_ring_entry_size; 62 enum ena_admin_llq_stride_ctrl llq_stride_ctrl; 63 enum ena_admin_llq_num_descs_before_header llq_num_decs_before_header; 64 u16 llq_ring_entry_size_value; 65 }; 66 67 enum queue_direction { 68 ENA_COM_IO_QUEUE_DIRECTION_TX, 69 ENA_COM_IO_QUEUE_DIRECTION_RX 70 }; 71 72 struct ena_com_buf { 73 dma_addr_t paddr; /**< Buffer physical address */ 74 u16 len; /**< Buffer length in bytes */ 75 }; 76 77 struct ena_com_rx_buf_info { 78 u16 len; 79 u16 req_id; 80 }; 81 82 struct ena_com_io_desc_addr { 83 u8 __iomem *pbuf_dev_addr; /* LLQ address */ 84 u8 *virt_addr; 85 dma_addr_t phys_addr; 86 }; 87 88 struct ena_com_tx_meta { 89 u16 mss; 90 u16 l3_hdr_len; 91 u16 l3_hdr_offset; 92 u16 l4_hdr_len; /* In words */ 93 }; 94 95 struct ena_com_llq_info { 96 u16 header_location_ctrl; 97 u16 desc_stride_ctrl; 98 u16 desc_list_entry_size_ctrl; 99 u16 desc_list_entry_size; 100 u16 descs_num_before_header; 101 u16 descs_per_entry; 102 u16 max_entries_in_tx_burst; 103 bool disable_meta_caching; 104 }; 105 106 struct ena_com_io_cq { 107 struct ena_com_io_desc_addr cdesc_addr; 108 109 /* Interrupt unmask register */ 110 u32 __iomem *unmask_reg; 111 112 /* The completion queue head doorbell register */ 113 u32 __iomem *cq_head_db_reg; 114 115 /* numa configuration register (for TPH) */ 116 u32 __iomem *numa_node_cfg_reg; 117 118 /* The value to write to the above register to unmask 119 * the interrupt of this queue 120 */ 121 u32 msix_vector; 122 123 enum queue_direction direction; 124 125 /* holds the number of cdesc of the current packet */ 126 u16 cur_rx_pkt_cdesc_count; 127 /* save the firt cdesc idx of the current packet */ 128 u16 cur_rx_pkt_cdesc_start_idx; 129 130 u16 q_depth; 131 /* Caller qid */ 132 u16 qid; 133 134 /* Device queue index */ 135 u16 idx; 136 u16 head; 137 u16 last_head_update; 138 u8 phase; 139 u8 cdesc_entry_size_in_bytes; 140 141 } ____cacheline_aligned; 142 143 struct ena_com_io_bounce_buffer_control { 144 u8 *base_buffer; 145 u16 next_to_use; 146 u16 buffer_size; 147 u16 buffers_num; /* Must be a power of 2 */ 148 }; 149 150 /* This struct is to keep tracking the current location of the next llq entry */ 151 struct ena_com_llq_pkt_ctrl { 152 u8 *curr_bounce_buf; 153 u16 idx; 154 u16 descs_left_in_line; 155 }; 156 157 struct ena_com_io_sq { 158 struct ena_com_io_desc_addr desc_addr; 159 160 u32 __iomem *db_addr; 161 u8 __iomem *header_addr; 162 163 enum queue_direction direction; 164 enum ena_admin_placement_policy_type mem_queue_type; 165 166 bool disable_meta_caching; 167 168 u32 msix_vector; 169 struct ena_com_tx_meta cached_tx_meta; 170 struct ena_com_llq_info llq_info; 171 struct ena_com_llq_pkt_ctrl llq_buf_ctrl; 172 struct ena_com_io_bounce_buffer_control bounce_buf_ctrl; 173 174 u16 q_depth; 175 u16 qid; 176 177 u16 idx; 178 u16 tail; 179 u16 next_to_comp; 180 u16 llq_last_copy_tail; 181 u32 tx_max_header_size; 182 u8 phase; 183 u8 desc_entry_size; 184 u8 dma_addr_bits; 185 u16 entries_in_tx_burst_left; 186 } ____cacheline_aligned; 187 188 struct ena_com_admin_cq { 189 struct ena_admin_acq_entry *entries; 190 dma_addr_t dma_addr; 191 192 u16 head; 193 u8 phase; 194 }; 195 196 struct ena_com_admin_sq { 197 struct ena_admin_aq_entry *entries; 198 dma_addr_t dma_addr; 199 200 u32 __iomem *db_addr; 201 202 u16 head; 203 u16 tail; 204 u8 phase; 205 206 }; 207 208 struct ena_com_stats_admin { 209 u64 aborted_cmd; 210 u64 submitted_cmd; 211 u64 completed_cmd; 212 u64 out_of_space; 213 u64 no_completion; 214 }; 215 216 struct ena_com_admin_queue { 217 void *q_dmadev; 218 struct ena_com_dev *ena_dev; 219 spinlock_t q_lock; /* spinlock for the admin queue */ 220 221 struct ena_comp_ctx *comp_ctx; 222 u32 completion_timeout; 223 u16 q_depth; 224 struct ena_com_admin_cq cq; 225 struct ena_com_admin_sq sq; 226 227 /* Indicate if the admin queue should poll for completion */ 228 bool polling; 229 230 /* Define if fallback to polling mode should occur */ 231 bool auto_polling; 232 233 u16 curr_cmd_id; 234 235 /* Indicate that the ena was initialized and can 236 * process new admin commands 237 */ 238 bool running_state; 239 240 /* Count the number of outstanding admin commands */ 241 atomic_t outstanding_cmds; 242 243 struct ena_com_stats_admin stats; 244 }; 245 246 struct ena_aenq_handlers; 247 248 struct ena_com_aenq { 249 u16 head; 250 u8 phase; 251 struct ena_admin_aenq_entry *entries; 252 dma_addr_t dma_addr; 253 u16 q_depth; 254 struct ena_aenq_handlers *aenq_handlers; 255 }; 256 257 struct ena_com_mmio_read { 258 struct ena_admin_ena_mmio_req_read_less_resp *read_resp; 259 dma_addr_t read_resp_dma_addr; 260 u32 reg_read_to; /* in us */ 261 u16 seq_num; 262 bool readless_supported; 263 /* spin lock to ensure a single outstanding read */ 264 spinlock_t lock; 265 }; 266 267 struct ena_rss { 268 /* Indirect table */ 269 u16 *host_rss_ind_tbl; 270 struct ena_admin_rss_ind_table_entry *rss_ind_tbl; 271 dma_addr_t rss_ind_tbl_dma_addr; 272 u16 tbl_log_size; 273 274 /* Hash key */ 275 enum ena_admin_hash_functions hash_func; 276 struct ena_admin_feature_rss_flow_hash_control *hash_key; 277 dma_addr_t hash_key_dma_addr; 278 u32 hash_init_val; 279 280 /* Flow Control */ 281 struct ena_admin_feature_rss_hash_control *hash_ctrl; 282 dma_addr_t hash_ctrl_dma_addr; 283 284 }; 285 286 struct ena_host_attribute { 287 /* Debug area */ 288 u8 *debug_area_virt_addr; 289 dma_addr_t debug_area_dma_addr; 290 u32 debug_area_size; 291 292 /* Host information */ 293 struct ena_admin_host_info *host_info; 294 dma_addr_t host_info_dma_addr; 295 }; 296 297 /* Each ena_dev is a PCI function. */ 298 struct ena_com_dev { 299 struct ena_com_admin_queue admin_queue; 300 struct ena_com_aenq aenq; 301 struct ena_com_io_cq io_cq_queues[ENA_TOTAL_NUM_QUEUES]; 302 struct ena_com_io_sq io_sq_queues[ENA_TOTAL_NUM_QUEUES]; 303 u8 __iomem *reg_bar; 304 void __iomem *mem_bar; 305 void *dmadev; 306 struct net_device *net_device; 307 308 enum ena_admin_placement_policy_type tx_mem_queue_type; 309 u32 tx_max_header_size; 310 u16 stats_func; /* Selected function for extended statistic dump */ 311 u16 stats_queue; /* Selected queue for extended statistic dump */ 312 313 struct ena_com_mmio_read mmio_read; 314 315 struct ena_rss rss; 316 u32 supported_features; 317 u32 dma_addr_bits; 318 319 struct ena_host_attribute host_attr; 320 bool adaptive_coalescing; 321 u16 intr_delay_resolution; 322 323 /* interrupt moderation intervals are in usec divided by 324 * intr_delay_resolution, which is supplied by the device. 325 */ 326 u32 intr_moder_tx_interval; 327 u32 intr_moder_rx_interval; 328 329 struct ena_intr_moder_entry *intr_moder_tbl; 330 331 struct ena_com_llq_info llq_info; 332 333 u32 ena_min_poll_delay_us; 334 }; 335 336 struct ena_com_dev_get_features_ctx { 337 struct ena_admin_queue_feature_desc max_queues; 338 struct ena_admin_queue_ext_feature_desc max_queue_ext; 339 struct ena_admin_device_attr_feature_desc dev_attr; 340 struct ena_admin_feature_aenq_desc aenq; 341 struct ena_admin_feature_offload_desc offload; 342 struct ena_admin_ena_hw_hints hw_hints; 343 struct ena_admin_feature_llq_desc llq; 344 }; 345 346 struct ena_com_create_io_ctx { 347 enum ena_admin_placement_policy_type mem_queue_type; 348 enum queue_direction direction; 349 int numa_node; 350 u32 msix_vector; 351 u16 queue_size; 352 u16 qid; 353 }; 354 355 typedef void (*ena_aenq_handler)(void *data, 356 struct ena_admin_aenq_entry *aenq_e); 357 358 /* Holds aenq handlers. Indexed by AENQ event group */ 359 struct ena_aenq_handlers { 360 ena_aenq_handler handlers[ENA_MAX_HANDLERS]; 361 ena_aenq_handler unimplemented_handler; 362 }; 363 364 /*****************************************************************************/ 365 /*****************************************************************************/ 366 367 /* ena_com_mmio_reg_read_request_init - Init the mmio reg read mechanism 368 * @ena_dev: ENA communication layer struct 369 * 370 * Initialize the register read mechanism. 371 * 372 * @note: This method must be the first stage in the initialization sequence. 373 * 374 * @return - 0 on success, negative value on failure. 375 */ 376 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev); 377 378 /* ena_com_set_mmio_read_mode - Enable/disable the indirect mmio reg read mechanism 379 * @ena_dev: ENA communication layer struct 380 * @readless_supported: readless mode (enable/disable) 381 */ 382 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, 383 bool readless_supported); 384 385 /* ena_com_mmio_reg_read_request_write_dev_addr - Write the mmio reg read return 386 * value physical address. 387 * @ena_dev: ENA communication layer struct 388 */ 389 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev); 390 391 /* ena_com_mmio_reg_read_request_destroy - Destroy the mmio reg read mechanism 392 * @ena_dev: ENA communication layer struct 393 */ 394 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev); 395 396 /* ena_com_admin_init - Init the admin and the async queues 397 * @ena_dev: ENA communication layer struct 398 * @aenq_handlers: Those handlers to be called upon event. 399 * 400 * Initialize the admin submission and completion queues. 401 * Initialize the asynchronous events notification queues. 402 * 403 * @return - 0 on success, negative value on failure. 404 */ 405 int ena_com_admin_init(struct ena_com_dev *ena_dev, 406 struct ena_aenq_handlers *aenq_handlers); 407 408 /* ena_com_admin_destroy - Destroy the admin and the async events queues. 409 * @ena_dev: ENA communication layer struct 410 * 411 * @note: Before calling this method, the caller must validate that the device 412 * won't send any additional admin completions/aenq. 413 * To achieve that, a FLR is recommended. 414 */ 415 void ena_com_admin_destroy(struct ena_com_dev *ena_dev); 416 417 /* ena_com_dev_reset - Perform device FLR to the device. 418 * @ena_dev: ENA communication layer struct 419 * @reset_reason: Specify what is the trigger for the reset in case of an error. 420 * 421 * @return - 0 on success, negative value on failure. 422 */ 423 int ena_com_dev_reset(struct ena_com_dev *ena_dev, 424 enum ena_regs_reset_reason_types reset_reason); 425 426 /* ena_com_create_io_queue - Create io queue. 427 * @ena_dev: ENA communication layer struct 428 * @ctx - create context structure 429 * 430 * Create the submission and the completion queues. 431 * 432 * @return - 0 on success, negative value on failure. 433 */ 434 int ena_com_create_io_queue(struct ena_com_dev *ena_dev, 435 struct ena_com_create_io_ctx *ctx); 436 437 /* ena_com_destroy_io_queue - Destroy IO queue with the queue id - qid. 438 * @ena_dev: ENA communication layer struct 439 * @qid - the caller virtual queue id. 440 */ 441 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid); 442 443 /* ena_com_get_io_handlers - Return the io queue handlers 444 * @ena_dev: ENA communication layer struct 445 * @qid - the caller virtual queue id. 446 * @io_sq - IO submission queue handler 447 * @io_cq - IO completion queue handler. 448 * 449 * @return - 0 on success, negative value on failure. 450 */ 451 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid, 452 struct ena_com_io_sq **io_sq, 453 struct ena_com_io_cq **io_cq); 454 455 /* ena_com_admin_aenq_enable - ENAble asynchronous event notifications 456 * @ena_dev: ENA communication layer struct 457 * 458 * After this method, aenq event can be received via AENQ. 459 */ 460 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev); 461 462 /* ena_com_set_admin_running_state - Set the state of the admin queue 463 * @ena_dev: ENA communication layer struct 464 * 465 * Change the state of the admin queue (enable/disable) 466 */ 467 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state); 468 469 /* ena_com_get_admin_running_state - Get the admin queue state 470 * @ena_dev: ENA communication layer struct 471 * 472 * Retrieve the state of the admin queue (enable/disable) 473 * 474 * @return - current polling mode (enable/disable) 475 */ 476 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev); 477 478 /* ena_com_set_admin_polling_mode - Set the admin completion queue polling mode 479 * @ena_dev: ENA communication layer struct 480 * @polling: ENAble/Disable polling mode 481 * 482 * Set the admin completion mode. 483 */ 484 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling); 485 486 /* ena_com_set_admin_auto_polling_mode - Enable autoswitch to polling mode 487 * @ena_dev: ENA communication layer struct 488 * @polling: Enable/Disable polling mode 489 * 490 * Set the autopolling mode. 491 * If autopolling is on: 492 * In case of missing interrupt when data is available switch to polling. 493 */ 494 void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, 495 bool polling); 496 497 /* ena_com_admin_q_comp_intr_handler - admin queue interrupt handler 498 * @ena_dev: ENA communication layer struct 499 * 500 * This method goes over the admin completion queue and wakes up all the pending 501 * threads that wait on the commands wait event. 502 * 503 * @note: Should be called after MSI-X interrupt. 504 */ 505 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev); 506 507 /* ena_com_aenq_intr_handler - AENQ interrupt handler 508 * @ena_dev: ENA communication layer struct 509 * 510 * This method goes over the async event notification queue and calls the proper 511 * aenq handler. 512 */ 513 void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data); 514 515 /* ena_com_abort_admin_commands - Abort all the outstanding admin commands. 516 * @ena_dev: ENA communication layer struct 517 * 518 * This method aborts all the outstanding admin commands. 519 * The caller should then call ena_com_wait_for_abort_completion to make sure 520 * all the commands were completed. 521 */ 522 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev); 523 524 /* ena_com_wait_for_abort_completion - Wait for admin commands abort. 525 * @ena_dev: ENA communication layer struct 526 * 527 * This method waits until all the outstanding admin commands are completed. 528 */ 529 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev); 530 531 /* ena_com_validate_version - Validate the device parameters 532 * @ena_dev: ENA communication layer struct 533 * 534 * This method verifies the device parameters are the same as the saved 535 * parameters in ena_dev. 536 * This method is useful after device reset, to validate the device mac address 537 * and the device offloads are the same as before the reset. 538 * 539 * @return - 0 on success negative value otherwise. 540 */ 541 int ena_com_validate_version(struct ena_com_dev *ena_dev); 542 543 /* ena_com_get_link_params - Retrieve physical link parameters. 544 * @ena_dev: ENA communication layer struct 545 * @resp: Link parameters 546 * 547 * Retrieve the physical link parameters, 548 * like speed, auto-negotiation and full duplex support. 549 * 550 * @return - 0 on Success negative value otherwise. 551 */ 552 int ena_com_get_link_params(struct ena_com_dev *ena_dev, 553 struct ena_admin_get_feat_resp *resp); 554 555 /* ena_com_get_dma_width - Retrieve physical dma address width the device 556 * supports. 557 * @ena_dev: ENA communication layer struct 558 * 559 * Retrieve the maximum physical address bits the device can handle. 560 * 561 * @return: > 0 on Success and negative value otherwise. 562 */ 563 int ena_com_get_dma_width(struct ena_com_dev *ena_dev); 564 565 /* ena_com_set_aenq_config - Set aenq groups configurations 566 * @ena_dev: ENA communication layer struct 567 * @groups flag: bit fields flags of enum ena_admin_aenq_group. 568 * 569 * Configure which aenq event group the driver would like to receive. 570 * 571 * @return: 0 on Success and negative value otherwise. 572 */ 573 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag); 574 575 /* ena_com_get_dev_attr_feat - Get device features 576 * @ena_dev: ENA communication layer struct 577 * @get_feat_ctx: returned context that contain the get features. 578 * 579 * @return: 0 on Success and negative value otherwise. 580 */ 581 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, 582 struct ena_com_dev_get_features_ctx *get_feat_ctx); 583 584 /* ena_com_get_dev_basic_stats - Get device basic statistics 585 * @ena_dev: ENA communication layer struct 586 * @stats: stats return value 587 * 588 * @return: 0 on Success and negative value otherwise. 589 */ 590 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, 591 struct ena_admin_basic_stats *stats); 592 593 /* ena_com_get_eni_stats - Get extended network interface statistics 594 * @ena_dev: ENA communication layer struct 595 * @stats: stats return value 596 * 597 * @return: 0 on Success and negative value otherwise. 598 */ 599 int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, 600 struct ena_admin_eni_stats *stats); 601 602 /* ena_com_set_dev_mtu - Configure the device mtu. 603 * @ena_dev: ENA communication layer struct 604 * @mtu: mtu value 605 * 606 * @return: 0 on Success and negative value otherwise. 607 */ 608 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu); 609 610 /* ena_com_get_offload_settings - Retrieve the device offloads capabilities 611 * @ena_dev: ENA communication layer struct 612 * @offlad: offload return value 613 * 614 * @return: 0 on Success and negative value otherwise. 615 */ 616 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, 617 struct ena_admin_feature_offload_desc *offload); 618 619 /* ena_com_rss_init - Init RSS 620 * @ena_dev: ENA communication layer struct 621 * @log_size: indirection log size 622 * 623 * Allocate RSS/RFS resources. 624 * The caller then can configure rss using ena_com_set_hash_function, 625 * ena_com_set_hash_ctrl and ena_com_indirect_table_set. 626 * 627 * @return: 0 on Success and negative value otherwise. 628 */ 629 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 log_size); 630 631 /* ena_com_rss_destroy - Destroy rss 632 * @ena_dev: ENA communication layer struct 633 * 634 * Free all the RSS/RFS resources. 635 */ 636 void ena_com_rss_destroy(struct ena_com_dev *ena_dev); 637 638 /* ena_com_get_current_hash_function - Get RSS hash function 639 * @ena_dev: ENA communication layer struct 640 * 641 * Return the current hash function. 642 * @return: 0 or one of the ena_admin_hash_functions values. 643 */ 644 int ena_com_get_current_hash_function(struct ena_com_dev *ena_dev); 645 646 /* ena_com_fill_hash_function - Fill RSS hash function 647 * @ena_dev: ENA communication layer struct 648 * @func: The hash function (Toeplitz or crc) 649 * @key: Hash key (for toeplitz hash) 650 * @key_len: key length (max length 10 DW) 651 * @init_val: initial value for the hash function 652 * 653 * Fill the ena_dev resources with the desire hash function, hash key, key_len 654 * and key initial value (if needed by the hash function). 655 * To flush the key into the device the caller should call 656 * ena_com_set_hash_function. 657 * 658 * @return: 0 on Success and negative value otherwise. 659 */ 660 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev, 661 enum ena_admin_hash_functions func, 662 const u8 *key, u16 key_len, u32 init_val); 663 664 /* ena_com_set_hash_function - Flush the hash function and it dependencies to 665 * the device. 666 * @ena_dev: ENA communication layer struct 667 * 668 * Flush the hash function and it dependencies (key, key length and 669 * initial value) if needed. 670 * 671 * @note: Prior to this method the caller should call ena_com_fill_hash_function 672 * 673 * @return: 0 on Success and negative value otherwise. 674 */ 675 int ena_com_set_hash_function(struct ena_com_dev *ena_dev); 676 677 /* ena_com_get_hash_function - Retrieve the hash function from the device. 678 * @ena_dev: ENA communication layer struct 679 * @func: hash function 680 * 681 * Retrieve the hash function from the device. 682 * 683 * @note: If the caller called ena_com_fill_hash_function but didn't flush 684 * it to the device, the new configuration will be lost. 685 * 686 * @return: 0 on Success and negative value otherwise. 687 */ 688 int ena_com_get_hash_function(struct ena_com_dev *ena_dev, 689 enum ena_admin_hash_functions *func); 690 691 /* ena_com_get_hash_key - Retrieve the hash key 692 * @ena_dev: ENA communication layer struct 693 * @key: hash key 694 * 695 * Retrieve the hash key. 696 * 697 * @note: If the caller called ena_com_fill_hash_key but didn't flush 698 * it to the device, the new configuration will be lost. 699 * 700 * @return: 0 on Success and negative value otherwise. 701 */ 702 int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key); 703 /* ena_com_fill_hash_ctrl - Fill RSS hash control 704 * @ena_dev: ENA communication layer struct. 705 * @proto: The protocol to configure. 706 * @hash_fields: bit mask of ena_admin_flow_hash_fields 707 * 708 * Fill the ena_dev resources with the desire hash control (the ethernet 709 * fields that take part of the hash) for a specific protocol. 710 * To flush the hash control to the device, the caller should call 711 * ena_com_set_hash_ctrl. 712 * 713 * @return: 0 on Success and negative value otherwise. 714 */ 715 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev, 716 enum ena_admin_flow_hash_proto proto, 717 u16 hash_fields); 718 719 /* ena_com_set_hash_ctrl - Flush the hash control resources to the device. 720 * @ena_dev: ENA communication layer struct 721 * 722 * Flush the hash control (the ethernet fields that take part of the hash) 723 * 724 * @note: Prior to this method the caller should call ena_com_fill_hash_ctrl. 725 * 726 * @return: 0 on Success and negative value otherwise. 727 */ 728 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev); 729 730 /* ena_com_get_hash_ctrl - Retrieve the hash control from the device. 731 * @ena_dev: ENA communication layer struct 732 * @proto: The protocol to retrieve. 733 * @fields: bit mask of ena_admin_flow_hash_fields. 734 * 735 * Retrieve the hash control from the device. 736 * 737 * @note: If the caller called ena_com_fill_hash_ctrl but didn't flush 738 * it to the device, the new configuration will be lost. 739 * 740 * @return: 0 on Success and negative value otherwise. 741 */ 742 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev, 743 enum ena_admin_flow_hash_proto proto, 744 u16 *fields); 745 746 /* ena_com_set_default_hash_ctrl - Set the hash control to a default 747 * configuration. 748 * @ena_dev: ENA communication layer struct 749 * 750 * Fill the ena_dev resources with the default hash control configuration. 751 * To flush the hash control to the device, the caller should call 752 * ena_com_set_hash_ctrl. 753 * 754 * @return: 0 on Success and negative value otherwise. 755 */ 756 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev); 757 758 /* ena_com_indirect_table_fill_entry - Fill a single entry in the RSS 759 * indirection table 760 * @ena_dev: ENA communication layer struct. 761 * @entry_idx - indirection table entry. 762 * @entry_value - redirection value 763 * 764 * Fill a single entry of the RSS indirection table in the ena_dev resources. 765 * To flush the indirection table to the device, the called should call 766 * ena_com_indirect_table_set. 767 * 768 * @return: 0 on Success and negative value otherwise. 769 */ 770 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev, 771 u16 entry_idx, u16 entry_value); 772 773 /* ena_com_indirect_table_set - Flush the indirection table to the device. 774 * @ena_dev: ENA communication layer struct 775 * 776 * Flush the indirection hash control to the device. 777 * Prior to this method the caller should call ena_com_indirect_table_fill_entry 778 * 779 * @return: 0 on Success and negative value otherwise. 780 */ 781 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev); 782 783 /* ena_com_indirect_table_get - Retrieve the indirection table from the device. 784 * @ena_dev: ENA communication layer struct 785 * @ind_tbl: indirection table 786 * 787 * Retrieve the RSS indirection table from the device. 788 * 789 * @note: If the caller called ena_com_indirect_table_fill_entry but didn't flush 790 * it to the device, the new configuration will be lost. 791 * 792 * @return: 0 on Success and negative value otherwise. 793 */ 794 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl); 795 796 /* ena_com_allocate_host_info - Allocate host info resources. 797 * @ena_dev: ENA communication layer struct 798 * 799 * @return: 0 on Success and negative value otherwise. 800 */ 801 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev); 802 803 /* ena_com_allocate_debug_area - Allocate debug area. 804 * @ena_dev: ENA communication layer struct 805 * @debug_area_size - debug area size. 806 * 807 * @return: 0 on Success and negative value otherwise. 808 */ 809 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, 810 u32 debug_area_size); 811 812 /* ena_com_delete_debug_area - Free the debug area resources. 813 * @ena_dev: ENA communication layer struct 814 * 815 * Free the allocated debug area. 816 */ 817 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev); 818 819 /* ena_com_delete_host_info - Free the host info resources. 820 * @ena_dev: ENA communication layer struct 821 * 822 * Free the allocated host info. 823 */ 824 void ena_com_delete_host_info(struct ena_com_dev *ena_dev); 825 826 /* ena_com_set_host_attributes - Update the device with the host 827 * attributes (debug area and host info) base address. 828 * @ena_dev: ENA communication layer struct 829 * 830 * @return: 0 on Success and negative value otherwise. 831 */ 832 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev); 833 834 /* ena_com_create_io_cq - Create io completion queue. 835 * @ena_dev: ENA communication layer struct 836 * @io_cq - io completion queue handler 837 838 * Create IO completion queue. 839 * 840 * @return - 0 on success, negative value on failure. 841 */ 842 int ena_com_create_io_cq(struct ena_com_dev *ena_dev, 843 struct ena_com_io_cq *io_cq); 844 845 /* ena_com_destroy_io_cq - Destroy io completion queue. 846 * @ena_dev: ENA communication layer struct 847 * @io_cq - io completion queue handler 848 849 * Destroy IO completion queue. 850 * 851 * @return - 0 on success, negative value on failure. 852 */ 853 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev, 854 struct ena_com_io_cq *io_cq); 855 856 /* ena_com_execute_admin_command - Execute admin command 857 * @admin_queue: admin queue. 858 * @cmd: the admin command to execute. 859 * @cmd_size: the command size. 860 * @cmd_completion: command completion return value. 861 * @cmd_comp_size: command completion size. 862 863 * Submit an admin command and then wait until the device returns a 864 * completion. 865 * The completion will be copied into cmd_comp. 866 * 867 * @return - 0 on success, negative value on failure. 868 */ 869 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue, 870 struct ena_admin_aq_entry *cmd, 871 size_t cmd_size, 872 struct ena_admin_acq_entry *cmd_comp, 873 size_t cmd_comp_size); 874 875 /* ena_com_init_interrupt_moderation - Init interrupt moderation 876 * @ena_dev: ENA communication layer struct 877 * 878 * @return - 0 on success, negative value on failure. 879 */ 880 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev); 881 882 /* ena_com_interrupt_moderation_supported - Return if interrupt moderation 883 * capability is supported by the device. 884 * 885 * @return - supported or not. 886 */ 887 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev); 888 889 /* ena_com_update_nonadaptive_moderation_interval_tx - Update the 890 * non-adaptive interval in Tx direction. 891 * @ena_dev: ENA communication layer struct 892 * @tx_coalesce_usecs: Interval in usec. 893 * 894 * @return - 0 on success, negative value on failure. 895 */ 896 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, 897 u32 tx_coalesce_usecs); 898 899 /* ena_com_update_nonadaptive_moderation_interval_rx - Update the 900 * non-adaptive interval in Rx direction. 901 * @ena_dev: ENA communication layer struct 902 * @rx_coalesce_usecs: Interval in usec. 903 * 904 * @return - 0 on success, negative value on failure. 905 */ 906 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, 907 u32 rx_coalesce_usecs); 908 909 /* ena_com_get_nonadaptive_moderation_interval_tx - Retrieve the 910 * non-adaptive interval in Tx direction. 911 * @ena_dev: ENA communication layer struct 912 * 913 * @return - interval in usec 914 */ 915 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev); 916 917 /* ena_com_get_nonadaptive_moderation_interval_rx - Retrieve the 918 * non-adaptive interval in Rx direction. 919 * @ena_dev: ENA communication layer struct 920 * 921 * @return - interval in usec 922 */ 923 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev); 924 925 /* ena_com_config_dev_mode - Configure the placement policy of the device. 926 * @ena_dev: ENA communication layer struct 927 * @llq_features: LLQ feature descriptor, retrieve via 928 * ena_com_get_dev_attr_feat. 929 * @ena_llq_config: The default driver LLQ parameters configurations 930 */ 931 int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, 932 struct ena_admin_feature_llq_desc *llq_features, 933 struct ena_llq_configurations *llq_default_config); 934 935 /* ena_com_io_sq_to_ena_dev - Extract ena_com_dev using contained field io_sq. 936 * @io_sq: IO submit queue struct 937 * 938 * @return - ena_com_dev struct extracted from io_sq 939 */ 940 static inline struct ena_com_dev *ena_com_io_sq_to_ena_dev(struct ena_com_io_sq *io_sq) 941 { 942 return container_of(io_sq, struct ena_com_dev, io_sq_queues[io_sq->qid]); 943 } 944 945 /* ena_com_io_cq_to_ena_dev - Extract ena_com_dev using contained field io_cq. 946 * @io_sq: IO submit queue struct 947 * 948 * @return - ena_com_dev struct extracted from io_sq 949 */ 950 static inline struct ena_com_dev *ena_com_io_cq_to_ena_dev(struct ena_com_io_cq *io_cq) 951 { 952 return container_of(io_cq, struct ena_com_dev, io_cq_queues[io_cq->qid]); 953 } 954 955 static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev) 956 { 957 return ena_dev->adaptive_coalescing; 958 } 959 960 static inline void ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev) 961 { 962 ena_dev->adaptive_coalescing = true; 963 } 964 965 static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev) 966 { 967 ena_dev->adaptive_coalescing = false; 968 } 969 970 /* ena_com_update_intr_reg - Prepare interrupt register 971 * @intr_reg: interrupt register to update. 972 * @rx_delay_interval: Rx interval in usecs 973 * @tx_delay_interval: Tx interval in usecs 974 * @unmask: unmask enable/disable 975 * 976 * Prepare interrupt update register with the supplied parameters. 977 */ 978 static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg, 979 u32 rx_delay_interval, 980 u32 tx_delay_interval, 981 bool unmask) 982 { 983 intr_reg->intr_control = 0; 984 intr_reg->intr_control |= rx_delay_interval & 985 ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; 986 987 intr_reg->intr_control |= 988 (tx_delay_interval << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) 989 & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK; 990 991 if (unmask) 992 intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; 993 } 994 995 static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl) 996 { 997 u16 size, buffers_num; 998 u8 *buf; 999 1000 size = bounce_buf_ctrl->buffer_size; 1001 buffers_num = bounce_buf_ctrl->buffers_num; 1002 1003 buf = bounce_buf_ctrl->base_buffer + 1004 (bounce_buf_ctrl->next_to_use++ & (buffers_num - 1)) * size; 1005 1006 prefetchw(bounce_buf_ctrl->base_buffer + 1007 (bounce_buf_ctrl->next_to_use & (buffers_num - 1)) * size); 1008 1009 return buf; 1010 } 1011 1012 #endif /* !(ENA_COM) */ 1013