11738cd3eSNetanel Belgazal /* 21738cd3eSNetanel Belgazal * Copyright 2015 Amazon.com, Inc. or its affiliates. 31738cd3eSNetanel Belgazal * 41738cd3eSNetanel Belgazal * This software is available to you under a choice of one of two 51738cd3eSNetanel Belgazal * licenses. You may choose to be licensed under the terms of the GNU 61738cd3eSNetanel Belgazal * General Public License (GPL) Version 2, available from the file 71738cd3eSNetanel Belgazal * COPYING in the main directory of this source tree, or the 81738cd3eSNetanel Belgazal * BSD license below: 91738cd3eSNetanel Belgazal * 101738cd3eSNetanel Belgazal * Redistribution and use in source and binary forms, with or 111738cd3eSNetanel Belgazal * without modification, are permitted provided that the following 121738cd3eSNetanel Belgazal * conditions are met: 131738cd3eSNetanel Belgazal * 141738cd3eSNetanel Belgazal * - Redistributions of source code must retain the above 151738cd3eSNetanel Belgazal * copyright notice, this list of conditions and the following 161738cd3eSNetanel Belgazal * disclaimer. 171738cd3eSNetanel Belgazal * 181738cd3eSNetanel Belgazal * - Redistributions in binary form must reproduce the above 191738cd3eSNetanel Belgazal * copyright notice, this list of conditions and the following 201738cd3eSNetanel Belgazal * disclaimer in the documentation and/or other materials 211738cd3eSNetanel Belgazal * provided with the distribution. 221738cd3eSNetanel Belgazal * 231738cd3eSNetanel Belgazal * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 241738cd3eSNetanel Belgazal * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 251738cd3eSNetanel Belgazal * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 261738cd3eSNetanel Belgazal * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 271738cd3eSNetanel Belgazal * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 281738cd3eSNetanel Belgazal * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 291738cd3eSNetanel Belgazal * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 301738cd3eSNetanel Belgazal * SOFTWARE. 311738cd3eSNetanel Belgazal */ 321738cd3eSNetanel Belgazal 331738cd3eSNetanel Belgazal #ifndef ENA_ETH_COM_H_ 341738cd3eSNetanel Belgazal #define ENA_ETH_COM_H_ 351738cd3eSNetanel Belgazal 361738cd3eSNetanel Belgazal #include "ena_com.h" 371738cd3eSNetanel Belgazal 381738cd3eSNetanel Belgazal /* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */ 391738cd3eSNetanel Belgazal #define ENA_COMP_HEAD_THRESH 4 401738cd3eSNetanel Belgazal 411738cd3eSNetanel Belgazal struct ena_com_tx_ctx { 421738cd3eSNetanel Belgazal struct ena_com_tx_meta ena_meta; 431738cd3eSNetanel Belgazal struct ena_com_buf *ena_bufs; 441738cd3eSNetanel Belgazal /* For LLQ, header buffer - pushed to the device mem space */ 451738cd3eSNetanel Belgazal void *push_header; 461738cd3eSNetanel Belgazal 471738cd3eSNetanel Belgazal enum ena_eth_io_l3_proto_index l3_proto; 481738cd3eSNetanel Belgazal enum ena_eth_io_l4_proto_index l4_proto; 491738cd3eSNetanel Belgazal u16 num_bufs; 501738cd3eSNetanel Belgazal u16 req_id; 511738cd3eSNetanel Belgazal /* For regular queue, indicate the size of the header 521738cd3eSNetanel Belgazal * For LLQ, indicate the size of the pushed buffer 531738cd3eSNetanel Belgazal */ 541738cd3eSNetanel Belgazal u16 header_len; 551738cd3eSNetanel Belgazal 561738cd3eSNetanel Belgazal u8 meta_valid; 571738cd3eSNetanel Belgazal u8 tso_enable; 581738cd3eSNetanel Belgazal u8 l3_csum_enable; 591738cd3eSNetanel Belgazal u8 l4_csum_enable; 601738cd3eSNetanel Belgazal u8 l4_csum_partial; 611738cd3eSNetanel Belgazal u8 df; /* Don't fragment */ 621738cd3eSNetanel Belgazal }; 631738cd3eSNetanel Belgazal 641738cd3eSNetanel Belgazal struct ena_com_rx_ctx { 651738cd3eSNetanel Belgazal struct ena_com_rx_buf_info *ena_bufs; 661738cd3eSNetanel Belgazal enum ena_eth_io_l3_proto_index l3_proto; 671738cd3eSNetanel Belgazal enum ena_eth_io_l4_proto_index l4_proto; 681738cd3eSNetanel Belgazal bool l3_csum_err; 691738cd3eSNetanel Belgazal bool l4_csum_err; 70cb36bb36SArthur Kiyanovski u8 l4_csum_checked; 711738cd3eSNetanel Belgazal /* fragmented packet */ 721738cd3eSNetanel Belgazal bool frag; 731738cd3eSNetanel Belgazal u32 hash; 741738cd3eSNetanel Belgazal u16 descs; 751738cd3eSNetanel Belgazal int max_bufs; 761738cd3eSNetanel Belgazal }; 771738cd3eSNetanel Belgazal 781738cd3eSNetanel Belgazal int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, 791738cd3eSNetanel Belgazal struct ena_com_tx_ctx *ena_tx_ctx, 801738cd3eSNetanel Belgazal int *nb_hw_desc); 811738cd3eSNetanel Belgazal 821738cd3eSNetanel Belgazal int ena_com_rx_pkt(struct ena_com_io_cq *io_cq, 831738cd3eSNetanel Belgazal struct ena_com_io_sq *io_sq, 841738cd3eSNetanel Belgazal struct ena_com_rx_ctx *ena_rx_ctx); 851738cd3eSNetanel Belgazal 861738cd3eSNetanel Belgazal int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq, 871738cd3eSNetanel Belgazal struct ena_com_buf *ena_buf, 881738cd3eSNetanel Belgazal u16 req_id); 891738cd3eSNetanel Belgazal 908510e1a3SNetanel Belgazal bool ena_com_cq_empty(struct ena_com_io_cq *io_cq); 918510e1a3SNetanel Belgazal 921738cd3eSNetanel Belgazal static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq, 931738cd3eSNetanel Belgazal struct ena_eth_io_intr_reg *intr_reg) 941738cd3eSNetanel Belgazal { 951738cd3eSNetanel Belgazal writel(intr_reg->intr_control, io_cq->unmask_reg); 961738cd3eSNetanel Belgazal } 971738cd3eSNetanel Belgazal 98689b2bdaSArthur Kiyanovski static inline int ena_com_free_desc(struct ena_com_io_sq *io_sq) 991738cd3eSNetanel Belgazal { 1001738cd3eSNetanel Belgazal u16 tail, next_to_comp, cnt; 1011738cd3eSNetanel Belgazal 1021738cd3eSNetanel Belgazal next_to_comp = io_sq->next_to_comp; 1031738cd3eSNetanel Belgazal tail = io_sq->tail; 1041738cd3eSNetanel Belgazal cnt = tail - next_to_comp; 1051738cd3eSNetanel Belgazal 1061738cd3eSNetanel Belgazal return io_sq->q_depth - 1 - cnt; 1071738cd3eSNetanel Belgazal } 1081738cd3eSNetanel Belgazal 109689b2bdaSArthur Kiyanovski /* Check if the submission queue has enough space to hold required_buffers */ 110689b2bdaSArthur Kiyanovski static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq, 111689b2bdaSArthur Kiyanovski u16 required_buffers) 112689b2bdaSArthur Kiyanovski { 113689b2bdaSArthur Kiyanovski int temp; 114689b2bdaSArthur Kiyanovski 115689b2bdaSArthur Kiyanovski if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 116689b2bdaSArthur Kiyanovski return ena_com_free_desc(io_sq) >= required_buffers; 117689b2bdaSArthur Kiyanovski 118689b2bdaSArthur Kiyanovski /* This calculation doesn't need to be 100% accurate. So to reduce 119689b2bdaSArthur Kiyanovski * the calculation overhead just Subtract 2 lines from the free descs 120689b2bdaSArthur Kiyanovski * (one for the header line and one to compensate the devision 121689b2bdaSArthur Kiyanovski * down calculation. 122689b2bdaSArthur Kiyanovski */ 123689b2bdaSArthur Kiyanovski temp = required_buffers / io_sq->llq_info.descs_per_entry + 2; 124689b2bdaSArthur Kiyanovski 125689b2bdaSArthur Kiyanovski return ena_com_free_desc(io_sq) > temp; 126689b2bdaSArthur Kiyanovski } 127689b2bdaSArthur Kiyanovski 12805d62ca2SSameeh Jubran static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq, 12905d62ca2SSameeh Jubran struct ena_com_tx_ctx *ena_tx_ctx) 13005d62ca2SSameeh Jubran { 13105d62ca2SSameeh Jubran if (!ena_tx_ctx->meta_valid) 13205d62ca2SSameeh Jubran return false; 13305d62ca2SSameeh Jubran 13405d62ca2SSameeh Jubran return !!memcmp(&io_sq->cached_tx_meta, 13505d62ca2SSameeh Jubran &ena_tx_ctx->ena_meta, 13605d62ca2SSameeh Jubran sizeof(struct ena_com_tx_meta)); 13705d62ca2SSameeh Jubran } 13805d62ca2SSameeh Jubran 13905d62ca2SSameeh Jubran static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq) 14005d62ca2SSameeh Jubran { 14105d62ca2SSameeh Jubran return (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) && 14205d62ca2SSameeh Jubran io_sq->llq_info.max_entries_in_tx_burst > 0; 14305d62ca2SSameeh Jubran } 14405d62ca2SSameeh Jubran 14505d62ca2SSameeh Jubran static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq, 14605d62ca2SSameeh Jubran struct ena_com_tx_ctx *ena_tx_ctx) 14705d62ca2SSameeh Jubran { 14805d62ca2SSameeh Jubran struct ena_com_llq_info *llq_info; 14905d62ca2SSameeh Jubran int descs_after_first_entry; 15005d62ca2SSameeh Jubran int num_entries_needed = 1; 15105d62ca2SSameeh Jubran u16 num_descs; 15205d62ca2SSameeh Jubran 15305d62ca2SSameeh Jubran if (!is_llq_max_tx_burst_exists(io_sq)) 15405d62ca2SSameeh Jubran return false; 15505d62ca2SSameeh Jubran 15605d62ca2SSameeh Jubran llq_info = &io_sq->llq_info; 15705d62ca2SSameeh Jubran num_descs = ena_tx_ctx->num_bufs; 15805d62ca2SSameeh Jubran 15905d62ca2SSameeh Jubran if (unlikely(ena_com_meta_desc_changed(io_sq, ena_tx_ctx))) 16005d62ca2SSameeh Jubran ++num_descs; 16105d62ca2SSameeh Jubran 16205d62ca2SSameeh Jubran if (num_descs > llq_info->descs_num_before_header) { 16305d62ca2SSameeh Jubran descs_after_first_entry = num_descs - llq_info->descs_num_before_header; 16405d62ca2SSameeh Jubran num_entries_needed += DIV_ROUND_UP(descs_after_first_entry, 16505d62ca2SSameeh Jubran llq_info->descs_per_entry); 16605d62ca2SSameeh Jubran } 16705d62ca2SSameeh Jubran 16805d62ca2SSameeh Jubran pr_debug("queue: %d num_descs: %d num_entries_needed: %d\n", io_sq->qid, 16905d62ca2SSameeh Jubran num_descs, num_entries_needed); 17005d62ca2SSameeh Jubran 17105d62ca2SSameeh Jubran return num_entries_needed > io_sq->entries_in_tx_burst_left; 17205d62ca2SSameeh Jubran } 17305d62ca2SSameeh Jubran 17437dff155SNetanel Belgazal static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq) 1751738cd3eSNetanel Belgazal { 17605d62ca2SSameeh Jubran u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst; 177689b2bdaSArthur Kiyanovski u16 tail = io_sq->tail; 1781738cd3eSNetanel Belgazal 1791738cd3eSNetanel Belgazal pr_debug("write submission queue doorbell for queue: %d tail: %d\n", 1801738cd3eSNetanel Belgazal io_sq->qid, tail); 1811738cd3eSNetanel Belgazal 1821738cd3eSNetanel Belgazal writel(tail, io_sq->db_addr); 1831738cd3eSNetanel Belgazal 18405d62ca2SSameeh Jubran if (is_llq_max_tx_burst_exists(io_sq)) { 18505d62ca2SSameeh Jubran pr_debug("reset available entries in tx burst for queue %d to %d\n", 18605d62ca2SSameeh Jubran io_sq->qid, max_entries_in_tx_burst); 18705d62ca2SSameeh Jubran io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst; 18805d62ca2SSameeh Jubran } 18905d62ca2SSameeh Jubran 1901738cd3eSNetanel Belgazal return 0; 1911738cd3eSNetanel Belgazal } 1921738cd3eSNetanel Belgazal 1931738cd3eSNetanel Belgazal static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq) 1941738cd3eSNetanel Belgazal { 1951738cd3eSNetanel Belgazal u16 unreported_comp, head; 1961738cd3eSNetanel Belgazal bool need_update; 1971738cd3eSNetanel Belgazal 198d9186098SSameeh Jubran if (unlikely(io_cq->cq_head_db_reg)) { 1991738cd3eSNetanel Belgazal head = io_cq->head; 2001738cd3eSNetanel Belgazal unreported_comp = head - io_cq->last_head_update; 2011738cd3eSNetanel Belgazal need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH); 2021738cd3eSNetanel Belgazal 203d9186098SSameeh Jubran if (unlikely(need_update)) { 2041738cd3eSNetanel Belgazal pr_debug("Write completion queue doorbell for queue %d: head: %d\n", 2051738cd3eSNetanel Belgazal io_cq->qid, head); 2061738cd3eSNetanel Belgazal writel(head, io_cq->cq_head_db_reg); 2071738cd3eSNetanel Belgazal io_cq->last_head_update = head; 2081738cd3eSNetanel Belgazal } 209d9186098SSameeh Jubran } 2101738cd3eSNetanel Belgazal 2111738cd3eSNetanel Belgazal return 0; 2121738cd3eSNetanel Belgazal } 2131738cd3eSNetanel Belgazal 2141738cd3eSNetanel Belgazal static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq, 2151738cd3eSNetanel Belgazal u8 numa_node) 2161738cd3eSNetanel Belgazal { 2171738cd3eSNetanel Belgazal struct ena_eth_io_numa_node_cfg_reg numa_cfg; 2181738cd3eSNetanel Belgazal 2191738cd3eSNetanel Belgazal if (!io_cq->numa_node_cfg_reg) 2201738cd3eSNetanel Belgazal return; 2211738cd3eSNetanel Belgazal 2221738cd3eSNetanel Belgazal numa_cfg.numa_cfg = (numa_node & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK) 2231738cd3eSNetanel Belgazal | ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK; 2241738cd3eSNetanel Belgazal 2251738cd3eSNetanel Belgazal writel(numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg); 2261738cd3eSNetanel Belgazal } 2271738cd3eSNetanel Belgazal 2281738cd3eSNetanel Belgazal static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem) 2291738cd3eSNetanel Belgazal { 2301738cd3eSNetanel Belgazal io_sq->next_to_comp += elem; 2311738cd3eSNetanel Belgazal } 2321738cd3eSNetanel Belgazal 2330e575f85SArthur Kiyanovski static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq) 2340e575f85SArthur Kiyanovski { 2350e575f85SArthur Kiyanovski io_cq->head++; 2360e575f85SArthur Kiyanovski 2370e575f85SArthur Kiyanovski /* Switch phase bit in case of wrap around */ 2380e575f85SArthur Kiyanovski if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0)) 2390e575f85SArthur Kiyanovski io_cq->phase ^= 1; 2400e575f85SArthur Kiyanovski } 2410e575f85SArthur Kiyanovski 2420e575f85SArthur Kiyanovski static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, 2430e575f85SArthur Kiyanovski u16 *req_id) 2440e575f85SArthur Kiyanovski { 2450e575f85SArthur Kiyanovski u8 expected_phase, cdesc_phase; 2460e575f85SArthur Kiyanovski struct ena_eth_io_tx_cdesc *cdesc; 2470e575f85SArthur Kiyanovski u16 masked_head; 2480e575f85SArthur Kiyanovski 2490e575f85SArthur Kiyanovski masked_head = io_cq->head & (io_cq->q_depth - 1); 2500e575f85SArthur Kiyanovski expected_phase = io_cq->phase; 2510e575f85SArthur Kiyanovski 2520e575f85SArthur Kiyanovski cdesc = (struct ena_eth_io_tx_cdesc *) 2530e575f85SArthur Kiyanovski ((uintptr_t)io_cq->cdesc_addr.virt_addr + 2540e575f85SArthur Kiyanovski (masked_head * io_cq->cdesc_entry_size_in_bytes)); 2550e575f85SArthur Kiyanovski 2560e575f85SArthur Kiyanovski /* When the current completion descriptor phase isn't the same as the 2570e575f85SArthur Kiyanovski * expected, it mean that the device still didn't update 2580e575f85SArthur Kiyanovski * this completion. 2590e575f85SArthur Kiyanovski */ 2600e575f85SArthur Kiyanovski cdesc_phase = READ_ONCE(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK; 2610e575f85SArthur Kiyanovski if (cdesc_phase != expected_phase) 2620e575f85SArthur Kiyanovski return -EAGAIN; 2630e575f85SArthur Kiyanovski 2640e575f85SArthur Kiyanovski dma_rmb(); 2650e575f85SArthur Kiyanovski 2660e575f85SArthur Kiyanovski *req_id = READ_ONCE(cdesc->req_id); 2670e575f85SArthur Kiyanovski if (unlikely(*req_id >= io_cq->q_depth)) { 2680e575f85SArthur Kiyanovski pr_err("Invalid req id %d\n", cdesc->req_id); 2690e575f85SArthur Kiyanovski return -EINVAL; 2700e575f85SArthur Kiyanovski } 2710e575f85SArthur Kiyanovski 2720e575f85SArthur Kiyanovski ena_com_cq_inc_head(io_cq); 2730e575f85SArthur Kiyanovski 2740e575f85SArthur Kiyanovski return 0; 2750e575f85SArthur Kiyanovski } 2760e575f85SArthur Kiyanovski 2771738cd3eSNetanel Belgazal #endif /* ENA_ETH_COM_H_ */ 278