1 /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */ 2 3 #ifndef _FUNETH_TXRX_H 4 #define _FUNETH_TXRX_H 5 6 #include <linux/netdevice.h> 7 #include <linux/u64_stats_sync.h> 8 9 /* Tx descriptor size */ 10 #define FUNETH_SQE_SIZE 64U 11 12 /* Size of device headers per Tx packet */ 13 #define FUNETH_FUNOS_HDR_SZ (sizeof(struct fun_eth_tx_req)) 14 15 /* Number of gather list entries per Tx descriptor */ 16 #define FUNETH_GLE_PER_DESC (FUNETH_SQE_SIZE / sizeof(struct fun_dataop_gl)) 17 18 /* Max gather list size in bytes for an sk_buff. */ 19 #define FUNETH_MAX_GL_SZ ((MAX_SKB_FRAGS + 1) * sizeof(struct fun_dataop_gl)) 20 21 #if IS_ENABLED(CONFIG_TLS_DEVICE) 22 # define FUNETH_TLS_SZ sizeof(struct fun_eth_tls) 23 #else 24 # define FUNETH_TLS_SZ 0 25 #endif 26 27 /* Max number of Tx descriptors for an sk_buff using a gather list. */ 28 #define FUNETH_MAX_GL_DESC \ 29 DIV_ROUND_UP((FUNETH_FUNOS_HDR_SZ + FUNETH_MAX_GL_SZ + FUNETH_TLS_SZ), \ 30 FUNETH_SQE_SIZE) 31 32 /* Max number of Tx descriptors for any packet. */ 33 #define FUNETH_MAX_PKT_DESC FUNETH_MAX_GL_DESC 34 35 /* Rx CQ descriptor size. */ 36 #define FUNETH_CQE_SIZE 64U 37 38 /* Offset of cqe_info within a CQE. */ 39 #define FUNETH_CQE_INFO_OFFSET (FUNETH_CQE_SIZE - sizeof(struct fun_cqe_info)) 40 41 /* Construct the IRQ portion of a CQ doorbell. The resulting value arms the 42 * interrupt with the supplied time delay and packet count moderation settings. 43 */ 44 #define FUN_IRQ_CQ_DB(usec, pkts) \ 45 (FUN_DB_IRQ_ARM_F | ((usec) << FUN_DB_INTCOAL_USEC_S) | \ 46 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S)) 47 48 /* As above for SQ doorbells. */ 49 #define FUN_IRQ_SQ_DB(usec, pkts) \ 50 (FUN_DB_IRQ_ARM_F | \ 51 ((usec) << FUN_DB_INTCOAL_USEC_S) | \ 52 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S)) 53 54 /* Per packet tailroom. Present only for 1-frag packets. */ 55 #define FUN_RX_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) 56 57 /* Per packet headroom for XDP. Preferred over XDP_PACKET_HEADROOM to 58 * accommodate two packets per buffer for 4K pages and 1500B MTUs. 59 */ 60 #define FUN_XDP_HEADROOM 192 61 62 /* Initialization state of a queue. */ 63 enum { 64 FUN_QSTATE_DESTROYED, /* what queue? */ 65 FUN_QSTATE_INIT_SW, /* exists in SW, not on the device */ 66 FUN_QSTATE_INIT_FULL, /* exists both in SW and on device */ 67 }; 68 69 /* Initialization state of an interrupt. */ 70 enum { 71 FUN_IRQ_INIT, /* initialized and in the XArray but inactive */ 72 FUN_IRQ_REQUESTED, /* request_irq() done */ 73 FUN_IRQ_ENABLED, /* processing enabled */ 74 FUN_IRQ_DISABLED, /* processing disabled */ 75 }; 76 77 struct bpf_prog; 78 79 struct funeth_txq_stats { /* per Tx queue SW counters */ 80 u64 tx_pkts; /* # of Tx packets */ 81 u64 tx_bytes; /* total bytes of Tx packets */ 82 u64 tx_cso; /* # of packets with checksum offload */ 83 u64 tx_tso; /* # of non-encapsulated TSO super-packets */ 84 u64 tx_encap_tso; /* # of encapsulated TSO super-packets */ 85 u64 tx_more; /* # of DBs elided due to xmit_more */ 86 u64 tx_nstops; /* # of times the queue has stopped */ 87 u64 tx_nrestarts; /* # of times the queue has restarted */ 88 u64 tx_map_err; /* # of packets dropped due to DMA mapping errors */ 89 u64 tx_xdp_full; /* # of XDP packets that could not be enqueued */ 90 u64 tx_tls_pkts; /* # of Tx TLS packets offloaded to HW */ 91 u64 tx_tls_bytes; /* Tx bytes of HW-handled TLS payload */ 92 u64 tx_tls_fallback; /* attempted Tx TLS offloads punted to SW */ 93 u64 tx_tls_drops; /* attempted Tx TLS offloads dropped */ 94 }; 95 96 struct funeth_tx_info { /* per Tx descriptor state */ 97 union { 98 struct sk_buff *skb; /* associated packet */ 99 void *vaddr; /* start address for XDP */ 100 }; 101 }; 102 103 struct funeth_txq { 104 /* RO cacheline of frequently accessed data */ 105 u32 mask; /* queue depth - 1 */ 106 u32 hw_qid; /* device ID of the queue */ 107 void *desc; /* base address of descriptor ring */ 108 struct funeth_tx_info *info; 109 struct device *dma_dev; /* device for DMA mappings */ 110 volatile __be64 *hw_wb; /* HW write-back location */ 111 u32 __iomem *db; /* SQ doorbell register address */ 112 struct netdev_queue *ndq; 113 dma_addr_t dma_addr; /* DMA address of descriptor ring */ 114 /* producer R/W cacheline */ 115 u16 qidx; /* queue index within net_device */ 116 u16 ethid; 117 u32 prod_cnt; /* producer counter */ 118 struct funeth_txq_stats stats; 119 /* shared R/W cacheline, primarily accessed by consumer */ 120 u32 irq_db_val; /* value written to IRQ doorbell */ 121 u32 cons_cnt; /* consumer (cleanup) counter */ 122 struct net_device *netdev; 123 struct fun_irq *irq; 124 int numa_node; 125 u8 init_state; /* queue initialization state */ 126 struct u64_stats_sync syncp; 127 }; 128 129 struct funeth_rxq_stats { /* per Rx queue SW counters */ 130 u64 rx_pkts; /* # of received packets, including SW drops */ 131 u64 rx_bytes; /* total size of received packets */ 132 u64 rx_cso; /* # of packets with checksum offload */ 133 u64 rx_bufs; /* total # of Rx buffers provided to device */ 134 u64 gro_pkts; /* # of GRO superpackets */ 135 u64 gro_merged; /* # of pkts merged into existing GRO superpackets */ 136 u64 rx_page_alloc; /* # of page allocations for Rx buffers */ 137 u64 rx_budget; /* NAPI iterations that exhausted their budget */ 138 u64 rx_mem_drops; /* # of packets dropped due to memory shortage */ 139 u64 rx_map_err; /* # of page DMA mapping errors */ 140 u64 xdp_drops; /* XDP_DROPped packets */ 141 u64 xdp_tx; /* successful XDP transmits */ 142 u64 xdp_redir; /* successful XDP redirects */ 143 u64 xdp_err; /* packets dropped due to XDP errors */ 144 }; 145 146 struct funeth_rxbuf { /* per Rx buffer state */ 147 struct page *page; /* associated page */ 148 dma_addr_t dma_addr; /* DMA address of page start */ 149 int pg_refs; /* page refs held by driver */ 150 int node; /* page node, or -1 if it is PF_MEMALLOC */ 151 }; 152 153 struct funeth_rx_cache { /* cache of DMA-mapped previously used buffers */ 154 struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */ 155 unsigned int prod_cnt; /* producer counter */ 156 unsigned int cons_cnt; /* consumer counter */ 157 unsigned int mask; /* depth - 1 */ 158 }; 159 160 /* An Rx queue consists of a CQ and an SQ used to provide Rx buffers. */ 161 struct funeth_rxq { 162 struct net_device *netdev; 163 struct napi_struct *napi; 164 struct device *dma_dev; /* device for DMA mappings */ 165 void *cqes; /* base of CQ descriptor ring */ 166 const void *next_cqe_info; /* fun_cqe_info of next CQE */ 167 u32 __iomem *cq_db; /* CQ doorbell register address */ 168 unsigned int cq_head; /* CQ head index */ 169 unsigned int cq_mask; /* CQ depth - 1 */ 170 u16 phase; /* CQ phase tag */ 171 u16 qidx; /* queue index within net_device */ 172 unsigned int irq_db_val; /* IRQ info for CQ doorbell */ 173 struct fun_eprq_rqbuf *rqes; /* base of RQ descriptor ring */ 174 struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */ 175 struct funeth_rxbuf *cur_buf; /* currently active buffer */ 176 u32 __iomem *rq_db; /* RQ doorbell register address */ 177 unsigned int rq_cons; /* RQ consumer counter */ 178 unsigned int rq_mask; /* RQ depth - 1 */ 179 unsigned int buf_offset; /* offset of next pkt in head buffer */ 180 u8 xdp_flush; /* XDP flush types needed at NAPI end */ 181 u8 init_state; /* queue initialization state */ 182 u16 headroom; /* per packet headroom */ 183 unsigned int rq_cons_db; /* value of rq_cons at last RQ db */ 184 unsigned int rq_db_thres; /* # of new buffers needed to write RQ db */ 185 struct funeth_rxbuf spare_buf; /* spare for next buffer replacement */ 186 struct funeth_rx_cache cache; /* used buffer cache */ 187 struct bpf_prog *xdp_prog; /* optional XDP BPF program */ 188 struct funeth_rxq_stats stats; 189 dma_addr_t cq_dma_addr; /* DMA address of CQE ring */ 190 dma_addr_t rq_dma_addr; /* DMA address of RQE ring */ 191 u16 irq_cnt; 192 u32 hw_cqid; /* device ID of the queue's CQ */ 193 u32 hw_sqid; /* device ID of the queue's SQ */ 194 int numa_node; 195 struct u64_stats_sync syncp; 196 struct xdp_rxq_info xdp_rxq; 197 }; 198 199 #define FUN_QSTAT_INC(q, counter) \ 200 do { \ 201 u64_stats_update_begin(&(q)->syncp); \ 202 (q)->stats.counter++; \ 203 u64_stats_update_end(&(q)->syncp); \ 204 } while (0) 205 206 #define FUN_QSTAT_READ(q, seq, stats_copy) \ 207 do { \ 208 seq = u64_stats_fetch_begin(&(q)->syncp); \ 209 stats_copy = (q)->stats; \ 210 } while (u64_stats_fetch_retry(&(q)->syncp, (seq))) 211 212 #define FUN_INT_NAME_LEN (IFNAMSIZ + 16) 213 214 struct fun_irq { 215 struct napi_struct napi; 216 struct funeth_txq *txq; 217 struct funeth_rxq *rxq; 218 u8 state; 219 u16 irq_idx; /* index of MSI-X interrupt */ 220 int irq; /* Linux IRQ vector */ 221 cpumask_t affinity_mask; /* IRQ affinity */ 222 struct irq_affinity_notify aff_notify; 223 char name[FUN_INT_NAME_LEN]; 224 } ____cacheline_internodealigned_in_smp; 225 226 /* Return the start address of the idx-th Tx descriptor. */ 227 static inline void *fun_tx_desc_addr(const struct funeth_txq *q, 228 unsigned int idx) 229 { 230 return q->desc + idx * FUNETH_SQE_SIZE; 231 } 232 233 static inline void fun_txq_wr_db(const struct funeth_txq *q) 234 { 235 unsigned int tail = q->prod_cnt & q->mask; 236 237 writel(tail, q->db); 238 } 239 240 static inline int fun_irq_node(const struct fun_irq *p) 241 { 242 return cpu_to_mem(cpumask_first(&p->affinity_mask)); 243 } 244 245 int fun_rxq_napi_poll(struct napi_struct *napi, int budget); 246 int fun_txq_napi_poll(struct napi_struct *napi, int budget); 247 netdev_tx_t fun_start_xmit(struct sk_buff *skb, struct net_device *netdev); 248 bool fun_xdp_tx(struct funeth_txq *q, void *data, unsigned int len); 249 int fun_xdp_xmit_frames(struct net_device *dev, int n, 250 struct xdp_frame **frames, u32 flags); 251 252 int funeth_txq_create(struct net_device *dev, unsigned int qidx, 253 unsigned int ndesc, struct fun_irq *irq, int state, 254 struct funeth_txq **qp); 255 int fun_txq_create_dev(struct funeth_txq *q, struct fun_irq *irq); 256 struct funeth_txq *funeth_txq_free(struct funeth_txq *q, int state); 257 int funeth_rxq_create(struct net_device *dev, unsigned int qidx, 258 unsigned int ncqe, unsigned int nrqe, struct fun_irq *irq, 259 int state, struct funeth_rxq **qp); 260 int fun_rxq_create_dev(struct funeth_rxq *q, struct fun_irq *irq); 261 struct funeth_rxq *funeth_rxq_free(struct funeth_rxq *q, int state); 262 int fun_rxq_set_bpf(struct funeth_rxq *q, struct bpf_prog *prog); 263 264 #endif /* _FUNETH_TXRX_H */ 265