1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #ifndef __MLX5_EN_H__ 33 #define __MLX5_EN_H__ 34 35 #include <linux/if_vlan.h> 36 #include <linux/etherdevice.h> 37 #include <linux/timecounter.h> 38 #include <linux/net_tstamp.h> 39 #include <linux/crash_dump.h> 40 #include <linux/mlx5/driver.h> 41 #include <linux/mlx5/qp.h> 42 #include <linux/mlx5/cq.h> 43 #include <linux/mlx5/port.h> 44 #include <linux/mlx5/vport.h> 45 #include <linux/mlx5/transobj.h> 46 #include <linux/mlx5/fs.h> 47 #include <linux/rhashtable.h> 48 #include <net/udp_tunnel.h> 49 #include <net/switchdev.h> 50 #include <net/xdp.h> 51 #include <linux/dim.h> 52 #include <linux/bits.h> 53 #include "wq.h" 54 #include "mlx5_core.h" 55 #include "en_stats.h" 56 #include "en/dcbnl.h" 57 #include "en/fs.h" 58 #include "lib/hv_vhca.h" 59 60 extern const struct net_device_ops mlx5e_netdev_ops; 61 struct page_pool; 62 63 #define MLX5E_METADATA_ETHER_TYPE (0x8CE4) 64 #define MLX5E_METADATA_ETHER_LEN 8 65 66 #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v) 67 68 #define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN) 69 70 #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu)) 71 #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu)) 72 73 #define MLX5E_MAX_NUM_TC 8 74 75 #define MLX5_RX_HEADROOM NET_SKB_PAD 76 #define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \ 77 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 78 79 #define MLX5E_RX_MAX_HEAD (256) 80 81 #define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \ 82 (6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */ 83 #define MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, req) \ 84 max_t(u32, MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev), req) 85 #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \ 86 MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD)) 87 88 #define MLX5_MPWRQ_LOG_WQE_SZ 18 89 #define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \ 90 MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0) 91 #define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER) 92 93 #define MLX5_MTT_OCTW(npages) (ALIGN(npages, 8) / 2) 94 /* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between 95 * WQEs, This page will absorb write overflow by the hardware, when 96 * receiving packets larger than MTU. These oversize packets are 97 * dropped by the driver at a later stage. 98 */ 99 #define MLX5E_REQUIRED_WQE_MTTS (ALIGN(MLX5_MPWRQ_PAGES_PER_WQE + 1, 8)) 100 #define MLX5E_LOG_ALIGNED_MPWQE_PPW (ilog2(MLX5E_REQUIRED_WQE_MTTS)) 101 #define MLX5E_REQUIRED_MTTS(wqes) (wqes * MLX5E_REQUIRED_WQE_MTTS) 102 #define MLX5E_MAX_RQ_NUM_MTTS \ 103 ((1 << 16) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */ 104 #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024)) 105 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW \ 106 (ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS)) 107 #define MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW \ 108 (MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \ 109 (MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU)) 110 111 #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM)) 112 #define MLX5E_LOG_MAX_RX_WQE_BULK \ 113 (ilog2(PAGE_SIZE / roundup_pow_of_two(MLX5E_MIN_SKB_FRAG_SZ))) 114 115 #define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6 116 #define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa 117 #define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd 118 119 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK) 120 #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa 121 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd, \ 122 MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW) 123 124 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2 125 126 #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024) 127 #define MLX5E_DEFAULT_LRO_TIMEOUT 32 128 #define MLX5E_LRO_TIMEOUT_ARR_SIZE 4 129 130 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10 131 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3 132 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20 133 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC 0x10 134 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE 0x10 135 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS 0x20 136 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES 0x80 137 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW 0x2 138 139 #define MLX5E_LOG_INDIR_RQT_SIZE 0x7 140 #define MLX5E_INDIR_RQT_SIZE BIT(MLX5E_LOG_INDIR_RQT_SIZE) 141 #define MLX5E_MIN_NUM_CHANNELS 0x1 142 #define MLX5E_MAX_NUM_CHANNELS MLX5E_INDIR_RQT_SIZE 143 #define MLX5E_MAX_NUM_SQS (MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC) 144 #define MLX5E_TX_CQ_POLL_BUDGET 128 145 #define MLX5E_TX_XSK_POLL_BUDGET 64 146 #define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */ 147 148 #define MLX5E_UMR_WQE_INLINE_SZ \ 149 (sizeof(struct mlx5e_umr_wqe) + \ 150 ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(struct mlx5_mtt), \ 151 MLX5_UMR_MTT_ALIGNMENT)) 152 #define MLX5E_UMR_WQEBBS \ 153 (DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB)) 154 155 #define MLX5E_MSG_LEVEL NETIF_MSG_LINK 156 157 #define mlx5e_dbg(mlevel, priv, format, ...) \ 158 do { \ 159 if (NETIF_MSG_##mlevel & (priv)->msglevel) \ 160 netdev_warn(priv->netdev, format, \ 161 ##__VA_ARGS__); \ 162 } while (0) 163 164 enum mlx5e_rq_group { 165 MLX5E_RQ_GROUP_REGULAR, 166 MLX5E_RQ_GROUP_XSK, 167 #define MLX5E_NUM_RQ_GROUPS(g) (1 + MLX5E_RQ_GROUP_##g) 168 }; 169 170 static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev) 171 { 172 if (mlx5_lag_is_lacp_owner(mdev)) 173 return 1; 174 175 return clamp_t(u8, MLX5_CAP_GEN(mdev, num_lag_ports), 1, MLX5_MAX_PORTS); 176 } 177 178 static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size) 179 { 180 switch (wq_type) { 181 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 182 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW, 183 wq_size / 2); 184 default: 185 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES, 186 wq_size / 2); 187 } 188 } 189 190 /* Use this function to get max num channels (rxqs/txqs) only to create netdev */ 191 static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) 192 { 193 return is_kdump_kernel() ? 194 MLX5E_MIN_NUM_CHANNELS : 195 min_t(int, mlx5_comp_vectors_count(mdev), MLX5E_MAX_NUM_CHANNELS); 196 } 197 198 struct mlx5e_tx_wqe { 199 struct mlx5_wqe_ctrl_seg ctrl; 200 struct mlx5_wqe_eth_seg eth; 201 struct mlx5_wqe_data_seg data[0]; 202 }; 203 204 struct mlx5e_rx_wqe_ll { 205 struct mlx5_wqe_srq_next_seg next; 206 struct mlx5_wqe_data_seg data[]; 207 }; 208 209 struct mlx5e_rx_wqe_cyc { 210 struct mlx5_wqe_data_seg data[0]; 211 }; 212 213 struct mlx5e_umr_wqe { 214 struct mlx5_wqe_ctrl_seg ctrl; 215 struct mlx5_wqe_umr_ctrl_seg uctrl; 216 struct mlx5_mkey_seg mkc; 217 struct mlx5_mtt inline_mtts[0]; 218 }; 219 220 extern const char mlx5e_self_tests[][ETH_GSTRING_LEN]; 221 222 enum mlx5e_priv_flag { 223 MLX5E_PFLAG_RX_CQE_BASED_MODER, 224 MLX5E_PFLAG_TX_CQE_BASED_MODER, 225 MLX5E_PFLAG_RX_CQE_COMPRESS, 226 MLX5E_PFLAG_RX_STRIDING_RQ, 227 MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, 228 MLX5E_PFLAG_XDP_TX_MPWQE, 229 MLX5E_PFLAG_SKB_TX_MPWQE, 230 MLX5E_NUM_PFLAGS, /* Keep last */ 231 }; 232 233 #define MLX5E_SET_PFLAG(params, pflag, enable) \ 234 do { \ 235 if (enable) \ 236 (params)->pflags |= BIT(pflag); \ 237 else \ 238 (params)->pflags &= ~(BIT(pflag)); \ 239 } while (0) 240 241 #define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag)))) 242 243 struct mlx5e_params { 244 u8 log_sq_size; 245 u8 rq_wq_type; 246 u8 log_rq_mtu_frames; 247 u16 num_channels; 248 u8 num_tc; 249 bool rx_cqe_compress_def; 250 bool tunneled_offload_en; 251 struct dim_cq_moder rx_cq_moderation; 252 struct dim_cq_moder tx_cq_moderation; 253 bool lro_en; 254 u8 tx_min_inline_mode; 255 bool vlan_strip_disable; 256 bool scatter_fcs_en; 257 bool rx_dim_enabled; 258 bool tx_dim_enabled; 259 u32 lro_timeout; 260 u32 pflags; 261 struct bpf_prog *xdp_prog; 262 struct mlx5e_xsk *xsk; 263 unsigned int sw_mtu; 264 int hard_mtu; 265 }; 266 267 enum { 268 MLX5E_RQ_STATE_ENABLED, 269 MLX5E_RQ_STATE_RECOVERING, 270 MLX5E_RQ_STATE_AM, 271 MLX5E_RQ_STATE_NO_CSUM_COMPLETE, 272 MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */ 273 MLX5E_RQ_STATE_FPGA_TLS, /* FPGA TLS enabled */ 274 MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX /* set when mini_cqe_resp_stride_index cap is used */ 275 }; 276 277 struct mlx5e_cq { 278 /* data path - accessed per cqe */ 279 struct mlx5_cqwq wq; 280 281 /* data path - accessed per napi poll */ 282 u16 event_ctr; 283 struct napi_struct *napi; 284 struct mlx5_core_cq mcq; 285 struct mlx5e_channel *channel; 286 287 /* control */ 288 struct mlx5_core_dev *mdev; 289 struct mlx5_wq_ctrl wq_ctrl; 290 } ____cacheline_aligned_in_smp; 291 292 struct mlx5e_cq_decomp { 293 /* cqe decompression */ 294 struct mlx5_cqe64 title; 295 struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE]; 296 u8 mini_arr_idx; 297 u16 left; 298 u16 wqe_counter; 299 } ____cacheline_aligned_in_smp; 300 301 enum mlx5e_dma_map_type { 302 MLX5E_DMA_MAP_SINGLE, 303 MLX5E_DMA_MAP_PAGE 304 }; 305 306 struct mlx5e_sq_dma { 307 dma_addr_t addr; 308 u32 size; 309 enum mlx5e_dma_map_type type; 310 }; 311 312 enum { 313 MLX5E_SQ_STATE_ENABLED, 314 MLX5E_SQ_STATE_MPWQE, 315 MLX5E_SQ_STATE_RECOVERING, 316 MLX5E_SQ_STATE_IPSEC, 317 MLX5E_SQ_STATE_AM, 318 MLX5E_SQ_STATE_TLS, 319 MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, 320 MLX5E_SQ_STATE_PENDING_XSK_TX, 321 }; 322 323 struct mlx5e_tx_mpwqe { 324 /* Current MPWQE session */ 325 struct mlx5e_tx_wqe *wqe; 326 u32 bytes_count; 327 u8 ds_count; 328 u8 pkt_count; 329 u8 inline_on; 330 }; 331 332 struct mlx5e_txqsq { 333 /* data path */ 334 335 /* dirtied @completion */ 336 u16 cc; 337 u16 skb_fifo_cc; 338 u32 dma_fifo_cc; 339 struct dim dim; /* Adaptive Moderation */ 340 341 /* dirtied @xmit */ 342 u16 pc ____cacheline_aligned_in_smp; 343 u16 skb_fifo_pc; 344 u32 dma_fifo_pc; 345 struct mlx5e_tx_mpwqe mpwqe; 346 347 struct mlx5e_cq cq; 348 349 /* read only */ 350 struct mlx5_wq_cyc wq; 351 u32 dma_fifo_mask; 352 u16 skb_fifo_mask; 353 struct mlx5e_sq_stats *stats; 354 struct { 355 struct mlx5e_sq_dma *dma_fifo; 356 struct sk_buff **skb_fifo; 357 struct mlx5e_tx_wqe_info *wqe_info; 358 } db; 359 void __iomem *uar_map; 360 struct netdev_queue *txq; 361 u32 sqn; 362 u16 stop_room; 363 u8 min_inline_mode; 364 struct device *pdev; 365 __be32 mkey_be; 366 unsigned long state; 367 unsigned int hw_mtu; 368 struct hwtstamp_config *tstamp; 369 struct mlx5_clock *clock; 370 371 /* control path */ 372 struct mlx5_wq_ctrl wq_ctrl; 373 struct mlx5e_channel *channel; 374 int ch_ix; 375 int txq_ix; 376 u32 rate_limit; 377 struct work_struct recover_work; 378 } ____cacheline_aligned_in_smp; 379 380 struct mlx5e_dma_info { 381 dma_addr_t addr; 382 union { 383 struct page *page; 384 struct xdp_buff *xsk; 385 }; 386 }; 387 388 /* XDP packets can be transmitted in different ways. On completion, we need to 389 * distinguish between them to clean up things in a proper way. 390 */ 391 enum mlx5e_xdp_xmit_mode { 392 /* An xdp_frame was transmitted due to either XDP_REDIRECT from another 393 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and 394 * returned. 395 */ 396 MLX5E_XDP_XMIT_MODE_FRAME, 397 398 /* The xdp_frame was created in place as a result of XDP_TX from a 399 * regular RQ. No DMA remapping happened, and the page belongs to us. 400 */ 401 MLX5E_XDP_XMIT_MODE_PAGE, 402 403 /* No xdp_frame was created at all, the transmit happened from a UMEM 404 * page. The UMEM Completion Ring producer pointer has to be increased. 405 */ 406 MLX5E_XDP_XMIT_MODE_XSK, 407 }; 408 409 struct mlx5e_xdp_info { 410 enum mlx5e_xdp_xmit_mode mode; 411 union { 412 struct { 413 struct xdp_frame *xdpf; 414 dma_addr_t dma_addr; 415 } frame; 416 struct { 417 struct mlx5e_rq *rq; 418 struct mlx5e_dma_info di; 419 } page; 420 }; 421 }; 422 423 struct mlx5e_xmit_data { 424 dma_addr_t dma_addr; 425 void *data; 426 u32 len; 427 }; 428 429 struct mlx5e_xdp_info_fifo { 430 struct mlx5e_xdp_info *xi; 431 u32 *cc; 432 u32 *pc; 433 u32 mask; 434 }; 435 436 struct mlx5e_xdpsq; 437 typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *); 438 typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *, 439 struct mlx5e_xmit_data *, 440 struct mlx5e_xdp_info *, 441 int); 442 443 struct mlx5e_xdpsq { 444 /* data path */ 445 446 /* dirtied @completion */ 447 u32 xdpi_fifo_cc; 448 u16 cc; 449 450 /* dirtied @xmit */ 451 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp; 452 u16 pc; 453 struct mlx5_wqe_ctrl_seg *doorbell_cseg; 454 struct mlx5e_tx_mpwqe mpwqe; 455 456 struct mlx5e_cq cq; 457 458 /* read only */ 459 struct xsk_buff_pool *xsk_pool; 460 struct mlx5_wq_cyc wq; 461 struct mlx5e_xdpsq_stats *stats; 462 mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check; 463 mlx5e_fp_xmit_xdp_frame xmit_xdp_frame; 464 struct { 465 struct mlx5e_xdp_wqe_info *wqe_info; 466 struct mlx5e_xdp_info_fifo xdpi_fifo; 467 } db; 468 void __iomem *uar_map; 469 u32 sqn; 470 struct device *pdev; 471 __be32 mkey_be; 472 u8 min_inline_mode; 473 unsigned long state; 474 unsigned int hw_mtu; 475 476 /* control path */ 477 struct mlx5_wq_ctrl wq_ctrl; 478 struct mlx5e_channel *channel; 479 } ____cacheline_aligned_in_smp; 480 481 struct mlx5e_icosq { 482 /* data path */ 483 u16 cc; 484 u16 pc; 485 486 struct mlx5_wqe_ctrl_seg *doorbell_cseg; 487 struct mlx5e_cq cq; 488 489 /* write@xmit, read@completion */ 490 struct { 491 struct mlx5e_icosq_wqe_info *wqe_info; 492 } db; 493 494 /* read only */ 495 struct mlx5_wq_cyc wq; 496 void __iomem *uar_map; 497 u32 sqn; 498 unsigned long state; 499 500 /* control path */ 501 struct mlx5_wq_ctrl wq_ctrl; 502 struct mlx5e_channel *channel; 503 504 struct work_struct recover_work; 505 } ____cacheline_aligned_in_smp; 506 507 struct mlx5e_wqe_frag_info { 508 struct mlx5e_dma_info *di; 509 u32 offset; 510 bool last_in_page; 511 }; 512 513 struct mlx5e_umr_dma_info { 514 struct mlx5e_dma_info dma_info[MLX5_MPWRQ_PAGES_PER_WQE]; 515 }; 516 517 struct mlx5e_mpw_info { 518 struct mlx5e_umr_dma_info umr; 519 u16 consumed_strides; 520 DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE); 521 }; 522 523 #define MLX5E_MAX_RX_FRAGS 4 524 525 /* a single cache unit is capable to serve one napi call (for non-striding rq) 526 * or a MPWQE (for striding rq). 527 */ 528 #define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \ 529 MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT) 530 #define MLX5E_CACHE_SIZE (4 * roundup_pow_of_two(MLX5E_CACHE_UNIT)) 531 struct mlx5e_page_cache { 532 u32 head; 533 u32 tail; 534 struct mlx5e_dma_info page_cache[MLX5E_CACHE_SIZE]; 535 }; 536 537 struct mlx5e_rq; 538 typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*); 539 typedef struct sk_buff * 540 (*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, 541 u16 cqe_bcnt, u32 head_offset, u32 page_idx); 542 typedef struct sk_buff * 543 (*mlx5e_fp_skb_from_cqe)(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, 544 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt); 545 typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); 546 typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16); 547 548 int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk); 549 550 enum mlx5e_rq_flag { 551 MLX5E_RQ_FLAG_XDP_XMIT, 552 MLX5E_RQ_FLAG_XDP_REDIRECT, 553 }; 554 555 struct mlx5e_rq_frag_info { 556 int frag_size; 557 int frag_stride; 558 }; 559 560 struct mlx5e_rq_frags_info { 561 struct mlx5e_rq_frag_info arr[MLX5E_MAX_RX_FRAGS]; 562 u8 num_frags; 563 u8 log_num_frags; 564 u8 wqe_bulk; 565 }; 566 567 struct mlx5e_rq { 568 /* data path */ 569 union { 570 struct { 571 struct mlx5_wq_cyc wq; 572 struct mlx5e_wqe_frag_info *frags; 573 struct mlx5e_dma_info *di; 574 struct mlx5e_rq_frags_info info; 575 mlx5e_fp_skb_from_cqe skb_from_cqe; 576 } wqe; 577 struct { 578 struct mlx5_wq_ll wq; 579 struct mlx5e_umr_wqe umr_wqe; 580 struct mlx5e_mpw_info *info; 581 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq; 582 u16 num_strides; 583 u16 actual_wq_head; 584 u8 log_stride_sz; 585 u8 umr_in_progress; 586 u8 umr_last_bulk; 587 u8 umr_completed; 588 } mpwqe; 589 }; 590 struct { 591 u16 headroom; 592 u32 frame0_sz; 593 u8 map_dir; /* dma map direction */ 594 } buff; 595 596 struct mlx5e_channel *channel; 597 struct device *pdev; 598 struct net_device *netdev; 599 struct mlx5e_rq_stats *stats; 600 struct mlx5e_cq cq; 601 struct mlx5e_cq_decomp cqd; 602 struct mlx5e_page_cache page_cache; 603 struct hwtstamp_config *tstamp; 604 struct mlx5_clock *clock; 605 606 mlx5e_fp_handle_rx_cqe handle_rx_cqe; 607 mlx5e_fp_post_rx_wqes post_wqes; 608 mlx5e_fp_dealloc_wqe dealloc_wqe; 609 610 unsigned long state; 611 int ix; 612 unsigned int hw_mtu; 613 614 struct dim dim; /* Dynamic Interrupt Moderation */ 615 616 /* XDP */ 617 struct bpf_prog __rcu *xdp_prog; 618 struct mlx5e_xdpsq *xdpsq; 619 DECLARE_BITMAP(flags, 8); 620 struct page_pool *page_pool; 621 622 /* AF_XDP zero-copy */ 623 struct xsk_buff_pool *xsk_pool; 624 625 struct work_struct recover_work; 626 627 /* control */ 628 struct mlx5_wq_ctrl wq_ctrl; 629 __be32 mkey_be; 630 u8 wq_type; 631 u32 rqn; 632 struct mlx5_core_dev *mdev; 633 struct mlx5_core_mkey umr_mkey; 634 struct mlx5e_dma_info wqe_overflow; 635 636 /* XDP read-mostly */ 637 struct xdp_rxq_info xdp_rxq; 638 } ____cacheline_aligned_in_smp; 639 640 enum mlx5e_channel_state { 641 MLX5E_CHANNEL_STATE_XSK, 642 MLX5E_CHANNEL_NUM_STATES 643 }; 644 645 struct mlx5e_channel { 646 /* data path */ 647 struct mlx5e_rq rq; 648 struct mlx5e_xdpsq rq_xdpsq; 649 struct mlx5e_txqsq sq[MLX5E_MAX_NUM_TC]; 650 struct mlx5e_icosq icosq; /* internal control operations */ 651 bool xdp; 652 struct napi_struct napi; 653 struct device *pdev; 654 struct net_device *netdev; 655 __be32 mkey_be; 656 u8 num_tc; 657 u8 lag_port; 658 659 /* XDP_REDIRECT */ 660 struct mlx5e_xdpsq xdpsq; 661 662 /* AF_XDP zero-copy */ 663 struct mlx5e_rq xskrq; 664 struct mlx5e_xdpsq xsksq; 665 666 /* Async ICOSQ */ 667 struct mlx5e_icosq async_icosq; 668 /* async_icosq can be accessed from any CPU - the spinlock protects it. */ 669 spinlock_t async_icosq_lock; 670 671 /* data path - accessed per napi poll */ 672 struct irq_desc *irq_desc; 673 struct mlx5e_ch_stats *stats; 674 675 /* control */ 676 struct mlx5e_priv *priv; 677 struct mlx5_core_dev *mdev; 678 struct hwtstamp_config *tstamp; 679 DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES); 680 int ix; 681 int cpu; 682 }; 683 684 struct mlx5e_channels { 685 struct mlx5e_channel **c; 686 unsigned int num; 687 struct mlx5e_params params; 688 }; 689 690 struct mlx5e_channel_stats { 691 struct mlx5e_ch_stats ch; 692 struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC]; 693 struct mlx5e_rq_stats rq; 694 struct mlx5e_rq_stats xskrq; 695 struct mlx5e_xdpsq_stats rq_xdpsq; 696 struct mlx5e_xdpsq_stats xdpsq; 697 struct mlx5e_xdpsq_stats xsksq; 698 } ____cacheline_aligned_in_smp; 699 700 enum { 701 MLX5E_STATE_OPENED, 702 MLX5E_STATE_DESTROYING, 703 MLX5E_STATE_XDP_TX_ENABLED, 704 MLX5E_STATE_XDP_ACTIVE, 705 }; 706 707 struct mlx5e_rqt { 708 u32 rqtn; 709 bool enabled; 710 }; 711 712 struct mlx5e_tir { 713 u32 tirn; 714 struct mlx5e_rqt rqt; 715 struct list_head list; 716 }; 717 718 enum { 719 MLX5E_TC_PRIO = 0, 720 MLX5E_NIC_PRIO 721 }; 722 723 struct mlx5e_rss_params { 724 u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE]; 725 u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS]; 726 u8 toeplitz_hash_key[40]; 727 u8 hfunc; 728 }; 729 730 struct mlx5e_modify_sq_param { 731 int curr_state; 732 int next_state; 733 int rl_update; 734 int rl_index; 735 }; 736 737 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE) 738 struct mlx5e_hv_vhca_stats_agent { 739 struct mlx5_hv_vhca_agent *agent; 740 struct delayed_work work; 741 u16 delay; 742 void *buf; 743 }; 744 #endif 745 746 struct mlx5e_xsk { 747 /* XSK buffer pools are stored separately from channels, 748 * because we don't want to lose them when channels are 749 * recreated. The kernel also stores buffer pool, but it doesn't 750 * distinguish between zero-copy and non-zero-copy UMEMs, so 751 * rely on our mechanism. 752 */ 753 struct xsk_buff_pool **pools; 754 u16 refcnt; 755 bool ever_used; 756 }; 757 758 /* Temporary storage for variables that are allocated when struct mlx5e_priv is 759 * initialized, and used where we can't allocate them because that functions 760 * must not fail. Use with care and make sure the same variable is not used 761 * simultaneously by multiple users. 762 */ 763 struct mlx5e_scratchpad { 764 cpumask_var_t cpumask; 765 }; 766 767 struct mlx5e_priv { 768 /* priv data path fields - start */ 769 struct mlx5e_txqsq *txq2sq[MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC]; 770 int channel_tc2realtxq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC]; 771 #ifdef CONFIG_MLX5_CORE_EN_DCB 772 struct mlx5e_dcbx_dp dcbx_dp; 773 #endif 774 /* priv data path fields - end */ 775 776 u32 msglevel; 777 unsigned long state; 778 struct mutex state_lock; /* Protects Interface state */ 779 struct mlx5e_rq drop_rq; 780 781 struct mlx5e_channels channels; 782 u32 tisn[MLX5_MAX_PORTS][MLX5E_MAX_NUM_TC]; 783 struct mlx5e_rqt indir_rqt; 784 struct mlx5e_tir indir_tir[MLX5E_NUM_INDIR_TIRS]; 785 struct mlx5e_tir inner_indir_tir[MLX5E_NUM_INDIR_TIRS]; 786 struct mlx5e_tir direct_tir[MLX5E_MAX_NUM_CHANNELS]; 787 struct mlx5e_tir xsk_tir[MLX5E_MAX_NUM_CHANNELS]; 788 struct mlx5e_rss_params rss_params; 789 u32 tx_rates[MLX5E_MAX_NUM_SQS]; 790 791 struct mlx5e_flow_steering fs; 792 793 struct workqueue_struct *wq; 794 struct work_struct update_carrier_work; 795 struct work_struct set_rx_mode_work; 796 struct work_struct tx_timeout_work; 797 struct work_struct update_stats_work; 798 struct work_struct monitor_counters_work; 799 struct mlx5_nb monitor_counters_nb; 800 801 struct mlx5_core_dev *mdev; 802 struct net_device *netdev; 803 struct mlx5e_stats stats; 804 struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS]; 805 u16 max_nch; 806 u8 max_opened_tc; 807 struct hwtstamp_config tstamp; 808 u16 q_counter; 809 u16 drop_rq_q_counter; 810 struct notifier_block events_nb; 811 812 struct udp_tunnel_nic_info nic_info; 813 #ifdef CONFIG_MLX5_CORE_EN_DCB 814 struct mlx5e_dcbx dcbx; 815 #endif 816 817 const struct mlx5e_profile *profile; 818 void *ppriv; 819 #ifdef CONFIG_MLX5_EN_IPSEC 820 struct mlx5e_ipsec *ipsec; 821 #endif 822 #ifdef CONFIG_MLX5_EN_TLS 823 struct mlx5e_tls *tls; 824 #endif 825 struct devlink_health_reporter *tx_reporter; 826 struct devlink_health_reporter *rx_reporter; 827 struct devlink_port dl_port; 828 struct mlx5e_xsk xsk; 829 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE) 830 struct mlx5e_hv_vhca_stats_agent stats_agent; 831 #endif 832 struct mlx5e_scratchpad scratchpad; 833 }; 834 835 struct mlx5e_rx_handlers { 836 mlx5e_fp_handle_rx_cqe handle_rx_cqe; 837 mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe; 838 }; 839 840 extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic; 841 842 struct mlx5e_profile { 843 int (*init)(struct mlx5_core_dev *mdev, 844 struct net_device *netdev, 845 const struct mlx5e_profile *profile, void *ppriv); 846 void (*cleanup)(struct mlx5e_priv *priv); 847 int (*init_rx)(struct mlx5e_priv *priv); 848 void (*cleanup_rx)(struct mlx5e_priv *priv); 849 int (*init_tx)(struct mlx5e_priv *priv); 850 void (*cleanup_tx)(struct mlx5e_priv *priv); 851 void (*enable)(struct mlx5e_priv *priv); 852 void (*disable)(struct mlx5e_priv *priv); 853 int (*update_rx)(struct mlx5e_priv *priv); 854 void (*update_stats)(struct mlx5e_priv *priv); 855 void (*update_carrier)(struct mlx5e_priv *priv); 856 unsigned int (*stats_grps_num)(struct mlx5e_priv *priv); 857 mlx5e_stats_grp_t *stats_grps; 858 const struct mlx5e_rx_handlers *rx_handlers; 859 int max_tc; 860 u8 rq_groups; 861 }; 862 863 void mlx5e_build_ptys2ethtool_map(void); 864 865 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev); 866 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, 867 struct mlx5e_params *params); 868 869 void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); 870 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s); 871 872 void mlx5e_init_l2_addr(struct mlx5e_priv *priv); 873 int mlx5e_self_test_num(struct mlx5e_priv *priv); 874 void mlx5e_self_test(struct net_device *ndev, struct ethtool_test *etest, 875 u64 *buf); 876 void mlx5e_set_rx_mode_work(struct work_struct *work); 877 878 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr); 879 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr); 880 int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val); 881 882 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __always_unused __be16 proto, 883 u16 vid); 884 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __always_unused __be16 proto, 885 u16 vid); 886 void mlx5e_timestamp_init(struct mlx5e_priv *priv); 887 888 struct mlx5e_redirect_rqt_param { 889 bool is_rss; 890 union { 891 u32 rqn; /* Direct RQN (Non-RSS) */ 892 struct { 893 u8 hfunc; 894 struct mlx5e_channels *channels; 895 } rss; /* RSS data */ 896 }; 897 }; 898 899 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, 900 struct mlx5e_redirect_rqt_param rrp); 901 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params, 902 const struct mlx5e_tirc_config *ttconfig, 903 void *tirc, bool inner); 904 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in); 905 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt); 906 907 struct mlx5e_xsk_param; 908 909 struct mlx5e_rq_param; 910 int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 911 struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk, 912 struct xsk_buff_pool *xsk_pool, struct mlx5e_rq *rq); 913 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); 914 void mlx5e_deactivate_rq(struct mlx5e_rq *rq); 915 void mlx5e_close_rq(struct mlx5e_rq *rq); 916 917 struct mlx5e_sq_param; 918 int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 919 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq); 920 void mlx5e_close_icosq(struct mlx5e_icosq *sq); 921 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 922 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool, 923 struct mlx5e_xdpsq *sq, bool is_redirect); 924 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq); 925 926 struct mlx5e_cq_param; 927 int mlx5e_open_cq(struct mlx5e_channel *c, struct dim_cq_moder moder, 928 struct mlx5e_cq_param *param, struct mlx5e_cq *cq); 929 void mlx5e_close_cq(struct mlx5e_cq *cq); 930 931 int mlx5e_open_locked(struct net_device *netdev); 932 int mlx5e_close_locked(struct net_device *netdev); 933 934 int mlx5e_open_channels(struct mlx5e_priv *priv, 935 struct mlx5e_channels *chs); 936 void mlx5e_close_channels(struct mlx5e_channels *chs); 937 938 /* Function pointer to be used to modify HW or kernel settings while 939 * switching channels 940 */ 941 typedef int (*mlx5e_fp_preactivate)(struct mlx5e_priv *priv, void *context); 942 #define MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(fn) \ 943 int fn##_ctx(struct mlx5e_priv *priv, void *context) \ 944 { \ 945 return fn(priv); \ 946 } 947 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv); 948 int mlx5e_safe_switch_channels(struct mlx5e_priv *priv, 949 struct mlx5e_channels *new_chs, 950 mlx5e_fp_preactivate preactivate, 951 void *context); 952 int mlx5e_num_channels_changed(struct mlx5e_priv *priv); 953 int mlx5e_num_channels_changed_ctx(struct mlx5e_priv *priv, void *context); 954 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv); 955 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv); 956 957 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len, 958 int num_channels); 959 960 void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode); 961 void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode); 962 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode); 963 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode); 964 965 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params); 966 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, 967 struct mlx5e_params *params); 968 int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state); 969 void mlx5e_activate_rq(struct mlx5e_rq *rq); 970 void mlx5e_deactivate_rq(struct mlx5e_rq *rq); 971 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq); 972 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq); 973 974 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 975 struct mlx5e_modify_sq_param *p); 976 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq); 977 void mlx5e_tx_disable_queue(struct netdev_queue *txq); 978 979 static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev) 980 { 981 return MLX5_CAP_ETH(mdev, swp) && 982 MLX5_CAP_ETH(mdev, swp_csum) && MLX5_CAP_ETH(mdev, swp_lso); 983 } 984 985 extern const struct ethtool_ops mlx5e_ethtool_ops; 986 987 int mlx5e_create_tir(struct mlx5_core_dev *mdev, struct mlx5e_tir *tir, 988 u32 *in); 989 void mlx5e_destroy_tir(struct mlx5_core_dev *mdev, 990 struct mlx5e_tir *tir); 991 int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev); 992 void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev); 993 int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb, 994 bool enable_mc_lb); 995 void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc); 996 997 /* common netdev helpers */ 998 void mlx5e_create_q_counters(struct mlx5e_priv *priv); 999 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv); 1000 int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 1001 struct mlx5e_rq *drop_rq); 1002 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq); 1003 1004 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv); 1005 1006 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc); 1007 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv); 1008 1009 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs); 1010 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs); 1011 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs); 1012 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs); 1013 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt); 1014 1015 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn); 1016 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn); 1017 1018 int mlx5e_create_tises(struct mlx5e_priv *priv); 1019 void mlx5e_destroy_tises(struct mlx5e_priv *priv); 1020 int mlx5e_update_nic_rx(struct mlx5e_priv *priv); 1021 void mlx5e_update_carrier(struct mlx5e_priv *priv); 1022 int mlx5e_close(struct net_device *netdev); 1023 int mlx5e_open(struct net_device *netdev); 1024 1025 void mlx5e_queue_update_stats(struct mlx5e_priv *priv); 1026 int mlx5e_bits_invert(unsigned long a, int size); 1027 1028 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv); 1029 int mlx5e_set_dev_port_mtu_ctx(struct mlx5e_priv *priv, void *context); 1030 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 1031 mlx5e_fp_preactivate preactivate); 1032 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv); 1033 1034 /* ethtool helpers */ 1035 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv, 1036 struct ethtool_drvinfo *drvinfo); 1037 void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv, 1038 uint32_t stringset, uint8_t *data); 1039 int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset); 1040 void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv, 1041 struct ethtool_stats *stats, u64 *data); 1042 void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, 1043 struct ethtool_ringparam *param); 1044 int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, 1045 struct ethtool_ringparam *param); 1046 void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv, 1047 struct ethtool_channels *ch); 1048 int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, 1049 struct ethtool_channels *ch); 1050 int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv, 1051 struct ethtool_coalesce *coal); 1052 int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, 1053 struct ethtool_coalesce *coal); 1054 int mlx5e_ethtool_get_link_ksettings(struct mlx5e_priv *priv, 1055 struct ethtool_link_ksettings *link_ksettings); 1056 int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv, 1057 const struct ethtool_link_ksettings *link_ksettings); 1058 int mlx5e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc); 1059 int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, 1060 const u8 hfunc); 1061 int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, 1062 u32 *rule_locs); 1063 int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd); 1064 u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv); 1065 u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv); 1066 int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv, 1067 struct ethtool_ts_info *info); 1068 int mlx5e_ethtool_flash_device(struct mlx5e_priv *priv, 1069 struct ethtool_flash *flash); 1070 void mlx5e_ethtool_get_pauseparam(struct mlx5e_priv *priv, 1071 struct ethtool_pauseparam *pauseparam); 1072 int mlx5e_ethtool_set_pauseparam(struct mlx5e_priv *priv, 1073 struct ethtool_pauseparam *pauseparam); 1074 1075 /* mlx5e generic netdev management API */ 1076 int mlx5e_netdev_init(struct net_device *netdev, 1077 struct mlx5e_priv *priv, 1078 struct mlx5_core_dev *mdev, 1079 const struct mlx5e_profile *profile, 1080 void *ppriv); 1081 void mlx5e_netdev_cleanup(struct net_device *netdev, struct mlx5e_priv *priv); 1082 struct net_device* 1083 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile, 1084 int nch, void *ppriv); 1085 int mlx5e_attach_netdev(struct mlx5e_priv *priv); 1086 void mlx5e_detach_netdev(struct mlx5e_priv *priv); 1087 void mlx5e_destroy_netdev(struct mlx5e_priv *priv); 1088 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv); 1089 void mlx5e_build_nic_params(struct mlx5e_priv *priv, 1090 struct mlx5e_xsk *xsk, 1091 struct mlx5e_rss_params *rss_params, 1092 struct mlx5e_params *params, 1093 u16 mtu); 1094 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, 1095 struct mlx5e_params *params); 1096 void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params, 1097 u16 num_channels); 1098 void mlx5e_rx_dim_work(struct work_struct *work); 1099 void mlx5e_tx_dim_work(struct work_struct *work); 1100 1101 netdev_features_t mlx5e_features_check(struct sk_buff *skb, 1102 struct net_device *netdev, 1103 netdev_features_t features); 1104 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features); 1105 #ifdef CONFIG_MLX5_ESWITCH 1106 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac); 1107 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate); 1108 int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi); 1109 int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats); 1110 #endif 1111 #endif /* __MLX5_EN_H__ */ 1112