15a2cc190SJeff Kirsher /*
25a2cc190SJeff Kirsher * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
35a2cc190SJeff Kirsher *
45a2cc190SJeff Kirsher * This software is available to you under a choice of one of two
55a2cc190SJeff Kirsher * licenses. You may choose to be licensed under the terms of the GNU
65a2cc190SJeff Kirsher * General Public License (GPL) Version 2, available from the file
75a2cc190SJeff Kirsher * COPYING in the main directory of this source tree, or the
85a2cc190SJeff Kirsher * OpenIB.org BSD license below:
95a2cc190SJeff Kirsher *
105a2cc190SJeff Kirsher * Redistribution and use in source and binary forms, with or
115a2cc190SJeff Kirsher * without modification, are permitted provided that the following
125a2cc190SJeff Kirsher * conditions are met:
135a2cc190SJeff Kirsher *
145a2cc190SJeff Kirsher * - Redistributions of source code must retain the above
155a2cc190SJeff Kirsher * copyright notice, this list of conditions and the following
165a2cc190SJeff Kirsher * disclaimer.
175a2cc190SJeff Kirsher *
185a2cc190SJeff Kirsher * - Redistributions in binary form must reproduce the above
195a2cc190SJeff Kirsher * copyright notice, this list of conditions and the following
205a2cc190SJeff Kirsher * disclaimer in the documentation and/or other materials
215a2cc190SJeff Kirsher * provided with the distribution.
225a2cc190SJeff Kirsher *
235a2cc190SJeff Kirsher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
245a2cc190SJeff Kirsher * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
255a2cc190SJeff Kirsher * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
265a2cc190SJeff Kirsher * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
275a2cc190SJeff Kirsher * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
285a2cc190SJeff Kirsher * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
295a2cc190SJeff Kirsher * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
305a2cc190SJeff Kirsher * SOFTWARE.
315a2cc190SJeff Kirsher *
325a2cc190SJeff Kirsher */
335a2cc190SJeff Kirsher
345a2cc190SJeff Kirsher #ifndef _MLX4_EN_H_
355a2cc190SJeff Kirsher #define _MLX4_EN_H_
365a2cc190SJeff Kirsher
375a2cc190SJeff Kirsher #include <linux/bitops.h>
385a2cc190SJeff Kirsher #include <linux/compiler.h>
39cc69837fSJakub Kicinski #include <linux/ethtool.h>
405a2cc190SJeff Kirsher #include <linux/list.h>
415a2cc190SJeff Kirsher #include <linux/mutex.h>
425a2cc190SJeff Kirsher #include <linux/netdevice.h>
435a2cc190SJeff Kirsher #include <linux/if_vlan.h>
44ec693d47SAmir Vadai #include <linux/net_tstamp.h>
45564c274cSAmir Vadai #ifdef CONFIG_MLX4_EN_DCB
46564c274cSAmir Vadai #include <linux/dcbnl.h>
47564c274cSAmir Vadai #endif
481eb8c695SAmir Vadai #include <linux/cpu_rmap.h>
49ad7d4eaeSShawn Bohrer #include <linux/ptp_clock_kernel.h>
5080a62deeSThomas Gleixner #include <linux/irq.h>
51ae75415dSJesper Dangaard Brouer #include <net/xdp.h>
52*73d68002SPetr Pavlu #include <linux/notifier.h>
535a2cc190SJeff Kirsher
545a2cc190SJeff Kirsher #include <linux/mlx4/device.h>
555a2cc190SJeff Kirsher #include <linux/mlx4/qp.h>
565a2cc190SJeff Kirsher #include <linux/mlx4/cq.h>
575a2cc190SJeff Kirsher #include <linux/mlx4/srq.h>
585a2cc190SJeff Kirsher #include <linux/mlx4/doorbell.h>
595a2cc190SJeff Kirsher #include <linux/mlx4/cmd.h>
605a2cc190SJeff Kirsher
615a2cc190SJeff Kirsher #include "en_port.h"
62b4b6e842SEran Ben Elisha #include "mlx4_stats.h"
635a2cc190SJeff Kirsher
645a2cc190SJeff Kirsher #define DRV_NAME "mlx4_en"
65808df6a2STariq Toukan #define DRV_VERSION "4.0-0"
665a2cc190SJeff Kirsher
675a2cc190SJeff Kirsher #define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN)
685a2cc190SJeff Kirsher
695a2cc190SJeff Kirsher /*
705a2cc190SJeff Kirsher * Device constants
715a2cc190SJeff Kirsher */
725a2cc190SJeff Kirsher
735a2cc190SJeff Kirsher
745a2cc190SJeff Kirsher #define MLX4_EN_PAGE_SHIFT 12
755a2cc190SJeff Kirsher #define MLX4_EN_PAGE_SIZE (1 << MLX4_EN_PAGE_SHIFT)
76d317966bSAmir Vadai #define DEF_RX_RINGS 16
77d317966bSAmir Vadai #define MAX_RX_RINGS 128
7827055454SAlaa Hleihel #define MIN_RX_RINGS 1
799573e0d3STariq Toukan #define LOG_TXBB_SIZE 6
809573e0d3STariq Toukan #define TXBB_SIZE BIT(LOG_TXBB_SIZE)
815a2cc190SJeff Kirsher #define HEADROOM (2048 / TXBB_SIZE + 1)
825a2cc190SJeff Kirsher #define STAMP_STRIDE 64
835a2cc190SJeff Kirsher #define STAMP_DWORDS (STAMP_STRIDE / 4)
845a2cc190SJeff Kirsher #define STAMP_SHIFT 31
855a2cc190SJeff Kirsher #define STAMP_VAL 0x7fffffff
865a2cc190SJeff Kirsher #define STATS_DELAY (HZ / 4)
87b6c39bfcSAmir Vadai #define SERVICE_TASK_DELAY (HZ / 4)
8882067281SHadar Hen Zion #define MAX_NUM_OF_FS_RULES 256
895a2cc190SJeff Kirsher
901eb8c695SAmir Vadai #define MLX4_EN_FILTER_HASH_SHIFT 4
911eb8c695SAmir Vadai #define MLX4_EN_FILTER_EXPIRY_QUOTA 60
921eb8c695SAmir Vadai
9326782aadSEric Dumazet #define CTRL_SIZE sizeof(struct mlx4_wqe_ctrl_seg)
9426782aadSEric Dumazet #define DS_SIZE sizeof(struct mlx4_wqe_data_seg)
9526782aadSEric Dumazet
9626782aadSEric Dumazet /* Maximal size of the bounce buffer:
9726782aadSEric Dumazet * 256 bytes for LSO headers.
9826782aadSEric Dumazet * CTRL_SIZE for control desc.
9926782aadSEric Dumazet * DS_SIZE if skb->head contains some payload.
10026782aadSEric Dumazet * MAX_SKB_FRAGS frags.
10126782aadSEric Dumazet */
10226782aadSEric Dumazet #define MLX4_TX_BOUNCE_BUFFER_SIZE \
10326782aadSEric Dumazet ALIGN(256 + CTRL_SIZE + DS_SIZE + MAX_SKB_FRAGS * DS_SIZE, TXBB_SIZE)
10426782aadSEric Dumazet
10535f31ff0SEric Dumazet #define MLX4_MAX_DESC_TXBBS (MLX4_TX_BOUNCE_BUFFER_SIZE / TXBB_SIZE)
1065a2cc190SJeff Kirsher
1075a2cc190SJeff Kirsher /*
1085a2cc190SJeff Kirsher * OS related constants and tunables
1095a2cc190SJeff Kirsher */
1105a2cc190SJeff Kirsher
1110fef9d03SAmir Vadai #define MLX4_EN_PRIV_FLAGS_BLUEFLAME 1
112e38af4faSHadar Hen Zion #define MLX4_EN_PRIV_FLAGS_PHV 2
1130fef9d03SAmir Vadai
1145a2cc190SJeff Kirsher #define MLX4_EN_WATCHDOG_TIMEOUT (15 * HZ)
1155a2cc190SJeff Kirsher
116117980c4SThadeu Lima de Souza Cascardo /* Use the maximum between 16384 and a single page */
117117980c4SThadeu Lima de Souza Cascardo #define MLX4_EN_ALLOC_SIZE PAGE_ALIGN(16384)
11851151a16SEric Dumazet
1195a2cc190SJeff Kirsher #define MLX4_EN_MAX_RX_FRAGS 4
1205a2cc190SJeff Kirsher
1215a2cc190SJeff Kirsher /* Maximum ring sizes */
1225a2cc190SJeff Kirsher #define MLX4_EN_MAX_TX_SIZE 8192
1235a2cc190SJeff Kirsher #define MLX4_EN_MAX_RX_SIZE 8192
1245a2cc190SJeff Kirsher
1254cce66cdSThadeu Lima de Souza Cascardo /* Minimum ring size for our page-allocation scheme to work */
1265a2cc190SJeff Kirsher #define MLX4_EN_MIN_RX_SIZE (MLX4_EN_ALLOC_SIZE / SMP_CACHE_BYTES)
1275a2cc190SJeff Kirsher #define MLX4_EN_MIN_TX_SIZE (4096 / TXBB_SIZE)
1285a2cc190SJeff Kirsher
1295a2cc190SJeff Kirsher #define MLX4_EN_SMALL_PKT_SIZE 64
130ea1c1af1SAmir Vadai #define MLX4_EN_MIN_TX_RING_P_UP 1
131bc6a4744SAmir Vadai #define MLX4_EN_MAX_TX_RING_P_UP 32
132ec327f7aSInbar Karmy #define MLX4_EN_NUM_UP_LOW 1
133f21ad614SInbar Karmy #define MLX4_EN_NUM_UP_HIGH 8
1345a2cc190SJeff Kirsher #define MLX4_EN_DEF_RX_RING_SIZE 1024
13577788b5bSTariq Toukan #define MLX4_EN_DEF_TX_RING_SIZE MLX4_EN_DEF_RX_RING_SIZE
136d317966bSAmir Vadai #define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \
137f21ad614SInbar Karmy MLX4_EN_NUM_UP_HIGH)
1385a2cc190SJeff Kirsher
139fbc6daf1SAmir Vadai #define MLX4_EN_DEFAULT_TX_WORK 256
140fbc6daf1SAmir Vadai
1415a2cc190SJeff Kirsher /* Target number of packets to coalesce with interrupt moderation */
1425a2cc190SJeff Kirsher #define MLX4_EN_RX_COAL_TARGET 44
1435a2cc190SJeff Kirsher #define MLX4_EN_RX_COAL_TIME 0x10
1445a2cc190SJeff Kirsher
145e22979d9SYevgeny Petrilin #define MLX4_EN_TX_COAL_PKTS 16
146ecfd2ce1SEric Dumazet #define MLX4_EN_TX_COAL_TIME 0x10
1475a2cc190SJeff Kirsher
1486ad4e91cSMoshe Shemesh #define MLX4_EN_MAX_COAL_PKTS U16_MAX
1496ad4e91cSMoshe Shemesh #define MLX4_EN_MAX_COAL_TIME U16_MAX
1506ad4e91cSMoshe Shemesh
1515a2cc190SJeff Kirsher #define MLX4_EN_RX_RATE_LOW 400000
1525a2cc190SJeff Kirsher #define MLX4_EN_RX_COAL_TIME_LOW 0
1535a2cc190SJeff Kirsher #define MLX4_EN_RX_RATE_HIGH 450000
1545a2cc190SJeff Kirsher #define MLX4_EN_RX_COAL_TIME_HIGH 128
1555a2cc190SJeff Kirsher #define MLX4_EN_RX_SIZE_THRESH 1024
1565a2cc190SJeff Kirsher #define MLX4_EN_RX_RATE_THRESH (1000000 / MLX4_EN_RX_COAL_TIME_HIGH)
1575a2cc190SJeff Kirsher #define MLX4_EN_SAMPLE_INTERVAL 0
1585a2cc190SJeff Kirsher #define MLX4_EN_AVG_PKT_SMALL 256
1595a2cc190SJeff Kirsher
1605a2cc190SJeff Kirsher #define MLX4_EN_AUTO_CONF 0xffff
1615a2cc190SJeff Kirsher
1625a2cc190SJeff Kirsher #define MLX4_EN_DEF_RX_PAUSE 1
1635a2cc190SJeff Kirsher #define MLX4_EN_DEF_TX_PAUSE 1
1645a2cc190SJeff Kirsher
1655a2cc190SJeff Kirsher /* Interval between successive polls in the Tx routine when polling is used
1665a2cc190SJeff Kirsher instead of interrupts (in per-core Tx rings) - should be power of 2 */
1675a2cc190SJeff Kirsher #define MLX4_EN_TX_POLL_MODER 16
1685a2cc190SJeff Kirsher #define MLX4_EN_TX_POLL_TIMEOUT (HZ / 4)
1695a2cc190SJeff Kirsher
1705a2cc190SJeff Kirsher #define SMALL_PACKET_SIZE (256 - NET_IP_ALIGN)
1715a2cc190SJeff Kirsher #define HEADER_COPY_SIZE (128 - NET_IP_ALIGN)
1725a2cc190SJeff Kirsher #define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETH_HLEN)
17378034f5fSEugenia Emantayev #define PREAMBLE_LEN 8
17478034f5fSEugenia Emantayev #define MLX4_SELFTEST_LB_MIN_MTU (MLX4_LOOPBACK_TEST_PAYLOAD + NET_IP_ALIGN + \
17578034f5fSEugenia Emantayev ETH_HLEN + PREAMBLE_LEN)
1765a2cc190SJeff Kirsher
17747a38e15SBrenden Blanco /* VLAN_HLEN is added twice,to support skb vlan tagged with multiple
17847a38e15SBrenden Blanco * headers. (For example: ETH_P_8021Q and ETH_P_8021AD).
17947a38e15SBrenden Blanco */
18047a38e15SBrenden Blanco #define MLX4_EN_EFF_MTU(mtu) ((mtu) + ETH_HLEN + (2 * VLAN_HLEN))
1815a2cc190SJeff Kirsher #define ETH_BCAST 0xffffffffffffULL
1825a2cc190SJeff Kirsher
1835a2cc190SJeff Kirsher #define MLX4_EN_LOOPBACK_RETRIES 5
1845a2cc190SJeff Kirsher #define MLX4_EN_LOOPBACK_TIMEOUT 100
1855a2cc190SJeff Kirsher
186b97b33a3SEugenia Emantayev /* Constants for TX flow */
187b97b33a3SEugenia Emantayev enum {
188b97b33a3SEugenia Emantayev MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */
189b97b33a3SEugenia Emantayev MAX_BF = 256,
190b97b33a3SEugenia Emantayev MIN_PKT_LEN = 17,
191b97b33a3SEugenia Emantayev };
192b97b33a3SEugenia Emantayev
1935a2cc190SJeff Kirsher /*
1945a2cc190SJeff Kirsher * Configurables
1955a2cc190SJeff Kirsher */
1965a2cc190SJeff Kirsher
1975a2cc190SJeff Kirsher enum cq_type {
19867f8b1dcSTariq Toukan /* keep tx types first */
199ccc109b8STariq Toukan TX,
200ccc109b8STariq Toukan TX_XDP,
20167f8b1dcSTariq Toukan #define MLX4_EN_NUM_TX_TYPES (TX_XDP + 1)
202ccc109b8STariq Toukan RX,
2035a2cc190SJeff Kirsher };
2045a2cc190SJeff Kirsher
2055a2cc190SJeff Kirsher
2065a2cc190SJeff Kirsher /*
2075a2cc190SJeff Kirsher * Useful macros
2085a2cc190SJeff Kirsher */
2095a2cc190SJeff Kirsher #define ROUNDUP_LOG2(x) ilog2(roundup_pow_of_two(x))
2105a2cc190SJeff Kirsher #define XNOR(x, y) (!(x) == !(y))
2115a2cc190SJeff Kirsher
2125a2cc190SJeff Kirsher
2135a2cc190SJeff Kirsher struct mlx4_en_tx_info {
2149ecc2d86SBrenden Blanco union {
2155a2cc190SJeff Kirsher struct sk_buff *skb;
2169ecc2d86SBrenden Blanco struct page *page;
2179ecc2d86SBrenden Blanco };
2183d03641cSEric Dumazet dma_addr_t map0_dma;
2193d03641cSEric Dumazet u32 map0_byte_count;
2205a2cc190SJeff Kirsher u32 nr_txbb;
2215b263f53SYevgeny Petrilin u32 nr_bytes;
2225a2cc190SJeff Kirsher u8 linear;
2235a2cc190SJeff Kirsher u8 data_offset;
2245a2cc190SJeff Kirsher u8 inl;
225ec693d47SAmir Vadai u8 ts_requested;
2263d03641cSEric Dumazet u8 nr_maps;
22798b16349SEric Dumazet } ____cacheline_aligned_in_smp;
2285a2cc190SJeff Kirsher
2295a2cc190SJeff Kirsher
2305a2cc190SJeff Kirsher #define MLX4_EN_BIT_DESC_OWN 0x80000000
2315a2cc190SJeff Kirsher #define MLX4_EN_MEMTYPE_PAD 0x100
2325a2cc190SJeff Kirsher
2335a2cc190SJeff Kirsher
2345a2cc190SJeff Kirsher struct mlx4_en_tx_desc {
2355a2cc190SJeff Kirsher struct mlx4_wqe_ctrl_seg ctrl;
2365a2cc190SJeff Kirsher union {
2375a2cc190SJeff Kirsher struct mlx4_wqe_data_seg data; /* at least one data segment */
2385a2cc190SJeff Kirsher struct mlx4_wqe_lso_seg lso;
2395a2cc190SJeff Kirsher struct mlx4_wqe_inline_seg inl;
2405a2cc190SJeff Kirsher };
2415a2cc190SJeff Kirsher };
2425a2cc190SJeff Kirsher
2435a2cc190SJeff Kirsher #define MLX4_EN_USE_SRQ 0x01000000
2445a2cc190SJeff Kirsher
2455a2cc190SJeff Kirsher #define MLX4_EN_CX3_LOW_ID 0x1000
2465a2cc190SJeff Kirsher #define MLX4_EN_CX3_HIGH_ID 0x1005
2475a2cc190SJeff Kirsher
2485a2cc190SJeff Kirsher struct mlx4_en_rx_alloc {
2495a2cc190SJeff Kirsher struct page *page;
2504cce66cdSThadeu Lima de Souza Cascardo dma_addr_t dma;
25170fbe079SAmir Vadai u32 page_offset;
2525a2cc190SJeff Kirsher };
2535a2cc190SJeff Kirsher
254d576acf0SBrenden Blanco #define MLX4_EN_CACHE_SIZE (2 * NAPI_POLL_WEIGHT)
255acd7628dSEric Dumazet
256d576acf0SBrenden Blanco struct mlx4_en_page_cache {
257d576acf0SBrenden Blanco u32 index;
258acd7628dSEric Dumazet struct {
259acd7628dSEric Dumazet struct page *page;
260acd7628dSEric Dumazet dma_addr_t dma;
261acd7628dSEric Dumazet } buf[MLX4_EN_CACHE_SIZE];
262d576acf0SBrenden Blanco };
263d576acf0SBrenden Blanco
264ba603d9dSMoshe Shemesh enum {
265ba603d9dSMoshe Shemesh MLX4_EN_TX_RING_STATE_RECOVERING,
266ba603d9dSMoshe Shemesh };
267ba603d9dSMoshe Shemesh
2689ecc2d86SBrenden Blanco struct mlx4_en_priv;
2699ecc2d86SBrenden Blanco
2705a2cc190SJeff Kirsher struct mlx4_en_tx_ring {
27198b16349SEric Dumazet /* cache line used and dirtied in tx completion
27298b16349SEric Dumazet * (mlx4_en_free_tx_buf())
27398b16349SEric Dumazet */
27498b16349SEric Dumazet u32 last_nr_txbb;
27598b16349SEric Dumazet u32 cons;
27698b16349SEric Dumazet unsigned long wake_queue;
277e3f42f84SEric Dumazet struct netdev_queue *tx_queue;
278e3f42f84SEric Dumazet u32 (*free_tx_desc)(struct mlx4_en_priv *priv,
279e3f42f84SEric Dumazet struct mlx4_en_tx_ring *ring,
280cf97050dSTariq Toukan int index,
281e3f42f84SEric Dumazet u64 timestamp, int napi_mode);
282e3f42f84SEric Dumazet struct mlx4_en_rx_ring *recycle_ring;
28398b16349SEric Dumazet
28498b16349SEric Dumazet /* cache line used and dirtied in mlx4_en_xmit() */
28598b16349SEric Dumazet u32 prod ____cacheline_aligned_in_smp;
286e3f42f84SEric Dumazet unsigned int tx_dropped;
28798b16349SEric Dumazet unsigned long bytes;
28898b16349SEric Dumazet unsigned long packets;
28998b16349SEric Dumazet unsigned long tx_csum;
29098b16349SEric Dumazet unsigned long tso_packets;
29198b16349SEric Dumazet unsigned long xmit_more;
29298b16349SEric Dumazet struct mlx4_bf bf;
29398b16349SEric Dumazet
29498b16349SEric Dumazet /* Following part should be mostly read */
2959ac93627SEric Dumazet void __iomem *doorbell_address;
2966a4e8121SEric Dumazet __be32 doorbell_qpn;
2976a4e8121SEric Dumazet __be32 mr_key;
298e3f42f84SEric Dumazet u32 size; /* number of TXBBs */
299e3f42f84SEric Dumazet u32 size_mask;
300e3f42f84SEric Dumazet u32 full_size;
301e3f42f84SEric Dumazet u32 buf_size;
3025a2cc190SJeff Kirsher void *buf;
3035a2cc190SJeff Kirsher struct mlx4_en_tx_info *tx_info;
3045a2cc190SJeff Kirsher int qpn;
30598b16349SEric Dumazet u8 queue_index;
3065a2cc190SJeff Kirsher bool bf_enabled;
3070fef9d03SAmir Vadai bool bf_alloced;
308e3f42f84SEric Dumazet u8 hwtstamp_tx_type;
309e3f42f84SEric Dumazet u8 *bounce_buf;
310e3f42f84SEric Dumazet
311e3f42f84SEric Dumazet /* Not used in fast path
312e3f42f84SEric Dumazet * Only queue_stopped might be used if BQL is not properly working.
313e3f42f84SEric Dumazet */
314e3f42f84SEric Dumazet unsigned long queue_stopped;
315ba603d9dSMoshe Shemesh unsigned long state;
316e3f42f84SEric Dumazet struct mlx4_hwq_resources sp_wqres;
317e3f42f84SEric Dumazet struct mlx4_qp sp_qp;
318e3f42f84SEric Dumazet struct mlx4_qp_context sp_context;
319e3f42f84SEric Dumazet cpumask_t sp_affinity_mask;
320e3f42f84SEric Dumazet enum mlx4_qp_state sp_qp_state;
321e3f42f84SEric Dumazet u16 sp_stride;
322e3f42f84SEric Dumazet u16 sp_cqn; /* index of port CQ associated with this ring */
32398b16349SEric Dumazet } ____cacheline_aligned_in_smp;
3245a2cc190SJeff Kirsher
3255a2cc190SJeff Kirsher struct mlx4_en_rx_desc {
3265a2cc190SJeff Kirsher /* actual number of entries depends on rx ring stride */
327966b6b80SGustavo A. R. Silva DECLARE_FLEX_ARRAY(struct mlx4_wqe_data_seg, data);
3285a2cc190SJeff Kirsher };
3295a2cc190SJeff Kirsher
3305a2cc190SJeff Kirsher struct mlx4_en_rx_ring {
3315a2cc190SJeff Kirsher struct mlx4_hwq_resources wqres;
3325a2cc190SJeff Kirsher u32 size ; /* number of Rx descs*/
3335a2cc190SJeff Kirsher u32 actual_size;
3345a2cc190SJeff Kirsher u32 size_mask;
3355a2cc190SJeff Kirsher u16 stride;
3365a2cc190SJeff Kirsher u16 log_stride;
3375a2cc190SJeff Kirsher u16 cqn; /* index of port CQ associated with this ring */
3385a2cc190SJeff Kirsher u32 prod;
3395a2cc190SJeff Kirsher u32 cons;
3405a2cc190SJeff Kirsher u32 buf_size;
3414a5f4dd8SYevgeny Petrilin u8 fcs_del;
3425a2cc190SJeff Kirsher void *buf;
3435a2cc190SJeff Kirsher void *rx_info;
344326fe02dSBrenden Blanco struct bpf_prog __rcu *xdp_prog;
345d576acf0SBrenden Blanco struct mlx4_en_page_cache page_cache;
3465a2cc190SJeff Kirsher unsigned long bytes;
3475a2cc190SJeff Kirsher unsigned long packets;
348ad04378cSYevgeny Petrilin unsigned long csum_ok;
349ad04378cSYevgeny Petrilin unsigned long csum_none;
350f8c6455bSShani Michaeli unsigned long csum_complete;
3517d7bfc6aSEric Dumazet unsigned long rx_alloc_pages;
35215fca2c8STariq Toukan unsigned long xdp_drop;
353dee3b2d0SJoshua Roys unsigned long xdp_redirect;
354dee3b2d0SJoshua Roys unsigned long xdp_redirect_fail;
35515fca2c8STariq Toukan unsigned long xdp_tx;
35615fca2c8STariq Toukan unsigned long xdp_tx_full;
357d21ed3a3SEran Ben Elisha unsigned long dropped;
358ec693d47SAmir Vadai int hwtstamp_rx_filter;
3599e311e77SYuval Atias cpumask_var_t affinity_mask;
360ae75415dSJesper Dangaard Brouer struct xdp_rxq_info xdp_rxq;
3615a2cc190SJeff Kirsher };
3625a2cc190SJeff Kirsher
3635a2cc190SJeff Kirsher struct mlx4_en_cq {
3645a2cc190SJeff Kirsher struct mlx4_cq mcq;
3655a2cc190SJeff Kirsher struct mlx4_hwq_resources wqres;
3665a2cc190SJeff Kirsher int ring;
3675a2cc190SJeff Kirsher struct net_device *dev;
3686c78511bSTariq Toukan union {
3695a2cc190SJeff Kirsher struct napi_struct napi;
3706c78511bSTariq Toukan bool xdp_busy;
3716c78511bSTariq Toukan };
3725a2cc190SJeff Kirsher int size;
3735a2cc190SJeff Kirsher int buf_size;
374c66fa19cSMatan Barak int vector;
375ccc109b8STariq Toukan enum cq_type type;
3765a2cc190SJeff Kirsher u16 moder_time;
3775a2cc190SJeff Kirsher u16 moder_cnt;
3785a2cc190SJeff Kirsher struct mlx4_cqe *buf;
3795a2cc190SJeff Kirsher #define MLX4_EN_OPCODE_ERROR 0x1e
3809e77a2b8SAmir Vadai
38180a62deeSThomas Gleixner const struct cpumask *aff_mask;
3825a2cc190SJeff Kirsher };
3835a2cc190SJeff Kirsher
3845a2cc190SJeff Kirsher struct mlx4_en_port_profile {
3855a2cc190SJeff Kirsher u32 flags;
38667f8b1dcSTariq Toukan u32 tx_ring_num[MLX4_EN_NUM_TX_TYPES];
3875a2cc190SJeff Kirsher u32 rx_ring_num;
3885a2cc190SJeff Kirsher u32 tx_ring_size;
3895a2cc190SJeff Kirsher u32 rx_ring_size;
390ec25bc04SEugenia Emantayev u8 num_tx_rings_p_up;
3915a2cc190SJeff Kirsher u8 rx_pause;
3925a2cc190SJeff Kirsher u8 rx_ppp;
3935a2cc190SJeff Kirsher u8 tx_pause;
3945a2cc190SJeff Kirsher u8 tx_ppp;
395f21ad614SInbar Karmy u8 num_up;
39693d3e367SYevgeny Petrilin int rss_rings;
397b97b33a3SEugenia Emantayev int inline_thold;
398ec25bc04SEugenia Emantayev struct hwtstamp_config hwtstamp_config;
3995a2cc190SJeff Kirsher };
4005a2cc190SJeff Kirsher
4015a2cc190SJeff Kirsher struct mlx4_en_profile {
4025a2cc190SJeff Kirsher int udp_rss;
4035a2cc190SJeff Kirsher u8 rss_mask;
4045a2cc190SJeff Kirsher u32 active_ports;
4055a2cc190SJeff Kirsher u32 small_pkt_int;
4065a2cc190SJeff Kirsher u8 no_reset;
4077e1dc5e9SInbar Karmy u8 max_num_tx_rings_p_up;
4085a2cc190SJeff Kirsher struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1];
4095a2cc190SJeff Kirsher };
4105a2cc190SJeff Kirsher
4115a2cc190SJeff Kirsher struct mlx4_en_dev {
4125a2cc190SJeff Kirsher struct mlx4_dev *dev;
4135a2cc190SJeff Kirsher struct pci_dev *pdev;
4145a2cc190SJeff Kirsher struct mutex state_lock;
4155a2cc190SJeff Kirsher struct net_device *pndev[MLX4_MAX_PORTS + 1];
4165da03547SMoni Shoua struct net_device *upper[MLX4_MAX_PORTS + 1];
4175a2cc190SJeff Kirsher u32 port_cnt;
4185a2cc190SJeff Kirsher bool device_up;
4195a2cc190SJeff Kirsher struct mlx4_en_profile profile;
4205a2cc190SJeff Kirsher u32 LSO_support;
4215a2cc190SJeff Kirsher struct workqueue_struct *workqueue;
4225a2cc190SJeff Kirsher struct device *dma_device;
4235a2cc190SJeff Kirsher void __iomem *uar_map;
4245a2cc190SJeff Kirsher struct mlx4_uar priv_uar;
4255a2cc190SJeff Kirsher struct mlx4_mr mr;
4265a2cc190SJeff Kirsher u32 priv_pdn;
4275a2cc190SJeff Kirsher spinlock_t uar_lock;
4285a2cc190SJeff Kirsher u8 mac_removed[MLX4_MAX_PORTS + 1];
429ad7d4eaeSShawn Bohrer u32 nominal_c_mult;
430ec693d47SAmir Vadai struct cyclecounter cycles;
43199f5711eSEric Dumazet seqlock_t clock_lock;
432ec693d47SAmir Vadai struct timecounter clock;
433ec693d47SAmir Vadai unsigned long last_overflow_check;
434ad7d4eaeSShawn Bohrer struct ptp_clock *ptp_clock;
435ad7d4eaeSShawn Bohrer struct ptp_clock_info ptp_clock_info;
436ef5617e3SPetr Pavlu struct notifier_block netdev_nb;
437*73d68002SPetr Pavlu struct notifier_block mlx_nb;
4385a2cc190SJeff Kirsher };
4395a2cc190SJeff Kirsher
4405a2cc190SJeff Kirsher
4415a2cc190SJeff Kirsher struct mlx4_en_rss_map {
4425a2cc190SJeff Kirsher int base_qpn;
4435a2cc190SJeff Kirsher struct mlx4_qp qps[MAX_RX_RINGS];
4445a2cc190SJeff Kirsher enum mlx4_qp_state state[MAX_RX_RINGS];
4454931c6efSSaeed Mahameed struct mlx4_qp *indir_qp;
4465a2cc190SJeff Kirsher enum mlx4_qp_state indir_state;
4475a2cc190SJeff Kirsher };
4485a2cc190SJeff Kirsher
4492c762679SSaeed Mahameed enum mlx4_en_port_flag {
4502c762679SSaeed Mahameed MLX4_EN_PORT_ANC = 1<<0, /* Auto-negotiation complete */
4512c762679SSaeed Mahameed MLX4_EN_PORT_ANE = 1<<1, /* Auto-negotiation enabled */
4522c762679SSaeed Mahameed };
4532c762679SSaeed Mahameed
4545a2cc190SJeff Kirsher struct mlx4_en_port_state {
4555a2cc190SJeff Kirsher int link_state;
4565a2cc190SJeff Kirsher int link_speed;
4572c762679SSaeed Mahameed int transceiver;
4582c762679SSaeed Mahameed u32 flags;
4595a2cc190SJeff Kirsher };
4605a2cc190SJeff Kirsher
4616d199937SYevgeny Petrilin enum mlx4_en_mclist_act {
4626d199937SYevgeny Petrilin MCLIST_NONE,
4636d199937SYevgeny Petrilin MCLIST_REM,
4646d199937SYevgeny Petrilin MCLIST_ADD,
4656d199937SYevgeny Petrilin };
4666d199937SYevgeny Petrilin
4676d199937SYevgeny Petrilin struct mlx4_en_mc_list {
4686d199937SYevgeny Petrilin struct list_head list;
4696d199937SYevgeny Petrilin enum mlx4_en_mclist_act action;
4706d199937SYevgeny Petrilin u8 addr[ETH_ALEN];
4710ff1fb65SHadar Hen Zion u64 reg_id;
472837052d0SOr Gerlitz u64 tunnel_reg_id;
4736d199937SYevgeny Petrilin };
4746d199937SYevgeny Petrilin
4755a2cc190SJeff Kirsher struct mlx4_en_frag_info {
4765a2cc190SJeff Kirsher u16 frag_size;
477aaca121dSEric Dumazet u32 frag_stride;
4785a2cc190SJeff Kirsher };
4795a2cc190SJeff Kirsher
480564c274cSAmir Vadai #ifdef CONFIG_MLX4_EN_DCB
481564c274cSAmir Vadai /* Minimal TC BW - setting to 0 will block traffic */
482564c274cSAmir Vadai #define MLX4_EN_BW_MIN 1
483564c274cSAmir Vadai #define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */
484564c274cSAmir Vadai
485a42b63c1SMoni Shoua #define MLX4_EN_TC_VENDOR 0
486564c274cSAmir Vadai #define MLX4_EN_TC_ETS 7
487564c274cSAmir Vadai
488af7d5185SRana Shahout enum dcb_pfc_type {
489af7d5185SRana Shahout pfc_disabled = 0,
490af7d5185SRana Shahout pfc_enabled_full,
491af7d5185SRana Shahout pfc_enabled_tx,
492af7d5185SRana Shahout pfc_enabled_rx
493af7d5185SRana Shahout };
494af7d5185SRana Shahout
495af7d5185SRana Shahout struct mlx4_en_cee_config {
496af7d5185SRana Shahout bool pfc_state;
497f21ad614SInbar Karmy enum dcb_pfc_type dcb_pfc[MLX4_EN_NUM_UP_HIGH];
498af7d5185SRana Shahout };
499564c274cSAmir Vadai #endif
500564c274cSAmir Vadai
50182067281SHadar Hen Zion struct ethtool_flow_id {
5020d256c0eSHadar Hen Zion struct list_head list;
50382067281SHadar Hen Zion struct ethtool_rx_flow_spec flow_spec;
50482067281SHadar Hen Zion u64 id;
50582067281SHadar Hen Zion };
50682067281SHadar Hen Zion
50779aeaccdSYan Burman enum {
50879aeaccdSYan Burman MLX4_EN_FLAG_PROMISC = (1 << 0),
50979aeaccdSYan Burman MLX4_EN_FLAG_MC_PROMISC = (1 << 1),
51079aeaccdSYan Burman /* whether we need to enable hardware loopback by putting dmac
51179aeaccdSYan Burman * in Tx WQE
51279aeaccdSYan Burman */
51379aeaccdSYan Burman MLX4_EN_FLAG_ENABLE_HW_LOOPBACK = (1 << 2),
51479aeaccdSYan Burman /* whether we need to drop packets that hardware loopback-ed */
515cc5387f7SYan Burman MLX4_EN_FLAG_RX_FILTER_NEEDED = (1 << 3),
516f8c6455bSShani Michaeli MLX4_EN_FLAG_FORCE_PROMISC = (1 << 4),
517f8c6455bSShani Michaeli MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP = (1 << 5),
518af7d5185SRana Shahout #ifdef CONFIG_MLX4_EN_DCB
519af7d5185SRana Shahout MLX4_EN_FLAG_DCB_ENABLED = (1 << 6),
520af7d5185SRana Shahout #endif
52179aeaccdSYan Burman };
52279aeaccdSYan Burman
52351af33cfSIdo Shamay #define PORT_BEACON_MAX_LIMIT (65535)
524c07cb4b0SYan Burman #define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE)
525c07cb4b0SYan Burman #define MLX4_EN_MAC_HASH_IDX 5
526c07cb4b0SYan Burman
5273da8a36cSEran Ben Elisha struct mlx4_en_stats_bitmap {
5283da8a36cSEran Ben Elisha DECLARE_BITMAP(bitmap, NUM_ALL_STATS);
5293da8a36cSEran Ben Elisha struct mutex mutex; /* for mutual access to stats bitmap */
5303da8a36cSEran Ben Elisha };
5313da8a36cSEran Ben Elisha
532fed91613SMoshe Shemesh enum {
533fed91613SMoshe Shemesh MLX4_EN_STATE_FLAG_RESTARTING,
534fed91613SMoshe Shemesh };
535fed91613SMoshe Shemesh
5365a2cc190SJeff Kirsher struct mlx4_en_priv {
5375a2cc190SJeff Kirsher struct mlx4_en_dev *mdev;
5385a2cc190SJeff Kirsher struct mlx4_en_port_profile *prof;
5395a2cc190SJeff Kirsher struct net_device *dev;
5405a2cc190SJeff Kirsher unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
5415a2cc190SJeff Kirsher struct mlx4_en_port_state port_state;
5425a2cc190SJeff Kirsher spinlock_t stats_lock;
54382067281SHadar Hen Zion struct ethtool_flow_id ethtool_rules[MAX_NUM_OF_FS_RULES];
5440d256c0eSHadar Hen Zion /* To allow rules removal while port is going down */
5450d256c0eSHadar Hen Zion struct list_head ethtool_list;
5465a2cc190SJeff Kirsher
5476b4d8d9fSAlexander Guller unsigned long last_moder_packets[MAX_RX_RINGS];
5485a2cc190SJeff Kirsher unsigned long last_moder_tx_packets;
5496b4d8d9fSAlexander Guller unsigned long last_moder_bytes[MAX_RX_RINGS];
5505a2cc190SJeff Kirsher unsigned long last_moder_jiffies;
5516b4d8d9fSAlexander Guller int last_moder_time[MAX_RX_RINGS];
5525a2cc190SJeff Kirsher u16 rx_usecs;
5535a2cc190SJeff Kirsher u16 rx_frames;
5545a2cc190SJeff Kirsher u16 tx_usecs;
5555a2cc190SJeff Kirsher u16 tx_frames;
5565a2cc190SJeff Kirsher u32 pkt_rate_low;
5575a2cc190SJeff Kirsher u16 rx_usecs_low;
5585a2cc190SJeff Kirsher u32 pkt_rate_high;
5595a2cc190SJeff Kirsher u16 rx_usecs_high;
5606ad4e91cSMoshe Shemesh u32 sample_interval;
5616ad4e91cSMoshe Shemesh u32 adaptive_rx_coal;
5625a2cc190SJeff Kirsher u32 msg_enable;
5635a2cc190SJeff Kirsher u32 loopback_ok;
5645a2cc190SJeff Kirsher u32 validate_loopback;
5655a2cc190SJeff Kirsher
5665a2cc190SJeff Kirsher struct mlx4_hwq_resources res;
5675a2cc190SJeff Kirsher int link_state;
5685a2cc190SJeff Kirsher bool port_up;
5695a2cc190SJeff Kirsher int port;
5705a2cc190SJeff Kirsher int registered;
5715a2cc190SJeff Kirsher int allocated;
5725a2cc190SJeff Kirsher int stride;
5732695bab2SNoa Osherovich unsigned char current_mac[ETH_ALEN + 2];
5745a2cc190SJeff Kirsher int mac_index;
5755a2cc190SJeff Kirsher unsigned max_mtu;
5765a2cc190SJeff Kirsher int base_qpn;
57708ff3235SOr Gerlitz int cqe_factor;
578b1b6b4daSIdo Shamay int cqe_size;
5795a2cc190SJeff Kirsher
5805a2cc190SJeff Kirsher struct mlx4_en_rss_map rss_map;
5814ef2a435SOr Gerlitz __be32 ctrl_flags;
5825a2cc190SJeff Kirsher u32 flags;
583d317966bSAmir Vadai u8 num_tx_rings_p_up;
584fbc6daf1SAmir Vadai u32 tx_work_limit;
58567f8b1dcSTariq Toukan u32 tx_ring_num[MLX4_EN_NUM_TX_TYPES];
5865a2cc190SJeff Kirsher u32 rx_ring_num;
5875a2cc190SJeff Kirsher u32 rx_skb_size;
5885a2cc190SJeff Kirsher struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS];
58969ba9431SEric Dumazet u8 num_frags;
59069ba9431SEric Dumazet u8 log_rx_info;
59169ba9431SEric Dumazet u8 dma_dir;
592d85f6c14SEric Dumazet u16 rx_headroom;
5935a2cc190SJeff Kirsher
59467f8b1dcSTariq Toukan struct mlx4_en_tx_ring **tx_ring[MLX4_EN_NUM_TX_TYPES];
59541d942d5SEugenia Emantayev struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS];
59667f8b1dcSTariq Toukan struct mlx4_en_cq **tx_cq[MLX4_EN_NUM_TX_TYPES];
59741d942d5SEugenia Emantayev struct mlx4_en_cq *rx_cq[MAX_RX_RINGS];
598cabdc8eeSHadar Hen Zion struct mlx4_qp drop_qp;
5990eb74fddSYan Burman struct work_struct rx_mode_task;
600fed91613SMoshe Shemesh struct work_struct restart_task;
6015a2cc190SJeff Kirsher struct work_struct linkstate_task;
6025a2cc190SJeff Kirsher struct delayed_work stats_task;
603b6c39bfcSAmir Vadai struct delayed_work service_task;
6045a2cc190SJeff Kirsher struct mlx4_en_pkt_stats pkstats;
605b42de4d0SEran Ben Elisha struct mlx4_en_counter_stats pf_stats;
6060b131561SMatan Barak struct mlx4_en_flow_stats_rx rx_priority_flowstats[MLX4_NUM_PRIORITIES];
6070b131561SMatan Barak struct mlx4_en_flow_stats_tx tx_priority_flowstats[MLX4_NUM_PRIORITIES];
6080b131561SMatan Barak struct mlx4_en_flow_stats_rx rx_flowstats;
6090b131561SMatan Barak struct mlx4_en_flow_stats_tx tx_flowstats;
6105a2cc190SJeff Kirsher struct mlx4_en_port_stats port_stats;
61115fca2c8STariq Toukan struct mlx4_en_xdp_stats xdp_stats;
612f26d0d25SEran Ben Elisha struct mlx4_en_phy_stats phy_stats;
6133da8a36cSEran Ben Elisha struct mlx4_en_stats_bitmap stats_bitmap;
6146d199937SYevgeny Petrilin struct list_head mc_list;
6156d199937SYevgeny Petrilin struct list_head curr_list;
6160ff1fb65SHadar Hen Zion u64 broadcast_id;
6175a2cc190SJeff Kirsher struct mlx4_en_stat_out_mbox hw_stats;
6185a2cc190SJeff Kirsher int vids[128];
6195a2cc190SJeff Kirsher bool wol;
620ebf8c9aaSYevgeny Petrilin struct device *ddev;
621c07cb4b0SYan Burman struct hlist_head mac_hash[MLX4_EN_MAC_HASH_SIZE];
622ec693d47SAmir Vadai struct hwtstamp_config hwtstamp_config;
6236de5f7f6SEran Ben Elisha u32 counter_index;
624564c274cSAmir Vadai
625564c274cSAmir Vadai #ifdef CONFIG_MLX4_EN_DCB
626af7d5185SRana Shahout #define MLX4_EN_DCB_ENABLED 0x3
627564c274cSAmir Vadai struct ieee_ets ets;
628109d2446SAmir Vadai u16 maxrate[IEEE_8021QAZ_MAX_TCS];
629708b869bSShani Michaeli enum dcbnl_cndd_states cndd_state[IEEE_8021QAZ_MAX_TCS];
630564ed9b1STariq Toukan struct mlx4_en_cee_config cee_config;
631564ed9b1STariq Toukan u8 dcbx_cap;
632564c274cSAmir Vadai #endif
6331eb8c695SAmir Vadai #ifdef CONFIG_RFS_ACCEL
6341eb8c695SAmir Vadai spinlock_t filters_lock;
6351eb8c695SAmir Vadai int last_filter_id;
6361eb8c695SAmir Vadai struct list_head filters;
6371eb8c695SAmir Vadai struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT];
6381eb8c695SAmir Vadai #endif
639837052d0SOr Gerlitz u64 tunnel_reg_id;
6401b136de1SOr Gerlitz __be16 vxlan_port;
6410fef9d03SAmir Vadai
6420fef9d03SAmir Vadai u32 pflags;
643bd635c35SEric Dumazet u8 rss_key[MLX4_EN_RSS_KEY_SIZE];
644947cbb0aSEyal Perry u8 rss_hash_fn;
645fed91613SMoshe Shemesh unsigned long state;
6465a2cc190SJeff Kirsher };
6475a2cc190SJeff Kirsher
6485a2cc190SJeff Kirsher enum mlx4_en_wol {
6495a2cc190SJeff Kirsher MLX4_EN_WOL_MAGIC = (1ULL << 61),
6505a2cc190SJeff Kirsher MLX4_EN_WOL_ENABLED = (1ULL << 62),
6515a2cc190SJeff Kirsher };
6525a2cc190SJeff Kirsher
65316a10ffdSYan Burman struct mlx4_mac_entry {
654c07cb4b0SYan Burman struct hlist_node hlist;
65516a10ffdSYan Burman unsigned char mac[ETH_ALEN + 2];
65616a10ffdSYan Burman u64 reg_id;
657c07cb4b0SYan Burman struct rcu_head rcu;
65816a10ffdSYan Burman };
65916a10ffdSYan Burman
mlx4_en_get_cqe(void * buf,int idx,int cqe_sz)660b1b6b4daSIdo Shamay static inline struct mlx4_cqe *mlx4_en_get_cqe(void *buf, int idx, int cqe_sz)
661b1b6b4daSIdo Shamay {
662b1b6b4daSIdo Shamay return buf + idx * cqe_sz;
663b1b6b4daSIdo Shamay }
664b1b6b4daSIdo Shamay
6650d9fdaa9SOr Gerlitz #define MLX4_EN_WOL_DO_MODIFY (1ULL << 63)
6665a2cc190SJeff Kirsher
6673d8f7cc7SDavid Decotigny void mlx4_en_init_ptys2ethtool_map(void);
66879aeaccdSYan Burman void mlx4_en_update_loopback_state(struct net_device *dev,
66979aeaccdSYan Burman netdev_features_t features);
67079aeaccdSYan Burman
6715a2cc190SJeff Kirsher void mlx4_en_destroy_netdev(struct net_device *dev);
6725a2cc190SJeff Kirsher int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
6735a2cc190SJeff Kirsher struct mlx4_en_port_profile *prof);
6745a2cc190SJeff Kirsher
6755a2cc190SJeff Kirsher int mlx4_en_start_port(struct net_device *dev);
6763484aac1SAmir Vadai void mlx4_en_stop_port(struct net_device *dev, int detach);
6775a2cc190SJeff Kirsher
6786fcd2735SEran Ben Elisha void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev,
6790b131561SMatan Barak struct mlx4_en_stats_bitmap *stats_bitmap,
6800b131561SMatan Barak u8 rx_ppp, u8 rx_pause,
6810b131561SMatan Barak u8 tx_ppp, u8 tx_pause);
682ffa88f37SEran Ben Elisha
683ec25bc04SEugenia Emantayev int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv,
684ec25bc04SEugenia Emantayev struct mlx4_en_priv *tmp,
685770f8225SMartin KaFai Lau struct mlx4_en_port_profile *prof,
686770f8225SMartin KaFai Lau bool carry_xdp_prog);
687ec25bc04SEugenia Emantayev void mlx4_en_safe_replace_resources(struct mlx4_en_priv *priv,
688ec25bc04SEugenia Emantayev struct mlx4_en_priv *tmp);
6895a2cc190SJeff Kirsher
69041d942d5SEugenia Emantayev int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq,
691163561a4SEugenia Emantayev int entries, int ring, enum cq_type mode, int node);
69241d942d5SEugenia Emantayev void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq);
69376532d0cSAlexander Guller int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
69476532d0cSAlexander Guller int cq_idx);
6955a2cc190SJeff Kirsher void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
6965a2cc190SJeff Kirsher int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
697f3eebe88SZhu Yanjun void mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
6985a2cc190SJeff Kirsher
6995a2cc190SJeff Kirsher void mlx4_en_tx_irq(struct mlx4_cq *mcq);
700f663dd9aSJason Wang u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
701a350ecceSPaolo Abeni struct net_device *sb_dev);
7025a2cc190SJeff Kirsher netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev);
70315fca2c8STariq Toukan netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring,
70415fca2c8STariq Toukan struct mlx4_en_rx_alloc *frame,
7055dad61b8STariq Toukan struct mlx4_en_priv *priv, unsigned int length,
70636ea7964STariq Toukan int tx_ind, bool *doorbell_pending);
7079ecc2d86SBrenden Blanco void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring *ring);
7089ecc2d86SBrenden Blanco bool mlx4_en_rx_recycle(struct mlx4_en_rx_ring *ring,
7099ecc2d86SBrenden Blanco struct mlx4_en_rx_alloc *frame);
7105a2cc190SJeff Kirsher
71141d942d5SEugenia Emantayev int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
71241d942d5SEugenia Emantayev struct mlx4_en_tx_ring **pring,
713ddae0349SEugenia Emantayev u32 size, u16 stride,
714d03a68f8SIdo Shamay int node, int queue_index);
71541d942d5SEugenia Emantayev void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
71641d942d5SEugenia Emantayev struct mlx4_en_tx_ring **pring);
717f025fd60STariq Toukan void mlx4_en_init_tx_xdp_ring_descs(struct mlx4_en_priv *priv,
718f025fd60STariq Toukan struct mlx4_en_tx_ring *ring);
7195a2cc190SJeff Kirsher int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
7205a2cc190SJeff Kirsher struct mlx4_en_tx_ring *ring,
7210e98b523SAmir Vadai int cq, int user_prio);
7225a2cc190SJeff Kirsher void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
7235a2cc190SJeff Kirsher struct mlx4_en_tx_ring *ring);
72402512482SIdo Shamay void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev);
72507841f9dSIdo Shamay void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv);
7265a2cc190SJeff Kirsher int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
72741d942d5SEugenia Emantayev struct mlx4_en_rx_ring **pring,
728ae75415dSJesper Dangaard Brouer u32 size, u16 stride, int node, int queue_index);
7295a2cc190SJeff Kirsher void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
73041d942d5SEugenia Emantayev struct mlx4_en_rx_ring **pring,
73168355f71SThadeu Lima de Souza Cascardo u32 size, u16 stride);
7325a2cc190SJeff Kirsher int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv);
7335a2cc190SJeff Kirsher void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
7345a2cc190SJeff Kirsher struct mlx4_en_rx_ring *ring);
7355a2cc190SJeff Kirsher int mlx4_en_process_rx_cq(struct net_device *dev,
7365a2cc190SJeff Kirsher struct mlx4_en_cq *cq,
7375a2cc190SJeff Kirsher int budget);
7385a2cc190SJeff Kirsher int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget);
7390276a330SEugenia Emantayev int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget);
740cf4058dbSEric Dumazet int mlx4_en_process_tx_cq(struct net_device *dev,
7416c78511bSTariq Toukan struct mlx4_en_cq *cq, int napi_budget);
7429ecc2d86SBrenden Blanco u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
7439ecc2d86SBrenden Blanco struct mlx4_en_tx_ring *ring,
744cf97050dSTariq Toukan int index, u64 timestamp,
7459ecc2d86SBrenden Blanco int napi_mode);
7469ecc2d86SBrenden Blanco u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv,
7479ecc2d86SBrenden Blanco struct mlx4_en_tx_ring *ring,
748cf97050dSTariq Toukan int index, u64 timestamp,
7499ecc2d86SBrenden Blanco int napi_mode);
7505a2cc190SJeff Kirsher void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
7510e98b523SAmir Vadai int is_tx, int rss, int qpn, int cqn, int user_prio,
7525a2cc190SJeff Kirsher struct mlx4_qp_context *context);
7535a2cc190SJeff Kirsher void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event);
75474194fb9SMaor Gottlieb int mlx4_en_change_mcast_lb(struct mlx4_en_priv *priv, struct mlx4_qp *qp,
75574194fb9SMaor Gottlieb int loopback);
7565a2cc190SJeff Kirsher void mlx4_en_calc_rx_buf(struct net_device *dev);
7575a2cc190SJeff Kirsher int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv);
7585a2cc190SJeff Kirsher void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv);
759cabdc8eeSHadar Hen Zion int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv);
760cabdc8eeSHadar Hen Zion void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv);
7615a2cc190SJeff Kirsher int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring);
7625a2cc190SJeff Kirsher void mlx4_en_rx_irq(struct mlx4_cq *mcq);
7635a2cc190SJeff Kirsher
7645a2cc190SJeff Kirsher int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode);
7655a2cc190SJeff Kirsher int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv);
7665a2cc190SJeff Kirsher
76740931b85SEric Dumazet void mlx4_en_fold_software_stats(struct net_device *dev);
7685a2cc190SJeff Kirsher int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset);
7695a2cc190SJeff Kirsher int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port);
7705a2cc190SJeff Kirsher
771564c274cSAmir Vadai #ifdef CONFIG_MLX4_EN_DCB
772564c274cSAmir Vadai extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops;
773540b3a39SOr Gerlitz extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops;
774564c274cSAmir Vadai #endif
775564c274cSAmir Vadai
776d317966bSAmir Vadai int mlx4_en_setup_tc(struct net_device *dev, u8 up);
777ec327f7aSInbar Karmy int mlx4_en_alloc_tx_queue_per_tc(struct net_device *dev, u8 tc);
778d317966bSAmir Vadai
7791eb8c695SAmir Vadai #ifdef CONFIG_RFS_ACCEL
78041d942d5SEugenia Emantayev void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv);
7811eb8c695SAmir Vadai #endif
7821eb8c695SAmir Vadai
7835a2cc190SJeff Kirsher #define MLX4_EN_NUM_SELF_TEST 5
7845a2cc190SJeff Kirsher void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf);
785b6c39bfcSAmir Vadai void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev);
7865a2cc190SJeff Kirsher
7877787fa66SSaeed Mahameed #define DEV_FEATURE_CHANGED(dev, new_features, feature) \
7887787fa66SSaeed Mahameed ((dev->features & feature) ^ (new_features & feature))
7897787fa66SSaeed Mahameed
79000ff801bSKevin(Yudong) Yang int mlx4_en_moderation_update(struct mlx4_en_priv *priv);
7917787fa66SSaeed Mahameed int mlx4_en_reset_config(struct net_device *dev,
7927787fa66SSaeed Mahameed struct hwtstamp_config ts_config,
7937787fa66SSaeed Mahameed netdev_features_t new_features);
7940b131561SMatan Barak void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev,
7950b131561SMatan Barak struct mlx4_en_stats_bitmap *stats_bitmap,
7960b131561SMatan Barak u8 rx_ppp, u8 rx_pause,
7970b131561SMatan Barak u8 tx_ppp, u8 tx_pause);
7985da03547SMoni Shoua int mlx4_en_netdev_event(struct notifier_block *this,
7995da03547SMoni Shoua unsigned long event, void *ptr);
8005da03547SMoni Shoua
801ab46182dSStanislav Fomichev struct xdp_md;
802ab46182dSStanislav Fomichev int mlx4_en_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp);
8030cd917a4SJesper Dangaard Brouer int mlx4_en_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash,
8040cd917a4SJesper Dangaard Brouer enum xdp_rss_hash_type *rss_type);
805ab46182dSStanislav Fomichev
8065a2cc190SJeff Kirsher /*
807ec693d47SAmir Vadai * Functions for time stamping
808ec693d47SAmir Vadai */
809ec693d47SAmir Vadai u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe);
810ab46182dSStanislav Fomichev u64 mlx4_en_get_hwtstamp(struct mlx4_en_dev *mdev, u64 timestamp);
811ec693d47SAmir Vadai void mlx4_en_fill_hwtstamps(struct mlx4_en_dev *mdev,
812ec693d47SAmir Vadai struct skb_shared_hwtstamps *hwts,
813ec693d47SAmir Vadai u64 timestamp);
814ec693d47SAmir Vadai void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev);
815ad7d4eaeSShawn Bohrer void mlx4_en_remove_timestamp(struct mlx4_en_dev *mdev);
816ec693d47SAmir Vadai
817ec693d47SAmir Vadai /* Globals
8185a2cc190SJeff Kirsher */
8195a2cc190SJeff Kirsher extern const struct ethtool_ops mlx4_en_ethtool_ops;
8205a2cc190SJeff Kirsher
8215a2cc190SJeff Kirsher
8225a2cc190SJeff Kirsher
8235a2cc190SJeff Kirsher /*
8245a2cc190SJeff Kirsher * printk / logging functions
8255a2cc190SJeff Kirsher */
8265a2cc190SJeff Kirsher
827b9075fa9SJoe Perches __printf(3, 4)
8280c87b29cSJoe Perches void en_print(const char *level, const struct mlx4_en_priv *priv,
829b9075fa9SJoe Perches const char *format, ...);
8305a2cc190SJeff Kirsher
8311a91de28SJoe Perches #define en_dbg(mlevel, priv, format, ...) \
8325a2cc190SJeff Kirsher do { \
8331a91de28SJoe Perches if (NETIF_MSG_##mlevel & (priv)->msg_enable) \
8341a91de28SJoe Perches en_print(KERN_DEBUG, priv, format, ##__VA_ARGS__); \
8355a2cc190SJeff Kirsher } while (0)
8361a91de28SJoe Perches #define en_warn(priv, format, ...) \
8371a91de28SJoe Perches en_print(KERN_WARNING, priv, format, ##__VA_ARGS__)
8381a91de28SJoe Perches #define en_err(priv, format, ...) \
8391a91de28SJoe Perches en_print(KERN_ERR, priv, format, ##__VA_ARGS__)
8401a91de28SJoe Perches #define en_info(priv, format, ...) \
8411a91de28SJoe Perches en_print(KERN_INFO, priv, format, ##__VA_ARGS__)
8425a2cc190SJeff Kirsher
8431a91de28SJoe Perches #define mlx4_err(mdev, format, ...) \
8441a91de28SJoe Perches pr_err(DRV_NAME " %s: " format, \
8451a91de28SJoe Perches dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
8461a91de28SJoe Perches #define mlx4_info(mdev, format, ...) \
8471a91de28SJoe Perches pr_info(DRV_NAME " %s: " format, \
8481a91de28SJoe Perches dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
8491a91de28SJoe Perches #define mlx4_warn(mdev, format, ...) \
8501a91de28SJoe Perches pr_warn(DRV_NAME " %s: " format, \
8511a91de28SJoe Perches dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
8525a2cc190SJeff Kirsher
8535a2cc190SJeff Kirsher #endif
854