1 /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
2 /* Copyright 2017-2019 NXP */
3 
4 #include <linux/timer.h>
5 #include <linux/pci.h>
6 #include <linux/netdevice.h>
7 #include <linux/etherdevice.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/skbuff.h>
10 #include <linux/ethtool.h>
11 #include <linux/if_vlan.h>
12 #include <linux/phylink.h>
13 #include <linux/dim.h>
14 
15 #include "enetc_hw.h"
16 
17 #define ENETC_MAC_MAXFRM_SIZE	9600
18 #define ENETC_MAX_MTU		(ENETC_MAC_MAXFRM_SIZE - \
19 				(ETH_FCS_LEN + ETH_HLEN + VLAN_HLEN))
20 
21 struct enetc_tx_swbd {
22 	struct sk_buff *skb;
23 	dma_addr_t dma;
24 	u16 len;
25 	u8 is_dma_page:1;
26 	u8 check_wb:1;
27 	u8 do_tstamp:1;
28 };
29 
30 #define ENETC_RX_MAXFRM_SIZE	ENETC_MAC_MAXFRM_SIZE
31 #define ENETC_RXB_TRUESIZE	2048 /* PAGE_SIZE >> 1 */
32 #define ENETC_RXB_PAD		NET_SKB_PAD /* add extra space if needed */
33 #define ENETC_RXB_DMA_SIZE	\
34 	(SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - ENETC_RXB_PAD)
35 
36 struct enetc_rx_swbd {
37 	dma_addr_t dma;
38 	struct page *page;
39 	u16 page_offset;
40 };
41 
42 struct enetc_ring_stats {
43 	unsigned int packets;
44 	unsigned int bytes;
45 	unsigned int rx_alloc_errs;
46 };
47 
48 #define ENETC_RX_RING_DEFAULT_SIZE	512
49 #define ENETC_TX_RING_DEFAULT_SIZE	256
50 #define ENETC_DEFAULT_TX_WORK		(ENETC_TX_RING_DEFAULT_SIZE / 2)
51 
52 struct enetc_bdr {
53 	struct device *dev; /* for DMA mapping */
54 	struct net_device *ndev;
55 	void *bd_base; /* points to Rx or Tx BD ring */
56 	union {
57 		void __iomem *tpir;
58 		void __iomem *rcir;
59 	};
60 	u16 index;
61 	int bd_count; /* # of BDs */
62 	int next_to_use;
63 	int next_to_clean;
64 	union {
65 		struct enetc_tx_swbd *tx_swbd;
66 		struct enetc_rx_swbd *rx_swbd;
67 	};
68 	union {
69 		void __iomem *tcir; /* Tx */
70 		int next_to_alloc; /* Rx */
71 	};
72 	void __iomem *idr; /* Interrupt Detect Register pointer */
73 
74 	struct enetc_ring_stats stats;
75 
76 	dma_addr_t bd_dma_base;
77 	u8 tsd_enable; /* Time specific departure */
78 	bool ext_en; /* enable h/w descriptor extensions */
79 } ____cacheline_aligned_in_smp;
80 
81 static inline void enetc_bdr_idx_inc(struct enetc_bdr *bdr, int *i)
82 {
83 	if (unlikely(++*i == bdr->bd_count))
84 		*i = 0;
85 }
86 
87 static inline int enetc_bd_unused(struct enetc_bdr *bdr)
88 {
89 	if (bdr->next_to_clean > bdr->next_to_use)
90 		return bdr->next_to_clean - bdr->next_to_use - 1;
91 
92 	return bdr->bd_count + bdr->next_to_clean - bdr->next_to_use - 1;
93 }
94 
95 /* Control BD ring */
96 #define ENETC_CBDR_DEFAULT_SIZE	64
97 struct enetc_cbdr {
98 	void *bd_base; /* points to Rx or Tx BD ring */
99 	void __iomem *pir;
100 	void __iomem *cir;
101 
102 	int bd_count; /* # of BDs */
103 	int next_to_use;
104 	int next_to_clean;
105 
106 	dma_addr_t bd_dma_base;
107 };
108 
109 #define ENETC_TXBD(BDR, i) (&(((union enetc_tx_bd *)((BDR).bd_base))[i]))
110 
111 static inline union enetc_rx_bd *enetc_rxbd(struct enetc_bdr *rx_ring, int i)
112 {
113 	int hw_idx = i;
114 
115 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
116 	if (rx_ring->ext_en)
117 		hw_idx = 2 * i;
118 #endif
119 	return &(((union enetc_rx_bd *)rx_ring->bd_base)[hw_idx]);
120 }
121 
122 static inline union enetc_rx_bd *enetc_rxbd_next(struct enetc_bdr *rx_ring,
123 						 union enetc_rx_bd *rxbd,
124 						 int i)
125 {
126 	rxbd++;
127 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
128 	if (rx_ring->ext_en)
129 		rxbd++;
130 #endif
131 	if (unlikely(++i == rx_ring->bd_count))
132 		rxbd = rx_ring->bd_base;
133 
134 	return rxbd;
135 }
136 
137 static inline union enetc_rx_bd *enetc_rxbd_ext(union enetc_rx_bd *rxbd)
138 {
139 	return ++rxbd;
140 }
141 
142 struct enetc_msg_swbd {
143 	void *vaddr;
144 	dma_addr_t dma;
145 	int size;
146 };
147 
148 #define ENETC_REV1	0x1
149 enum enetc_errata {
150 	ENETC_ERR_TXCSUM	= BIT(0),
151 	ENETC_ERR_VLAN_ISOL	= BIT(1),
152 	ENETC_ERR_UCMCSWP	= BIT(2),
153 };
154 
155 #define ENETC_SI_F_QBV BIT(0)
156 #define ENETC_SI_F_PSFP BIT(1)
157 
158 /* PCI IEP device data */
159 struct enetc_si {
160 	struct pci_dev *pdev;
161 	struct enetc_hw hw;
162 	enum enetc_errata errata;
163 
164 	struct net_device *ndev; /* back ref. */
165 
166 	struct enetc_cbdr cbd_ring;
167 
168 	int num_rx_rings; /* how many rings are available in the SI */
169 	int num_tx_rings;
170 	int num_fs_entries;
171 	int num_rss; /* number of RSS buckets */
172 	unsigned short pad;
173 	int hw_features;
174 };
175 
176 #define ENETC_SI_ALIGN	32
177 
178 static inline void *enetc_si_priv(const struct enetc_si *si)
179 {
180 	return (char *)si + ALIGN(sizeof(struct enetc_si), ENETC_SI_ALIGN);
181 }
182 
183 static inline bool enetc_si_is_pf(struct enetc_si *si)
184 {
185 	return !!(si->hw.port);
186 }
187 
188 #define ENETC_MAX_NUM_TXQS	8
189 #define ENETC_INT_NAME_MAX	(IFNAMSIZ + 8)
190 
191 struct enetc_int_vector {
192 	void __iomem *rbier;
193 	void __iomem *tbier_base;
194 	void __iomem *ricr1;
195 	unsigned long tx_rings_map;
196 	int count_tx_rings;
197 	u32 rx_ictt;
198 	u16 comp_cnt;
199 	bool rx_dim_en, rx_napi_work;
200 	struct napi_struct napi ____cacheline_aligned_in_smp;
201 	struct dim rx_dim ____cacheline_aligned_in_smp;
202 	char name[ENETC_INT_NAME_MAX];
203 
204 	struct enetc_bdr rx_ring;
205 	struct enetc_bdr tx_ring[];
206 } ____cacheline_aligned_in_smp;
207 
208 struct enetc_cls_rule {
209 	struct ethtool_rx_flow_spec fs;
210 	int used;
211 };
212 
213 #define ENETC_MAX_BDR_INT	2 /* fixed to max # of available cpus */
214 struct psfp_cap {
215 	u32 max_streamid;
216 	u32 max_psfp_filter;
217 	u32 max_psfp_gate;
218 	u32 max_psfp_gatelist;
219 	u32 max_psfp_meter;
220 };
221 
222 /* TODO: more hardware offloads */
223 enum enetc_active_offloads {
224 	ENETC_F_RX_TSTAMP	= BIT(0),
225 	ENETC_F_TX_TSTAMP	= BIT(1),
226 	ENETC_F_QBV             = BIT(2),
227 	ENETC_F_QCI		= BIT(3),
228 };
229 
230 /* interrupt coalescing modes */
231 enum enetc_ic_mode {
232 	/* one interrupt per frame */
233 	ENETC_IC_NONE = 0,
234 	/* activated when int coalescing time is set to a non-0 value */
235 	ENETC_IC_RX_MANUAL = BIT(0),
236 	ENETC_IC_TX_MANUAL = BIT(1),
237 	/* use dynamic interrupt moderation */
238 	ENETC_IC_RX_ADAPTIVE = BIT(2),
239 };
240 
241 #define ENETC_RXIC_PKTTHR	min_t(u32, 256, ENETC_RX_RING_DEFAULT_SIZE / 2)
242 #define ENETC_TXIC_PKTTHR	min_t(u32, 128, ENETC_TX_RING_DEFAULT_SIZE / 2)
243 #define ENETC_TXIC_TIMETHR	enetc_usecs_to_cycles(600)
244 
245 struct enetc_ndev_priv {
246 	struct net_device *ndev;
247 	struct device *dev; /* dma-mapping device */
248 	struct enetc_si *si;
249 
250 	int bdr_int_num; /* number of Rx/Tx ring interrupts */
251 	struct enetc_int_vector *int_vector[ENETC_MAX_BDR_INT];
252 	u16 num_rx_rings, num_tx_rings;
253 	u16 rx_bd_count, tx_bd_count;
254 
255 	u16 msg_enable;
256 	int active_offloads;
257 
258 	u32 speed; /* store speed for compare update pspeed */
259 
260 	struct enetc_bdr *tx_ring[16];
261 	struct enetc_bdr *rx_ring[16];
262 
263 	struct enetc_cls_rule *cls_rules;
264 
265 	struct psfp_cap psfp_cap;
266 
267 	struct phylink *phylink;
268 	int ic_mode;
269 	u32 tx_ictt;
270 };
271 
272 /* Messaging */
273 
274 /* VF-PF set primary MAC address message format */
275 struct enetc_msg_cmd_set_primary_mac {
276 	struct enetc_msg_cmd_header header;
277 	struct sockaddr mac;
278 };
279 
280 #define ENETC_CBD(R, i)	(&(((struct enetc_cbd *)((R).bd_base))[i]))
281 
282 #define ENETC_CBDR_TIMEOUT	1000 /* usecs */
283 
284 /* PTP driver exports */
285 extern int enetc_phc_index;
286 
287 /* SI common */
288 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv);
289 void enetc_pci_remove(struct pci_dev *pdev);
290 int enetc_alloc_msix(struct enetc_ndev_priv *priv);
291 void enetc_free_msix(struct enetc_ndev_priv *priv);
292 void enetc_get_si_caps(struct enetc_si *si);
293 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv);
294 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv);
295 void enetc_free_si_resources(struct enetc_ndev_priv *priv);
296 
297 int enetc_open(struct net_device *ndev);
298 int enetc_close(struct net_device *ndev);
299 void enetc_start(struct net_device *ndev);
300 void enetc_stop(struct net_device *ndev);
301 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev);
302 struct net_device_stats *enetc_get_stats(struct net_device *ndev);
303 int enetc_set_features(struct net_device *ndev,
304 		       netdev_features_t features);
305 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd);
306 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
307 		   void *type_data);
308 
309 /* ethtool */
310 void enetc_set_ethtool_ops(struct net_device *ndev);
311 
312 /* control buffer descriptor ring (CBDR) */
313 int enetc_set_mac_flt_entry(struct enetc_si *si, int index,
314 			    char *mac_addr, int si_map);
315 int enetc_clear_mac_flt_entry(struct enetc_si *si, int index);
316 int enetc_set_fs_entry(struct enetc_si *si, struct enetc_cmd_rfse *rfse,
317 		       int index);
318 void enetc_set_rss_key(struct enetc_hw *hw, const u8 *bytes);
319 int enetc_get_rss_table(struct enetc_si *si, u32 *table, int count);
320 int enetc_set_rss_table(struct enetc_si *si, const u32 *table, int count);
321 int enetc_send_cmd(struct enetc_si *si, struct enetc_cbd *cbd);
322 
323 #ifdef CONFIG_FSL_ENETC_QOS
324 int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data);
325 void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed);
326 int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data);
327 int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data);
328 int enetc_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
329 			    void *cb_priv);
330 int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data);
331 int enetc_psfp_init(struct enetc_ndev_priv *priv);
332 int enetc_psfp_clean(struct enetc_ndev_priv *priv);
333 
334 static inline void enetc_get_max_cap(struct enetc_ndev_priv *priv)
335 {
336 	u32 reg;
337 
338 	reg = enetc_port_rd(&priv->si->hw, ENETC_PSIDCAPR);
339 	priv->psfp_cap.max_streamid = reg & ENETC_PSIDCAPR_MSK;
340 	/* Port stream filter capability */
341 	reg = enetc_port_rd(&priv->si->hw, ENETC_PSFCAPR);
342 	priv->psfp_cap.max_psfp_filter = reg & ENETC_PSFCAPR_MSK;
343 	/* Port stream gate capability */
344 	reg = enetc_port_rd(&priv->si->hw, ENETC_PSGCAPR);
345 	priv->psfp_cap.max_psfp_gate = (reg & ENETC_PSGCAPR_SGIT_MSK);
346 	priv->psfp_cap.max_psfp_gatelist = (reg & ENETC_PSGCAPR_GCL_MSK) >> 16;
347 	/* Port flow meter capability */
348 	reg = enetc_port_rd(&priv->si->hw, ENETC_PFMCAPR);
349 	priv->psfp_cap.max_psfp_meter = reg & ENETC_PFMCAPR_MSK;
350 }
351 
352 static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
353 {
354 	struct enetc_hw *hw = &priv->si->hw;
355 	int err;
356 
357 	enetc_get_max_cap(priv);
358 
359 	err = enetc_psfp_init(priv);
360 	if (err)
361 		return err;
362 
363 	enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) |
364 		 ENETC_PPSFPMR_PSFPEN | ENETC_PPSFPMR_VS |
365 		 ENETC_PPSFPMR_PVC | ENETC_PPSFPMR_PVZC);
366 
367 	return 0;
368 }
369 
370 static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
371 {
372 	struct enetc_hw *hw = &priv->si->hw;
373 	int err;
374 
375 	err = enetc_psfp_clean(priv);
376 	if (err)
377 		return err;
378 
379 	enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) &
380 		 ~ENETC_PPSFPMR_PSFPEN & ~ENETC_PPSFPMR_VS &
381 		 ~ENETC_PPSFPMR_PVC & ~ENETC_PPSFPMR_PVZC);
382 
383 	memset(&priv->psfp_cap, 0, sizeof(struct psfp_cap));
384 
385 	return 0;
386 }
387 
388 #else
389 #define enetc_setup_tc_taprio(ndev, type_data) -EOPNOTSUPP
390 #define enetc_sched_speed_set(priv, speed) (void)0
391 #define enetc_setup_tc_cbs(ndev, type_data) -EOPNOTSUPP
392 #define enetc_setup_tc_txtime(ndev, type_data) -EOPNOTSUPP
393 #define enetc_setup_tc_psfp(ndev, type_data) -EOPNOTSUPP
394 #define enetc_setup_tc_block_cb NULL
395 
396 #define enetc_get_max_cap(p)		\
397 	memset(&((p)->psfp_cap), 0, sizeof(struct psfp_cap))
398 
399 static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
400 {
401 	return 0;
402 }
403 
404 static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
405 {
406 	return 0;
407 }
408 #endif
409