1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3 
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/tcp.h>
7 #include <linux/udp.h>
8 #include <linux/vmalloc.h>
9 #include <linux/ptp_classify.h>
10 #include <net/ip6_checksum.h>
11 #include <net/pkt_sched.h>
12 #include <net/tso.h>
13 
14 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
15 {
16 	int num_tx_rings = priv->num_tx_rings;
17 	int i;
18 
19 	for (i = 0; i < priv->num_rx_rings; i++)
20 		if (priv->rx_ring[i]->xdp.prog)
21 			return num_tx_rings - num_possible_cpus();
22 
23 	return num_tx_rings;
24 }
25 
26 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
27 							struct enetc_bdr *tx_ring)
28 {
29 	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
30 
31 	return priv->rx_ring[index];
32 }
33 
34 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
35 {
36 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
37 		return NULL;
38 
39 	return tx_swbd->skb;
40 }
41 
42 static struct xdp_frame *
43 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
44 {
45 	if (tx_swbd->is_xdp_redirect)
46 		return tx_swbd->xdp_frame;
47 
48 	return NULL;
49 }
50 
51 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
52 				struct enetc_tx_swbd *tx_swbd)
53 {
54 	/* For XDP_TX, pages come from RX, whereas for the other contexts where
55 	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
56 	 * to match the DMA mapping length, so we need to differentiate those.
57 	 */
58 	if (tx_swbd->is_dma_page)
59 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
60 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
61 			       tx_swbd->dir);
62 	else
63 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
64 				 tx_swbd->len, tx_swbd->dir);
65 	tx_swbd->dma = 0;
66 }
67 
68 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
69 				struct enetc_tx_swbd *tx_swbd)
70 {
71 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
72 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
73 
74 	if (tx_swbd->dma)
75 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
76 
77 	if (xdp_frame) {
78 		xdp_return_frame(tx_swbd->xdp_frame);
79 		tx_swbd->xdp_frame = NULL;
80 	} else if (skb) {
81 		dev_kfree_skb_any(skb);
82 		tx_swbd->skb = NULL;
83 	}
84 }
85 
86 /* Let H/W know BD ring has been updated */
87 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
88 {
89 	/* includes wmb() */
90 	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
91 }
92 
93 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
94 			   u8 *msgtype, u8 *twostep,
95 			   u16 *correction_offset, u16 *body_offset)
96 {
97 	unsigned int ptp_class;
98 	struct ptp_header *hdr;
99 	unsigned int type;
100 	u8 *base;
101 
102 	ptp_class = ptp_classify_raw(skb);
103 	if (ptp_class == PTP_CLASS_NONE)
104 		return -EINVAL;
105 
106 	hdr = ptp_parse_header(skb, ptp_class);
107 	if (!hdr)
108 		return -EINVAL;
109 
110 	type = ptp_class & PTP_CLASS_PMASK;
111 	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
112 		*udp = 1;
113 	else
114 		*udp = 0;
115 
116 	*msgtype = ptp_get_msgtype(hdr, ptp_class);
117 	*twostep = hdr->flag_field[0] & 0x2;
118 
119 	base = skb_mac_header(skb);
120 	*correction_offset = (u8 *)&hdr->correction - base;
121 	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
122 
123 	return 0;
124 }
125 
126 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
127 {
128 	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
129 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
130 	struct enetc_hw *hw = &priv->si->hw;
131 	struct enetc_tx_swbd *tx_swbd;
132 	int len = skb_headlen(skb);
133 	union enetc_tx_bd temp_bd;
134 	u8 msgtype, twostep, udp;
135 	union enetc_tx_bd *txbd;
136 	u16 offset1, offset2;
137 	int i, count = 0;
138 	skb_frag_t *frag;
139 	unsigned int f;
140 	dma_addr_t dma;
141 	u8 flags = 0;
142 
143 	i = tx_ring->next_to_use;
144 	txbd = ENETC_TXBD(*tx_ring, i);
145 	prefetchw(txbd);
146 
147 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
148 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
149 		goto dma_err;
150 
151 	temp_bd.addr = cpu_to_le64(dma);
152 	temp_bd.buf_len = cpu_to_le16(len);
153 	temp_bd.lstatus = 0;
154 
155 	tx_swbd = &tx_ring->tx_swbd[i];
156 	tx_swbd->dma = dma;
157 	tx_swbd->len = len;
158 	tx_swbd->is_dma_page = 0;
159 	tx_swbd->dir = DMA_TO_DEVICE;
160 	count++;
161 
162 	do_vlan = skb_vlan_tag_present(skb);
163 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
164 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
165 				    &offset2) ||
166 		    msgtype != PTP_MSGTYPE_SYNC || twostep)
167 			WARN_ONCE(1, "Bad packet for one-step timestamping\n");
168 		else
169 			do_onestep_tstamp = true;
170 	} else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
171 		do_twostep_tstamp = true;
172 	}
173 
174 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
175 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
176 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
177 
178 	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
179 		flags |= ENETC_TXBD_FLAGS_EX;
180 
181 	if (tx_ring->tsd_enable)
182 		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
183 
184 	/* first BD needs frm_len and offload flags set */
185 	temp_bd.frm_len = cpu_to_le16(skb->len);
186 	temp_bd.flags = flags;
187 
188 	if (flags & ENETC_TXBD_FLAGS_TSE)
189 		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
190 							  flags);
191 
192 	if (flags & ENETC_TXBD_FLAGS_EX) {
193 		u8 e_flags = 0;
194 		*txbd = temp_bd;
195 		enetc_clear_tx_bd(&temp_bd);
196 
197 		/* add extension BD for VLAN and/or timestamping */
198 		flags = 0;
199 		tx_swbd++;
200 		txbd++;
201 		i++;
202 		if (unlikely(i == tx_ring->bd_count)) {
203 			i = 0;
204 			tx_swbd = tx_ring->tx_swbd;
205 			txbd = ENETC_TXBD(*tx_ring, 0);
206 		}
207 		prefetchw(txbd);
208 
209 		if (do_vlan) {
210 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
211 			temp_bd.ext.tpid = 0; /* < C-TAG */
212 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
213 		}
214 
215 		if (do_onestep_tstamp) {
216 			u32 lo, hi, val;
217 			u64 sec, nsec;
218 			u8 *data;
219 
220 			lo = enetc_rd_hot(hw, ENETC_SICTR0);
221 			hi = enetc_rd_hot(hw, ENETC_SICTR1);
222 			sec = (u64)hi << 32 | lo;
223 			nsec = do_div(sec, 1000000000);
224 
225 			/* Configure extension BD */
226 			temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
227 			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
228 
229 			/* Update originTimestamp field of Sync packet
230 			 * - 48 bits seconds field
231 			 * - 32 bits nanseconds field
232 			 */
233 			data = skb_mac_header(skb);
234 			*(__be16 *)(data + offset2) =
235 				htons((sec >> 32) & 0xffff);
236 			*(__be32 *)(data + offset2 + 2) =
237 				htonl(sec & 0xffffffff);
238 			*(__be32 *)(data + offset2 + 6) = htonl(nsec);
239 
240 			/* Configure single-step register */
241 			val = ENETC_PM0_SINGLE_STEP_EN;
242 			val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
243 			if (udp)
244 				val |= ENETC_PM0_SINGLE_STEP_CH;
245 
246 			enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val);
247 			enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val);
248 		} else if (do_twostep_tstamp) {
249 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
250 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
251 		}
252 
253 		temp_bd.ext.e_flags = e_flags;
254 		count++;
255 	}
256 
257 	frag = &skb_shinfo(skb)->frags[0];
258 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
259 		len = skb_frag_size(frag);
260 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
261 				       DMA_TO_DEVICE);
262 		if (dma_mapping_error(tx_ring->dev, dma))
263 			goto dma_err;
264 
265 		*txbd = temp_bd;
266 		enetc_clear_tx_bd(&temp_bd);
267 
268 		flags = 0;
269 		tx_swbd++;
270 		txbd++;
271 		i++;
272 		if (unlikely(i == tx_ring->bd_count)) {
273 			i = 0;
274 			tx_swbd = tx_ring->tx_swbd;
275 			txbd = ENETC_TXBD(*tx_ring, 0);
276 		}
277 		prefetchw(txbd);
278 
279 		temp_bd.addr = cpu_to_le64(dma);
280 		temp_bd.buf_len = cpu_to_le16(len);
281 
282 		tx_swbd->dma = dma;
283 		tx_swbd->len = len;
284 		tx_swbd->is_dma_page = 1;
285 		tx_swbd->dir = DMA_TO_DEVICE;
286 		count++;
287 	}
288 
289 	/* last BD needs 'F' bit set */
290 	flags |= ENETC_TXBD_FLAGS_F;
291 	temp_bd.flags = flags;
292 	*txbd = temp_bd;
293 
294 	tx_ring->tx_swbd[i].is_eof = true;
295 	tx_ring->tx_swbd[i].skb = skb;
296 
297 	enetc_bdr_idx_inc(tx_ring, &i);
298 	tx_ring->next_to_use = i;
299 
300 	skb_tx_timestamp(skb);
301 
302 	enetc_update_tx_ring_tail(tx_ring);
303 
304 	return count;
305 
306 dma_err:
307 	dev_err(tx_ring->dev, "DMA map error");
308 
309 	do {
310 		tx_swbd = &tx_ring->tx_swbd[i];
311 		enetc_free_tx_frame(tx_ring, tx_swbd);
312 		if (i == 0)
313 			i = tx_ring->bd_count;
314 		i--;
315 	} while (count--);
316 
317 	return 0;
318 }
319 
320 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
321 				 struct enetc_tx_swbd *tx_swbd,
322 				 union enetc_tx_bd *txbd, int *i, int hdr_len,
323 				 int data_len)
324 {
325 	union enetc_tx_bd txbd_tmp;
326 	u8 flags = 0, e_flags = 0;
327 	dma_addr_t addr;
328 
329 	enetc_clear_tx_bd(&txbd_tmp);
330 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
331 
332 	if (skb_vlan_tag_present(skb))
333 		flags |= ENETC_TXBD_FLAGS_EX;
334 
335 	txbd_tmp.addr = cpu_to_le64(addr);
336 	txbd_tmp.buf_len = cpu_to_le16(hdr_len);
337 
338 	/* first BD needs frm_len and offload flags set */
339 	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
340 	txbd_tmp.flags = flags;
341 
342 	/* For the TSO header we do not set the dma address since we do not
343 	 * want it unmapped when we do cleanup. We still set len so that we
344 	 * count the bytes sent.
345 	 */
346 	tx_swbd->len = hdr_len;
347 	tx_swbd->do_twostep_tstamp = false;
348 	tx_swbd->check_wb = false;
349 
350 	/* Actually write the header in the BD */
351 	*txbd = txbd_tmp;
352 
353 	/* Add extension BD for VLAN */
354 	if (flags & ENETC_TXBD_FLAGS_EX) {
355 		/* Get the next BD */
356 		enetc_bdr_idx_inc(tx_ring, i);
357 		txbd = ENETC_TXBD(*tx_ring, *i);
358 		tx_swbd = &tx_ring->tx_swbd[*i];
359 		prefetchw(txbd);
360 
361 		/* Setup the VLAN fields */
362 		enetc_clear_tx_bd(&txbd_tmp);
363 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
364 		txbd_tmp.ext.tpid = 0; /* < C-TAG */
365 		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
366 
367 		/* Write the BD */
368 		txbd_tmp.ext.e_flags = e_flags;
369 		*txbd = txbd_tmp;
370 	}
371 }
372 
373 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
374 				 struct enetc_tx_swbd *tx_swbd,
375 				 union enetc_tx_bd *txbd, char *data,
376 				 int size, bool last_bd)
377 {
378 	union enetc_tx_bd txbd_tmp;
379 	dma_addr_t addr;
380 	u8 flags = 0;
381 
382 	enetc_clear_tx_bd(&txbd_tmp);
383 
384 	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
385 	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
386 		netdev_err(tx_ring->ndev, "DMA map error\n");
387 		return -ENOMEM;
388 	}
389 
390 	if (last_bd) {
391 		flags |= ENETC_TXBD_FLAGS_F;
392 		tx_swbd->is_eof = 1;
393 	}
394 
395 	txbd_tmp.addr = cpu_to_le64(addr);
396 	txbd_tmp.buf_len = cpu_to_le16(size);
397 	txbd_tmp.flags = flags;
398 
399 	tx_swbd->dma = addr;
400 	tx_swbd->len = size;
401 	tx_swbd->dir = DMA_TO_DEVICE;
402 
403 	*txbd = txbd_tmp;
404 
405 	return 0;
406 }
407 
408 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
409 				 char *hdr, int hdr_len, int *l4_hdr_len)
410 {
411 	char *l4_hdr = hdr + skb_transport_offset(skb);
412 	int mac_hdr_len = skb_network_offset(skb);
413 
414 	if (tso->tlen != sizeof(struct udphdr)) {
415 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
416 
417 		tcph->check = 0;
418 	} else {
419 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
420 
421 		udph->check = 0;
422 	}
423 
424 	/* Compute the IP checksum. This is necessary since tso_build_hdr()
425 	 * already incremented the IP ID field.
426 	 */
427 	if (!tso->ipv6) {
428 		struct iphdr *iph = (void *)(hdr + mac_hdr_len);
429 
430 		iph->check = 0;
431 		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
432 	}
433 
434 	/* Compute the checksum over the L4 header. */
435 	*l4_hdr_len = hdr_len - skb_transport_offset(skb);
436 	return csum_partial(l4_hdr, *l4_hdr_len, 0);
437 }
438 
439 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
440 				    struct sk_buff *skb, char *hdr, int len,
441 				    __wsum sum)
442 {
443 	char *l4_hdr = hdr + skb_transport_offset(skb);
444 	__sum16 csum_final;
445 
446 	/* Complete the L4 checksum by appending the pseudo-header to the
447 	 * already computed checksum.
448 	 */
449 	if (!tso->ipv6)
450 		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
451 					       ip_hdr(skb)->daddr,
452 					       len, ip_hdr(skb)->protocol, sum);
453 	else
454 		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
455 					     &ipv6_hdr(skb)->daddr,
456 					     len, ipv6_hdr(skb)->nexthdr, sum);
457 
458 	if (tso->tlen != sizeof(struct udphdr)) {
459 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
460 
461 		tcph->check = csum_final;
462 	} else {
463 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
464 
465 		udph->check = csum_final;
466 	}
467 }
468 
469 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
470 {
471 	int hdr_len, total_len, data_len;
472 	struct enetc_tx_swbd *tx_swbd;
473 	union enetc_tx_bd *txbd;
474 	struct tso_t tso;
475 	__wsum csum, csum2;
476 	int count = 0, pos;
477 	int err, i, bd_data_num;
478 
479 	/* Initialize the TSO handler, and prepare the first payload */
480 	hdr_len = tso_start(skb, &tso);
481 	total_len = skb->len - hdr_len;
482 	i = tx_ring->next_to_use;
483 
484 	while (total_len > 0) {
485 		char *hdr;
486 
487 		/* Get the BD */
488 		txbd = ENETC_TXBD(*tx_ring, i);
489 		tx_swbd = &tx_ring->tx_swbd[i];
490 		prefetchw(txbd);
491 
492 		/* Determine the length of this packet */
493 		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
494 		total_len -= data_len;
495 
496 		/* prepare packet headers: MAC + IP + TCP */
497 		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
498 		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
499 
500 		/* compute the csum over the L4 header */
501 		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
502 		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
503 		bd_data_num = 0;
504 		count++;
505 
506 		while (data_len > 0) {
507 			int size;
508 
509 			size = min_t(int, tso.size, data_len);
510 
511 			/* Advance the index in the BDR */
512 			enetc_bdr_idx_inc(tx_ring, &i);
513 			txbd = ENETC_TXBD(*tx_ring, i);
514 			tx_swbd = &tx_ring->tx_swbd[i];
515 			prefetchw(txbd);
516 
517 			/* Compute the checksum over this segment of data and
518 			 * add it to the csum already computed (over the L4
519 			 * header and possible other data segments).
520 			 */
521 			csum2 = csum_partial(tso.data, size, 0);
522 			csum = csum_block_add(csum, csum2, pos);
523 			pos += size;
524 
525 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
526 						    tso.data, size,
527 						    size == data_len);
528 			if (err)
529 				goto err_map_data;
530 
531 			data_len -= size;
532 			count++;
533 			bd_data_num++;
534 			tso_build_data(skb, &tso, size);
535 
536 			if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
537 				goto err_chained_bd;
538 		}
539 
540 		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
541 
542 		if (total_len == 0)
543 			tx_swbd->skb = skb;
544 
545 		/* Go to the next BD */
546 		enetc_bdr_idx_inc(tx_ring, &i);
547 	}
548 
549 	tx_ring->next_to_use = i;
550 	enetc_update_tx_ring_tail(tx_ring);
551 
552 	return count;
553 
554 err_map_data:
555 	dev_err(tx_ring->dev, "DMA map error");
556 
557 err_chained_bd:
558 	do {
559 		tx_swbd = &tx_ring->tx_swbd[i];
560 		enetc_free_tx_frame(tx_ring, tx_swbd);
561 		if (i == 0)
562 			i = tx_ring->bd_count;
563 		i--;
564 	} while (count--);
565 
566 	return 0;
567 }
568 
569 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
570 				    struct net_device *ndev)
571 {
572 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
573 	struct enetc_bdr *tx_ring;
574 	int count, err;
575 
576 	/* Queue one-step Sync packet if already locked */
577 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
578 		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
579 					  &priv->flags)) {
580 			skb_queue_tail(&priv->tx_skbs, skb);
581 			return NETDEV_TX_OK;
582 		}
583 	}
584 
585 	tx_ring = priv->tx_ring[skb->queue_mapping];
586 
587 	if (skb_is_gso(skb)) {
588 		if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
589 			netif_stop_subqueue(ndev, tx_ring->index);
590 			return NETDEV_TX_BUSY;
591 		}
592 
593 		enetc_lock_mdio();
594 		count = enetc_map_tx_tso_buffs(tx_ring, skb);
595 		enetc_unlock_mdio();
596 	} else {
597 		if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
598 			if (unlikely(skb_linearize(skb)))
599 				goto drop_packet_err;
600 
601 		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
602 		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
603 			netif_stop_subqueue(ndev, tx_ring->index);
604 			return NETDEV_TX_BUSY;
605 		}
606 
607 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
608 			err = skb_checksum_help(skb);
609 			if (err)
610 				goto drop_packet_err;
611 		}
612 		enetc_lock_mdio();
613 		count = enetc_map_tx_buffs(tx_ring, skb);
614 		enetc_unlock_mdio();
615 	}
616 
617 	if (unlikely(!count))
618 		goto drop_packet_err;
619 
620 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
621 		netif_stop_subqueue(ndev, tx_ring->index);
622 
623 	return NETDEV_TX_OK;
624 
625 drop_packet_err:
626 	dev_kfree_skb_any(skb);
627 	return NETDEV_TX_OK;
628 }
629 
630 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
631 {
632 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
633 	u8 udp, msgtype, twostep;
634 	u16 offset1, offset2;
635 
636 	/* Mark tx timestamp type on skb->cb[0] if requires */
637 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
638 	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
639 		skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
640 	} else {
641 		skb->cb[0] = 0;
642 	}
643 
644 	/* Fall back to two-step timestamp if not one-step Sync packet */
645 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
646 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
647 				    &offset1, &offset2) ||
648 		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
649 			skb->cb[0] = ENETC_F_TX_TSTAMP;
650 	}
651 
652 	return enetc_start_xmit(skb, ndev);
653 }
654 
655 static irqreturn_t enetc_msix(int irq, void *data)
656 {
657 	struct enetc_int_vector	*v = data;
658 	int i;
659 
660 	enetc_lock_mdio();
661 
662 	/* disable interrupts */
663 	enetc_wr_reg_hot(v->rbier, 0);
664 	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
665 
666 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
667 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
668 
669 	enetc_unlock_mdio();
670 
671 	napi_schedule(&v->napi);
672 
673 	return IRQ_HANDLED;
674 }
675 
676 static void enetc_rx_dim_work(struct work_struct *w)
677 {
678 	struct dim *dim = container_of(w, struct dim, work);
679 	struct dim_cq_moder moder =
680 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
681 	struct enetc_int_vector	*v =
682 		container_of(dim, struct enetc_int_vector, rx_dim);
683 
684 	v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
685 	dim->state = DIM_START_MEASURE;
686 }
687 
688 static void enetc_rx_net_dim(struct enetc_int_vector *v)
689 {
690 	struct dim_sample dim_sample = {};
691 
692 	v->comp_cnt++;
693 
694 	if (!v->rx_napi_work)
695 		return;
696 
697 	dim_update_sample(v->comp_cnt,
698 			  v->rx_ring.stats.packets,
699 			  v->rx_ring.stats.bytes,
700 			  &dim_sample);
701 	net_dim(&v->rx_dim, dim_sample);
702 }
703 
704 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
705 {
706 	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
707 
708 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
709 }
710 
711 static bool enetc_page_reusable(struct page *page)
712 {
713 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
714 }
715 
716 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
717 			     struct enetc_rx_swbd *old)
718 {
719 	struct enetc_rx_swbd *new;
720 
721 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
722 
723 	/* next buf that may reuse a page */
724 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
725 
726 	/* copy page reference */
727 	*new = *old;
728 }
729 
730 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
731 				u64 *tstamp)
732 {
733 	u32 lo, hi, tstamp_lo;
734 
735 	lo = enetc_rd_hot(hw, ENETC_SICTR0);
736 	hi = enetc_rd_hot(hw, ENETC_SICTR1);
737 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
738 	if (lo <= tstamp_lo)
739 		hi -= 1;
740 	*tstamp = (u64)hi << 32 | tstamp_lo;
741 }
742 
743 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
744 {
745 	struct skb_shared_hwtstamps shhwtstamps;
746 
747 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
748 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
749 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
750 		skb_txtime_consumed(skb);
751 		skb_tstamp_tx(skb, &shhwtstamps);
752 	}
753 }
754 
755 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
756 				      struct enetc_tx_swbd *tx_swbd)
757 {
758 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
759 	struct enetc_rx_swbd rx_swbd = {
760 		.dma = tx_swbd->dma,
761 		.page = tx_swbd->page,
762 		.page_offset = tx_swbd->page_offset,
763 		.dir = tx_swbd->dir,
764 		.len = tx_swbd->len,
765 	};
766 	struct enetc_bdr *rx_ring;
767 
768 	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
769 
770 	if (likely(enetc_swbd_unused(rx_ring))) {
771 		enetc_reuse_page(rx_ring, &rx_swbd);
772 
773 		/* sync for use by the device */
774 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
775 						 rx_swbd.page_offset,
776 						 ENETC_RXB_DMA_SIZE_XDP,
777 						 rx_swbd.dir);
778 
779 		rx_ring->stats.recycles++;
780 	} else {
781 		/* RX ring is already full, we need to unmap and free the
782 		 * page, since there's nothing useful we can do with it.
783 		 */
784 		rx_ring->stats.recycle_failures++;
785 
786 		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
787 			       rx_swbd.dir);
788 		__free_page(rx_swbd.page);
789 	}
790 
791 	rx_ring->xdp.xdp_tx_in_flight--;
792 }
793 
794 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
795 {
796 	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
797 	struct net_device *ndev = tx_ring->ndev;
798 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
799 	struct enetc_tx_swbd *tx_swbd;
800 	int i, bds_to_clean;
801 	bool do_twostep_tstamp;
802 	u64 tstamp = 0;
803 
804 	i = tx_ring->next_to_clean;
805 	tx_swbd = &tx_ring->tx_swbd[i];
806 
807 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
808 
809 	do_twostep_tstamp = false;
810 
811 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
812 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
813 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
814 		bool is_eof = tx_swbd->is_eof;
815 
816 		if (unlikely(tx_swbd->check_wb)) {
817 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
818 
819 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
820 			    tx_swbd->do_twostep_tstamp) {
821 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
822 						    &tstamp);
823 				do_twostep_tstamp = true;
824 			}
825 
826 			if (tx_swbd->qbv_en &&
827 			    txbd->wb.status & ENETC_TXBD_STATS_WIN)
828 				tx_win_drop++;
829 		}
830 
831 		if (tx_swbd->is_xdp_tx)
832 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
833 		else if (likely(tx_swbd->dma))
834 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
835 
836 		if (xdp_frame) {
837 			xdp_return_frame(xdp_frame);
838 		} else if (skb) {
839 			if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
840 				/* Start work to release lock for next one-step
841 				 * timestamping packet. And send one skb in
842 				 * tx_skbs queue if has.
843 				 */
844 				schedule_work(&priv->tx_onestep_tstamp);
845 			} else if (unlikely(do_twostep_tstamp)) {
846 				enetc_tstamp_tx(skb, tstamp);
847 				do_twostep_tstamp = false;
848 			}
849 			napi_consume_skb(skb, napi_budget);
850 		}
851 
852 		tx_byte_cnt += tx_swbd->len;
853 		/* Scrub the swbd here so we don't have to do that
854 		 * when we reuse it during xmit
855 		 */
856 		memset(tx_swbd, 0, sizeof(*tx_swbd));
857 
858 		bds_to_clean--;
859 		tx_swbd++;
860 		i++;
861 		if (unlikely(i == tx_ring->bd_count)) {
862 			i = 0;
863 			tx_swbd = tx_ring->tx_swbd;
864 		}
865 
866 		/* BD iteration loop end */
867 		if (is_eof) {
868 			tx_frm_cnt++;
869 			/* re-arm interrupt source */
870 			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
871 					 BIT(16 + tx_ring->index));
872 		}
873 
874 		if (unlikely(!bds_to_clean))
875 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
876 	}
877 
878 	tx_ring->next_to_clean = i;
879 	tx_ring->stats.packets += tx_frm_cnt;
880 	tx_ring->stats.bytes += tx_byte_cnt;
881 	tx_ring->stats.win_drop += tx_win_drop;
882 
883 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
884 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
885 		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
886 		netif_wake_subqueue(ndev, tx_ring->index);
887 	}
888 
889 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
890 }
891 
892 static bool enetc_new_page(struct enetc_bdr *rx_ring,
893 			   struct enetc_rx_swbd *rx_swbd)
894 {
895 	bool xdp = !!(rx_ring->xdp.prog);
896 	struct page *page;
897 	dma_addr_t addr;
898 
899 	page = dev_alloc_page();
900 	if (unlikely(!page))
901 		return false;
902 
903 	/* For XDP_TX, we forgo dma_unmap -> dma_map */
904 	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
905 
906 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
907 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
908 		__free_page(page);
909 
910 		return false;
911 	}
912 
913 	rx_swbd->dma = addr;
914 	rx_swbd->page = page;
915 	rx_swbd->page_offset = rx_ring->buffer_offset;
916 
917 	return true;
918 }
919 
920 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
921 {
922 	struct enetc_rx_swbd *rx_swbd;
923 	union enetc_rx_bd *rxbd;
924 	int i, j;
925 
926 	i = rx_ring->next_to_use;
927 	rx_swbd = &rx_ring->rx_swbd[i];
928 	rxbd = enetc_rxbd(rx_ring, i);
929 
930 	for (j = 0; j < buff_cnt; j++) {
931 		/* try reuse page */
932 		if (unlikely(!rx_swbd->page)) {
933 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
934 				rx_ring->stats.rx_alloc_errs++;
935 				break;
936 			}
937 		}
938 
939 		/* update RxBD */
940 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
941 					   rx_swbd->page_offset);
942 		/* clear 'R" as well */
943 		rxbd->r.lstatus = 0;
944 
945 		enetc_rxbd_next(rx_ring, &rxbd, &i);
946 		rx_swbd = &rx_ring->rx_swbd[i];
947 	}
948 
949 	if (likely(j)) {
950 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
951 		rx_ring->next_to_use = i;
952 
953 		/* update ENETC's consumer index */
954 		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
955 	}
956 
957 	return j;
958 }
959 
960 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
961 static void enetc_get_rx_tstamp(struct net_device *ndev,
962 				union enetc_rx_bd *rxbd,
963 				struct sk_buff *skb)
964 {
965 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
966 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
967 	struct enetc_hw *hw = &priv->si->hw;
968 	u32 lo, hi, tstamp_lo;
969 	u64 tstamp;
970 
971 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
972 		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
973 		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
974 		rxbd = enetc_rxbd_ext(rxbd);
975 		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
976 		if (lo <= tstamp_lo)
977 			hi -= 1;
978 
979 		tstamp = (u64)hi << 32 | tstamp_lo;
980 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
981 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
982 	}
983 }
984 #endif
985 
986 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
987 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
988 {
989 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
990 
991 	/* TODO: hashing */
992 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
993 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
994 
995 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
996 		skb->ip_summed = CHECKSUM_COMPLETE;
997 	}
998 
999 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1000 		__be16 tpid = 0;
1001 
1002 		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1003 		case 0:
1004 			tpid = htons(ETH_P_8021Q);
1005 			break;
1006 		case 1:
1007 			tpid = htons(ETH_P_8021AD);
1008 			break;
1009 		case 2:
1010 			tpid = htons(enetc_port_rd(&priv->si->hw,
1011 						   ENETC_PCVLANR1));
1012 			break;
1013 		case 3:
1014 			tpid = htons(enetc_port_rd(&priv->si->hw,
1015 						   ENETC_PCVLANR2));
1016 			break;
1017 		default:
1018 			break;
1019 		}
1020 
1021 		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1022 	}
1023 
1024 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1025 	if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1026 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1027 #endif
1028 }
1029 
1030 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1031  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1032  * mapped buffers.
1033  */
1034 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1035 					       int i, u16 size)
1036 {
1037 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1038 
1039 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1040 				      rx_swbd->page_offset,
1041 				      size, rx_swbd->dir);
1042 	return rx_swbd;
1043 }
1044 
1045 /* Reuse the current page without performing half-page buffer flipping */
1046 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1047 			      struct enetc_rx_swbd *rx_swbd)
1048 {
1049 	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1050 
1051 	enetc_reuse_page(rx_ring, rx_swbd);
1052 
1053 	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1054 					 rx_swbd->page_offset,
1055 					 buffer_size, rx_swbd->dir);
1056 
1057 	rx_swbd->page = NULL;
1058 }
1059 
1060 /* Reuse the current page by performing half-page buffer flipping */
1061 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1062 			       struct enetc_rx_swbd *rx_swbd)
1063 {
1064 	if (likely(enetc_page_reusable(rx_swbd->page))) {
1065 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1066 		page_ref_inc(rx_swbd->page);
1067 
1068 		enetc_put_rx_buff(rx_ring, rx_swbd);
1069 	} else {
1070 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1071 			       rx_swbd->dir);
1072 		rx_swbd->page = NULL;
1073 	}
1074 }
1075 
1076 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1077 						int i, u16 size)
1078 {
1079 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1080 	struct sk_buff *skb;
1081 	void *ba;
1082 
1083 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1084 	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1085 	if (unlikely(!skb)) {
1086 		rx_ring->stats.rx_alloc_errs++;
1087 		return NULL;
1088 	}
1089 
1090 	skb_reserve(skb, rx_ring->buffer_offset);
1091 	__skb_put(skb, size);
1092 
1093 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1094 
1095 	return skb;
1096 }
1097 
1098 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1099 				     u16 size, struct sk_buff *skb)
1100 {
1101 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1102 
1103 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1104 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1105 
1106 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1107 }
1108 
1109 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1110 					      u32 bd_status,
1111 					      union enetc_rx_bd **rxbd, int *i)
1112 {
1113 	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1114 		return false;
1115 
1116 	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1117 	enetc_rxbd_next(rx_ring, rxbd, i);
1118 
1119 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1120 		dma_rmb();
1121 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1122 
1123 		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1124 		enetc_rxbd_next(rx_ring, rxbd, i);
1125 	}
1126 
1127 	rx_ring->ndev->stats.rx_dropped++;
1128 	rx_ring->ndev->stats.rx_errors++;
1129 
1130 	return true;
1131 }
1132 
1133 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1134 				       u32 bd_status, union enetc_rx_bd **rxbd,
1135 				       int *i, int *cleaned_cnt, int buffer_size)
1136 {
1137 	struct sk_buff *skb;
1138 	u16 size;
1139 
1140 	size = le16_to_cpu((*rxbd)->r.buf_len);
1141 	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1142 	if (!skb)
1143 		return NULL;
1144 
1145 	enetc_get_offloads(rx_ring, *rxbd, skb);
1146 
1147 	(*cleaned_cnt)++;
1148 
1149 	enetc_rxbd_next(rx_ring, rxbd, i);
1150 
1151 	/* not last BD in frame? */
1152 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1153 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1154 		size = buffer_size;
1155 
1156 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1157 			dma_rmb();
1158 			size = le16_to_cpu((*rxbd)->r.buf_len);
1159 		}
1160 
1161 		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1162 
1163 		(*cleaned_cnt)++;
1164 
1165 		enetc_rxbd_next(rx_ring, rxbd, i);
1166 	}
1167 
1168 	skb_record_rx_queue(skb, rx_ring->index);
1169 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1170 
1171 	return skb;
1172 }
1173 
1174 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1175 
1176 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1177 			       struct napi_struct *napi, int work_limit)
1178 {
1179 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1180 	int cleaned_cnt, i;
1181 
1182 	cleaned_cnt = enetc_bd_unused(rx_ring);
1183 	/* next descriptor to process */
1184 	i = rx_ring->next_to_clean;
1185 
1186 	while (likely(rx_frm_cnt < work_limit)) {
1187 		union enetc_rx_bd *rxbd;
1188 		struct sk_buff *skb;
1189 		u32 bd_status;
1190 
1191 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1192 			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1193 							    cleaned_cnt);
1194 
1195 		rxbd = enetc_rxbd(rx_ring, i);
1196 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1197 		if (!bd_status)
1198 			break;
1199 
1200 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1201 		dma_rmb(); /* for reading other rxbd fields */
1202 
1203 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1204 						      &rxbd, &i))
1205 			break;
1206 
1207 		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1208 				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1209 		if (!skb)
1210 			break;
1211 
1212 		rx_byte_cnt += skb->len;
1213 		rx_frm_cnt++;
1214 
1215 		napi_gro_receive(napi, skb);
1216 	}
1217 
1218 	rx_ring->next_to_clean = i;
1219 
1220 	rx_ring->stats.packets += rx_frm_cnt;
1221 	rx_ring->stats.bytes += rx_byte_cnt;
1222 
1223 	return rx_frm_cnt;
1224 }
1225 
1226 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1227 				  struct enetc_tx_swbd *tx_swbd,
1228 				  int frm_len)
1229 {
1230 	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1231 
1232 	prefetchw(txbd);
1233 
1234 	enetc_clear_tx_bd(txbd);
1235 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1236 	txbd->buf_len = cpu_to_le16(tx_swbd->len);
1237 	txbd->frm_len = cpu_to_le16(frm_len);
1238 
1239 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1240 }
1241 
1242 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1243  * descriptors.
1244  */
1245 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1246 			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1247 {
1248 	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1249 	int i, k, frm_len = tmp_tx_swbd->len;
1250 
1251 	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1252 		return false;
1253 
1254 	while (unlikely(!tmp_tx_swbd->is_eof)) {
1255 		tmp_tx_swbd++;
1256 		frm_len += tmp_tx_swbd->len;
1257 	}
1258 
1259 	i = tx_ring->next_to_use;
1260 
1261 	for (k = 0; k < num_tx_swbd; k++) {
1262 		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1263 
1264 		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1265 
1266 		/* last BD needs 'F' bit set */
1267 		if (xdp_tx_swbd->is_eof) {
1268 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1269 
1270 			txbd->flags = ENETC_TXBD_FLAGS_F;
1271 		}
1272 
1273 		enetc_bdr_idx_inc(tx_ring, &i);
1274 	}
1275 
1276 	tx_ring->next_to_use = i;
1277 
1278 	return true;
1279 }
1280 
1281 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1282 					  struct enetc_tx_swbd *xdp_tx_arr,
1283 					  struct xdp_frame *xdp_frame)
1284 {
1285 	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1286 	struct skb_shared_info *shinfo;
1287 	void *data = xdp_frame->data;
1288 	int len = xdp_frame->len;
1289 	skb_frag_t *frag;
1290 	dma_addr_t dma;
1291 	unsigned int f;
1292 	int n = 0;
1293 
1294 	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1295 	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1296 		netdev_err(tx_ring->ndev, "DMA map error\n");
1297 		return -1;
1298 	}
1299 
1300 	xdp_tx_swbd->dma = dma;
1301 	xdp_tx_swbd->dir = DMA_TO_DEVICE;
1302 	xdp_tx_swbd->len = len;
1303 	xdp_tx_swbd->is_xdp_redirect = true;
1304 	xdp_tx_swbd->is_eof = false;
1305 	xdp_tx_swbd->xdp_frame = NULL;
1306 
1307 	n++;
1308 	xdp_tx_swbd = &xdp_tx_arr[n];
1309 
1310 	shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1311 
1312 	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1313 	     f++, frag++) {
1314 		data = skb_frag_address(frag);
1315 		len = skb_frag_size(frag);
1316 
1317 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1318 		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1319 			/* Undo the DMA mapping for all fragments */
1320 			while (--n >= 0)
1321 				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1322 
1323 			netdev_err(tx_ring->ndev, "DMA map error\n");
1324 			return -1;
1325 		}
1326 
1327 		xdp_tx_swbd->dma = dma;
1328 		xdp_tx_swbd->dir = DMA_TO_DEVICE;
1329 		xdp_tx_swbd->len = len;
1330 		xdp_tx_swbd->is_xdp_redirect = true;
1331 		xdp_tx_swbd->is_eof = false;
1332 		xdp_tx_swbd->xdp_frame = NULL;
1333 
1334 		n++;
1335 		xdp_tx_swbd = &xdp_tx_arr[n];
1336 	}
1337 
1338 	xdp_tx_arr[n - 1].is_eof = true;
1339 	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1340 
1341 	return n;
1342 }
1343 
1344 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1345 		   struct xdp_frame **frames, u32 flags)
1346 {
1347 	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1348 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1349 	struct enetc_bdr *tx_ring;
1350 	int xdp_tx_bd_cnt, i, k;
1351 	int xdp_tx_frm_cnt = 0;
1352 
1353 	enetc_lock_mdio();
1354 
1355 	tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1356 
1357 	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1358 
1359 	for (k = 0; k < num_frames; k++) {
1360 		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1361 							       xdp_redirect_arr,
1362 							       frames[k]);
1363 		if (unlikely(xdp_tx_bd_cnt < 0))
1364 			break;
1365 
1366 		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1367 					   xdp_tx_bd_cnt))) {
1368 			for (i = 0; i < xdp_tx_bd_cnt; i++)
1369 				enetc_unmap_tx_buff(tx_ring,
1370 						    &xdp_redirect_arr[i]);
1371 			tx_ring->stats.xdp_tx_drops++;
1372 			break;
1373 		}
1374 
1375 		xdp_tx_frm_cnt++;
1376 	}
1377 
1378 	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1379 		enetc_update_tx_ring_tail(tx_ring);
1380 
1381 	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1382 
1383 	enetc_unlock_mdio();
1384 
1385 	return xdp_tx_frm_cnt;
1386 }
1387 
1388 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1389 				     struct xdp_buff *xdp_buff, u16 size)
1390 {
1391 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1392 	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1393 	struct skb_shared_info *shinfo;
1394 
1395 	/* To be used for XDP_TX */
1396 	rx_swbd->len = size;
1397 
1398 	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1399 			 rx_ring->buffer_offset, size, false);
1400 
1401 	shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1402 	shinfo->nr_frags = 0;
1403 }
1404 
1405 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1406 				     u16 size, struct xdp_buff *xdp_buff)
1407 {
1408 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1409 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1410 	skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags];
1411 
1412 	/* To be used for XDP_TX */
1413 	rx_swbd->len = size;
1414 
1415 	skb_frag_off_set(frag, rx_swbd->page_offset);
1416 	skb_frag_size_set(frag, size);
1417 	__skb_frag_set_page(frag, rx_swbd->page);
1418 
1419 	shinfo->nr_frags++;
1420 }
1421 
1422 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1423 				 union enetc_rx_bd **rxbd, int *i,
1424 				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1425 {
1426 	u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1427 
1428 	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1429 
1430 	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1431 	(*cleaned_cnt)++;
1432 	enetc_rxbd_next(rx_ring, rxbd, i);
1433 
1434 	/* not last BD in frame? */
1435 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1436 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1437 		size = ENETC_RXB_DMA_SIZE_XDP;
1438 
1439 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1440 			dma_rmb();
1441 			size = le16_to_cpu((*rxbd)->r.buf_len);
1442 		}
1443 
1444 		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1445 		(*cleaned_cnt)++;
1446 		enetc_rxbd_next(rx_ring, rxbd, i);
1447 	}
1448 }
1449 
1450 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1451  * recycled back into the RX ring in enetc_clean_tx_ring.
1452  */
1453 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1454 					struct enetc_bdr *rx_ring,
1455 					int rx_ring_first, int rx_ring_last)
1456 {
1457 	int n = 0;
1458 
1459 	for (; rx_ring_first != rx_ring_last;
1460 	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1461 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1462 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1463 
1464 		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1465 		tx_swbd->dma = rx_swbd->dma;
1466 		tx_swbd->dir = rx_swbd->dir;
1467 		tx_swbd->page = rx_swbd->page;
1468 		tx_swbd->page_offset = rx_swbd->page_offset;
1469 		tx_swbd->len = rx_swbd->len;
1470 		tx_swbd->is_dma_page = true;
1471 		tx_swbd->is_xdp_tx = true;
1472 		tx_swbd->is_eof = false;
1473 	}
1474 
1475 	/* We rely on caller providing an rx_ring_last > rx_ring_first */
1476 	xdp_tx_arr[n - 1].is_eof = true;
1477 
1478 	return n;
1479 }
1480 
1481 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1482 			   int rx_ring_last)
1483 {
1484 	while (rx_ring_first != rx_ring_last) {
1485 		enetc_put_rx_buff(rx_ring,
1486 				  &rx_ring->rx_swbd[rx_ring_first]);
1487 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1488 	}
1489 	rx_ring->stats.xdp_drops++;
1490 }
1491 
1492 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1493 				   struct napi_struct *napi, int work_limit,
1494 				   struct bpf_prog *prog)
1495 {
1496 	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1497 	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1498 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1499 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1500 	struct enetc_bdr *tx_ring;
1501 	int cleaned_cnt, i;
1502 	u32 xdp_act;
1503 
1504 	cleaned_cnt = enetc_bd_unused(rx_ring);
1505 	/* next descriptor to process */
1506 	i = rx_ring->next_to_clean;
1507 
1508 	while (likely(rx_frm_cnt < work_limit)) {
1509 		union enetc_rx_bd *rxbd, *orig_rxbd;
1510 		int orig_i, orig_cleaned_cnt;
1511 		struct xdp_buff xdp_buff;
1512 		struct sk_buff *skb;
1513 		u32 bd_status;
1514 		int err;
1515 
1516 		rxbd = enetc_rxbd(rx_ring, i);
1517 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1518 		if (!bd_status)
1519 			break;
1520 
1521 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1522 		dma_rmb(); /* for reading other rxbd fields */
1523 
1524 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1525 						      &rxbd, &i))
1526 			break;
1527 
1528 		orig_rxbd = rxbd;
1529 		orig_cleaned_cnt = cleaned_cnt;
1530 		orig_i = i;
1531 
1532 		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1533 				     &cleaned_cnt, &xdp_buff);
1534 
1535 		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1536 
1537 		switch (xdp_act) {
1538 		default:
1539 			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1540 			fallthrough;
1541 		case XDP_ABORTED:
1542 			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1543 			fallthrough;
1544 		case XDP_DROP:
1545 			enetc_xdp_drop(rx_ring, orig_i, i);
1546 			break;
1547 		case XDP_PASS:
1548 			rxbd = orig_rxbd;
1549 			cleaned_cnt = orig_cleaned_cnt;
1550 			i = orig_i;
1551 
1552 			skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1553 					      &i, &cleaned_cnt,
1554 					      ENETC_RXB_DMA_SIZE_XDP);
1555 			if (unlikely(!skb))
1556 				goto out;
1557 
1558 			napi_gro_receive(napi, skb);
1559 			break;
1560 		case XDP_TX:
1561 			tx_ring = priv->xdp_tx_ring[rx_ring->index];
1562 			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1563 								     rx_ring,
1564 								     orig_i, i);
1565 
1566 			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1567 				enetc_xdp_drop(rx_ring, orig_i, i);
1568 				tx_ring->stats.xdp_tx_drops++;
1569 			} else {
1570 				tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1571 				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1572 				xdp_tx_frm_cnt++;
1573 				/* The XDP_TX enqueue was successful, so we
1574 				 * need to scrub the RX software BDs because
1575 				 * the ownership of the buffers no longer
1576 				 * belongs to the RX ring, and we must prevent
1577 				 * enetc_refill_rx_ring() from reusing
1578 				 * rx_swbd->page.
1579 				 */
1580 				while (orig_i != i) {
1581 					rx_ring->rx_swbd[orig_i].page = NULL;
1582 					enetc_bdr_idx_inc(rx_ring, &orig_i);
1583 				}
1584 			}
1585 			break;
1586 		case XDP_REDIRECT:
1587 			/* xdp_return_frame does not support S/G in the sense
1588 			 * that it leaks the fragments (__xdp_return should not
1589 			 * call page_frag_free only for the initial buffer).
1590 			 * Until XDP_REDIRECT gains support for S/G let's keep
1591 			 * the code structure in place, but dead. We drop the
1592 			 * S/G frames ourselves to avoid memory leaks which
1593 			 * would otherwise leave the kernel OOM.
1594 			 */
1595 			if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) {
1596 				enetc_xdp_drop(rx_ring, orig_i, i);
1597 				rx_ring->stats.xdp_redirect_sg++;
1598 				break;
1599 			}
1600 
1601 			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1602 			if (unlikely(err)) {
1603 				enetc_xdp_drop(rx_ring, orig_i, i);
1604 				rx_ring->stats.xdp_redirect_failures++;
1605 			} else {
1606 				while (orig_i != i) {
1607 					enetc_flip_rx_buff(rx_ring,
1608 							   &rx_ring->rx_swbd[orig_i]);
1609 					enetc_bdr_idx_inc(rx_ring, &orig_i);
1610 				}
1611 				xdp_redirect_frm_cnt++;
1612 				rx_ring->stats.xdp_redirect++;
1613 			}
1614 		}
1615 
1616 		rx_frm_cnt++;
1617 	}
1618 
1619 out:
1620 	rx_ring->next_to_clean = i;
1621 
1622 	rx_ring->stats.packets += rx_frm_cnt;
1623 	rx_ring->stats.bytes += rx_byte_cnt;
1624 
1625 	if (xdp_redirect_frm_cnt)
1626 		xdp_do_flush_map();
1627 
1628 	if (xdp_tx_frm_cnt)
1629 		enetc_update_tx_ring_tail(tx_ring);
1630 
1631 	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1632 		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1633 				     rx_ring->xdp.xdp_tx_in_flight);
1634 
1635 	return rx_frm_cnt;
1636 }
1637 
1638 static int enetc_poll(struct napi_struct *napi, int budget)
1639 {
1640 	struct enetc_int_vector
1641 		*v = container_of(napi, struct enetc_int_vector, napi);
1642 	struct enetc_bdr *rx_ring = &v->rx_ring;
1643 	struct bpf_prog *prog;
1644 	bool complete = true;
1645 	int work_done;
1646 	int i;
1647 
1648 	enetc_lock_mdio();
1649 
1650 	for (i = 0; i < v->count_tx_rings; i++)
1651 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1652 			complete = false;
1653 
1654 	prog = rx_ring->xdp.prog;
1655 	if (prog)
1656 		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1657 	else
1658 		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1659 	if (work_done == budget)
1660 		complete = false;
1661 	if (work_done)
1662 		v->rx_napi_work = true;
1663 
1664 	if (!complete) {
1665 		enetc_unlock_mdio();
1666 		return budget;
1667 	}
1668 
1669 	napi_complete_done(napi, work_done);
1670 
1671 	if (likely(v->rx_dim_en))
1672 		enetc_rx_net_dim(v);
1673 
1674 	v->rx_napi_work = false;
1675 
1676 	/* enable interrupts */
1677 	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1678 
1679 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1680 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1681 				 ENETC_TBIER_TXTIE);
1682 
1683 	enetc_unlock_mdio();
1684 
1685 	return work_done;
1686 }
1687 
1688 /* Probing and Init */
1689 #define ENETC_MAX_RFS_SIZE 64
1690 void enetc_get_si_caps(struct enetc_si *si)
1691 {
1692 	struct enetc_hw *hw = &si->hw;
1693 	u32 val;
1694 
1695 	/* find out how many of various resources we have to work with */
1696 	val = enetc_rd(hw, ENETC_SICAPR0);
1697 	si->num_rx_rings = (val >> 16) & 0xff;
1698 	si->num_tx_rings = val & 0xff;
1699 
1700 	val = enetc_rd(hw, ENETC_SIRFSCAPR);
1701 	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1702 	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1703 
1704 	si->num_rss = 0;
1705 	val = enetc_rd(hw, ENETC_SIPCAPR0);
1706 	if (val & ENETC_SIPCAPR0_RSS) {
1707 		u32 rss;
1708 
1709 		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1710 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1711 	}
1712 
1713 	if (val & ENETC_SIPCAPR0_QBV)
1714 		si->hw_features |= ENETC_SI_F_QBV;
1715 
1716 	if (val & ENETC_SIPCAPR0_PSFP)
1717 		si->hw_features |= ENETC_SI_F_PSFP;
1718 }
1719 
1720 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
1721 {
1722 	r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
1723 					&r->bd_dma_base, GFP_KERNEL);
1724 	if (!r->bd_base)
1725 		return -ENOMEM;
1726 
1727 	/* h/w requires 128B alignment */
1728 	if (!IS_ALIGNED(r->bd_dma_base, 128)) {
1729 		dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
1730 				  r->bd_dma_base);
1731 		return -EINVAL;
1732 	}
1733 
1734 	return 0;
1735 }
1736 
1737 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
1738 {
1739 	int err;
1740 
1741 	txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
1742 	if (!txr->tx_swbd)
1743 		return -ENOMEM;
1744 
1745 	err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
1746 	if (err)
1747 		goto err_alloc_bdr;
1748 
1749 	txr->tso_headers = dma_alloc_coherent(txr->dev,
1750 					      txr->bd_count * TSO_HEADER_SIZE,
1751 					      &txr->tso_headers_dma,
1752 					      GFP_KERNEL);
1753 	if (!txr->tso_headers) {
1754 		err = -ENOMEM;
1755 		goto err_alloc_tso;
1756 	}
1757 
1758 	txr->next_to_clean = 0;
1759 	txr->next_to_use = 0;
1760 
1761 	return 0;
1762 
1763 err_alloc_tso:
1764 	dma_free_coherent(txr->dev, txr->bd_count * sizeof(union enetc_tx_bd),
1765 			  txr->bd_base, txr->bd_dma_base);
1766 	txr->bd_base = NULL;
1767 err_alloc_bdr:
1768 	vfree(txr->tx_swbd);
1769 	txr->tx_swbd = NULL;
1770 
1771 	return err;
1772 }
1773 
1774 static void enetc_free_txbdr(struct enetc_bdr *txr)
1775 {
1776 	int size, i;
1777 
1778 	for (i = 0; i < txr->bd_count; i++)
1779 		enetc_free_tx_frame(txr, &txr->tx_swbd[i]);
1780 
1781 	size = txr->bd_count * sizeof(union enetc_tx_bd);
1782 
1783 	dma_free_coherent(txr->dev, txr->bd_count * TSO_HEADER_SIZE,
1784 			  txr->tso_headers, txr->tso_headers_dma);
1785 	txr->tso_headers = NULL;
1786 
1787 	dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
1788 	txr->bd_base = NULL;
1789 
1790 	vfree(txr->tx_swbd);
1791 	txr->tx_swbd = NULL;
1792 }
1793 
1794 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1795 {
1796 	int i, err;
1797 
1798 	for (i = 0; i < priv->num_tx_rings; i++) {
1799 		err = enetc_alloc_txbdr(priv->tx_ring[i]);
1800 
1801 		if (err)
1802 			goto fail;
1803 	}
1804 
1805 	return 0;
1806 
1807 fail:
1808 	while (i-- > 0)
1809 		enetc_free_txbdr(priv->tx_ring[i]);
1810 
1811 	return err;
1812 }
1813 
1814 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
1815 {
1816 	int i;
1817 
1818 	for (i = 0; i < priv->num_tx_rings; i++)
1819 		enetc_free_txbdr(priv->tx_ring[i]);
1820 }
1821 
1822 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
1823 {
1824 	size_t size = sizeof(union enetc_rx_bd);
1825 	int err;
1826 
1827 	rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
1828 	if (!rxr->rx_swbd)
1829 		return -ENOMEM;
1830 
1831 	if (extended)
1832 		size *= 2;
1833 
1834 	err = enetc_dma_alloc_bdr(rxr, size);
1835 	if (err) {
1836 		vfree(rxr->rx_swbd);
1837 		return err;
1838 	}
1839 
1840 	rxr->next_to_clean = 0;
1841 	rxr->next_to_use = 0;
1842 	rxr->next_to_alloc = 0;
1843 	rxr->ext_en = extended;
1844 
1845 	return 0;
1846 }
1847 
1848 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
1849 {
1850 	int size;
1851 
1852 	size = rxr->bd_count * sizeof(union enetc_rx_bd);
1853 
1854 	dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
1855 	rxr->bd_base = NULL;
1856 
1857 	vfree(rxr->rx_swbd);
1858 	rxr->rx_swbd = NULL;
1859 }
1860 
1861 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
1862 {
1863 	bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
1864 	int i, err;
1865 
1866 	for (i = 0; i < priv->num_rx_rings; i++) {
1867 		err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
1868 
1869 		if (err)
1870 			goto fail;
1871 	}
1872 
1873 	return 0;
1874 
1875 fail:
1876 	while (i-- > 0)
1877 		enetc_free_rxbdr(priv->rx_ring[i]);
1878 
1879 	return err;
1880 }
1881 
1882 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
1883 {
1884 	int i;
1885 
1886 	for (i = 0; i < priv->num_rx_rings; i++)
1887 		enetc_free_rxbdr(priv->rx_ring[i]);
1888 }
1889 
1890 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1891 {
1892 	int i;
1893 
1894 	if (!tx_ring->tx_swbd)
1895 		return;
1896 
1897 	for (i = 0; i < tx_ring->bd_count; i++) {
1898 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1899 
1900 		enetc_free_tx_frame(tx_ring, tx_swbd);
1901 	}
1902 
1903 	tx_ring->next_to_clean = 0;
1904 	tx_ring->next_to_use = 0;
1905 }
1906 
1907 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1908 {
1909 	int i;
1910 
1911 	if (!rx_ring->rx_swbd)
1912 		return;
1913 
1914 	for (i = 0; i < rx_ring->bd_count; i++) {
1915 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1916 
1917 		if (!rx_swbd->page)
1918 			continue;
1919 
1920 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1921 			       rx_swbd->dir);
1922 		__free_page(rx_swbd->page);
1923 		rx_swbd->page = NULL;
1924 	}
1925 
1926 	rx_ring->next_to_clean = 0;
1927 	rx_ring->next_to_use = 0;
1928 	rx_ring->next_to_alloc = 0;
1929 }
1930 
1931 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1932 {
1933 	int i;
1934 
1935 	for (i = 0; i < priv->num_rx_rings; i++)
1936 		enetc_free_rx_ring(priv->rx_ring[i]);
1937 
1938 	for (i = 0; i < priv->num_tx_rings; i++)
1939 		enetc_free_tx_ring(priv->tx_ring[i]);
1940 }
1941 
1942 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1943 {
1944 	int *rss_table;
1945 	int i;
1946 
1947 	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1948 	if (!rss_table)
1949 		return -ENOMEM;
1950 
1951 	/* Set up RSS table defaults */
1952 	for (i = 0; i < si->num_rss; i++)
1953 		rss_table[i] = i % num_groups;
1954 
1955 	enetc_set_rss_table(si, rss_table, si->num_rss);
1956 
1957 	kfree(rss_table);
1958 
1959 	return 0;
1960 }
1961 
1962 int enetc_configure_si(struct enetc_ndev_priv *priv)
1963 {
1964 	struct enetc_si *si = priv->si;
1965 	struct enetc_hw *hw = &si->hw;
1966 	int err;
1967 
1968 	/* set SI cache attributes */
1969 	enetc_wr(hw, ENETC_SICAR0,
1970 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1971 	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1972 	/* enable SI */
1973 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1974 
1975 	if (si->num_rss) {
1976 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1977 		if (err)
1978 			return err;
1979 	}
1980 
1981 	return 0;
1982 }
1983 
1984 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1985 {
1986 	struct enetc_si *si = priv->si;
1987 	int cpus = num_online_cpus();
1988 
1989 	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1990 	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1991 
1992 	/* Enable all available TX rings in order to configure as many
1993 	 * priorities as possible, when needed.
1994 	 * TODO: Make # of TX rings run-time configurable
1995 	 */
1996 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1997 	priv->num_tx_rings = si->num_tx_rings;
1998 	priv->bdr_int_num = cpus;
1999 	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2000 	priv->tx_ictt = ENETC_TXIC_TIMETHR;
2001 }
2002 
2003 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2004 {
2005 	struct enetc_si *si = priv->si;
2006 
2007 	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2008 				  GFP_KERNEL);
2009 	if (!priv->cls_rules)
2010 		return -ENOMEM;
2011 
2012 	return 0;
2013 }
2014 
2015 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2016 {
2017 	kfree(priv->cls_rules);
2018 }
2019 
2020 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2021 {
2022 	int idx = tx_ring->index;
2023 	u32 tbmr;
2024 
2025 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2026 		       lower_32_bits(tx_ring->bd_dma_base));
2027 
2028 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2029 		       upper_32_bits(tx_ring->bd_dma_base));
2030 
2031 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2032 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2033 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
2034 
2035 	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2036 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2037 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2038 
2039 	/* enable Tx ints by setting pkt thr to 1 */
2040 	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2041 
2042 	tbmr = ENETC_TBMR_EN | ENETC_TBMR_SET_PRIO(tx_ring->prio);
2043 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2044 		tbmr |= ENETC_TBMR_VIH;
2045 
2046 	/* enable ring */
2047 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2048 
2049 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2050 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2051 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
2052 }
2053 
2054 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2055 {
2056 	int idx = rx_ring->index;
2057 	u32 rbmr;
2058 
2059 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2060 		       lower_32_bits(rx_ring->bd_dma_base));
2061 
2062 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2063 		       upper_32_bits(rx_ring->bd_dma_base));
2064 
2065 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2066 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2067 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
2068 
2069 	if (rx_ring->xdp.prog)
2070 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2071 	else
2072 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2073 
2074 	/* Also prepare the consumer index in case page allocation never
2075 	 * succeeds. In that case, hardware will never advance producer index
2076 	 * to match consumer index, and will drop all frames.
2077 	 */
2078 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2079 	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2080 
2081 	/* enable Rx ints by setting pkt thr to 1 */
2082 	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2083 
2084 	rbmr = ENETC_RBMR_EN;
2085 
2086 	if (rx_ring->ext_en)
2087 		rbmr |= ENETC_RBMR_BDS;
2088 
2089 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2090 		rbmr |= ENETC_RBMR_VTE;
2091 
2092 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2093 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2094 
2095 	enetc_lock_mdio();
2096 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2097 	enetc_unlock_mdio();
2098 
2099 	/* enable ring */
2100 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2101 }
2102 
2103 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
2104 {
2105 	struct enetc_hw *hw = &priv->si->hw;
2106 	int i;
2107 
2108 	for (i = 0; i < priv->num_tx_rings; i++)
2109 		enetc_setup_txbdr(hw, priv->tx_ring[i]);
2110 
2111 	for (i = 0; i < priv->num_rx_rings; i++)
2112 		enetc_setup_rxbdr(hw, priv->rx_ring[i]);
2113 }
2114 
2115 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2116 {
2117 	int idx = rx_ring->index;
2118 
2119 	/* disable EN bit on ring */
2120 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2121 }
2122 
2123 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2124 {
2125 	int delay = 8, timeout = 100;
2126 	int idx = tx_ring->index;
2127 
2128 	/* disable EN bit on ring */
2129 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2130 
2131 	/* wait for busy to clear */
2132 	while (delay < timeout &&
2133 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2134 		msleep(delay);
2135 		delay *= 2;
2136 	}
2137 
2138 	if (delay >= timeout)
2139 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2140 			    idx);
2141 }
2142 
2143 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
2144 {
2145 	struct enetc_hw *hw = &priv->si->hw;
2146 	int i;
2147 
2148 	for (i = 0; i < priv->num_tx_rings; i++)
2149 		enetc_clear_txbdr(hw, priv->tx_ring[i]);
2150 
2151 	for (i = 0; i < priv->num_rx_rings; i++)
2152 		enetc_clear_rxbdr(hw, priv->rx_ring[i]);
2153 
2154 	udelay(1);
2155 }
2156 
2157 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2158 {
2159 	struct pci_dev *pdev = priv->si->pdev;
2160 	struct enetc_hw *hw = &priv->si->hw;
2161 	int i, j, err;
2162 
2163 	for (i = 0; i < priv->bdr_int_num; i++) {
2164 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2165 		struct enetc_int_vector *v = priv->int_vector[i];
2166 		int entry = ENETC_BDR_INT_BASE_IDX + i;
2167 
2168 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2169 			 priv->ndev->name, i);
2170 		err = request_irq(irq, enetc_msix, 0, v->name, v);
2171 		if (err) {
2172 			dev_err(priv->dev, "request_irq() failed!\n");
2173 			goto irq_err;
2174 		}
2175 		disable_irq(irq);
2176 
2177 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2178 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2179 		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2180 
2181 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2182 
2183 		for (j = 0; j < v->count_tx_rings; j++) {
2184 			int idx = v->tx_ring[j].index;
2185 
2186 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2187 		}
2188 		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2189 	}
2190 
2191 	return 0;
2192 
2193 irq_err:
2194 	while (i--) {
2195 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2196 
2197 		irq_set_affinity_hint(irq, NULL);
2198 		free_irq(irq, priv->int_vector[i]);
2199 	}
2200 
2201 	return err;
2202 }
2203 
2204 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2205 {
2206 	struct pci_dev *pdev = priv->si->pdev;
2207 	int i;
2208 
2209 	for (i = 0; i < priv->bdr_int_num; i++) {
2210 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2211 
2212 		irq_set_affinity_hint(irq, NULL);
2213 		free_irq(irq, priv->int_vector[i]);
2214 	}
2215 }
2216 
2217 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2218 {
2219 	struct enetc_hw *hw = &priv->si->hw;
2220 	u32 icpt, ictt;
2221 	int i;
2222 
2223 	/* enable Tx & Rx event indication */
2224 	if (priv->ic_mode &
2225 	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2226 		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2227 		/* init to non-0 minimum, will be adjusted later */
2228 		ictt = 0x1;
2229 	} else {
2230 		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2231 		ictt = 0;
2232 	}
2233 
2234 	for (i = 0; i < priv->num_rx_rings; i++) {
2235 		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2236 		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2237 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2238 	}
2239 
2240 	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2241 		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2242 	else
2243 		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2244 
2245 	for (i = 0; i < priv->num_tx_rings; i++) {
2246 		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2247 		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2248 		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2249 	}
2250 }
2251 
2252 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2253 {
2254 	struct enetc_hw *hw = &priv->si->hw;
2255 	int i;
2256 
2257 	for (i = 0; i < priv->num_tx_rings; i++)
2258 		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2259 
2260 	for (i = 0; i < priv->num_rx_rings; i++)
2261 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2262 }
2263 
2264 static int enetc_phylink_connect(struct net_device *ndev)
2265 {
2266 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2267 	struct ethtool_eee edata;
2268 	int err;
2269 
2270 	if (!priv->phylink)
2271 		return 0; /* phy-less mode */
2272 
2273 	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2274 	if (err) {
2275 		dev_err(&ndev->dev, "could not attach to PHY\n");
2276 		return err;
2277 	}
2278 
2279 	/* disable EEE autoneg, until ENETC driver supports it */
2280 	memset(&edata, 0, sizeof(struct ethtool_eee));
2281 	phylink_ethtool_set_eee(priv->phylink, &edata);
2282 
2283 	return 0;
2284 }
2285 
2286 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2287 {
2288 	struct enetc_ndev_priv *priv;
2289 	struct sk_buff *skb;
2290 
2291 	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2292 
2293 	netif_tx_lock(priv->ndev);
2294 
2295 	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2296 	skb = skb_dequeue(&priv->tx_skbs);
2297 	if (skb)
2298 		enetc_start_xmit(skb, priv->ndev);
2299 
2300 	netif_tx_unlock(priv->ndev);
2301 }
2302 
2303 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2304 {
2305 	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2306 	skb_queue_head_init(&priv->tx_skbs);
2307 }
2308 
2309 void enetc_start(struct net_device *ndev)
2310 {
2311 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2312 	int i;
2313 
2314 	enetc_setup_interrupts(priv);
2315 
2316 	for (i = 0; i < priv->bdr_int_num; i++) {
2317 		int irq = pci_irq_vector(priv->si->pdev,
2318 					 ENETC_BDR_INT_BASE_IDX + i);
2319 
2320 		napi_enable(&priv->int_vector[i]->napi);
2321 		enable_irq(irq);
2322 	}
2323 
2324 	if (priv->phylink)
2325 		phylink_start(priv->phylink);
2326 	else
2327 		netif_carrier_on(ndev);
2328 
2329 	netif_tx_start_all_queues(ndev);
2330 }
2331 
2332 int enetc_open(struct net_device *ndev)
2333 {
2334 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2335 	int num_stack_tx_queues;
2336 	int err;
2337 
2338 	err = enetc_setup_irqs(priv);
2339 	if (err)
2340 		return err;
2341 
2342 	err = enetc_phylink_connect(ndev);
2343 	if (err)
2344 		goto err_phy_connect;
2345 
2346 	err = enetc_alloc_tx_resources(priv);
2347 	if (err)
2348 		goto err_alloc_tx;
2349 
2350 	err = enetc_alloc_rx_resources(priv);
2351 	if (err)
2352 		goto err_alloc_rx;
2353 
2354 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2355 
2356 	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2357 	if (err)
2358 		goto err_set_queues;
2359 
2360 	err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
2361 	if (err)
2362 		goto err_set_queues;
2363 
2364 	enetc_tx_onestep_tstamp_init(priv);
2365 	enetc_setup_bdrs(priv);
2366 	enetc_start(ndev);
2367 
2368 	return 0;
2369 
2370 err_set_queues:
2371 	enetc_free_rx_resources(priv);
2372 err_alloc_rx:
2373 	enetc_free_tx_resources(priv);
2374 err_alloc_tx:
2375 	if (priv->phylink)
2376 		phylink_disconnect_phy(priv->phylink);
2377 err_phy_connect:
2378 	enetc_free_irqs(priv);
2379 
2380 	return err;
2381 }
2382 
2383 void enetc_stop(struct net_device *ndev)
2384 {
2385 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2386 	int i;
2387 
2388 	netif_tx_stop_all_queues(ndev);
2389 
2390 	for (i = 0; i < priv->bdr_int_num; i++) {
2391 		int irq = pci_irq_vector(priv->si->pdev,
2392 					 ENETC_BDR_INT_BASE_IDX + i);
2393 
2394 		disable_irq(irq);
2395 		napi_synchronize(&priv->int_vector[i]->napi);
2396 		napi_disable(&priv->int_vector[i]->napi);
2397 	}
2398 
2399 	if (priv->phylink)
2400 		phylink_stop(priv->phylink);
2401 	else
2402 		netif_carrier_off(ndev);
2403 
2404 	enetc_clear_interrupts(priv);
2405 }
2406 
2407 int enetc_close(struct net_device *ndev)
2408 {
2409 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2410 
2411 	enetc_stop(ndev);
2412 	enetc_clear_bdrs(priv);
2413 
2414 	if (priv->phylink)
2415 		phylink_disconnect_phy(priv->phylink);
2416 	enetc_free_rxtx_rings(priv);
2417 	enetc_free_rx_resources(priv);
2418 	enetc_free_tx_resources(priv);
2419 	enetc_free_irqs(priv);
2420 
2421 	return 0;
2422 }
2423 
2424 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2425 {
2426 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2427 	struct tc_mqprio_qopt *mqprio = type_data;
2428 	struct enetc_hw *hw = &priv->si->hw;
2429 	struct enetc_bdr *tx_ring;
2430 	int num_stack_tx_queues;
2431 	u8 num_tc;
2432 	int i;
2433 
2434 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2435 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
2436 	num_tc = mqprio->num_tc;
2437 
2438 	if (!num_tc) {
2439 		netdev_reset_tc(ndev);
2440 		netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2441 
2442 		/* Reset all ring priorities to 0 */
2443 		for (i = 0; i < priv->num_tx_rings; i++) {
2444 			tx_ring = priv->tx_ring[i];
2445 			tx_ring->prio = 0;
2446 			enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2447 		}
2448 
2449 		return 0;
2450 	}
2451 
2452 	/* Check if we have enough BD rings available to accommodate all TCs */
2453 	if (num_tc > num_stack_tx_queues) {
2454 		netdev_err(ndev, "Max %d traffic classes supported\n",
2455 			   priv->num_tx_rings);
2456 		return -EINVAL;
2457 	}
2458 
2459 	/* For the moment, we use only one BD ring per TC.
2460 	 *
2461 	 * Configure num_tc BD rings with increasing priorities.
2462 	 */
2463 	for (i = 0; i < num_tc; i++) {
2464 		tx_ring = priv->tx_ring[i];
2465 		tx_ring->prio = i;
2466 		enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2467 	}
2468 
2469 	/* Reset the number of netdev queues based on the TC count */
2470 	netif_set_real_num_tx_queues(ndev, num_tc);
2471 
2472 	netdev_set_num_tc(ndev, num_tc);
2473 
2474 	/* Each TC is associated with one netdev queue */
2475 	for (i = 0; i < num_tc; i++)
2476 		netdev_set_tc_queue(ndev, i, 1, i);
2477 
2478 	return 0;
2479 }
2480 
2481 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog,
2482 				struct netlink_ext_ack *extack)
2483 {
2484 	struct enetc_ndev_priv *priv = netdev_priv(dev);
2485 	struct bpf_prog *old_prog;
2486 	bool is_up;
2487 	int i;
2488 
2489 	/* The buffer layout is changing, so we need to drain the old
2490 	 * RX buffers and seed new ones.
2491 	 */
2492 	is_up = netif_running(dev);
2493 	if (is_up)
2494 		dev_close(dev);
2495 
2496 	old_prog = xchg(&priv->xdp_prog, prog);
2497 	if (old_prog)
2498 		bpf_prog_put(old_prog);
2499 
2500 	for (i = 0; i < priv->num_rx_rings; i++) {
2501 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
2502 
2503 		rx_ring->xdp.prog = prog;
2504 
2505 		if (prog)
2506 			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2507 		else
2508 			rx_ring->buffer_offset = ENETC_RXB_PAD;
2509 	}
2510 
2511 	if (is_up)
2512 		return dev_open(dev, extack);
2513 
2514 	return 0;
2515 }
2516 
2517 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp)
2518 {
2519 	switch (xdp->command) {
2520 	case XDP_SETUP_PROG:
2521 		return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack);
2522 	default:
2523 		return -EINVAL;
2524 	}
2525 
2526 	return 0;
2527 }
2528 
2529 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2530 {
2531 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2532 	struct net_device_stats *stats = &ndev->stats;
2533 	unsigned long packets = 0, bytes = 0;
2534 	unsigned long tx_dropped = 0;
2535 	int i;
2536 
2537 	for (i = 0; i < priv->num_rx_rings; i++) {
2538 		packets += priv->rx_ring[i]->stats.packets;
2539 		bytes	+= priv->rx_ring[i]->stats.bytes;
2540 	}
2541 
2542 	stats->rx_packets = packets;
2543 	stats->rx_bytes = bytes;
2544 	bytes = 0;
2545 	packets = 0;
2546 
2547 	for (i = 0; i < priv->num_tx_rings; i++) {
2548 		packets += priv->tx_ring[i]->stats.packets;
2549 		bytes	+= priv->tx_ring[i]->stats.bytes;
2550 		tx_dropped += priv->tx_ring[i]->stats.win_drop;
2551 	}
2552 
2553 	stats->tx_packets = packets;
2554 	stats->tx_bytes = bytes;
2555 	stats->tx_dropped = tx_dropped;
2556 
2557 	return stats;
2558 }
2559 
2560 static int enetc_set_rss(struct net_device *ndev, int en)
2561 {
2562 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2563 	struct enetc_hw *hw = &priv->si->hw;
2564 	u32 reg;
2565 
2566 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2567 
2568 	reg = enetc_rd(hw, ENETC_SIMR);
2569 	reg &= ~ENETC_SIMR_RSSE;
2570 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
2571 	enetc_wr(hw, ENETC_SIMR, reg);
2572 
2573 	return 0;
2574 }
2575 
2576 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2577 {
2578 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2579 	struct enetc_hw *hw = &priv->si->hw;
2580 	int i;
2581 
2582 	for (i = 0; i < priv->num_rx_rings; i++)
2583 		enetc_bdr_enable_rxvlan(hw, i, en);
2584 }
2585 
2586 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2587 {
2588 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2589 	struct enetc_hw *hw = &priv->si->hw;
2590 	int i;
2591 
2592 	for (i = 0; i < priv->num_tx_rings; i++)
2593 		enetc_bdr_enable_txvlan(hw, i, en);
2594 }
2595 
2596 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2597 {
2598 	netdev_features_t changed = ndev->features ^ features;
2599 
2600 	if (changed & NETIF_F_RXHASH)
2601 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2602 
2603 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2604 		enetc_enable_rxvlan(ndev,
2605 				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2606 
2607 	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2608 		enetc_enable_txvlan(ndev,
2609 				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2610 }
2611 
2612 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2613 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2614 {
2615 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2616 	struct hwtstamp_config config;
2617 	int ao;
2618 
2619 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2620 		return -EFAULT;
2621 
2622 	switch (config.tx_type) {
2623 	case HWTSTAMP_TX_OFF:
2624 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2625 		break;
2626 	case HWTSTAMP_TX_ON:
2627 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2628 		priv->active_offloads |= ENETC_F_TX_TSTAMP;
2629 		break;
2630 	case HWTSTAMP_TX_ONESTEP_SYNC:
2631 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2632 		priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2633 		break;
2634 	default:
2635 		return -ERANGE;
2636 	}
2637 
2638 	ao = priv->active_offloads;
2639 	switch (config.rx_filter) {
2640 	case HWTSTAMP_FILTER_NONE:
2641 		priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
2642 		break;
2643 	default:
2644 		priv->active_offloads |= ENETC_F_RX_TSTAMP;
2645 		config.rx_filter = HWTSTAMP_FILTER_ALL;
2646 	}
2647 
2648 	if (netif_running(ndev) && ao != priv->active_offloads) {
2649 		enetc_close(ndev);
2650 		enetc_open(ndev);
2651 	}
2652 
2653 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2654 	       -EFAULT : 0;
2655 }
2656 
2657 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2658 {
2659 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2660 	struct hwtstamp_config config;
2661 
2662 	config.flags = 0;
2663 
2664 	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2665 		config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2666 	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2667 		config.tx_type = HWTSTAMP_TX_ON;
2668 	else
2669 		config.tx_type = HWTSTAMP_TX_OFF;
2670 
2671 	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2672 			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2673 
2674 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2675 	       -EFAULT : 0;
2676 }
2677 #endif
2678 
2679 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2680 {
2681 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2682 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2683 	if (cmd == SIOCSHWTSTAMP)
2684 		return enetc_hwtstamp_set(ndev, rq);
2685 	if (cmd == SIOCGHWTSTAMP)
2686 		return enetc_hwtstamp_get(ndev, rq);
2687 #endif
2688 
2689 	if (!priv->phylink)
2690 		return -EOPNOTSUPP;
2691 
2692 	return phylink_mii_ioctl(priv->phylink, rq, cmd);
2693 }
2694 
2695 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2696 {
2697 	struct pci_dev *pdev = priv->si->pdev;
2698 	int first_xdp_tx_ring;
2699 	int i, n, err, nvec;
2700 	int v_tx_rings;
2701 
2702 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2703 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
2704 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2705 
2706 	if (n < 0)
2707 		return n;
2708 
2709 	if (n != nvec)
2710 		return -EPERM;
2711 
2712 	/* # of tx rings per int vector */
2713 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2714 
2715 	for (i = 0; i < priv->bdr_int_num; i++) {
2716 		struct enetc_int_vector *v;
2717 		struct enetc_bdr *bdr;
2718 		int j;
2719 
2720 		v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
2721 		if (!v) {
2722 			err = -ENOMEM;
2723 			goto fail;
2724 		}
2725 
2726 		priv->int_vector[i] = v;
2727 
2728 		bdr = &v->rx_ring;
2729 		bdr->index = i;
2730 		bdr->ndev = priv->ndev;
2731 		bdr->dev = priv->dev;
2732 		bdr->bd_count = priv->rx_bd_count;
2733 		bdr->buffer_offset = ENETC_RXB_PAD;
2734 		priv->rx_ring[i] = bdr;
2735 
2736 		err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
2737 		if (err) {
2738 			kfree(v);
2739 			goto fail;
2740 		}
2741 
2742 		err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
2743 						 MEM_TYPE_PAGE_SHARED, NULL);
2744 		if (err) {
2745 			xdp_rxq_info_unreg(&bdr->xdp.rxq);
2746 			kfree(v);
2747 			goto fail;
2748 		}
2749 
2750 		/* init defaults for adaptive IC */
2751 		if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
2752 			v->rx_ictt = 0x1;
2753 			v->rx_dim_en = true;
2754 		}
2755 		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
2756 		netif_napi_add(priv->ndev, &v->napi, enetc_poll);
2757 		v->count_tx_rings = v_tx_rings;
2758 
2759 		for (j = 0; j < v_tx_rings; j++) {
2760 			int idx;
2761 
2762 			/* default tx ring mapping policy */
2763 			idx = priv->bdr_int_num * j + i;
2764 			__set_bit(idx, &v->tx_rings_map);
2765 			bdr = &v->tx_ring[j];
2766 			bdr->index = idx;
2767 			bdr->ndev = priv->ndev;
2768 			bdr->dev = priv->dev;
2769 			bdr->bd_count = priv->tx_bd_count;
2770 			priv->tx_ring[idx] = bdr;
2771 		}
2772 	}
2773 
2774 	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
2775 	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
2776 
2777 	return 0;
2778 
2779 fail:
2780 	while (i--) {
2781 		struct enetc_int_vector *v = priv->int_vector[i];
2782 		struct enetc_bdr *rx_ring = &v->rx_ring;
2783 
2784 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2785 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2786 		netif_napi_del(&v->napi);
2787 		cancel_work_sync(&v->rx_dim.work);
2788 		kfree(v);
2789 	}
2790 
2791 	pci_free_irq_vectors(pdev);
2792 
2793 	return err;
2794 }
2795 
2796 void enetc_free_msix(struct enetc_ndev_priv *priv)
2797 {
2798 	int i;
2799 
2800 	for (i = 0; i < priv->bdr_int_num; i++) {
2801 		struct enetc_int_vector *v = priv->int_vector[i];
2802 		struct enetc_bdr *rx_ring = &v->rx_ring;
2803 
2804 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2805 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2806 		netif_napi_del(&v->napi);
2807 		cancel_work_sync(&v->rx_dim.work);
2808 	}
2809 
2810 	for (i = 0; i < priv->num_rx_rings; i++)
2811 		priv->rx_ring[i] = NULL;
2812 
2813 	for (i = 0; i < priv->num_tx_rings; i++)
2814 		priv->tx_ring[i] = NULL;
2815 
2816 	for (i = 0; i < priv->bdr_int_num; i++) {
2817 		kfree(priv->int_vector[i]);
2818 		priv->int_vector[i] = NULL;
2819 	}
2820 
2821 	/* disable all MSIX for this device */
2822 	pci_free_irq_vectors(priv->si->pdev);
2823 }
2824 
2825 static void enetc_kfree_si(struct enetc_si *si)
2826 {
2827 	char *p = (char *)si - si->pad;
2828 
2829 	kfree(p);
2830 }
2831 
2832 static void enetc_detect_errata(struct enetc_si *si)
2833 {
2834 	if (si->pdev->revision == ENETC_REV1)
2835 		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
2836 }
2837 
2838 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
2839 {
2840 	struct enetc_si *si, *p;
2841 	struct enetc_hw *hw;
2842 	size_t alloc_size;
2843 	int err, len;
2844 
2845 	pcie_flr(pdev);
2846 	err = pci_enable_device_mem(pdev);
2847 	if (err)
2848 		return dev_err_probe(&pdev->dev, err, "device enable failed\n");
2849 
2850 	/* set up for high or low dma */
2851 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2852 	if (err) {
2853 		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
2854 		goto err_dma;
2855 	}
2856 
2857 	err = pci_request_mem_regions(pdev, name);
2858 	if (err) {
2859 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
2860 		goto err_pci_mem_reg;
2861 	}
2862 
2863 	pci_set_master(pdev);
2864 
2865 	alloc_size = sizeof(struct enetc_si);
2866 	if (sizeof_priv) {
2867 		/* align priv to 32B */
2868 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
2869 		alloc_size += sizeof_priv;
2870 	}
2871 	/* force 32B alignment for enetc_si */
2872 	alloc_size += ENETC_SI_ALIGN - 1;
2873 
2874 	p = kzalloc(alloc_size, GFP_KERNEL);
2875 	if (!p) {
2876 		err = -ENOMEM;
2877 		goto err_alloc_si;
2878 	}
2879 
2880 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
2881 	si->pad = (char *)si - (char *)p;
2882 
2883 	pci_set_drvdata(pdev, si);
2884 	si->pdev = pdev;
2885 	hw = &si->hw;
2886 
2887 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
2888 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
2889 	if (!hw->reg) {
2890 		err = -ENXIO;
2891 		dev_err(&pdev->dev, "ioremap() failed\n");
2892 		goto err_ioremap;
2893 	}
2894 	if (len > ENETC_PORT_BASE)
2895 		hw->port = hw->reg + ENETC_PORT_BASE;
2896 	if (len > ENETC_GLOBAL_BASE)
2897 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
2898 
2899 	enetc_detect_errata(si);
2900 
2901 	return 0;
2902 
2903 err_ioremap:
2904 	enetc_kfree_si(si);
2905 err_alloc_si:
2906 	pci_release_mem_regions(pdev);
2907 err_pci_mem_reg:
2908 err_dma:
2909 	pci_disable_device(pdev);
2910 
2911 	return err;
2912 }
2913 
2914 void enetc_pci_remove(struct pci_dev *pdev)
2915 {
2916 	struct enetc_si *si = pci_get_drvdata(pdev);
2917 	struct enetc_hw *hw = &si->hw;
2918 
2919 	iounmap(hw->reg);
2920 	enetc_kfree_si(si);
2921 	pci_release_mem_regions(pdev);
2922 	pci_disable_device(pdev);
2923 }
2924