1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3 
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/tcp.h>
7 #include <linux/udp.h>
8 #include <linux/vmalloc.h>
9 #include <linux/ptp_classify.h>
10 #include <net/ip6_checksum.h>
11 #include <net/pkt_sched.h>
12 #include <net/tso.h>
13 
14 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
15 {
16 	int num_tx_rings = priv->num_tx_rings;
17 	int i;
18 
19 	for (i = 0; i < priv->num_rx_rings; i++)
20 		if (priv->rx_ring[i]->xdp.prog)
21 			return num_tx_rings - num_possible_cpus();
22 
23 	return num_tx_rings;
24 }
25 
26 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
27 							struct enetc_bdr *tx_ring)
28 {
29 	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
30 
31 	return priv->rx_ring[index];
32 }
33 
34 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
35 {
36 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
37 		return NULL;
38 
39 	return tx_swbd->skb;
40 }
41 
42 static struct xdp_frame *
43 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
44 {
45 	if (tx_swbd->is_xdp_redirect)
46 		return tx_swbd->xdp_frame;
47 
48 	return NULL;
49 }
50 
51 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
52 				struct enetc_tx_swbd *tx_swbd)
53 {
54 	/* For XDP_TX, pages come from RX, whereas for the other contexts where
55 	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
56 	 * to match the DMA mapping length, so we need to differentiate those.
57 	 */
58 	if (tx_swbd->is_dma_page)
59 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
60 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
61 			       tx_swbd->dir);
62 	else
63 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
64 				 tx_swbd->len, tx_swbd->dir);
65 	tx_swbd->dma = 0;
66 }
67 
68 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
69 				struct enetc_tx_swbd *tx_swbd)
70 {
71 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
72 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
73 
74 	if (tx_swbd->dma)
75 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
76 
77 	if (xdp_frame) {
78 		xdp_return_frame(tx_swbd->xdp_frame);
79 		tx_swbd->xdp_frame = NULL;
80 	} else if (skb) {
81 		dev_kfree_skb_any(skb);
82 		tx_swbd->skb = NULL;
83 	}
84 }
85 
86 /* Let H/W know BD ring has been updated */
87 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
88 {
89 	/* includes wmb() */
90 	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
91 }
92 
93 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
94 			   u8 *msgtype, u8 *twostep,
95 			   u16 *correction_offset, u16 *body_offset)
96 {
97 	unsigned int ptp_class;
98 	struct ptp_header *hdr;
99 	unsigned int type;
100 	u8 *base;
101 
102 	ptp_class = ptp_classify_raw(skb);
103 	if (ptp_class == PTP_CLASS_NONE)
104 		return -EINVAL;
105 
106 	hdr = ptp_parse_header(skb, ptp_class);
107 	if (!hdr)
108 		return -EINVAL;
109 
110 	type = ptp_class & PTP_CLASS_PMASK;
111 	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
112 		*udp = 1;
113 	else
114 		*udp = 0;
115 
116 	*msgtype = ptp_get_msgtype(hdr, ptp_class);
117 	*twostep = hdr->flag_field[0] & 0x2;
118 
119 	base = skb_mac_header(skb);
120 	*correction_offset = (u8 *)&hdr->correction - base;
121 	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
122 
123 	return 0;
124 }
125 
126 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
127 {
128 	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
129 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
130 	struct enetc_hw *hw = &priv->si->hw;
131 	struct enetc_tx_swbd *tx_swbd;
132 	int len = skb_headlen(skb);
133 	union enetc_tx_bd temp_bd;
134 	u8 msgtype, twostep, udp;
135 	union enetc_tx_bd *txbd;
136 	u16 offset1, offset2;
137 	int i, count = 0;
138 	skb_frag_t *frag;
139 	unsigned int f;
140 	dma_addr_t dma;
141 	u8 flags = 0;
142 
143 	i = tx_ring->next_to_use;
144 	txbd = ENETC_TXBD(*tx_ring, i);
145 	prefetchw(txbd);
146 
147 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
148 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
149 		goto dma_err;
150 
151 	temp_bd.addr = cpu_to_le64(dma);
152 	temp_bd.buf_len = cpu_to_le16(len);
153 	temp_bd.lstatus = 0;
154 
155 	tx_swbd = &tx_ring->tx_swbd[i];
156 	tx_swbd->dma = dma;
157 	tx_swbd->len = len;
158 	tx_swbd->is_dma_page = 0;
159 	tx_swbd->dir = DMA_TO_DEVICE;
160 	count++;
161 
162 	do_vlan = skb_vlan_tag_present(skb);
163 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
164 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
165 				    &offset2) ||
166 		    msgtype != PTP_MSGTYPE_SYNC || twostep)
167 			WARN_ONCE(1, "Bad packet for one-step timestamping\n");
168 		else
169 			do_onestep_tstamp = true;
170 	} else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
171 		do_twostep_tstamp = true;
172 	}
173 
174 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
175 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
176 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
177 
178 	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
179 		flags |= ENETC_TXBD_FLAGS_EX;
180 
181 	if (tx_ring->tsd_enable)
182 		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
183 
184 	/* first BD needs frm_len and offload flags set */
185 	temp_bd.frm_len = cpu_to_le16(skb->len);
186 	temp_bd.flags = flags;
187 
188 	if (flags & ENETC_TXBD_FLAGS_TSE)
189 		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
190 							  flags);
191 
192 	if (flags & ENETC_TXBD_FLAGS_EX) {
193 		u8 e_flags = 0;
194 		*txbd = temp_bd;
195 		enetc_clear_tx_bd(&temp_bd);
196 
197 		/* add extension BD for VLAN and/or timestamping */
198 		flags = 0;
199 		tx_swbd++;
200 		txbd++;
201 		i++;
202 		if (unlikely(i == tx_ring->bd_count)) {
203 			i = 0;
204 			tx_swbd = tx_ring->tx_swbd;
205 			txbd = ENETC_TXBD(*tx_ring, 0);
206 		}
207 		prefetchw(txbd);
208 
209 		if (do_vlan) {
210 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
211 			temp_bd.ext.tpid = 0; /* < C-TAG */
212 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
213 		}
214 
215 		if (do_onestep_tstamp) {
216 			u32 lo, hi, val;
217 			u64 sec, nsec;
218 			u8 *data;
219 
220 			lo = enetc_rd_hot(hw, ENETC_SICTR0);
221 			hi = enetc_rd_hot(hw, ENETC_SICTR1);
222 			sec = (u64)hi << 32 | lo;
223 			nsec = do_div(sec, 1000000000);
224 
225 			/* Configure extension BD */
226 			temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
227 			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
228 
229 			/* Update originTimestamp field of Sync packet
230 			 * - 48 bits seconds field
231 			 * - 32 bits nanseconds field
232 			 */
233 			data = skb_mac_header(skb);
234 			*(__be16 *)(data + offset2) =
235 				htons((sec >> 32) & 0xffff);
236 			*(__be32 *)(data + offset2 + 2) =
237 				htonl(sec & 0xffffffff);
238 			*(__be32 *)(data + offset2 + 6) = htonl(nsec);
239 
240 			/* Configure single-step register */
241 			val = ENETC_PM0_SINGLE_STEP_EN;
242 			val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
243 			if (udp)
244 				val |= ENETC_PM0_SINGLE_STEP_CH;
245 
246 			enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val);
247 			enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val);
248 		} else if (do_twostep_tstamp) {
249 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
250 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
251 		}
252 
253 		temp_bd.ext.e_flags = e_flags;
254 		count++;
255 	}
256 
257 	frag = &skb_shinfo(skb)->frags[0];
258 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
259 		len = skb_frag_size(frag);
260 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
261 				       DMA_TO_DEVICE);
262 		if (dma_mapping_error(tx_ring->dev, dma))
263 			goto dma_err;
264 
265 		*txbd = temp_bd;
266 		enetc_clear_tx_bd(&temp_bd);
267 
268 		flags = 0;
269 		tx_swbd++;
270 		txbd++;
271 		i++;
272 		if (unlikely(i == tx_ring->bd_count)) {
273 			i = 0;
274 			tx_swbd = tx_ring->tx_swbd;
275 			txbd = ENETC_TXBD(*tx_ring, 0);
276 		}
277 		prefetchw(txbd);
278 
279 		temp_bd.addr = cpu_to_le64(dma);
280 		temp_bd.buf_len = cpu_to_le16(len);
281 
282 		tx_swbd->dma = dma;
283 		tx_swbd->len = len;
284 		tx_swbd->is_dma_page = 1;
285 		tx_swbd->dir = DMA_TO_DEVICE;
286 		count++;
287 	}
288 
289 	/* last BD needs 'F' bit set */
290 	flags |= ENETC_TXBD_FLAGS_F;
291 	temp_bd.flags = flags;
292 	*txbd = temp_bd;
293 
294 	tx_ring->tx_swbd[i].is_eof = true;
295 	tx_ring->tx_swbd[i].skb = skb;
296 
297 	enetc_bdr_idx_inc(tx_ring, &i);
298 	tx_ring->next_to_use = i;
299 
300 	skb_tx_timestamp(skb);
301 
302 	enetc_update_tx_ring_tail(tx_ring);
303 
304 	return count;
305 
306 dma_err:
307 	dev_err(tx_ring->dev, "DMA map error");
308 
309 	do {
310 		tx_swbd = &tx_ring->tx_swbd[i];
311 		enetc_free_tx_frame(tx_ring, tx_swbd);
312 		if (i == 0)
313 			i = tx_ring->bd_count;
314 		i--;
315 	} while (count--);
316 
317 	return 0;
318 }
319 
320 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
321 				 struct enetc_tx_swbd *tx_swbd,
322 				 union enetc_tx_bd *txbd, int *i, int hdr_len,
323 				 int data_len)
324 {
325 	union enetc_tx_bd txbd_tmp;
326 	u8 flags = 0, e_flags = 0;
327 	dma_addr_t addr;
328 
329 	enetc_clear_tx_bd(&txbd_tmp);
330 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
331 
332 	if (skb_vlan_tag_present(skb))
333 		flags |= ENETC_TXBD_FLAGS_EX;
334 
335 	txbd_tmp.addr = cpu_to_le64(addr);
336 	txbd_tmp.buf_len = cpu_to_le16(hdr_len);
337 
338 	/* first BD needs frm_len and offload flags set */
339 	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
340 	txbd_tmp.flags = flags;
341 
342 	/* For the TSO header we do not set the dma address since we do not
343 	 * want it unmapped when we do cleanup. We still set len so that we
344 	 * count the bytes sent.
345 	 */
346 	tx_swbd->len = hdr_len;
347 	tx_swbd->do_twostep_tstamp = false;
348 	tx_swbd->check_wb = false;
349 
350 	/* Actually write the header in the BD */
351 	*txbd = txbd_tmp;
352 
353 	/* Add extension BD for VLAN */
354 	if (flags & ENETC_TXBD_FLAGS_EX) {
355 		/* Get the next BD */
356 		enetc_bdr_idx_inc(tx_ring, i);
357 		txbd = ENETC_TXBD(*tx_ring, *i);
358 		tx_swbd = &tx_ring->tx_swbd[*i];
359 		prefetchw(txbd);
360 
361 		/* Setup the VLAN fields */
362 		enetc_clear_tx_bd(&txbd_tmp);
363 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
364 		txbd_tmp.ext.tpid = 0; /* < C-TAG */
365 		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
366 
367 		/* Write the BD */
368 		txbd_tmp.ext.e_flags = e_flags;
369 		*txbd = txbd_tmp;
370 	}
371 }
372 
373 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
374 				 struct enetc_tx_swbd *tx_swbd,
375 				 union enetc_tx_bd *txbd, char *data,
376 				 int size, bool last_bd)
377 {
378 	union enetc_tx_bd txbd_tmp;
379 	dma_addr_t addr;
380 	u8 flags = 0;
381 
382 	enetc_clear_tx_bd(&txbd_tmp);
383 
384 	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
385 	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
386 		netdev_err(tx_ring->ndev, "DMA map error\n");
387 		return -ENOMEM;
388 	}
389 
390 	if (last_bd) {
391 		flags |= ENETC_TXBD_FLAGS_F;
392 		tx_swbd->is_eof = 1;
393 	}
394 
395 	txbd_tmp.addr = cpu_to_le64(addr);
396 	txbd_tmp.buf_len = cpu_to_le16(size);
397 	txbd_tmp.flags = flags;
398 
399 	tx_swbd->dma = addr;
400 	tx_swbd->len = size;
401 	tx_swbd->dir = DMA_TO_DEVICE;
402 
403 	*txbd = txbd_tmp;
404 
405 	return 0;
406 }
407 
408 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
409 				 char *hdr, int hdr_len, int *l4_hdr_len)
410 {
411 	char *l4_hdr = hdr + skb_transport_offset(skb);
412 	int mac_hdr_len = skb_network_offset(skb);
413 
414 	if (tso->tlen != sizeof(struct udphdr)) {
415 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
416 
417 		tcph->check = 0;
418 	} else {
419 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
420 
421 		udph->check = 0;
422 	}
423 
424 	/* Compute the IP checksum. This is necessary since tso_build_hdr()
425 	 * already incremented the IP ID field.
426 	 */
427 	if (!tso->ipv6) {
428 		struct iphdr *iph = (void *)(hdr + mac_hdr_len);
429 
430 		iph->check = 0;
431 		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
432 	}
433 
434 	/* Compute the checksum over the L4 header. */
435 	*l4_hdr_len = hdr_len - skb_transport_offset(skb);
436 	return csum_partial(l4_hdr, *l4_hdr_len, 0);
437 }
438 
439 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
440 				    struct sk_buff *skb, char *hdr, int len,
441 				    __wsum sum)
442 {
443 	char *l4_hdr = hdr + skb_transport_offset(skb);
444 	__sum16 csum_final;
445 
446 	/* Complete the L4 checksum by appending the pseudo-header to the
447 	 * already computed checksum.
448 	 */
449 	if (!tso->ipv6)
450 		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
451 					       ip_hdr(skb)->daddr,
452 					       len, ip_hdr(skb)->protocol, sum);
453 	else
454 		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
455 					     &ipv6_hdr(skb)->daddr,
456 					     len, ipv6_hdr(skb)->nexthdr, sum);
457 
458 	if (tso->tlen != sizeof(struct udphdr)) {
459 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
460 
461 		tcph->check = csum_final;
462 	} else {
463 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
464 
465 		udph->check = csum_final;
466 	}
467 }
468 
469 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
470 {
471 	int hdr_len, total_len, data_len;
472 	struct enetc_tx_swbd *tx_swbd;
473 	union enetc_tx_bd *txbd;
474 	struct tso_t tso;
475 	__wsum csum, csum2;
476 	int count = 0, pos;
477 	int err, i, bd_data_num;
478 
479 	/* Initialize the TSO handler, and prepare the first payload */
480 	hdr_len = tso_start(skb, &tso);
481 	total_len = skb->len - hdr_len;
482 	i = tx_ring->next_to_use;
483 
484 	while (total_len > 0) {
485 		char *hdr;
486 
487 		/* Get the BD */
488 		txbd = ENETC_TXBD(*tx_ring, i);
489 		tx_swbd = &tx_ring->tx_swbd[i];
490 		prefetchw(txbd);
491 
492 		/* Determine the length of this packet */
493 		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
494 		total_len -= data_len;
495 
496 		/* prepare packet headers: MAC + IP + TCP */
497 		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
498 		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
499 
500 		/* compute the csum over the L4 header */
501 		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
502 		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
503 		bd_data_num = 0;
504 		count++;
505 
506 		while (data_len > 0) {
507 			int size;
508 
509 			size = min_t(int, tso.size, data_len);
510 
511 			/* Advance the index in the BDR */
512 			enetc_bdr_idx_inc(tx_ring, &i);
513 			txbd = ENETC_TXBD(*tx_ring, i);
514 			tx_swbd = &tx_ring->tx_swbd[i];
515 			prefetchw(txbd);
516 
517 			/* Compute the checksum over this segment of data and
518 			 * add it to the csum already computed (over the L4
519 			 * header and possible other data segments).
520 			 */
521 			csum2 = csum_partial(tso.data, size, 0);
522 			csum = csum_block_add(csum, csum2, pos);
523 			pos += size;
524 
525 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
526 						    tso.data, size,
527 						    size == data_len);
528 			if (err)
529 				goto err_map_data;
530 
531 			data_len -= size;
532 			count++;
533 			bd_data_num++;
534 			tso_build_data(skb, &tso, size);
535 
536 			if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
537 				goto err_chained_bd;
538 		}
539 
540 		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
541 
542 		if (total_len == 0)
543 			tx_swbd->skb = skb;
544 
545 		/* Go to the next BD */
546 		enetc_bdr_idx_inc(tx_ring, &i);
547 	}
548 
549 	tx_ring->next_to_use = i;
550 	enetc_update_tx_ring_tail(tx_ring);
551 
552 	return count;
553 
554 err_map_data:
555 	dev_err(tx_ring->dev, "DMA map error");
556 
557 err_chained_bd:
558 	do {
559 		tx_swbd = &tx_ring->tx_swbd[i];
560 		enetc_free_tx_frame(tx_ring, tx_swbd);
561 		if (i == 0)
562 			i = tx_ring->bd_count;
563 		i--;
564 	} while (count--);
565 
566 	return 0;
567 }
568 
569 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
570 				    struct net_device *ndev)
571 {
572 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
573 	struct enetc_bdr *tx_ring;
574 	int count, err;
575 
576 	/* Queue one-step Sync packet if already locked */
577 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
578 		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
579 					  &priv->flags)) {
580 			skb_queue_tail(&priv->tx_skbs, skb);
581 			return NETDEV_TX_OK;
582 		}
583 	}
584 
585 	tx_ring = priv->tx_ring[skb->queue_mapping];
586 
587 	if (skb_is_gso(skb)) {
588 		if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
589 			netif_stop_subqueue(ndev, tx_ring->index);
590 			return NETDEV_TX_BUSY;
591 		}
592 
593 		enetc_lock_mdio();
594 		count = enetc_map_tx_tso_buffs(tx_ring, skb);
595 		enetc_unlock_mdio();
596 	} else {
597 		if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
598 			if (unlikely(skb_linearize(skb)))
599 				goto drop_packet_err;
600 
601 		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
602 		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
603 			netif_stop_subqueue(ndev, tx_ring->index);
604 			return NETDEV_TX_BUSY;
605 		}
606 
607 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
608 			err = skb_checksum_help(skb);
609 			if (err)
610 				goto drop_packet_err;
611 		}
612 		enetc_lock_mdio();
613 		count = enetc_map_tx_buffs(tx_ring, skb);
614 		enetc_unlock_mdio();
615 	}
616 
617 	if (unlikely(!count))
618 		goto drop_packet_err;
619 
620 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
621 		netif_stop_subqueue(ndev, tx_ring->index);
622 
623 	return NETDEV_TX_OK;
624 
625 drop_packet_err:
626 	dev_kfree_skb_any(skb);
627 	return NETDEV_TX_OK;
628 }
629 
630 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
631 {
632 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
633 	u8 udp, msgtype, twostep;
634 	u16 offset1, offset2;
635 
636 	/* Mark tx timestamp type on skb->cb[0] if requires */
637 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
638 	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
639 		skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
640 	} else {
641 		skb->cb[0] = 0;
642 	}
643 
644 	/* Fall back to two-step timestamp if not one-step Sync packet */
645 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
646 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
647 				    &offset1, &offset2) ||
648 		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
649 			skb->cb[0] = ENETC_F_TX_TSTAMP;
650 	}
651 
652 	return enetc_start_xmit(skb, ndev);
653 }
654 
655 static irqreturn_t enetc_msix(int irq, void *data)
656 {
657 	struct enetc_int_vector	*v = data;
658 	int i;
659 
660 	enetc_lock_mdio();
661 
662 	/* disable interrupts */
663 	enetc_wr_reg_hot(v->rbier, 0);
664 	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
665 
666 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
667 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
668 
669 	enetc_unlock_mdio();
670 
671 	napi_schedule(&v->napi);
672 
673 	return IRQ_HANDLED;
674 }
675 
676 static void enetc_rx_dim_work(struct work_struct *w)
677 {
678 	struct dim *dim = container_of(w, struct dim, work);
679 	struct dim_cq_moder moder =
680 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
681 	struct enetc_int_vector	*v =
682 		container_of(dim, struct enetc_int_vector, rx_dim);
683 
684 	v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
685 	dim->state = DIM_START_MEASURE;
686 }
687 
688 static void enetc_rx_net_dim(struct enetc_int_vector *v)
689 {
690 	struct dim_sample dim_sample = {};
691 
692 	v->comp_cnt++;
693 
694 	if (!v->rx_napi_work)
695 		return;
696 
697 	dim_update_sample(v->comp_cnt,
698 			  v->rx_ring.stats.packets,
699 			  v->rx_ring.stats.bytes,
700 			  &dim_sample);
701 	net_dim(&v->rx_dim, dim_sample);
702 }
703 
704 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
705 {
706 	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
707 
708 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
709 }
710 
711 static bool enetc_page_reusable(struct page *page)
712 {
713 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
714 }
715 
716 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
717 			     struct enetc_rx_swbd *old)
718 {
719 	struct enetc_rx_swbd *new;
720 
721 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
722 
723 	/* next buf that may reuse a page */
724 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
725 
726 	/* copy page reference */
727 	*new = *old;
728 }
729 
730 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
731 				u64 *tstamp)
732 {
733 	u32 lo, hi, tstamp_lo;
734 
735 	lo = enetc_rd_hot(hw, ENETC_SICTR0);
736 	hi = enetc_rd_hot(hw, ENETC_SICTR1);
737 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
738 	if (lo <= tstamp_lo)
739 		hi -= 1;
740 	*tstamp = (u64)hi << 32 | tstamp_lo;
741 }
742 
743 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
744 {
745 	struct skb_shared_hwtstamps shhwtstamps;
746 
747 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
748 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
749 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
750 		skb_txtime_consumed(skb);
751 		skb_tstamp_tx(skb, &shhwtstamps);
752 	}
753 }
754 
755 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
756 				      struct enetc_tx_swbd *tx_swbd)
757 {
758 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
759 	struct enetc_rx_swbd rx_swbd = {
760 		.dma = tx_swbd->dma,
761 		.page = tx_swbd->page,
762 		.page_offset = tx_swbd->page_offset,
763 		.dir = tx_swbd->dir,
764 		.len = tx_swbd->len,
765 	};
766 	struct enetc_bdr *rx_ring;
767 
768 	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
769 
770 	if (likely(enetc_swbd_unused(rx_ring))) {
771 		enetc_reuse_page(rx_ring, &rx_swbd);
772 
773 		/* sync for use by the device */
774 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
775 						 rx_swbd.page_offset,
776 						 ENETC_RXB_DMA_SIZE_XDP,
777 						 rx_swbd.dir);
778 
779 		rx_ring->stats.recycles++;
780 	} else {
781 		/* RX ring is already full, we need to unmap and free the
782 		 * page, since there's nothing useful we can do with it.
783 		 */
784 		rx_ring->stats.recycle_failures++;
785 
786 		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
787 			       rx_swbd.dir);
788 		__free_page(rx_swbd.page);
789 	}
790 
791 	rx_ring->xdp.xdp_tx_in_flight--;
792 }
793 
794 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
795 {
796 	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
797 	struct net_device *ndev = tx_ring->ndev;
798 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
799 	struct enetc_tx_swbd *tx_swbd;
800 	int i, bds_to_clean;
801 	bool do_twostep_tstamp;
802 	u64 tstamp = 0;
803 
804 	i = tx_ring->next_to_clean;
805 	tx_swbd = &tx_ring->tx_swbd[i];
806 
807 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
808 
809 	do_twostep_tstamp = false;
810 
811 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
812 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
813 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
814 		bool is_eof = tx_swbd->is_eof;
815 
816 		if (unlikely(tx_swbd->check_wb)) {
817 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
818 
819 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
820 			    tx_swbd->do_twostep_tstamp) {
821 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
822 						    &tstamp);
823 				do_twostep_tstamp = true;
824 			}
825 
826 			if (tx_swbd->qbv_en &&
827 			    txbd->wb.status & ENETC_TXBD_STATS_WIN)
828 				tx_win_drop++;
829 		}
830 
831 		if (tx_swbd->is_xdp_tx)
832 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
833 		else if (likely(tx_swbd->dma))
834 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
835 
836 		if (xdp_frame) {
837 			xdp_return_frame(xdp_frame);
838 		} else if (skb) {
839 			if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
840 				/* Start work to release lock for next one-step
841 				 * timestamping packet. And send one skb in
842 				 * tx_skbs queue if has.
843 				 */
844 				schedule_work(&priv->tx_onestep_tstamp);
845 			} else if (unlikely(do_twostep_tstamp)) {
846 				enetc_tstamp_tx(skb, tstamp);
847 				do_twostep_tstamp = false;
848 			}
849 			napi_consume_skb(skb, napi_budget);
850 		}
851 
852 		tx_byte_cnt += tx_swbd->len;
853 		/* Scrub the swbd here so we don't have to do that
854 		 * when we reuse it during xmit
855 		 */
856 		memset(tx_swbd, 0, sizeof(*tx_swbd));
857 
858 		bds_to_clean--;
859 		tx_swbd++;
860 		i++;
861 		if (unlikely(i == tx_ring->bd_count)) {
862 			i = 0;
863 			tx_swbd = tx_ring->tx_swbd;
864 		}
865 
866 		/* BD iteration loop end */
867 		if (is_eof) {
868 			tx_frm_cnt++;
869 			/* re-arm interrupt source */
870 			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
871 					 BIT(16 + tx_ring->index));
872 		}
873 
874 		if (unlikely(!bds_to_clean))
875 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
876 	}
877 
878 	tx_ring->next_to_clean = i;
879 	tx_ring->stats.packets += tx_frm_cnt;
880 	tx_ring->stats.bytes += tx_byte_cnt;
881 	tx_ring->stats.win_drop += tx_win_drop;
882 
883 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
884 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
885 		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
886 		netif_wake_subqueue(ndev, tx_ring->index);
887 	}
888 
889 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
890 }
891 
892 static bool enetc_new_page(struct enetc_bdr *rx_ring,
893 			   struct enetc_rx_swbd *rx_swbd)
894 {
895 	bool xdp = !!(rx_ring->xdp.prog);
896 	struct page *page;
897 	dma_addr_t addr;
898 
899 	page = dev_alloc_page();
900 	if (unlikely(!page))
901 		return false;
902 
903 	/* For XDP_TX, we forgo dma_unmap -> dma_map */
904 	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
905 
906 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
907 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
908 		__free_page(page);
909 
910 		return false;
911 	}
912 
913 	rx_swbd->dma = addr;
914 	rx_swbd->page = page;
915 	rx_swbd->page_offset = rx_ring->buffer_offset;
916 
917 	return true;
918 }
919 
920 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
921 {
922 	struct enetc_rx_swbd *rx_swbd;
923 	union enetc_rx_bd *rxbd;
924 	int i, j;
925 
926 	i = rx_ring->next_to_use;
927 	rx_swbd = &rx_ring->rx_swbd[i];
928 	rxbd = enetc_rxbd(rx_ring, i);
929 
930 	for (j = 0; j < buff_cnt; j++) {
931 		/* try reuse page */
932 		if (unlikely(!rx_swbd->page)) {
933 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
934 				rx_ring->stats.rx_alloc_errs++;
935 				break;
936 			}
937 		}
938 
939 		/* update RxBD */
940 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
941 					   rx_swbd->page_offset);
942 		/* clear 'R" as well */
943 		rxbd->r.lstatus = 0;
944 
945 		enetc_rxbd_next(rx_ring, &rxbd, &i);
946 		rx_swbd = &rx_ring->rx_swbd[i];
947 	}
948 
949 	if (likely(j)) {
950 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
951 		rx_ring->next_to_use = i;
952 
953 		/* update ENETC's consumer index */
954 		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
955 	}
956 
957 	return j;
958 }
959 
960 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
961 static void enetc_get_rx_tstamp(struct net_device *ndev,
962 				union enetc_rx_bd *rxbd,
963 				struct sk_buff *skb)
964 {
965 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
966 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
967 	struct enetc_hw *hw = &priv->si->hw;
968 	u32 lo, hi, tstamp_lo;
969 	u64 tstamp;
970 
971 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
972 		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
973 		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
974 		rxbd = enetc_rxbd_ext(rxbd);
975 		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
976 		if (lo <= tstamp_lo)
977 			hi -= 1;
978 
979 		tstamp = (u64)hi << 32 | tstamp_lo;
980 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
981 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
982 	}
983 }
984 #endif
985 
986 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
987 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
988 {
989 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
990 
991 	/* TODO: hashing */
992 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
993 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
994 
995 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
996 		skb->ip_summed = CHECKSUM_COMPLETE;
997 	}
998 
999 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1000 		__be16 tpid = 0;
1001 
1002 		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1003 		case 0:
1004 			tpid = htons(ETH_P_8021Q);
1005 			break;
1006 		case 1:
1007 			tpid = htons(ETH_P_8021AD);
1008 			break;
1009 		case 2:
1010 			tpid = htons(enetc_port_rd(&priv->si->hw,
1011 						   ENETC_PCVLANR1));
1012 			break;
1013 		case 3:
1014 			tpid = htons(enetc_port_rd(&priv->si->hw,
1015 						   ENETC_PCVLANR2));
1016 			break;
1017 		default:
1018 			break;
1019 		}
1020 
1021 		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1022 	}
1023 
1024 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1025 	if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1026 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1027 #endif
1028 }
1029 
1030 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1031  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1032  * mapped buffers.
1033  */
1034 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1035 					       int i, u16 size)
1036 {
1037 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1038 
1039 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1040 				      rx_swbd->page_offset,
1041 				      size, rx_swbd->dir);
1042 	return rx_swbd;
1043 }
1044 
1045 /* Reuse the current page without performing half-page buffer flipping */
1046 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1047 			      struct enetc_rx_swbd *rx_swbd)
1048 {
1049 	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1050 
1051 	enetc_reuse_page(rx_ring, rx_swbd);
1052 
1053 	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1054 					 rx_swbd->page_offset,
1055 					 buffer_size, rx_swbd->dir);
1056 
1057 	rx_swbd->page = NULL;
1058 }
1059 
1060 /* Reuse the current page by performing half-page buffer flipping */
1061 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1062 			       struct enetc_rx_swbd *rx_swbd)
1063 {
1064 	if (likely(enetc_page_reusable(rx_swbd->page))) {
1065 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1066 		page_ref_inc(rx_swbd->page);
1067 
1068 		enetc_put_rx_buff(rx_ring, rx_swbd);
1069 	} else {
1070 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1071 			       rx_swbd->dir);
1072 		rx_swbd->page = NULL;
1073 	}
1074 }
1075 
1076 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1077 						int i, u16 size)
1078 {
1079 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1080 	struct sk_buff *skb;
1081 	void *ba;
1082 
1083 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1084 	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1085 	if (unlikely(!skb)) {
1086 		rx_ring->stats.rx_alloc_errs++;
1087 		return NULL;
1088 	}
1089 
1090 	skb_reserve(skb, rx_ring->buffer_offset);
1091 	__skb_put(skb, size);
1092 
1093 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1094 
1095 	return skb;
1096 }
1097 
1098 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1099 				     u16 size, struct sk_buff *skb)
1100 {
1101 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1102 
1103 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1104 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1105 
1106 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1107 }
1108 
1109 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1110 					      u32 bd_status,
1111 					      union enetc_rx_bd **rxbd, int *i)
1112 {
1113 	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1114 		return false;
1115 
1116 	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1117 	enetc_rxbd_next(rx_ring, rxbd, i);
1118 
1119 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1120 		dma_rmb();
1121 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1122 
1123 		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1124 		enetc_rxbd_next(rx_ring, rxbd, i);
1125 	}
1126 
1127 	rx_ring->ndev->stats.rx_dropped++;
1128 	rx_ring->ndev->stats.rx_errors++;
1129 
1130 	return true;
1131 }
1132 
1133 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1134 				       u32 bd_status, union enetc_rx_bd **rxbd,
1135 				       int *i, int *cleaned_cnt, int buffer_size)
1136 {
1137 	struct sk_buff *skb;
1138 	u16 size;
1139 
1140 	size = le16_to_cpu((*rxbd)->r.buf_len);
1141 	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1142 	if (!skb)
1143 		return NULL;
1144 
1145 	enetc_get_offloads(rx_ring, *rxbd, skb);
1146 
1147 	(*cleaned_cnt)++;
1148 
1149 	enetc_rxbd_next(rx_ring, rxbd, i);
1150 
1151 	/* not last BD in frame? */
1152 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1153 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1154 		size = buffer_size;
1155 
1156 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1157 			dma_rmb();
1158 			size = le16_to_cpu((*rxbd)->r.buf_len);
1159 		}
1160 
1161 		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1162 
1163 		(*cleaned_cnt)++;
1164 
1165 		enetc_rxbd_next(rx_ring, rxbd, i);
1166 	}
1167 
1168 	skb_record_rx_queue(skb, rx_ring->index);
1169 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1170 
1171 	return skb;
1172 }
1173 
1174 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1175 
1176 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1177 			       struct napi_struct *napi, int work_limit)
1178 {
1179 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1180 	int cleaned_cnt, i;
1181 
1182 	cleaned_cnt = enetc_bd_unused(rx_ring);
1183 	/* next descriptor to process */
1184 	i = rx_ring->next_to_clean;
1185 
1186 	while (likely(rx_frm_cnt < work_limit)) {
1187 		union enetc_rx_bd *rxbd;
1188 		struct sk_buff *skb;
1189 		u32 bd_status;
1190 
1191 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1192 			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1193 							    cleaned_cnt);
1194 
1195 		rxbd = enetc_rxbd(rx_ring, i);
1196 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1197 		if (!bd_status)
1198 			break;
1199 
1200 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1201 		dma_rmb(); /* for reading other rxbd fields */
1202 
1203 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1204 						      &rxbd, &i))
1205 			break;
1206 
1207 		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1208 				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1209 		if (!skb)
1210 			break;
1211 
1212 		rx_byte_cnt += skb->len;
1213 		rx_frm_cnt++;
1214 
1215 		napi_gro_receive(napi, skb);
1216 	}
1217 
1218 	rx_ring->next_to_clean = i;
1219 
1220 	rx_ring->stats.packets += rx_frm_cnt;
1221 	rx_ring->stats.bytes += rx_byte_cnt;
1222 
1223 	return rx_frm_cnt;
1224 }
1225 
1226 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1227 				  struct enetc_tx_swbd *tx_swbd,
1228 				  int frm_len)
1229 {
1230 	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1231 
1232 	prefetchw(txbd);
1233 
1234 	enetc_clear_tx_bd(txbd);
1235 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1236 	txbd->buf_len = cpu_to_le16(tx_swbd->len);
1237 	txbd->frm_len = cpu_to_le16(frm_len);
1238 
1239 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1240 }
1241 
1242 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1243  * descriptors.
1244  */
1245 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1246 			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1247 {
1248 	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1249 	int i, k, frm_len = tmp_tx_swbd->len;
1250 
1251 	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1252 		return false;
1253 
1254 	while (unlikely(!tmp_tx_swbd->is_eof)) {
1255 		tmp_tx_swbd++;
1256 		frm_len += tmp_tx_swbd->len;
1257 	}
1258 
1259 	i = tx_ring->next_to_use;
1260 
1261 	for (k = 0; k < num_tx_swbd; k++) {
1262 		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1263 
1264 		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1265 
1266 		/* last BD needs 'F' bit set */
1267 		if (xdp_tx_swbd->is_eof) {
1268 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1269 
1270 			txbd->flags = ENETC_TXBD_FLAGS_F;
1271 		}
1272 
1273 		enetc_bdr_idx_inc(tx_ring, &i);
1274 	}
1275 
1276 	tx_ring->next_to_use = i;
1277 
1278 	return true;
1279 }
1280 
1281 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1282 					  struct enetc_tx_swbd *xdp_tx_arr,
1283 					  struct xdp_frame *xdp_frame)
1284 {
1285 	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1286 	struct skb_shared_info *shinfo;
1287 	void *data = xdp_frame->data;
1288 	int len = xdp_frame->len;
1289 	skb_frag_t *frag;
1290 	dma_addr_t dma;
1291 	unsigned int f;
1292 	int n = 0;
1293 
1294 	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1295 	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1296 		netdev_err(tx_ring->ndev, "DMA map error\n");
1297 		return -1;
1298 	}
1299 
1300 	xdp_tx_swbd->dma = dma;
1301 	xdp_tx_swbd->dir = DMA_TO_DEVICE;
1302 	xdp_tx_swbd->len = len;
1303 	xdp_tx_swbd->is_xdp_redirect = true;
1304 	xdp_tx_swbd->is_eof = false;
1305 	xdp_tx_swbd->xdp_frame = NULL;
1306 
1307 	n++;
1308 	xdp_tx_swbd = &xdp_tx_arr[n];
1309 
1310 	shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1311 
1312 	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1313 	     f++, frag++) {
1314 		data = skb_frag_address(frag);
1315 		len = skb_frag_size(frag);
1316 
1317 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1318 		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1319 			/* Undo the DMA mapping for all fragments */
1320 			while (--n >= 0)
1321 				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1322 
1323 			netdev_err(tx_ring->ndev, "DMA map error\n");
1324 			return -1;
1325 		}
1326 
1327 		xdp_tx_swbd->dma = dma;
1328 		xdp_tx_swbd->dir = DMA_TO_DEVICE;
1329 		xdp_tx_swbd->len = len;
1330 		xdp_tx_swbd->is_xdp_redirect = true;
1331 		xdp_tx_swbd->is_eof = false;
1332 		xdp_tx_swbd->xdp_frame = NULL;
1333 
1334 		n++;
1335 		xdp_tx_swbd = &xdp_tx_arr[n];
1336 	}
1337 
1338 	xdp_tx_arr[n - 1].is_eof = true;
1339 	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1340 
1341 	return n;
1342 }
1343 
1344 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1345 		   struct xdp_frame **frames, u32 flags)
1346 {
1347 	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1348 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1349 	struct enetc_bdr *tx_ring;
1350 	int xdp_tx_bd_cnt, i, k;
1351 	int xdp_tx_frm_cnt = 0;
1352 
1353 	enetc_lock_mdio();
1354 
1355 	tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1356 
1357 	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1358 
1359 	for (k = 0; k < num_frames; k++) {
1360 		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1361 							       xdp_redirect_arr,
1362 							       frames[k]);
1363 		if (unlikely(xdp_tx_bd_cnt < 0))
1364 			break;
1365 
1366 		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1367 					   xdp_tx_bd_cnt))) {
1368 			for (i = 0; i < xdp_tx_bd_cnt; i++)
1369 				enetc_unmap_tx_buff(tx_ring,
1370 						    &xdp_redirect_arr[i]);
1371 			tx_ring->stats.xdp_tx_drops++;
1372 			break;
1373 		}
1374 
1375 		xdp_tx_frm_cnt++;
1376 	}
1377 
1378 	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1379 		enetc_update_tx_ring_tail(tx_ring);
1380 
1381 	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1382 
1383 	enetc_unlock_mdio();
1384 
1385 	return xdp_tx_frm_cnt;
1386 }
1387 
1388 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1389 				     struct xdp_buff *xdp_buff, u16 size)
1390 {
1391 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1392 	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1393 	struct skb_shared_info *shinfo;
1394 
1395 	/* To be used for XDP_TX */
1396 	rx_swbd->len = size;
1397 
1398 	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1399 			 rx_ring->buffer_offset, size, false);
1400 
1401 	shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1402 	shinfo->nr_frags = 0;
1403 }
1404 
1405 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1406 				     u16 size, struct xdp_buff *xdp_buff)
1407 {
1408 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1409 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1410 	skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags];
1411 
1412 	/* To be used for XDP_TX */
1413 	rx_swbd->len = size;
1414 
1415 	skb_frag_off_set(frag, rx_swbd->page_offset);
1416 	skb_frag_size_set(frag, size);
1417 	__skb_frag_set_page(frag, rx_swbd->page);
1418 
1419 	shinfo->nr_frags++;
1420 }
1421 
1422 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1423 				 union enetc_rx_bd **rxbd, int *i,
1424 				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1425 {
1426 	u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1427 
1428 	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1429 
1430 	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1431 	(*cleaned_cnt)++;
1432 	enetc_rxbd_next(rx_ring, rxbd, i);
1433 
1434 	/* not last BD in frame? */
1435 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1436 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1437 		size = ENETC_RXB_DMA_SIZE_XDP;
1438 
1439 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1440 			dma_rmb();
1441 			size = le16_to_cpu((*rxbd)->r.buf_len);
1442 		}
1443 
1444 		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1445 		(*cleaned_cnt)++;
1446 		enetc_rxbd_next(rx_ring, rxbd, i);
1447 	}
1448 }
1449 
1450 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1451  * recycled back into the RX ring in enetc_clean_tx_ring.
1452  */
1453 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1454 					struct enetc_bdr *rx_ring,
1455 					int rx_ring_first, int rx_ring_last)
1456 {
1457 	int n = 0;
1458 
1459 	for (; rx_ring_first != rx_ring_last;
1460 	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1461 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1462 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1463 
1464 		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1465 		tx_swbd->dma = rx_swbd->dma;
1466 		tx_swbd->dir = rx_swbd->dir;
1467 		tx_swbd->page = rx_swbd->page;
1468 		tx_swbd->page_offset = rx_swbd->page_offset;
1469 		tx_swbd->len = rx_swbd->len;
1470 		tx_swbd->is_dma_page = true;
1471 		tx_swbd->is_xdp_tx = true;
1472 		tx_swbd->is_eof = false;
1473 	}
1474 
1475 	/* We rely on caller providing an rx_ring_last > rx_ring_first */
1476 	xdp_tx_arr[n - 1].is_eof = true;
1477 
1478 	return n;
1479 }
1480 
1481 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1482 			   int rx_ring_last)
1483 {
1484 	while (rx_ring_first != rx_ring_last) {
1485 		enetc_put_rx_buff(rx_ring,
1486 				  &rx_ring->rx_swbd[rx_ring_first]);
1487 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1488 	}
1489 	rx_ring->stats.xdp_drops++;
1490 }
1491 
1492 static void enetc_xdp_free(struct enetc_bdr *rx_ring, int rx_ring_first,
1493 			   int rx_ring_last)
1494 {
1495 	while (rx_ring_first != rx_ring_last) {
1496 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1497 
1498 		if (rx_swbd->page) {
1499 			dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1500 				       rx_swbd->dir);
1501 			__free_page(rx_swbd->page);
1502 			rx_swbd->page = NULL;
1503 		}
1504 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1505 	}
1506 	rx_ring->stats.xdp_redirect_failures++;
1507 }
1508 
1509 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1510 				   struct napi_struct *napi, int work_limit,
1511 				   struct bpf_prog *prog)
1512 {
1513 	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1514 	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1515 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1516 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1517 	struct enetc_bdr *tx_ring;
1518 	int cleaned_cnt, i;
1519 	u32 xdp_act;
1520 
1521 	cleaned_cnt = enetc_bd_unused(rx_ring);
1522 	/* next descriptor to process */
1523 	i = rx_ring->next_to_clean;
1524 
1525 	while (likely(rx_frm_cnt < work_limit)) {
1526 		union enetc_rx_bd *rxbd, *orig_rxbd;
1527 		int orig_i, orig_cleaned_cnt;
1528 		struct xdp_buff xdp_buff;
1529 		struct sk_buff *skb;
1530 		int tmp_orig_i, err;
1531 		u32 bd_status;
1532 
1533 		rxbd = enetc_rxbd(rx_ring, i);
1534 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1535 		if (!bd_status)
1536 			break;
1537 
1538 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1539 		dma_rmb(); /* for reading other rxbd fields */
1540 
1541 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1542 						      &rxbd, &i))
1543 			break;
1544 
1545 		orig_rxbd = rxbd;
1546 		orig_cleaned_cnt = cleaned_cnt;
1547 		orig_i = i;
1548 
1549 		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1550 				     &cleaned_cnt, &xdp_buff);
1551 
1552 		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1553 
1554 		switch (xdp_act) {
1555 		default:
1556 			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1557 			fallthrough;
1558 		case XDP_ABORTED:
1559 			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1560 			fallthrough;
1561 		case XDP_DROP:
1562 			enetc_xdp_drop(rx_ring, orig_i, i);
1563 			break;
1564 		case XDP_PASS:
1565 			rxbd = orig_rxbd;
1566 			cleaned_cnt = orig_cleaned_cnt;
1567 			i = orig_i;
1568 
1569 			skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1570 					      &i, &cleaned_cnt,
1571 					      ENETC_RXB_DMA_SIZE_XDP);
1572 			if (unlikely(!skb))
1573 				goto out;
1574 
1575 			napi_gro_receive(napi, skb);
1576 			break;
1577 		case XDP_TX:
1578 			tx_ring = priv->xdp_tx_ring[rx_ring->index];
1579 			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1580 								     rx_ring,
1581 								     orig_i, i);
1582 
1583 			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1584 				enetc_xdp_drop(rx_ring, orig_i, i);
1585 				tx_ring->stats.xdp_tx_drops++;
1586 			} else {
1587 				tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1588 				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1589 				xdp_tx_frm_cnt++;
1590 				/* The XDP_TX enqueue was successful, so we
1591 				 * need to scrub the RX software BDs because
1592 				 * the ownership of the buffers no longer
1593 				 * belongs to the RX ring, and we must prevent
1594 				 * enetc_refill_rx_ring() from reusing
1595 				 * rx_swbd->page.
1596 				 */
1597 				while (orig_i != i) {
1598 					rx_ring->rx_swbd[orig_i].page = NULL;
1599 					enetc_bdr_idx_inc(rx_ring, &orig_i);
1600 				}
1601 			}
1602 			break;
1603 		case XDP_REDIRECT:
1604 			/* xdp_return_frame does not support S/G in the sense
1605 			 * that it leaks the fragments (__xdp_return should not
1606 			 * call page_frag_free only for the initial buffer).
1607 			 * Until XDP_REDIRECT gains support for S/G let's keep
1608 			 * the code structure in place, but dead. We drop the
1609 			 * S/G frames ourselves to avoid memory leaks which
1610 			 * would otherwise leave the kernel OOM.
1611 			 */
1612 			if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) {
1613 				enetc_xdp_drop(rx_ring, orig_i, i);
1614 				rx_ring->stats.xdp_redirect_sg++;
1615 				break;
1616 			}
1617 
1618 			tmp_orig_i = orig_i;
1619 
1620 			while (orig_i != i) {
1621 				enetc_flip_rx_buff(rx_ring,
1622 						   &rx_ring->rx_swbd[orig_i]);
1623 				enetc_bdr_idx_inc(rx_ring, &orig_i);
1624 			}
1625 
1626 			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1627 			if (unlikely(err)) {
1628 				enetc_xdp_free(rx_ring, tmp_orig_i, i);
1629 			} else {
1630 				xdp_redirect_frm_cnt++;
1631 				rx_ring->stats.xdp_redirect++;
1632 			}
1633 		}
1634 
1635 		rx_frm_cnt++;
1636 	}
1637 
1638 out:
1639 	rx_ring->next_to_clean = i;
1640 
1641 	rx_ring->stats.packets += rx_frm_cnt;
1642 	rx_ring->stats.bytes += rx_byte_cnt;
1643 
1644 	if (xdp_redirect_frm_cnt)
1645 		xdp_do_flush_map();
1646 
1647 	if (xdp_tx_frm_cnt)
1648 		enetc_update_tx_ring_tail(tx_ring);
1649 
1650 	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1651 		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1652 				     rx_ring->xdp.xdp_tx_in_flight);
1653 
1654 	return rx_frm_cnt;
1655 }
1656 
1657 static int enetc_poll(struct napi_struct *napi, int budget)
1658 {
1659 	struct enetc_int_vector
1660 		*v = container_of(napi, struct enetc_int_vector, napi);
1661 	struct enetc_bdr *rx_ring = &v->rx_ring;
1662 	struct bpf_prog *prog;
1663 	bool complete = true;
1664 	int work_done;
1665 	int i;
1666 
1667 	enetc_lock_mdio();
1668 
1669 	for (i = 0; i < v->count_tx_rings; i++)
1670 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1671 			complete = false;
1672 
1673 	prog = rx_ring->xdp.prog;
1674 	if (prog)
1675 		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1676 	else
1677 		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1678 	if (work_done == budget)
1679 		complete = false;
1680 	if (work_done)
1681 		v->rx_napi_work = true;
1682 
1683 	if (!complete) {
1684 		enetc_unlock_mdio();
1685 		return budget;
1686 	}
1687 
1688 	napi_complete_done(napi, work_done);
1689 
1690 	if (likely(v->rx_dim_en))
1691 		enetc_rx_net_dim(v);
1692 
1693 	v->rx_napi_work = false;
1694 
1695 	/* enable interrupts */
1696 	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1697 
1698 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1699 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1700 				 ENETC_TBIER_TXTIE);
1701 
1702 	enetc_unlock_mdio();
1703 
1704 	return work_done;
1705 }
1706 
1707 /* Probing and Init */
1708 #define ENETC_MAX_RFS_SIZE 64
1709 void enetc_get_si_caps(struct enetc_si *si)
1710 {
1711 	struct enetc_hw *hw = &si->hw;
1712 	u32 val;
1713 
1714 	/* find out how many of various resources we have to work with */
1715 	val = enetc_rd(hw, ENETC_SICAPR0);
1716 	si->num_rx_rings = (val >> 16) & 0xff;
1717 	si->num_tx_rings = val & 0xff;
1718 
1719 	val = enetc_rd(hw, ENETC_SIRFSCAPR);
1720 	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1721 	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1722 
1723 	si->num_rss = 0;
1724 	val = enetc_rd(hw, ENETC_SIPCAPR0);
1725 	if (val & ENETC_SIPCAPR0_RSS) {
1726 		u32 rss;
1727 
1728 		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1729 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1730 	}
1731 
1732 	if (val & ENETC_SIPCAPR0_QBV)
1733 		si->hw_features |= ENETC_SI_F_QBV;
1734 
1735 	if (val & ENETC_SIPCAPR0_PSFP)
1736 		si->hw_features |= ENETC_SI_F_PSFP;
1737 }
1738 
1739 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
1740 {
1741 	r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
1742 					&r->bd_dma_base, GFP_KERNEL);
1743 	if (!r->bd_base)
1744 		return -ENOMEM;
1745 
1746 	/* h/w requires 128B alignment */
1747 	if (!IS_ALIGNED(r->bd_dma_base, 128)) {
1748 		dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
1749 				  r->bd_dma_base);
1750 		return -EINVAL;
1751 	}
1752 
1753 	return 0;
1754 }
1755 
1756 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
1757 {
1758 	int err;
1759 
1760 	txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
1761 	if (!txr->tx_swbd)
1762 		return -ENOMEM;
1763 
1764 	err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
1765 	if (err)
1766 		goto err_alloc_bdr;
1767 
1768 	txr->tso_headers = dma_alloc_coherent(txr->dev,
1769 					      txr->bd_count * TSO_HEADER_SIZE,
1770 					      &txr->tso_headers_dma,
1771 					      GFP_KERNEL);
1772 	if (!txr->tso_headers) {
1773 		err = -ENOMEM;
1774 		goto err_alloc_tso;
1775 	}
1776 
1777 	txr->next_to_clean = 0;
1778 	txr->next_to_use = 0;
1779 
1780 	return 0;
1781 
1782 err_alloc_tso:
1783 	dma_free_coherent(txr->dev, txr->bd_count * sizeof(union enetc_tx_bd),
1784 			  txr->bd_base, txr->bd_dma_base);
1785 	txr->bd_base = NULL;
1786 err_alloc_bdr:
1787 	vfree(txr->tx_swbd);
1788 	txr->tx_swbd = NULL;
1789 
1790 	return err;
1791 }
1792 
1793 static void enetc_free_txbdr(struct enetc_bdr *txr)
1794 {
1795 	int size, i;
1796 
1797 	for (i = 0; i < txr->bd_count; i++)
1798 		enetc_free_tx_frame(txr, &txr->tx_swbd[i]);
1799 
1800 	size = txr->bd_count * sizeof(union enetc_tx_bd);
1801 
1802 	dma_free_coherent(txr->dev, txr->bd_count * TSO_HEADER_SIZE,
1803 			  txr->tso_headers, txr->tso_headers_dma);
1804 	txr->tso_headers = NULL;
1805 
1806 	dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
1807 	txr->bd_base = NULL;
1808 
1809 	vfree(txr->tx_swbd);
1810 	txr->tx_swbd = NULL;
1811 }
1812 
1813 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1814 {
1815 	int i, err;
1816 
1817 	for (i = 0; i < priv->num_tx_rings; i++) {
1818 		err = enetc_alloc_txbdr(priv->tx_ring[i]);
1819 
1820 		if (err)
1821 			goto fail;
1822 	}
1823 
1824 	return 0;
1825 
1826 fail:
1827 	while (i-- > 0)
1828 		enetc_free_txbdr(priv->tx_ring[i]);
1829 
1830 	return err;
1831 }
1832 
1833 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
1834 {
1835 	int i;
1836 
1837 	for (i = 0; i < priv->num_tx_rings; i++)
1838 		enetc_free_txbdr(priv->tx_ring[i]);
1839 }
1840 
1841 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
1842 {
1843 	size_t size = sizeof(union enetc_rx_bd);
1844 	int err;
1845 
1846 	rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
1847 	if (!rxr->rx_swbd)
1848 		return -ENOMEM;
1849 
1850 	if (extended)
1851 		size *= 2;
1852 
1853 	err = enetc_dma_alloc_bdr(rxr, size);
1854 	if (err) {
1855 		vfree(rxr->rx_swbd);
1856 		return err;
1857 	}
1858 
1859 	rxr->next_to_clean = 0;
1860 	rxr->next_to_use = 0;
1861 	rxr->next_to_alloc = 0;
1862 	rxr->ext_en = extended;
1863 
1864 	return 0;
1865 }
1866 
1867 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
1868 {
1869 	int size;
1870 
1871 	size = rxr->bd_count * sizeof(union enetc_rx_bd);
1872 
1873 	dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
1874 	rxr->bd_base = NULL;
1875 
1876 	vfree(rxr->rx_swbd);
1877 	rxr->rx_swbd = NULL;
1878 }
1879 
1880 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
1881 {
1882 	bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
1883 	int i, err;
1884 
1885 	for (i = 0; i < priv->num_rx_rings; i++) {
1886 		err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
1887 
1888 		if (err)
1889 			goto fail;
1890 	}
1891 
1892 	return 0;
1893 
1894 fail:
1895 	while (i-- > 0)
1896 		enetc_free_rxbdr(priv->rx_ring[i]);
1897 
1898 	return err;
1899 }
1900 
1901 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
1902 {
1903 	int i;
1904 
1905 	for (i = 0; i < priv->num_rx_rings; i++)
1906 		enetc_free_rxbdr(priv->rx_ring[i]);
1907 }
1908 
1909 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1910 {
1911 	int i;
1912 
1913 	if (!tx_ring->tx_swbd)
1914 		return;
1915 
1916 	for (i = 0; i < tx_ring->bd_count; i++) {
1917 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1918 
1919 		enetc_free_tx_frame(tx_ring, tx_swbd);
1920 	}
1921 
1922 	tx_ring->next_to_clean = 0;
1923 	tx_ring->next_to_use = 0;
1924 }
1925 
1926 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1927 {
1928 	int i;
1929 
1930 	if (!rx_ring->rx_swbd)
1931 		return;
1932 
1933 	for (i = 0; i < rx_ring->bd_count; i++) {
1934 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1935 
1936 		if (!rx_swbd->page)
1937 			continue;
1938 
1939 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1940 			       rx_swbd->dir);
1941 		__free_page(rx_swbd->page);
1942 		rx_swbd->page = NULL;
1943 	}
1944 
1945 	rx_ring->next_to_clean = 0;
1946 	rx_ring->next_to_use = 0;
1947 	rx_ring->next_to_alloc = 0;
1948 }
1949 
1950 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1951 {
1952 	int i;
1953 
1954 	for (i = 0; i < priv->num_rx_rings; i++)
1955 		enetc_free_rx_ring(priv->rx_ring[i]);
1956 
1957 	for (i = 0; i < priv->num_tx_rings; i++)
1958 		enetc_free_tx_ring(priv->tx_ring[i]);
1959 }
1960 
1961 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1962 {
1963 	int *rss_table;
1964 	int i;
1965 
1966 	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1967 	if (!rss_table)
1968 		return -ENOMEM;
1969 
1970 	/* Set up RSS table defaults */
1971 	for (i = 0; i < si->num_rss; i++)
1972 		rss_table[i] = i % num_groups;
1973 
1974 	enetc_set_rss_table(si, rss_table, si->num_rss);
1975 
1976 	kfree(rss_table);
1977 
1978 	return 0;
1979 }
1980 
1981 int enetc_configure_si(struct enetc_ndev_priv *priv)
1982 {
1983 	struct enetc_si *si = priv->si;
1984 	struct enetc_hw *hw = &si->hw;
1985 	int err;
1986 
1987 	/* set SI cache attributes */
1988 	enetc_wr(hw, ENETC_SICAR0,
1989 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1990 	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1991 	/* enable SI */
1992 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1993 
1994 	if (si->num_rss) {
1995 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1996 		if (err)
1997 			return err;
1998 	}
1999 
2000 	return 0;
2001 }
2002 
2003 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2004 {
2005 	struct enetc_si *si = priv->si;
2006 	int cpus = num_online_cpus();
2007 
2008 	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2009 	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2010 
2011 	/* Enable all available TX rings in order to configure as many
2012 	 * priorities as possible, when needed.
2013 	 * TODO: Make # of TX rings run-time configurable
2014 	 */
2015 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2016 	priv->num_tx_rings = si->num_tx_rings;
2017 	priv->bdr_int_num = cpus;
2018 	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2019 	priv->tx_ictt = ENETC_TXIC_TIMETHR;
2020 }
2021 
2022 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2023 {
2024 	struct enetc_si *si = priv->si;
2025 
2026 	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2027 				  GFP_KERNEL);
2028 	if (!priv->cls_rules)
2029 		return -ENOMEM;
2030 
2031 	return 0;
2032 }
2033 
2034 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2035 {
2036 	kfree(priv->cls_rules);
2037 }
2038 
2039 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2040 {
2041 	int idx = tx_ring->index;
2042 	u32 tbmr;
2043 
2044 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2045 		       lower_32_bits(tx_ring->bd_dma_base));
2046 
2047 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2048 		       upper_32_bits(tx_ring->bd_dma_base));
2049 
2050 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2051 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2052 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
2053 
2054 	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2055 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2056 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2057 
2058 	/* enable Tx ints by setting pkt thr to 1 */
2059 	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2060 
2061 	tbmr = ENETC_TBMR_EN;
2062 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2063 		tbmr |= ENETC_TBMR_VIH;
2064 
2065 	/* enable ring */
2066 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2067 
2068 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2069 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2070 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
2071 }
2072 
2073 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2074 {
2075 	int idx = rx_ring->index;
2076 	u32 rbmr;
2077 
2078 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2079 		       lower_32_bits(rx_ring->bd_dma_base));
2080 
2081 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2082 		       upper_32_bits(rx_ring->bd_dma_base));
2083 
2084 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2085 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2086 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
2087 
2088 	if (rx_ring->xdp.prog)
2089 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2090 	else
2091 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2092 
2093 	/* Also prepare the consumer index in case page allocation never
2094 	 * succeeds. In that case, hardware will never advance producer index
2095 	 * to match consumer index, and will drop all frames.
2096 	 */
2097 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2098 	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2099 
2100 	/* enable Rx ints by setting pkt thr to 1 */
2101 	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2102 
2103 	rbmr = ENETC_RBMR_EN;
2104 
2105 	if (rx_ring->ext_en)
2106 		rbmr |= ENETC_RBMR_BDS;
2107 
2108 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2109 		rbmr |= ENETC_RBMR_VTE;
2110 
2111 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2112 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2113 
2114 	enetc_lock_mdio();
2115 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2116 	enetc_unlock_mdio();
2117 
2118 	/* enable ring */
2119 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2120 }
2121 
2122 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
2123 {
2124 	struct enetc_hw *hw = &priv->si->hw;
2125 	int i;
2126 
2127 	for (i = 0; i < priv->num_tx_rings; i++)
2128 		enetc_setup_txbdr(hw, priv->tx_ring[i]);
2129 
2130 	for (i = 0; i < priv->num_rx_rings; i++)
2131 		enetc_setup_rxbdr(hw, priv->rx_ring[i]);
2132 }
2133 
2134 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2135 {
2136 	int idx = rx_ring->index;
2137 
2138 	/* disable EN bit on ring */
2139 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2140 }
2141 
2142 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2143 {
2144 	int delay = 8, timeout = 100;
2145 	int idx = tx_ring->index;
2146 
2147 	/* disable EN bit on ring */
2148 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2149 
2150 	/* wait for busy to clear */
2151 	while (delay < timeout &&
2152 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2153 		msleep(delay);
2154 		delay *= 2;
2155 	}
2156 
2157 	if (delay >= timeout)
2158 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2159 			    idx);
2160 }
2161 
2162 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
2163 {
2164 	struct enetc_hw *hw = &priv->si->hw;
2165 	int i;
2166 
2167 	for (i = 0; i < priv->num_tx_rings; i++)
2168 		enetc_clear_txbdr(hw, priv->tx_ring[i]);
2169 
2170 	for (i = 0; i < priv->num_rx_rings; i++)
2171 		enetc_clear_rxbdr(hw, priv->rx_ring[i]);
2172 
2173 	udelay(1);
2174 }
2175 
2176 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2177 {
2178 	struct pci_dev *pdev = priv->si->pdev;
2179 	struct enetc_hw *hw = &priv->si->hw;
2180 	int i, j, err;
2181 
2182 	for (i = 0; i < priv->bdr_int_num; i++) {
2183 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2184 		struct enetc_int_vector *v = priv->int_vector[i];
2185 		int entry = ENETC_BDR_INT_BASE_IDX + i;
2186 
2187 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2188 			 priv->ndev->name, i);
2189 		err = request_irq(irq, enetc_msix, 0, v->name, v);
2190 		if (err) {
2191 			dev_err(priv->dev, "request_irq() failed!\n");
2192 			goto irq_err;
2193 		}
2194 		disable_irq(irq);
2195 
2196 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2197 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2198 		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2199 
2200 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2201 
2202 		for (j = 0; j < v->count_tx_rings; j++) {
2203 			int idx = v->tx_ring[j].index;
2204 
2205 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2206 		}
2207 		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2208 	}
2209 
2210 	return 0;
2211 
2212 irq_err:
2213 	while (i--) {
2214 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2215 
2216 		irq_set_affinity_hint(irq, NULL);
2217 		free_irq(irq, priv->int_vector[i]);
2218 	}
2219 
2220 	return err;
2221 }
2222 
2223 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2224 {
2225 	struct pci_dev *pdev = priv->si->pdev;
2226 	int i;
2227 
2228 	for (i = 0; i < priv->bdr_int_num; i++) {
2229 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2230 
2231 		irq_set_affinity_hint(irq, NULL);
2232 		free_irq(irq, priv->int_vector[i]);
2233 	}
2234 }
2235 
2236 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2237 {
2238 	struct enetc_hw *hw = &priv->si->hw;
2239 	u32 icpt, ictt;
2240 	int i;
2241 
2242 	/* enable Tx & Rx event indication */
2243 	if (priv->ic_mode &
2244 	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2245 		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2246 		/* init to non-0 minimum, will be adjusted later */
2247 		ictt = 0x1;
2248 	} else {
2249 		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2250 		ictt = 0;
2251 	}
2252 
2253 	for (i = 0; i < priv->num_rx_rings; i++) {
2254 		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2255 		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2256 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2257 	}
2258 
2259 	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2260 		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2261 	else
2262 		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2263 
2264 	for (i = 0; i < priv->num_tx_rings; i++) {
2265 		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2266 		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2267 		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2268 	}
2269 }
2270 
2271 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2272 {
2273 	struct enetc_hw *hw = &priv->si->hw;
2274 	int i;
2275 
2276 	for (i = 0; i < priv->num_tx_rings; i++)
2277 		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2278 
2279 	for (i = 0; i < priv->num_rx_rings; i++)
2280 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2281 }
2282 
2283 static int enetc_phylink_connect(struct net_device *ndev)
2284 {
2285 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2286 	struct ethtool_eee edata;
2287 	int err;
2288 
2289 	if (!priv->phylink)
2290 		return 0; /* phy-less mode */
2291 
2292 	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2293 	if (err) {
2294 		dev_err(&ndev->dev, "could not attach to PHY\n");
2295 		return err;
2296 	}
2297 
2298 	/* disable EEE autoneg, until ENETC driver supports it */
2299 	memset(&edata, 0, sizeof(struct ethtool_eee));
2300 	phylink_ethtool_set_eee(priv->phylink, &edata);
2301 
2302 	return 0;
2303 }
2304 
2305 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2306 {
2307 	struct enetc_ndev_priv *priv;
2308 	struct sk_buff *skb;
2309 
2310 	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2311 
2312 	netif_tx_lock(priv->ndev);
2313 
2314 	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2315 	skb = skb_dequeue(&priv->tx_skbs);
2316 	if (skb)
2317 		enetc_start_xmit(skb, priv->ndev);
2318 
2319 	netif_tx_unlock(priv->ndev);
2320 }
2321 
2322 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2323 {
2324 	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2325 	skb_queue_head_init(&priv->tx_skbs);
2326 }
2327 
2328 void enetc_start(struct net_device *ndev)
2329 {
2330 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2331 	int i;
2332 
2333 	enetc_setup_interrupts(priv);
2334 
2335 	for (i = 0; i < priv->bdr_int_num; i++) {
2336 		int irq = pci_irq_vector(priv->si->pdev,
2337 					 ENETC_BDR_INT_BASE_IDX + i);
2338 
2339 		napi_enable(&priv->int_vector[i]->napi);
2340 		enable_irq(irq);
2341 	}
2342 
2343 	if (priv->phylink)
2344 		phylink_start(priv->phylink);
2345 	else
2346 		netif_carrier_on(ndev);
2347 
2348 	netif_tx_start_all_queues(ndev);
2349 }
2350 
2351 int enetc_open(struct net_device *ndev)
2352 {
2353 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2354 	int num_stack_tx_queues;
2355 	int err;
2356 
2357 	err = enetc_setup_irqs(priv);
2358 	if (err)
2359 		return err;
2360 
2361 	err = enetc_phylink_connect(ndev);
2362 	if (err)
2363 		goto err_phy_connect;
2364 
2365 	err = enetc_alloc_tx_resources(priv);
2366 	if (err)
2367 		goto err_alloc_tx;
2368 
2369 	err = enetc_alloc_rx_resources(priv);
2370 	if (err)
2371 		goto err_alloc_rx;
2372 
2373 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2374 
2375 	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2376 	if (err)
2377 		goto err_set_queues;
2378 
2379 	err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
2380 	if (err)
2381 		goto err_set_queues;
2382 
2383 	enetc_tx_onestep_tstamp_init(priv);
2384 	enetc_setup_bdrs(priv);
2385 	enetc_start(ndev);
2386 
2387 	return 0;
2388 
2389 err_set_queues:
2390 	enetc_free_rx_resources(priv);
2391 err_alloc_rx:
2392 	enetc_free_tx_resources(priv);
2393 err_alloc_tx:
2394 	if (priv->phylink)
2395 		phylink_disconnect_phy(priv->phylink);
2396 err_phy_connect:
2397 	enetc_free_irqs(priv);
2398 
2399 	return err;
2400 }
2401 
2402 void enetc_stop(struct net_device *ndev)
2403 {
2404 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2405 	int i;
2406 
2407 	netif_tx_stop_all_queues(ndev);
2408 
2409 	for (i = 0; i < priv->bdr_int_num; i++) {
2410 		int irq = pci_irq_vector(priv->si->pdev,
2411 					 ENETC_BDR_INT_BASE_IDX + i);
2412 
2413 		disable_irq(irq);
2414 		napi_synchronize(&priv->int_vector[i]->napi);
2415 		napi_disable(&priv->int_vector[i]->napi);
2416 	}
2417 
2418 	if (priv->phylink)
2419 		phylink_stop(priv->phylink);
2420 	else
2421 		netif_carrier_off(ndev);
2422 
2423 	enetc_clear_interrupts(priv);
2424 }
2425 
2426 int enetc_close(struct net_device *ndev)
2427 {
2428 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2429 
2430 	enetc_stop(ndev);
2431 	enetc_clear_bdrs(priv);
2432 
2433 	if (priv->phylink)
2434 		phylink_disconnect_phy(priv->phylink);
2435 	enetc_free_rxtx_rings(priv);
2436 	enetc_free_rx_resources(priv);
2437 	enetc_free_tx_resources(priv);
2438 	enetc_free_irqs(priv);
2439 
2440 	return 0;
2441 }
2442 
2443 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2444 {
2445 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2446 	struct tc_mqprio_qopt *mqprio = type_data;
2447 	struct enetc_hw *hw = &priv->si->hw;
2448 	struct enetc_bdr *tx_ring;
2449 	int num_stack_tx_queues;
2450 	u8 num_tc;
2451 	int i;
2452 
2453 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2454 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
2455 	num_tc = mqprio->num_tc;
2456 
2457 	if (!num_tc) {
2458 		netdev_reset_tc(ndev);
2459 		netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2460 
2461 		/* Reset all ring priorities to 0 */
2462 		for (i = 0; i < priv->num_tx_rings; i++) {
2463 			tx_ring = priv->tx_ring[i];
2464 			enetc_set_bdr_prio(hw, tx_ring->index, 0);
2465 		}
2466 
2467 		return 0;
2468 	}
2469 
2470 	/* Check if we have enough BD rings available to accommodate all TCs */
2471 	if (num_tc > num_stack_tx_queues) {
2472 		netdev_err(ndev, "Max %d traffic classes supported\n",
2473 			   priv->num_tx_rings);
2474 		return -EINVAL;
2475 	}
2476 
2477 	/* For the moment, we use only one BD ring per TC.
2478 	 *
2479 	 * Configure num_tc BD rings with increasing priorities.
2480 	 */
2481 	for (i = 0; i < num_tc; i++) {
2482 		tx_ring = priv->tx_ring[i];
2483 		enetc_set_bdr_prio(hw, tx_ring->index, i);
2484 	}
2485 
2486 	/* Reset the number of netdev queues based on the TC count */
2487 	netif_set_real_num_tx_queues(ndev, num_tc);
2488 
2489 	netdev_set_num_tc(ndev, num_tc);
2490 
2491 	/* Each TC is associated with one netdev queue */
2492 	for (i = 0; i < num_tc; i++)
2493 		netdev_set_tc_queue(ndev, i, 1, i);
2494 
2495 	return 0;
2496 }
2497 
2498 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog,
2499 				struct netlink_ext_ack *extack)
2500 {
2501 	struct enetc_ndev_priv *priv = netdev_priv(dev);
2502 	struct bpf_prog *old_prog;
2503 	bool is_up;
2504 	int i;
2505 
2506 	/* The buffer layout is changing, so we need to drain the old
2507 	 * RX buffers and seed new ones.
2508 	 */
2509 	is_up = netif_running(dev);
2510 	if (is_up)
2511 		dev_close(dev);
2512 
2513 	old_prog = xchg(&priv->xdp_prog, prog);
2514 	if (old_prog)
2515 		bpf_prog_put(old_prog);
2516 
2517 	for (i = 0; i < priv->num_rx_rings; i++) {
2518 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
2519 
2520 		rx_ring->xdp.prog = prog;
2521 
2522 		if (prog)
2523 			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2524 		else
2525 			rx_ring->buffer_offset = ENETC_RXB_PAD;
2526 	}
2527 
2528 	if (is_up)
2529 		return dev_open(dev, extack);
2530 
2531 	return 0;
2532 }
2533 
2534 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp)
2535 {
2536 	switch (xdp->command) {
2537 	case XDP_SETUP_PROG:
2538 		return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack);
2539 	default:
2540 		return -EINVAL;
2541 	}
2542 
2543 	return 0;
2544 }
2545 
2546 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2547 {
2548 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2549 	struct net_device_stats *stats = &ndev->stats;
2550 	unsigned long packets = 0, bytes = 0;
2551 	unsigned long tx_dropped = 0;
2552 	int i;
2553 
2554 	for (i = 0; i < priv->num_rx_rings; i++) {
2555 		packets += priv->rx_ring[i]->stats.packets;
2556 		bytes	+= priv->rx_ring[i]->stats.bytes;
2557 	}
2558 
2559 	stats->rx_packets = packets;
2560 	stats->rx_bytes = bytes;
2561 	bytes = 0;
2562 	packets = 0;
2563 
2564 	for (i = 0; i < priv->num_tx_rings; i++) {
2565 		packets += priv->tx_ring[i]->stats.packets;
2566 		bytes	+= priv->tx_ring[i]->stats.bytes;
2567 		tx_dropped += priv->tx_ring[i]->stats.win_drop;
2568 	}
2569 
2570 	stats->tx_packets = packets;
2571 	stats->tx_bytes = bytes;
2572 	stats->tx_dropped = tx_dropped;
2573 
2574 	return stats;
2575 }
2576 
2577 static int enetc_set_rss(struct net_device *ndev, int en)
2578 {
2579 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2580 	struct enetc_hw *hw = &priv->si->hw;
2581 	u32 reg;
2582 
2583 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2584 
2585 	reg = enetc_rd(hw, ENETC_SIMR);
2586 	reg &= ~ENETC_SIMR_RSSE;
2587 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
2588 	enetc_wr(hw, ENETC_SIMR, reg);
2589 
2590 	return 0;
2591 }
2592 
2593 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2594 {
2595 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2596 	struct enetc_hw *hw = &priv->si->hw;
2597 	int i;
2598 
2599 	for (i = 0; i < priv->num_rx_rings; i++)
2600 		enetc_bdr_enable_rxvlan(hw, i, en);
2601 }
2602 
2603 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2604 {
2605 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2606 	struct enetc_hw *hw = &priv->si->hw;
2607 	int i;
2608 
2609 	for (i = 0; i < priv->num_tx_rings; i++)
2610 		enetc_bdr_enable_txvlan(hw, i, en);
2611 }
2612 
2613 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2614 {
2615 	netdev_features_t changed = ndev->features ^ features;
2616 
2617 	if (changed & NETIF_F_RXHASH)
2618 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2619 
2620 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2621 		enetc_enable_rxvlan(ndev,
2622 				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2623 
2624 	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2625 		enetc_enable_txvlan(ndev,
2626 				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2627 }
2628 
2629 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2630 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2631 {
2632 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2633 	struct hwtstamp_config config;
2634 	int ao;
2635 
2636 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2637 		return -EFAULT;
2638 
2639 	switch (config.tx_type) {
2640 	case HWTSTAMP_TX_OFF:
2641 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2642 		break;
2643 	case HWTSTAMP_TX_ON:
2644 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2645 		priv->active_offloads |= ENETC_F_TX_TSTAMP;
2646 		break;
2647 	case HWTSTAMP_TX_ONESTEP_SYNC:
2648 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2649 		priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2650 		break;
2651 	default:
2652 		return -ERANGE;
2653 	}
2654 
2655 	ao = priv->active_offloads;
2656 	switch (config.rx_filter) {
2657 	case HWTSTAMP_FILTER_NONE:
2658 		priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
2659 		break;
2660 	default:
2661 		priv->active_offloads |= ENETC_F_RX_TSTAMP;
2662 		config.rx_filter = HWTSTAMP_FILTER_ALL;
2663 	}
2664 
2665 	if (netif_running(ndev) && ao != priv->active_offloads) {
2666 		enetc_close(ndev);
2667 		enetc_open(ndev);
2668 	}
2669 
2670 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2671 	       -EFAULT : 0;
2672 }
2673 
2674 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2675 {
2676 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2677 	struct hwtstamp_config config;
2678 
2679 	config.flags = 0;
2680 
2681 	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2682 		config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2683 	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2684 		config.tx_type = HWTSTAMP_TX_ON;
2685 	else
2686 		config.tx_type = HWTSTAMP_TX_OFF;
2687 
2688 	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2689 			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2690 
2691 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2692 	       -EFAULT : 0;
2693 }
2694 #endif
2695 
2696 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2697 {
2698 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2699 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2700 	if (cmd == SIOCSHWTSTAMP)
2701 		return enetc_hwtstamp_set(ndev, rq);
2702 	if (cmd == SIOCGHWTSTAMP)
2703 		return enetc_hwtstamp_get(ndev, rq);
2704 #endif
2705 
2706 	if (!priv->phylink)
2707 		return -EOPNOTSUPP;
2708 
2709 	return phylink_mii_ioctl(priv->phylink, rq, cmd);
2710 }
2711 
2712 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2713 {
2714 	struct pci_dev *pdev = priv->si->pdev;
2715 	int first_xdp_tx_ring;
2716 	int i, n, err, nvec;
2717 	int v_tx_rings;
2718 
2719 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2720 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
2721 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2722 
2723 	if (n < 0)
2724 		return n;
2725 
2726 	if (n != nvec)
2727 		return -EPERM;
2728 
2729 	/* # of tx rings per int vector */
2730 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2731 
2732 	for (i = 0; i < priv->bdr_int_num; i++) {
2733 		struct enetc_int_vector *v;
2734 		struct enetc_bdr *bdr;
2735 		int j;
2736 
2737 		v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
2738 		if (!v) {
2739 			err = -ENOMEM;
2740 			goto fail;
2741 		}
2742 
2743 		priv->int_vector[i] = v;
2744 
2745 		bdr = &v->rx_ring;
2746 		bdr->index = i;
2747 		bdr->ndev = priv->ndev;
2748 		bdr->dev = priv->dev;
2749 		bdr->bd_count = priv->rx_bd_count;
2750 		bdr->buffer_offset = ENETC_RXB_PAD;
2751 		priv->rx_ring[i] = bdr;
2752 
2753 		err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
2754 		if (err) {
2755 			kfree(v);
2756 			goto fail;
2757 		}
2758 
2759 		err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
2760 						 MEM_TYPE_PAGE_SHARED, NULL);
2761 		if (err) {
2762 			xdp_rxq_info_unreg(&bdr->xdp.rxq);
2763 			kfree(v);
2764 			goto fail;
2765 		}
2766 
2767 		/* init defaults for adaptive IC */
2768 		if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
2769 			v->rx_ictt = 0x1;
2770 			v->rx_dim_en = true;
2771 		}
2772 		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
2773 		netif_napi_add(priv->ndev, &v->napi, enetc_poll);
2774 		v->count_tx_rings = v_tx_rings;
2775 
2776 		for (j = 0; j < v_tx_rings; j++) {
2777 			int idx;
2778 
2779 			/* default tx ring mapping policy */
2780 			idx = priv->bdr_int_num * j + i;
2781 			__set_bit(idx, &v->tx_rings_map);
2782 			bdr = &v->tx_ring[j];
2783 			bdr->index = idx;
2784 			bdr->ndev = priv->ndev;
2785 			bdr->dev = priv->dev;
2786 			bdr->bd_count = priv->tx_bd_count;
2787 			priv->tx_ring[idx] = bdr;
2788 		}
2789 	}
2790 
2791 	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
2792 	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
2793 
2794 	return 0;
2795 
2796 fail:
2797 	while (i--) {
2798 		struct enetc_int_vector *v = priv->int_vector[i];
2799 		struct enetc_bdr *rx_ring = &v->rx_ring;
2800 
2801 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2802 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2803 		netif_napi_del(&v->napi);
2804 		cancel_work_sync(&v->rx_dim.work);
2805 		kfree(v);
2806 	}
2807 
2808 	pci_free_irq_vectors(pdev);
2809 
2810 	return err;
2811 }
2812 
2813 void enetc_free_msix(struct enetc_ndev_priv *priv)
2814 {
2815 	int i;
2816 
2817 	for (i = 0; i < priv->bdr_int_num; i++) {
2818 		struct enetc_int_vector *v = priv->int_vector[i];
2819 		struct enetc_bdr *rx_ring = &v->rx_ring;
2820 
2821 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2822 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2823 		netif_napi_del(&v->napi);
2824 		cancel_work_sync(&v->rx_dim.work);
2825 	}
2826 
2827 	for (i = 0; i < priv->num_rx_rings; i++)
2828 		priv->rx_ring[i] = NULL;
2829 
2830 	for (i = 0; i < priv->num_tx_rings; i++)
2831 		priv->tx_ring[i] = NULL;
2832 
2833 	for (i = 0; i < priv->bdr_int_num; i++) {
2834 		kfree(priv->int_vector[i]);
2835 		priv->int_vector[i] = NULL;
2836 	}
2837 
2838 	/* disable all MSIX for this device */
2839 	pci_free_irq_vectors(priv->si->pdev);
2840 }
2841 
2842 static void enetc_kfree_si(struct enetc_si *si)
2843 {
2844 	char *p = (char *)si - si->pad;
2845 
2846 	kfree(p);
2847 }
2848 
2849 static void enetc_detect_errata(struct enetc_si *si)
2850 {
2851 	if (si->pdev->revision == ENETC_REV1)
2852 		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
2853 }
2854 
2855 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
2856 {
2857 	struct enetc_si *si, *p;
2858 	struct enetc_hw *hw;
2859 	size_t alloc_size;
2860 	int err, len;
2861 
2862 	pcie_flr(pdev);
2863 	err = pci_enable_device_mem(pdev);
2864 	if (err)
2865 		return dev_err_probe(&pdev->dev, err, "device enable failed\n");
2866 
2867 	/* set up for high or low dma */
2868 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2869 	if (err) {
2870 		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
2871 		goto err_dma;
2872 	}
2873 
2874 	err = pci_request_mem_regions(pdev, name);
2875 	if (err) {
2876 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
2877 		goto err_pci_mem_reg;
2878 	}
2879 
2880 	pci_set_master(pdev);
2881 
2882 	alloc_size = sizeof(struct enetc_si);
2883 	if (sizeof_priv) {
2884 		/* align priv to 32B */
2885 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
2886 		alloc_size += sizeof_priv;
2887 	}
2888 	/* force 32B alignment for enetc_si */
2889 	alloc_size += ENETC_SI_ALIGN - 1;
2890 
2891 	p = kzalloc(alloc_size, GFP_KERNEL);
2892 	if (!p) {
2893 		err = -ENOMEM;
2894 		goto err_alloc_si;
2895 	}
2896 
2897 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
2898 	si->pad = (char *)si - (char *)p;
2899 
2900 	pci_set_drvdata(pdev, si);
2901 	si->pdev = pdev;
2902 	hw = &si->hw;
2903 
2904 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
2905 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
2906 	if (!hw->reg) {
2907 		err = -ENXIO;
2908 		dev_err(&pdev->dev, "ioremap() failed\n");
2909 		goto err_ioremap;
2910 	}
2911 	if (len > ENETC_PORT_BASE)
2912 		hw->port = hw->reg + ENETC_PORT_BASE;
2913 	if (len > ENETC_GLOBAL_BASE)
2914 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
2915 
2916 	enetc_detect_errata(si);
2917 
2918 	return 0;
2919 
2920 err_ioremap:
2921 	enetc_kfree_si(si);
2922 err_alloc_si:
2923 	pci_release_mem_regions(pdev);
2924 err_pci_mem_reg:
2925 err_dma:
2926 	pci_disable_device(pdev);
2927 
2928 	return err;
2929 }
2930 
2931 void enetc_pci_remove(struct pci_dev *pdev)
2932 {
2933 	struct enetc_si *si = pci_get_drvdata(pdev);
2934 	struct enetc_hw *hw = &si->hw;
2935 
2936 	iounmap(hw->reg);
2937 	enetc_kfree_si(si);
2938 	pci_release_mem_regions(pdev);
2939 	pci_disable_device(pdev);
2940 }
2941