xref: /openbmc/linux/drivers/net/ethernet/freescale/enetc/enetc.c (revision 278002edb19bce2c628fafb0af936e77000f3a5b)
1d4fd0404SClaudiu Manoil // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2d4fd0404SClaudiu Manoil /* Copyright 2017-2019 NXP */
3d4fd0404SClaudiu Manoil 
4d4fd0404SClaudiu Manoil #include "enetc.h"
5d1b15102SVladimir Oltean #include <linux/bpf_trace.h>
6d4fd0404SClaudiu Manoil #include <linux/tcp.h>
7d4fd0404SClaudiu Manoil #include <linux/udp.h>
8bbcbf2eeSStephen Rothwell #include <linux/vmalloc.h>
97294380cSYangbo Lu #include <linux/ptp_classify.h>
10edce2a93SIoana Ciornei #include <net/ip6_checksum.h>
11847cbfc0SVladimir Oltean #include <net/pkt_sched.h>
12fb8629e2SIoana Ciornei #include <net/tso.h>
13d4fd0404SClaudiu Manoil 
enetc_port_mac_rd(struct enetc_si * si,u32 reg)1412717decSVladimir Oltean u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
1512717decSVladimir Oltean {
1612717decSVladimir Oltean 	return enetc_port_rd(&si->hw, reg);
1712717decSVladimir Oltean }
1812717decSVladimir Oltean EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
1912717decSVladimir Oltean 
enetc_port_mac_wr(struct enetc_si * si,u32 reg,u32 val)2012717decSVladimir Oltean void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
2112717decSVladimir Oltean {
2212717decSVladimir Oltean 	enetc_port_wr(&si->hw, reg, val);
2312717decSVladimir Oltean 	if (si->hw_features & ENETC_SI_F_QBU)
2412717decSVladimir Oltean 		enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val);
2512717decSVladimir Oltean }
2612717decSVladimir Oltean EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
2712717decSVladimir Oltean 
enetc_change_preemptible_tcs(struct enetc_ndev_priv * priv,u8 preemptible_tcs)2882714539SVladimir Oltean static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
2982714539SVladimir Oltean 					 u8 preemptible_tcs)
3001e23b2bSVladimir Oltean {
31*66127f0dSWei Fang 	if (!(priv->si->hw_features & ENETC_SI_F_QBU))
32*66127f0dSWei Fang 		return;
33*66127f0dSWei Fang 
3482714539SVladimir Oltean 	priv->preemptible_tcs = preemptible_tcs;
3582714539SVladimir Oltean 	enetc_mm_commit_preemptible_tcs(priv);
3601e23b2bSVladimir Oltean }
3701e23b2bSVladimir Oltean 
enetc_num_stack_tx_queues(struct enetc_ndev_priv * priv)387eab503bSVladimir Oltean static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
397eab503bSVladimir Oltean {
407eab503bSVladimir Oltean 	int num_tx_rings = priv->num_tx_rings;
417eab503bSVladimir Oltean 
421c81a9b3SVladimir Oltean 	if (priv->xdp_prog)
437eab503bSVladimir Oltean 		return num_tx_rings - num_possible_cpus();
447eab503bSVladimir Oltean 
457eab503bSVladimir Oltean 	return num_tx_rings;
467eab503bSVladimir Oltean }
477eab503bSVladimir Oltean 
enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv * priv,struct enetc_bdr * tx_ring)487eab503bSVladimir Oltean static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
497eab503bSVladimir Oltean 							struct enetc_bdr *tx_ring)
507eab503bSVladimir Oltean {
517eab503bSVladimir Oltean 	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
527eab503bSVladimir Oltean 
537eab503bSVladimir Oltean 	return priv->rx_ring[index];
547eab503bSVladimir Oltean }
557eab503bSVladimir Oltean 
enetc_tx_swbd_get_skb(struct enetc_tx_swbd * tx_swbd)569d2b68ccSVladimir Oltean static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
579d2b68ccSVladimir Oltean {
589d2b68ccSVladimir Oltean 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
599d2b68ccSVladimir Oltean 		return NULL;
609d2b68ccSVladimir Oltean 
619d2b68ccSVladimir Oltean 	return tx_swbd->skb;
629d2b68ccSVladimir Oltean }
639d2b68ccSVladimir Oltean 
649d2b68ccSVladimir Oltean static struct xdp_frame *
enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd * tx_swbd)659d2b68ccSVladimir Oltean enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
669d2b68ccSVladimir Oltean {
679d2b68ccSVladimir Oltean 	if (tx_swbd->is_xdp_redirect)
689d2b68ccSVladimir Oltean 		return tx_swbd->xdp_frame;
699d2b68ccSVladimir Oltean 
709d2b68ccSVladimir Oltean 	return NULL;
719d2b68ccSVladimir Oltean }
729d2b68ccSVladimir Oltean 
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)73d4fd0404SClaudiu Manoil static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
74d4fd0404SClaudiu Manoil 				struct enetc_tx_swbd *tx_swbd)
75d4fd0404SClaudiu Manoil {
767ed2bc80SVladimir Oltean 	/* For XDP_TX, pages come from RX, whereas for the other contexts where
777ed2bc80SVladimir Oltean 	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
787ed2bc80SVladimir Oltean 	 * to match the DMA mapping length, so we need to differentiate those.
797ed2bc80SVladimir Oltean 	 */
80d4fd0404SClaudiu Manoil 	if (tx_swbd->is_dma_page)
81d4fd0404SClaudiu Manoil 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
827ed2bc80SVladimir Oltean 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
837ed2bc80SVladimir Oltean 			       tx_swbd->dir);
84d4fd0404SClaudiu Manoil 	else
85d4fd0404SClaudiu Manoil 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
867ed2bc80SVladimir Oltean 				 tx_swbd->len, tx_swbd->dir);
87d4fd0404SClaudiu Manoil 	tx_swbd->dma = 0;
88d4fd0404SClaudiu Manoil }
89d4fd0404SClaudiu Manoil 
enetc_free_tx_frame(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)909d2b68ccSVladimir Oltean static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
91d4fd0404SClaudiu Manoil 				struct enetc_tx_swbd *tx_swbd)
92d4fd0404SClaudiu Manoil {
939d2b68ccSVladimir Oltean 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
949d2b68ccSVladimir Oltean 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
959d2b68ccSVladimir Oltean 
96d4fd0404SClaudiu Manoil 	if (tx_swbd->dma)
97d4fd0404SClaudiu Manoil 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
98d4fd0404SClaudiu Manoil 
999d2b68ccSVladimir Oltean 	if (xdp_frame) {
1009d2b68ccSVladimir Oltean 		xdp_return_frame(tx_swbd->xdp_frame);
1019d2b68ccSVladimir Oltean 		tx_swbd->xdp_frame = NULL;
1029d2b68ccSVladimir Oltean 	} else if (skb) {
1039d2b68ccSVladimir Oltean 		dev_kfree_skb_any(skb);
104d4fd0404SClaudiu Manoil 		tx_swbd->skb = NULL;
105d4fd0404SClaudiu Manoil 	}
106d4fd0404SClaudiu Manoil }
107d4fd0404SClaudiu Manoil 
1087ed2bc80SVladimir Oltean /* Let H/W know BD ring has been updated */
enetc_update_tx_ring_tail(struct enetc_bdr * tx_ring)1097ed2bc80SVladimir Oltean static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
1107ed2bc80SVladimir Oltean {
1117ed2bc80SVladimir Oltean 	/* includes wmb() */
1127ed2bc80SVladimir Oltean 	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
1137ed2bc80SVladimir Oltean }
1147ed2bc80SVladimir Oltean 
enetc_ptp_parse(struct sk_buff * skb,u8 * udp,u8 * msgtype,u8 * twostep,u16 * correction_offset,u16 * body_offset)1157294380cSYangbo Lu static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
1167294380cSYangbo Lu 			   u8 *msgtype, u8 *twostep,
1177294380cSYangbo Lu 			   u16 *correction_offset, u16 *body_offset)
1187294380cSYangbo Lu {
1197294380cSYangbo Lu 	unsigned int ptp_class;
1207294380cSYangbo Lu 	struct ptp_header *hdr;
1217294380cSYangbo Lu 	unsigned int type;
1227294380cSYangbo Lu 	u8 *base;
1237294380cSYangbo Lu 
1247294380cSYangbo Lu 	ptp_class = ptp_classify_raw(skb);
1257294380cSYangbo Lu 	if (ptp_class == PTP_CLASS_NONE)
1267294380cSYangbo Lu 		return -EINVAL;
1277294380cSYangbo Lu 
1287294380cSYangbo Lu 	hdr = ptp_parse_header(skb, ptp_class);
1297294380cSYangbo Lu 	if (!hdr)
1307294380cSYangbo Lu 		return -EINVAL;
1317294380cSYangbo Lu 
1327294380cSYangbo Lu 	type = ptp_class & PTP_CLASS_PMASK;
1337294380cSYangbo Lu 	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
1347294380cSYangbo Lu 		*udp = 1;
1357294380cSYangbo Lu 	else
1367294380cSYangbo Lu 		*udp = 0;
1377294380cSYangbo Lu 
1387294380cSYangbo Lu 	*msgtype = ptp_get_msgtype(hdr, ptp_class);
1397294380cSYangbo Lu 	*twostep = hdr->flag_field[0] & 0x2;
1407294380cSYangbo Lu 
1417294380cSYangbo Lu 	base = skb_mac_header(skb);
1427294380cSYangbo Lu 	*correction_offset = (u8 *)&hdr->correction - base;
1437294380cSYangbo Lu 	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
1447294380cSYangbo Lu 
1457294380cSYangbo Lu 	return 0;
1467294380cSYangbo Lu }
1477294380cSYangbo Lu 
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)148f768e751SYangbo Lu static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
149d4fd0404SClaudiu Manoil {
1507294380cSYangbo Lu 	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
1517294380cSYangbo Lu 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
1527294380cSYangbo Lu 	struct enetc_hw *hw = &priv->si->hw;
153d4fd0404SClaudiu Manoil 	struct enetc_tx_swbd *tx_swbd;
154d4fd0404SClaudiu Manoil 	int len = skb_headlen(skb);
155d4fd0404SClaudiu Manoil 	union enetc_tx_bd temp_bd;
1567294380cSYangbo Lu 	u8 msgtype, twostep, udp;
157d4fd0404SClaudiu Manoil 	union enetc_tx_bd *txbd;
1587294380cSYangbo Lu 	u16 offset1, offset2;
159d4fd0404SClaudiu Manoil 	int i, count = 0;
1607294380cSYangbo Lu 	skb_frag_t *frag;
161d4fd0404SClaudiu Manoil 	unsigned int f;
162d4fd0404SClaudiu Manoil 	dma_addr_t dma;
163d4fd0404SClaudiu Manoil 	u8 flags = 0;
164d4fd0404SClaudiu Manoil 
165d4fd0404SClaudiu Manoil 	i = tx_ring->next_to_use;
166d4fd0404SClaudiu Manoil 	txbd = ENETC_TXBD(*tx_ring, i);
167d4fd0404SClaudiu Manoil 	prefetchw(txbd);
168d4fd0404SClaudiu Manoil 
169d4fd0404SClaudiu Manoil 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
170d4fd0404SClaudiu Manoil 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
171d4fd0404SClaudiu Manoil 		goto dma_err;
172d4fd0404SClaudiu Manoil 
173d4fd0404SClaudiu Manoil 	temp_bd.addr = cpu_to_le64(dma);
174d4fd0404SClaudiu Manoil 	temp_bd.buf_len = cpu_to_le16(len);
175d4fd0404SClaudiu Manoil 	temp_bd.lstatus = 0;
176d4fd0404SClaudiu Manoil 
177d4fd0404SClaudiu Manoil 	tx_swbd = &tx_ring->tx_swbd[i];
178d4fd0404SClaudiu Manoil 	tx_swbd->dma = dma;
179d4fd0404SClaudiu Manoil 	tx_swbd->len = len;
180d4fd0404SClaudiu Manoil 	tx_swbd->is_dma_page = 0;
1817ed2bc80SVladimir Oltean 	tx_swbd->dir = DMA_TO_DEVICE;
182d4fd0404SClaudiu Manoil 	count++;
183d4fd0404SClaudiu Manoil 
184d4fd0404SClaudiu Manoil 	do_vlan = skb_vlan_tag_present(skb);
1857294380cSYangbo Lu 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
1867294380cSYangbo Lu 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
1877294380cSYangbo Lu 				    &offset2) ||
1887294380cSYangbo Lu 		    msgtype != PTP_MSGTYPE_SYNC || twostep)
1897294380cSYangbo Lu 			WARN_ONCE(1, "Bad packet for one-step timestamping\n");
1907294380cSYangbo Lu 		else
1917294380cSYangbo Lu 			do_onestep_tstamp = true;
1927294380cSYangbo Lu 	} else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
1937294380cSYangbo Lu 		do_twostep_tstamp = true;
1947294380cSYangbo Lu 	}
195d4fd0404SClaudiu Manoil 
1967294380cSYangbo Lu 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
197285e8dedSPo Liu 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
198285e8dedSPo Liu 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
1997294380cSYangbo Lu 
2007294380cSYangbo Lu 	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
201d4fd0404SClaudiu Manoil 		flags |= ENETC_TXBD_FLAGS_EX;
202d4fd0404SClaudiu Manoil 
20382728b91SClaudiu Manoil 	if (tx_ring->tsd_enable)
2040d08c9ecSPo Liu 		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
205d4fd0404SClaudiu Manoil 
206d4fd0404SClaudiu Manoil 	/* first BD needs frm_len and offload flags set */
207d4fd0404SClaudiu Manoil 	temp_bd.frm_len = cpu_to_le16(skb->len);
208d4fd0404SClaudiu Manoil 	temp_bd.flags = flags;
209d4fd0404SClaudiu Manoil 
21082728b91SClaudiu Manoil 	if (flags & ENETC_TXBD_FLAGS_TSE)
21182728b91SClaudiu Manoil 		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
21282728b91SClaudiu Manoil 							  flags);
2130d08c9ecSPo Liu 
214d4fd0404SClaudiu Manoil 	if (flags & ENETC_TXBD_FLAGS_EX) {
215d4fd0404SClaudiu Manoil 		u8 e_flags = 0;
216d4fd0404SClaudiu Manoil 		*txbd = temp_bd;
217d4fd0404SClaudiu Manoil 		enetc_clear_tx_bd(&temp_bd);
218d4fd0404SClaudiu Manoil 
219d4fd0404SClaudiu Manoil 		/* add extension BD for VLAN and/or timestamping */
220d4fd0404SClaudiu Manoil 		flags = 0;
221d4fd0404SClaudiu Manoil 		tx_swbd++;
222d4fd0404SClaudiu Manoil 		txbd++;
223d4fd0404SClaudiu Manoil 		i++;
224d4fd0404SClaudiu Manoil 		if (unlikely(i == tx_ring->bd_count)) {
225d4fd0404SClaudiu Manoil 			i = 0;
226d4fd0404SClaudiu Manoil 			tx_swbd = tx_ring->tx_swbd;
227d4fd0404SClaudiu Manoil 			txbd = ENETC_TXBD(*tx_ring, 0);
228d4fd0404SClaudiu Manoil 		}
229d4fd0404SClaudiu Manoil 		prefetchw(txbd);
230d4fd0404SClaudiu Manoil 
231d4fd0404SClaudiu Manoil 		if (do_vlan) {
232d4fd0404SClaudiu Manoil 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
233d4fd0404SClaudiu Manoil 			temp_bd.ext.tpid = 0; /* < C-TAG */
234d4fd0404SClaudiu Manoil 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
235d4fd0404SClaudiu Manoil 		}
236d4fd0404SClaudiu Manoil 
2377294380cSYangbo Lu 		if (do_onestep_tstamp) {
2387294380cSYangbo Lu 			u32 lo, hi, val;
2397294380cSYangbo Lu 			u64 sec, nsec;
2407294380cSYangbo Lu 			u8 *data;
2417294380cSYangbo Lu 
2427294380cSYangbo Lu 			lo = enetc_rd_hot(hw, ENETC_SICTR0);
2437294380cSYangbo Lu 			hi = enetc_rd_hot(hw, ENETC_SICTR1);
2447294380cSYangbo Lu 			sec = (u64)hi << 32 | lo;
2457294380cSYangbo Lu 			nsec = do_div(sec, 1000000000);
2467294380cSYangbo Lu 
2477294380cSYangbo Lu 			/* Configure extension BD */
2487294380cSYangbo Lu 			temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
2497294380cSYangbo Lu 			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
2507294380cSYangbo Lu 
2517294380cSYangbo Lu 			/* Update originTimestamp field of Sync packet
2527294380cSYangbo Lu 			 * - 48 bits seconds field
2537294380cSYangbo Lu 			 * - 32 bits nanseconds field
2547294380cSYangbo Lu 			 */
2557294380cSYangbo Lu 			data = skb_mac_header(skb);
2567294380cSYangbo Lu 			*(__be16 *)(data + offset2) =
2577294380cSYangbo Lu 				htons((sec >> 32) & 0xffff);
2587294380cSYangbo Lu 			*(__be32 *)(data + offset2 + 2) =
2597294380cSYangbo Lu 				htonl(sec & 0xffffffff);
2607294380cSYangbo Lu 			*(__be32 *)(data + offset2 + 6) = htonl(nsec);
2617294380cSYangbo Lu 
2627294380cSYangbo Lu 			/* Configure single-step register */
2637294380cSYangbo Lu 			val = ENETC_PM0_SINGLE_STEP_EN;
2647294380cSYangbo Lu 			val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
2657294380cSYangbo Lu 			if (udp)
2667294380cSYangbo Lu 				val |= ENETC_PM0_SINGLE_STEP_CH;
2677294380cSYangbo Lu 
26812717decSVladimir Oltean 			enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
26912717decSVladimir Oltean 					  val);
2707294380cSYangbo Lu 		} else if (do_twostep_tstamp) {
271d4fd0404SClaudiu Manoil 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
272d4fd0404SClaudiu Manoil 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
273d4fd0404SClaudiu Manoil 		}
274d4fd0404SClaudiu Manoil 
275d4fd0404SClaudiu Manoil 		temp_bd.ext.e_flags = e_flags;
276d4fd0404SClaudiu Manoil 		count++;
277d4fd0404SClaudiu Manoil 	}
278d4fd0404SClaudiu Manoil 
279d4fd0404SClaudiu Manoil 	frag = &skb_shinfo(skb)->frags[0];
280d4fd0404SClaudiu Manoil 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
281d4fd0404SClaudiu Manoil 		len = skb_frag_size(frag);
282d4fd0404SClaudiu Manoil 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
283d4fd0404SClaudiu Manoil 				       DMA_TO_DEVICE);
284d4fd0404SClaudiu Manoil 		if (dma_mapping_error(tx_ring->dev, dma))
285d4fd0404SClaudiu Manoil 			goto dma_err;
286d4fd0404SClaudiu Manoil 
287d4fd0404SClaudiu Manoil 		*txbd = temp_bd;
288d4fd0404SClaudiu Manoil 		enetc_clear_tx_bd(&temp_bd);
289d4fd0404SClaudiu Manoil 
290d4fd0404SClaudiu Manoil 		flags = 0;
291d4fd0404SClaudiu Manoil 		tx_swbd++;
292d4fd0404SClaudiu Manoil 		txbd++;
293d4fd0404SClaudiu Manoil 		i++;
294d4fd0404SClaudiu Manoil 		if (unlikely(i == tx_ring->bd_count)) {
295d4fd0404SClaudiu Manoil 			i = 0;
296d4fd0404SClaudiu Manoil 			tx_swbd = tx_ring->tx_swbd;
297d4fd0404SClaudiu Manoil 			txbd = ENETC_TXBD(*tx_ring, 0);
298d4fd0404SClaudiu Manoil 		}
299d4fd0404SClaudiu Manoil 		prefetchw(txbd);
300d4fd0404SClaudiu Manoil 
301d4fd0404SClaudiu Manoil 		temp_bd.addr = cpu_to_le64(dma);
302d4fd0404SClaudiu Manoil 		temp_bd.buf_len = cpu_to_le16(len);
303d4fd0404SClaudiu Manoil 
304d4fd0404SClaudiu Manoil 		tx_swbd->dma = dma;
305d4fd0404SClaudiu Manoil 		tx_swbd->len = len;
306d4fd0404SClaudiu Manoil 		tx_swbd->is_dma_page = 1;
3077ed2bc80SVladimir Oltean 		tx_swbd->dir = DMA_TO_DEVICE;
308d4fd0404SClaudiu Manoil 		count++;
309d4fd0404SClaudiu Manoil 	}
310d4fd0404SClaudiu Manoil 
311d4fd0404SClaudiu Manoil 	/* last BD needs 'F' bit set */
312d4fd0404SClaudiu Manoil 	flags |= ENETC_TXBD_FLAGS_F;
313d4fd0404SClaudiu Manoil 	temp_bd.flags = flags;
314d4fd0404SClaudiu Manoil 	*txbd = temp_bd;
315d4fd0404SClaudiu Manoil 
316d504498dSVladimir Oltean 	tx_ring->tx_swbd[i].is_eof = true;
317d4fd0404SClaudiu Manoil 	tx_ring->tx_swbd[i].skb = skb;
318d4fd0404SClaudiu Manoil 
319d4fd0404SClaudiu Manoil 	enetc_bdr_idx_inc(tx_ring, &i);
320d4fd0404SClaudiu Manoil 	tx_ring->next_to_use = i;
321d4fd0404SClaudiu Manoil 
3224caefbceSMichael Walle 	skb_tx_timestamp(skb);
3234caefbceSMichael Walle 
3247ed2bc80SVladimir Oltean 	enetc_update_tx_ring_tail(tx_ring);
325d4fd0404SClaudiu Manoil 
326d4fd0404SClaudiu Manoil 	return count;
327d4fd0404SClaudiu Manoil 
328d4fd0404SClaudiu Manoil dma_err:
329d4fd0404SClaudiu Manoil 	dev_err(tx_ring->dev, "DMA map error");
330d4fd0404SClaudiu Manoil 
331d4fd0404SClaudiu Manoil 	do {
332d4fd0404SClaudiu Manoil 		tx_swbd = &tx_ring->tx_swbd[i];
3339d2b68ccSVladimir Oltean 		enetc_free_tx_frame(tx_ring, tx_swbd);
334d4fd0404SClaudiu Manoil 		if (i == 0)
335d4fd0404SClaudiu Manoil 			i = tx_ring->bd_count;
336d4fd0404SClaudiu Manoil 		i--;
337d4fd0404SClaudiu Manoil 	} while (count--);
338d4fd0404SClaudiu Manoil 
339d4fd0404SClaudiu Manoil 	return 0;
340d4fd0404SClaudiu Manoil }
341d4fd0404SClaudiu Manoil 
enetc_map_tx_tso_hdr(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,int * i,int hdr_len,int data_len)342fb8629e2SIoana Ciornei static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
343fb8629e2SIoana Ciornei 				 struct enetc_tx_swbd *tx_swbd,
344fb8629e2SIoana Ciornei 				 union enetc_tx_bd *txbd, int *i, int hdr_len,
345fb8629e2SIoana Ciornei 				 int data_len)
346fb8629e2SIoana Ciornei {
347fb8629e2SIoana Ciornei 	union enetc_tx_bd txbd_tmp;
348fb8629e2SIoana Ciornei 	u8 flags = 0, e_flags = 0;
349fb8629e2SIoana Ciornei 	dma_addr_t addr;
350fb8629e2SIoana Ciornei 
351fb8629e2SIoana Ciornei 	enetc_clear_tx_bd(&txbd_tmp);
352fb8629e2SIoana Ciornei 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
353fb8629e2SIoana Ciornei 
354fb8629e2SIoana Ciornei 	if (skb_vlan_tag_present(skb))
355fb8629e2SIoana Ciornei 		flags |= ENETC_TXBD_FLAGS_EX;
356fb8629e2SIoana Ciornei 
357fb8629e2SIoana Ciornei 	txbd_tmp.addr = cpu_to_le64(addr);
358fb8629e2SIoana Ciornei 	txbd_tmp.buf_len = cpu_to_le16(hdr_len);
359fb8629e2SIoana Ciornei 
360fb8629e2SIoana Ciornei 	/* first BD needs frm_len and offload flags set */
361fb8629e2SIoana Ciornei 	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
362fb8629e2SIoana Ciornei 	txbd_tmp.flags = flags;
363fb8629e2SIoana Ciornei 
364fb8629e2SIoana Ciornei 	/* For the TSO header we do not set the dma address since we do not
365fb8629e2SIoana Ciornei 	 * want it unmapped when we do cleanup. We still set len so that we
366fb8629e2SIoana Ciornei 	 * count the bytes sent.
367fb8629e2SIoana Ciornei 	 */
368fb8629e2SIoana Ciornei 	tx_swbd->len = hdr_len;
369fb8629e2SIoana Ciornei 	tx_swbd->do_twostep_tstamp = false;
370fb8629e2SIoana Ciornei 	tx_swbd->check_wb = false;
371fb8629e2SIoana Ciornei 
372fb8629e2SIoana Ciornei 	/* Actually write the header in the BD */
373fb8629e2SIoana Ciornei 	*txbd = txbd_tmp;
374fb8629e2SIoana Ciornei 
375fb8629e2SIoana Ciornei 	/* Add extension BD for VLAN */
376fb8629e2SIoana Ciornei 	if (flags & ENETC_TXBD_FLAGS_EX) {
377fb8629e2SIoana Ciornei 		/* Get the next BD */
378fb8629e2SIoana Ciornei 		enetc_bdr_idx_inc(tx_ring, i);
379fb8629e2SIoana Ciornei 		txbd = ENETC_TXBD(*tx_ring, *i);
380fb8629e2SIoana Ciornei 		tx_swbd = &tx_ring->tx_swbd[*i];
381fb8629e2SIoana Ciornei 		prefetchw(txbd);
382fb8629e2SIoana Ciornei 
383fb8629e2SIoana Ciornei 		/* Setup the VLAN fields */
384fb8629e2SIoana Ciornei 		enetc_clear_tx_bd(&txbd_tmp);
385fb8629e2SIoana Ciornei 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
386fb8629e2SIoana Ciornei 		txbd_tmp.ext.tpid = 0; /* < C-TAG */
387fb8629e2SIoana Ciornei 		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
388fb8629e2SIoana Ciornei 
389fb8629e2SIoana Ciornei 		/* Write the BD */
390fb8629e2SIoana Ciornei 		txbd_tmp.ext.e_flags = e_flags;
391fb8629e2SIoana Ciornei 		*txbd = txbd_tmp;
392fb8629e2SIoana Ciornei 	}
393fb8629e2SIoana Ciornei }
394fb8629e2SIoana Ciornei 
enetc_map_tx_tso_data(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,char * data,int size,bool last_bd)395fb8629e2SIoana Ciornei static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
396fb8629e2SIoana Ciornei 				 struct enetc_tx_swbd *tx_swbd,
397fb8629e2SIoana Ciornei 				 union enetc_tx_bd *txbd, char *data,
398fb8629e2SIoana Ciornei 				 int size, bool last_bd)
399fb8629e2SIoana Ciornei {
400fb8629e2SIoana Ciornei 	union enetc_tx_bd txbd_tmp;
401fb8629e2SIoana Ciornei 	dma_addr_t addr;
402fb8629e2SIoana Ciornei 	u8 flags = 0;
403fb8629e2SIoana Ciornei 
404fb8629e2SIoana Ciornei 	enetc_clear_tx_bd(&txbd_tmp);
405fb8629e2SIoana Ciornei 
406fb8629e2SIoana Ciornei 	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
407fb8629e2SIoana Ciornei 	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
408fb8629e2SIoana Ciornei 		netdev_err(tx_ring->ndev, "DMA map error\n");
409fb8629e2SIoana Ciornei 		return -ENOMEM;
410fb8629e2SIoana Ciornei 	}
411fb8629e2SIoana Ciornei 
412fb8629e2SIoana Ciornei 	if (last_bd) {
413fb8629e2SIoana Ciornei 		flags |= ENETC_TXBD_FLAGS_F;
414fb8629e2SIoana Ciornei 		tx_swbd->is_eof = 1;
415fb8629e2SIoana Ciornei 	}
416fb8629e2SIoana Ciornei 
417fb8629e2SIoana Ciornei 	txbd_tmp.addr = cpu_to_le64(addr);
418fb8629e2SIoana Ciornei 	txbd_tmp.buf_len = cpu_to_le16(size);
419fb8629e2SIoana Ciornei 	txbd_tmp.flags = flags;
420fb8629e2SIoana Ciornei 
421fb8629e2SIoana Ciornei 	tx_swbd->dma = addr;
422fb8629e2SIoana Ciornei 	tx_swbd->len = size;
423fb8629e2SIoana Ciornei 	tx_swbd->dir = DMA_TO_DEVICE;
424fb8629e2SIoana Ciornei 
425fb8629e2SIoana Ciornei 	*txbd = txbd_tmp;
426fb8629e2SIoana Ciornei 
427fb8629e2SIoana Ciornei 	return 0;
428fb8629e2SIoana Ciornei }
429fb8629e2SIoana Ciornei 
enetc_tso_hdr_csum(struct tso_t * tso,struct sk_buff * skb,char * hdr,int hdr_len,int * l4_hdr_len)430fb8629e2SIoana Ciornei static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
431fb8629e2SIoana Ciornei 				 char *hdr, int hdr_len, int *l4_hdr_len)
432fb8629e2SIoana Ciornei {
433fb8629e2SIoana Ciornei 	char *l4_hdr = hdr + skb_transport_offset(skb);
434fb8629e2SIoana Ciornei 	int mac_hdr_len = skb_network_offset(skb);
435fb8629e2SIoana Ciornei 
436fb8629e2SIoana Ciornei 	if (tso->tlen != sizeof(struct udphdr)) {
437fb8629e2SIoana Ciornei 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
438fb8629e2SIoana Ciornei 
439fb8629e2SIoana Ciornei 		tcph->check = 0;
440fb8629e2SIoana Ciornei 	} else {
441fb8629e2SIoana Ciornei 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
442fb8629e2SIoana Ciornei 
443fb8629e2SIoana Ciornei 		udph->check = 0;
444fb8629e2SIoana Ciornei 	}
445fb8629e2SIoana Ciornei 
446fb8629e2SIoana Ciornei 	/* Compute the IP checksum. This is necessary since tso_build_hdr()
447fb8629e2SIoana Ciornei 	 * already incremented the IP ID field.
448fb8629e2SIoana Ciornei 	 */
449fb8629e2SIoana Ciornei 	if (!tso->ipv6) {
450fb8629e2SIoana Ciornei 		struct iphdr *iph = (void *)(hdr + mac_hdr_len);
451fb8629e2SIoana Ciornei 
452fb8629e2SIoana Ciornei 		iph->check = 0;
453fb8629e2SIoana Ciornei 		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
454fb8629e2SIoana Ciornei 	}
455fb8629e2SIoana Ciornei 
456fb8629e2SIoana Ciornei 	/* Compute the checksum over the L4 header. */
457fb8629e2SIoana Ciornei 	*l4_hdr_len = hdr_len - skb_transport_offset(skb);
458fb8629e2SIoana Ciornei 	return csum_partial(l4_hdr, *l4_hdr_len, 0);
459fb8629e2SIoana Ciornei }
460fb8629e2SIoana Ciornei 
enetc_tso_complete_csum(struct enetc_bdr * tx_ring,struct tso_t * tso,struct sk_buff * skb,char * hdr,int len,__wsum sum)461fb8629e2SIoana Ciornei static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
462fb8629e2SIoana Ciornei 				    struct sk_buff *skb, char *hdr, int len,
463fb8629e2SIoana Ciornei 				    __wsum sum)
464fb8629e2SIoana Ciornei {
465fb8629e2SIoana Ciornei 	char *l4_hdr = hdr + skb_transport_offset(skb);
466fb8629e2SIoana Ciornei 	__sum16 csum_final;
467fb8629e2SIoana Ciornei 
468fb8629e2SIoana Ciornei 	/* Complete the L4 checksum by appending the pseudo-header to the
469fb8629e2SIoana Ciornei 	 * already computed checksum.
470fb8629e2SIoana Ciornei 	 */
471fb8629e2SIoana Ciornei 	if (!tso->ipv6)
472fb8629e2SIoana Ciornei 		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
473fb8629e2SIoana Ciornei 					       ip_hdr(skb)->daddr,
474fb8629e2SIoana Ciornei 					       len, ip_hdr(skb)->protocol, sum);
475fb8629e2SIoana Ciornei 	else
476fb8629e2SIoana Ciornei 		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
477fb8629e2SIoana Ciornei 					     &ipv6_hdr(skb)->daddr,
478fb8629e2SIoana Ciornei 					     len, ipv6_hdr(skb)->nexthdr, sum);
479fb8629e2SIoana Ciornei 
480fb8629e2SIoana Ciornei 	if (tso->tlen != sizeof(struct udphdr)) {
481fb8629e2SIoana Ciornei 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
482fb8629e2SIoana Ciornei 
483fb8629e2SIoana Ciornei 		tcph->check = csum_final;
484fb8629e2SIoana Ciornei 	} else {
485fb8629e2SIoana Ciornei 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
486fb8629e2SIoana Ciornei 
487fb8629e2SIoana Ciornei 		udph->check = csum_final;
488fb8629e2SIoana Ciornei 	}
489fb8629e2SIoana Ciornei }
490fb8629e2SIoana Ciornei 
enetc_map_tx_tso_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)491fb8629e2SIoana Ciornei static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
492fb8629e2SIoana Ciornei {
493fb8629e2SIoana Ciornei 	int hdr_len, total_len, data_len;
494fb8629e2SIoana Ciornei 	struct enetc_tx_swbd *tx_swbd;
495fb8629e2SIoana Ciornei 	union enetc_tx_bd *txbd;
496fb8629e2SIoana Ciornei 	struct tso_t tso;
497fb8629e2SIoana Ciornei 	__wsum csum, csum2;
498fb8629e2SIoana Ciornei 	int count = 0, pos;
499fb8629e2SIoana Ciornei 	int err, i, bd_data_num;
500fb8629e2SIoana Ciornei 
501fb8629e2SIoana Ciornei 	/* Initialize the TSO handler, and prepare the first payload */
502fb8629e2SIoana Ciornei 	hdr_len = tso_start(skb, &tso);
503fb8629e2SIoana Ciornei 	total_len = skb->len - hdr_len;
504fb8629e2SIoana Ciornei 	i = tx_ring->next_to_use;
505fb8629e2SIoana Ciornei 
506fb8629e2SIoana Ciornei 	while (total_len > 0) {
507fb8629e2SIoana Ciornei 		char *hdr;
508fb8629e2SIoana Ciornei 
509fb8629e2SIoana Ciornei 		/* Get the BD */
510fb8629e2SIoana Ciornei 		txbd = ENETC_TXBD(*tx_ring, i);
511fb8629e2SIoana Ciornei 		tx_swbd = &tx_ring->tx_swbd[i];
512fb8629e2SIoana Ciornei 		prefetchw(txbd);
513fb8629e2SIoana Ciornei 
514fb8629e2SIoana Ciornei 		/* Determine the length of this packet */
515fb8629e2SIoana Ciornei 		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
516fb8629e2SIoana Ciornei 		total_len -= data_len;
517fb8629e2SIoana Ciornei 
518fb8629e2SIoana Ciornei 		/* prepare packet headers: MAC + IP + TCP */
519fb8629e2SIoana Ciornei 		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
520fb8629e2SIoana Ciornei 		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
521fb8629e2SIoana Ciornei 
522fb8629e2SIoana Ciornei 		/* compute the csum over the L4 header */
523fb8629e2SIoana Ciornei 		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
524fb8629e2SIoana Ciornei 		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
525fb8629e2SIoana Ciornei 		bd_data_num = 0;
526fb8629e2SIoana Ciornei 		count++;
527fb8629e2SIoana Ciornei 
528fb8629e2SIoana Ciornei 		while (data_len > 0) {
529fb8629e2SIoana Ciornei 			int size;
530fb8629e2SIoana Ciornei 
531fb8629e2SIoana Ciornei 			size = min_t(int, tso.size, data_len);
532fb8629e2SIoana Ciornei 
533fb8629e2SIoana Ciornei 			/* Advance the index in the BDR */
534fb8629e2SIoana Ciornei 			enetc_bdr_idx_inc(tx_ring, &i);
535fb8629e2SIoana Ciornei 			txbd = ENETC_TXBD(*tx_ring, i);
536fb8629e2SIoana Ciornei 			tx_swbd = &tx_ring->tx_swbd[i];
537fb8629e2SIoana Ciornei 			prefetchw(txbd);
538fb8629e2SIoana Ciornei 
539fb8629e2SIoana Ciornei 			/* Compute the checksum over this segment of data and
540fb8629e2SIoana Ciornei 			 * add it to the csum already computed (over the L4
541fb8629e2SIoana Ciornei 			 * header and possible other data segments).
542fb8629e2SIoana Ciornei 			 */
543fb8629e2SIoana Ciornei 			csum2 = csum_partial(tso.data, size, 0);
544fb8629e2SIoana Ciornei 			csum = csum_block_add(csum, csum2, pos);
545fb8629e2SIoana Ciornei 			pos += size;
546fb8629e2SIoana Ciornei 
547fb8629e2SIoana Ciornei 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
548fb8629e2SIoana Ciornei 						    tso.data, size,
549fb8629e2SIoana Ciornei 						    size == data_len);
550fb8629e2SIoana Ciornei 			if (err)
551fb8629e2SIoana Ciornei 				goto err_map_data;
552fb8629e2SIoana Ciornei 
553fb8629e2SIoana Ciornei 			data_len -= size;
554fb8629e2SIoana Ciornei 			count++;
555fb8629e2SIoana Ciornei 			bd_data_num++;
556fb8629e2SIoana Ciornei 			tso_build_data(skb, &tso, size);
557fb8629e2SIoana Ciornei 
558fb8629e2SIoana Ciornei 			if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
559fb8629e2SIoana Ciornei 				goto err_chained_bd;
560fb8629e2SIoana Ciornei 		}
561fb8629e2SIoana Ciornei 
562fb8629e2SIoana Ciornei 		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
563fb8629e2SIoana Ciornei 
564fb8629e2SIoana Ciornei 		if (total_len == 0)
565fb8629e2SIoana Ciornei 			tx_swbd->skb = skb;
566fb8629e2SIoana Ciornei 
567fb8629e2SIoana Ciornei 		/* Go to the next BD */
568fb8629e2SIoana Ciornei 		enetc_bdr_idx_inc(tx_ring, &i);
569fb8629e2SIoana Ciornei 	}
570fb8629e2SIoana Ciornei 
571fb8629e2SIoana Ciornei 	tx_ring->next_to_use = i;
572fb8629e2SIoana Ciornei 	enetc_update_tx_ring_tail(tx_ring);
573fb8629e2SIoana Ciornei 
574fb8629e2SIoana Ciornei 	return count;
575fb8629e2SIoana Ciornei 
576fb8629e2SIoana Ciornei err_map_data:
577fb8629e2SIoana Ciornei 	dev_err(tx_ring->dev, "DMA map error");
578fb8629e2SIoana Ciornei 
579fb8629e2SIoana Ciornei err_chained_bd:
580fb8629e2SIoana Ciornei 	do {
581fb8629e2SIoana Ciornei 		tx_swbd = &tx_ring->tx_swbd[i];
582fb8629e2SIoana Ciornei 		enetc_free_tx_frame(tx_ring, tx_swbd);
583fb8629e2SIoana Ciornei 		if (i == 0)
584fb8629e2SIoana Ciornei 			i = tx_ring->bd_count;
585fb8629e2SIoana Ciornei 		i--;
586fb8629e2SIoana Ciornei 	} while (count--);
587fb8629e2SIoana Ciornei 
588fb8629e2SIoana Ciornei 	return 0;
589fb8629e2SIoana Ciornei }
590fb8629e2SIoana Ciornei 
enetc_start_xmit(struct sk_buff * skb,struct net_device * ndev)5917294380cSYangbo Lu static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
5927294380cSYangbo Lu 				    struct net_device *ndev)
5930486185eSVladimir Oltean {
5940486185eSVladimir Oltean 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
5950486185eSVladimir Oltean 	struct enetc_bdr *tx_ring;
596acede3c5SIoana Ciornei 	int count, err;
5970486185eSVladimir Oltean 
5987ce9c3d3SYangbo Lu 	/* Queue one-step Sync packet if already locked */
5997ce9c3d3SYangbo Lu 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
6007ce9c3d3SYangbo Lu 		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
6017ce9c3d3SYangbo Lu 					  &priv->flags)) {
6027ce9c3d3SYangbo Lu 			skb_queue_tail(&priv->tx_skbs, skb);
6037ce9c3d3SYangbo Lu 			return NETDEV_TX_OK;
6047ce9c3d3SYangbo Lu 		}
6057ce9c3d3SYangbo Lu 	}
6067ce9c3d3SYangbo Lu 
6070486185eSVladimir Oltean 	tx_ring = priv->tx_ring[skb->queue_mapping];
6080486185eSVladimir Oltean 
609fb8629e2SIoana Ciornei 	if (skb_is_gso(skb)) {
610fb8629e2SIoana Ciornei 		if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
611fb8629e2SIoana Ciornei 			netif_stop_subqueue(ndev, tx_ring->index);
612fb8629e2SIoana Ciornei 			return NETDEV_TX_BUSY;
613fb8629e2SIoana Ciornei 		}
614fb8629e2SIoana Ciornei 
615fb8629e2SIoana Ciornei 		enetc_lock_mdio();
616fb8629e2SIoana Ciornei 		count = enetc_map_tx_tso_buffs(tx_ring, skb);
617fb8629e2SIoana Ciornei 		enetc_unlock_mdio();
618fb8629e2SIoana Ciornei 	} else {
6190486185eSVladimir Oltean 		if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
6200486185eSVladimir Oltean 			if (unlikely(skb_linearize(skb)))
6210486185eSVladimir Oltean 				goto drop_packet_err;
6220486185eSVladimir Oltean 
6230486185eSVladimir Oltean 		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
6240486185eSVladimir Oltean 		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
6250486185eSVladimir Oltean 			netif_stop_subqueue(ndev, tx_ring->index);
6260486185eSVladimir Oltean 			return NETDEV_TX_BUSY;
6270486185eSVladimir Oltean 		}
6280486185eSVladimir Oltean 
629acede3c5SIoana Ciornei 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
630acede3c5SIoana Ciornei 			err = skb_checksum_help(skb);
631acede3c5SIoana Ciornei 			if (err)
632acede3c5SIoana Ciornei 				goto drop_packet_err;
633acede3c5SIoana Ciornei 		}
6340486185eSVladimir Oltean 		enetc_lock_mdio();
635f768e751SYangbo Lu 		count = enetc_map_tx_buffs(tx_ring, skb);
6360486185eSVladimir Oltean 		enetc_unlock_mdio();
637fb8629e2SIoana Ciornei 	}
6380486185eSVladimir Oltean 
6390486185eSVladimir Oltean 	if (unlikely(!count))
6400486185eSVladimir Oltean 		goto drop_packet_err;
6410486185eSVladimir Oltean 
6420486185eSVladimir Oltean 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
6430486185eSVladimir Oltean 		netif_stop_subqueue(ndev, tx_ring->index);
6440486185eSVladimir Oltean 
6450486185eSVladimir Oltean 	return NETDEV_TX_OK;
6460486185eSVladimir Oltean 
6470486185eSVladimir Oltean drop_packet_err:
6480486185eSVladimir Oltean 	dev_kfree_skb_any(skb);
6490486185eSVladimir Oltean 	return NETDEV_TX_OK;
6500486185eSVladimir Oltean }
6510486185eSVladimir Oltean 
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)6527294380cSYangbo Lu netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
6537294380cSYangbo Lu {
6547294380cSYangbo Lu 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
6557294380cSYangbo Lu 	u8 udp, msgtype, twostep;
6567294380cSYangbo Lu 	u16 offset1, offset2;
6577294380cSYangbo Lu 
6587294380cSYangbo Lu 	/* Mark tx timestamp type on skb->cb[0] if requires */
6597294380cSYangbo Lu 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
6607294380cSYangbo Lu 	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
6617294380cSYangbo Lu 		skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
6627294380cSYangbo Lu 	} else {
6637294380cSYangbo Lu 		skb->cb[0] = 0;
6647294380cSYangbo Lu 	}
6657294380cSYangbo Lu 
6667294380cSYangbo Lu 	/* Fall back to two-step timestamp if not one-step Sync packet */
6677294380cSYangbo Lu 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
6687294380cSYangbo Lu 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
6697294380cSYangbo Lu 				    &offset1, &offset2) ||
6707294380cSYangbo Lu 		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
6717294380cSYangbo Lu 			skb->cb[0] = ENETC_F_TX_TSTAMP;
6727294380cSYangbo Lu 	}
6737294380cSYangbo Lu 
6747294380cSYangbo Lu 	return enetc_start_xmit(skb, ndev);
6757294380cSYangbo Lu }
676e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_xmit);
6777294380cSYangbo Lu 
enetc_msix(int irq,void * data)678d4fd0404SClaudiu Manoil static irqreturn_t enetc_msix(int irq, void *data)
679d4fd0404SClaudiu Manoil {
680d4fd0404SClaudiu Manoil 	struct enetc_int_vector	*v = data;
681d4fd0404SClaudiu Manoil 	int i;
682d4fd0404SClaudiu Manoil 
683fd5736bfSAlex Marginean 	enetc_lock_mdio();
684fd5736bfSAlex Marginean 
685d4fd0404SClaudiu Manoil 	/* disable interrupts */
686fd5736bfSAlex Marginean 	enetc_wr_reg_hot(v->rbier, 0);
687fd5736bfSAlex Marginean 	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
688d4fd0404SClaudiu Manoil 
6890574e200SClaudiu Manoil 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
690fd5736bfSAlex Marginean 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
691fd5736bfSAlex Marginean 
692fd5736bfSAlex Marginean 	enetc_unlock_mdio();
693d4fd0404SClaudiu Manoil 
694215602a8SJiafei Pan 	napi_schedule(&v->napi);
695d4fd0404SClaudiu Manoil 
696d4fd0404SClaudiu Manoil 	return IRQ_HANDLED;
697d4fd0404SClaudiu Manoil }
698d4fd0404SClaudiu Manoil 
enetc_rx_dim_work(struct work_struct * w)699ae0e6a5dSClaudiu Manoil static void enetc_rx_dim_work(struct work_struct *w)
700ae0e6a5dSClaudiu Manoil {
701ae0e6a5dSClaudiu Manoil 	struct dim *dim = container_of(w, struct dim, work);
702ae0e6a5dSClaudiu Manoil 	struct dim_cq_moder moder =
703ae0e6a5dSClaudiu Manoil 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
704ae0e6a5dSClaudiu Manoil 	struct enetc_int_vector	*v =
705ae0e6a5dSClaudiu Manoil 		container_of(dim, struct enetc_int_vector, rx_dim);
706ae0e6a5dSClaudiu Manoil 
707ae0e6a5dSClaudiu Manoil 	v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
708ae0e6a5dSClaudiu Manoil 	dim->state = DIM_START_MEASURE;
709ae0e6a5dSClaudiu Manoil }
710ae0e6a5dSClaudiu Manoil 
enetc_rx_net_dim(struct enetc_int_vector * v)711ae0e6a5dSClaudiu Manoil static void enetc_rx_net_dim(struct enetc_int_vector *v)
712ae0e6a5dSClaudiu Manoil {
7139f7afa05SClaudiu Manoil 	struct dim_sample dim_sample = {};
714ae0e6a5dSClaudiu Manoil 
715ae0e6a5dSClaudiu Manoil 	v->comp_cnt++;
716ae0e6a5dSClaudiu Manoil 
717ae0e6a5dSClaudiu Manoil 	if (!v->rx_napi_work)
718ae0e6a5dSClaudiu Manoil 		return;
719ae0e6a5dSClaudiu Manoil 
720ae0e6a5dSClaudiu Manoil 	dim_update_sample(v->comp_cnt,
721ae0e6a5dSClaudiu Manoil 			  v->rx_ring.stats.packets,
722ae0e6a5dSClaudiu Manoil 			  v->rx_ring.stats.bytes,
723ae0e6a5dSClaudiu Manoil 			  &dim_sample);
724ae0e6a5dSClaudiu Manoil 	net_dim(&v->rx_dim, dim_sample);
725ae0e6a5dSClaudiu Manoil }
726ae0e6a5dSClaudiu Manoil 
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)727d4fd0404SClaudiu Manoil static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
728d4fd0404SClaudiu Manoil {
729fd5736bfSAlex Marginean 	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
730d4fd0404SClaudiu Manoil 
731d4fd0404SClaudiu Manoil 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
732d4fd0404SClaudiu Manoil }
733d4fd0404SClaudiu Manoil 
enetc_page_reusable(struct page * page)73465d0cbb4SVladimir Oltean static bool enetc_page_reusable(struct page *page)
73565d0cbb4SVladimir Oltean {
73665d0cbb4SVladimir Oltean 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
73765d0cbb4SVladimir Oltean }
73865d0cbb4SVladimir Oltean 
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)73965d0cbb4SVladimir Oltean static void enetc_reuse_page(struct enetc_bdr *rx_ring,
74065d0cbb4SVladimir Oltean 			     struct enetc_rx_swbd *old)
74165d0cbb4SVladimir Oltean {
74265d0cbb4SVladimir Oltean 	struct enetc_rx_swbd *new;
74365d0cbb4SVladimir Oltean 
74465d0cbb4SVladimir Oltean 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
74565d0cbb4SVladimir Oltean 
74665d0cbb4SVladimir Oltean 	/* next buf that may reuse a page */
74765d0cbb4SVladimir Oltean 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
74865d0cbb4SVladimir Oltean 
74965d0cbb4SVladimir Oltean 	/* copy page reference */
75065d0cbb4SVladimir Oltean 	*new = *old;
75165d0cbb4SVladimir Oltean }
75265d0cbb4SVladimir Oltean 
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)753d3982312SY.b. Lu static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
754d3982312SY.b. Lu 				u64 *tstamp)
755d3982312SY.b. Lu {
756cec4f328SY.b. Lu 	u32 lo, hi, tstamp_lo;
757d3982312SY.b. Lu 
7586d36ecdbSVladimir Oltean 	lo = enetc_rd_hot(hw, ENETC_SICTR0);
7596d36ecdbSVladimir Oltean 	hi = enetc_rd_hot(hw, ENETC_SICTR1);
760cec4f328SY.b. Lu 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
761cec4f328SY.b. Lu 	if (lo <= tstamp_lo)
762d3982312SY.b. Lu 		hi -= 1;
763cec4f328SY.b. Lu 	*tstamp = (u64)hi << 32 | tstamp_lo;
764d3982312SY.b. Lu }
765d3982312SY.b. Lu 
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)766d3982312SY.b. Lu static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
767d3982312SY.b. Lu {
768d3982312SY.b. Lu 	struct skb_shared_hwtstamps shhwtstamps;
769d3982312SY.b. Lu 
770d3982312SY.b. Lu 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
771d3982312SY.b. Lu 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
772d3982312SY.b. Lu 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
773847cbfc0SVladimir Oltean 		skb_txtime_consumed(skb);
774d3982312SY.b. Lu 		skb_tstamp_tx(skb, &shhwtstamps);
775d3982312SY.b. Lu 	}
776d3982312SY.b. Lu }
777d3982312SY.b. Lu 
enetc_recycle_xdp_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)7787ed2bc80SVladimir Oltean static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
7797ed2bc80SVladimir Oltean 				      struct enetc_tx_swbd *tx_swbd)
7807ed2bc80SVladimir Oltean {
7817ed2bc80SVladimir Oltean 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
7827ed2bc80SVladimir Oltean 	struct enetc_rx_swbd rx_swbd = {
7837ed2bc80SVladimir Oltean 		.dma = tx_swbd->dma,
7847ed2bc80SVladimir Oltean 		.page = tx_swbd->page,
7857ed2bc80SVladimir Oltean 		.page_offset = tx_swbd->page_offset,
7867ed2bc80SVladimir Oltean 		.dir = tx_swbd->dir,
7877ed2bc80SVladimir Oltean 		.len = tx_swbd->len,
7887ed2bc80SVladimir Oltean 	};
7897eab503bSVladimir Oltean 	struct enetc_bdr *rx_ring;
7907eab503bSVladimir Oltean 
7917eab503bSVladimir Oltean 	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
7927ed2bc80SVladimir Oltean 
7937ed2bc80SVladimir Oltean 	if (likely(enetc_swbd_unused(rx_ring))) {
7947ed2bc80SVladimir Oltean 		enetc_reuse_page(rx_ring, &rx_swbd);
7957ed2bc80SVladimir Oltean 
7967ed2bc80SVladimir Oltean 		/* sync for use by the device */
7977ed2bc80SVladimir Oltean 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
7987ed2bc80SVladimir Oltean 						 rx_swbd.page_offset,
7997ed2bc80SVladimir Oltean 						 ENETC_RXB_DMA_SIZE_XDP,
8007ed2bc80SVladimir Oltean 						 rx_swbd.dir);
8017ed2bc80SVladimir Oltean 
8027ed2bc80SVladimir Oltean 		rx_ring->stats.recycles++;
8037ed2bc80SVladimir Oltean 	} else {
8047ed2bc80SVladimir Oltean 		/* RX ring is already full, we need to unmap and free the
8057ed2bc80SVladimir Oltean 		 * page, since there's nothing useful we can do with it.
8067ed2bc80SVladimir Oltean 		 */
8077ed2bc80SVladimir Oltean 		rx_ring->stats.recycle_failures++;
8087ed2bc80SVladimir Oltean 
8097ed2bc80SVladimir Oltean 		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
8107ed2bc80SVladimir Oltean 			       rx_swbd.dir);
8117ed2bc80SVladimir Oltean 		__free_page(rx_swbd.page);
8127ed2bc80SVladimir Oltean 	}
8137ed2bc80SVladimir Oltean 
8147ed2bc80SVladimir Oltean 	rx_ring->xdp.xdp_tx_in_flight--;
8157ed2bc80SVladimir Oltean }
8167ed2bc80SVladimir Oltean 
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)817d4fd0404SClaudiu Manoil static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
818d4fd0404SClaudiu Manoil {
819285e8dedSPo Liu 	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
820d4fd0404SClaudiu Manoil 	struct net_device *ndev = tx_ring->ndev;
8217294380cSYangbo Lu 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
822d4fd0404SClaudiu Manoil 	struct enetc_tx_swbd *tx_swbd;
823d4fd0404SClaudiu Manoil 	int i, bds_to_clean;
8247294380cSYangbo Lu 	bool do_twostep_tstamp;
825d3982312SY.b. Lu 	u64 tstamp = 0;
826d4fd0404SClaudiu Manoil 
827d4fd0404SClaudiu Manoil 	i = tx_ring->next_to_clean;
828d4fd0404SClaudiu Manoil 	tx_swbd = &tx_ring->tx_swbd[i];
829fd5736bfSAlex Marginean 
830d4fd0404SClaudiu Manoil 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
831d4fd0404SClaudiu Manoil 
8327294380cSYangbo Lu 	do_twostep_tstamp = false;
833d3982312SY.b. Lu 
834d4fd0404SClaudiu Manoil 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
8359d2b68ccSVladimir Oltean 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
8369d2b68ccSVladimir Oltean 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
837a93580a0SVladimir Oltean 		bool is_eof = tx_swbd->is_eof;
8389d2b68ccSVladimir Oltean 
839d3982312SY.b. Lu 		if (unlikely(tx_swbd->check_wb)) {
840ae77bdbcSVladimir Oltean 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
841d3982312SY.b. Lu 
842d3982312SY.b. Lu 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
8437294380cSYangbo Lu 			    tx_swbd->do_twostep_tstamp) {
844d3982312SY.b. Lu 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
845d3982312SY.b. Lu 						    &tstamp);
8467294380cSYangbo Lu 				do_twostep_tstamp = true;
847d3982312SY.b. Lu 			}
848285e8dedSPo Liu 
849285e8dedSPo Liu 			if (tx_swbd->qbv_en &&
850285e8dedSPo Liu 			    txbd->wb.status & ENETC_TXBD_STATS_WIN)
851285e8dedSPo Liu 				tx_win_drop++;
852d3982312SY.b. Lu 		}
853d3982312SY.b. Lu 
8547ed2bc80SVladimir Oltean 		if (tx_swbd->is_xdp_tx)
8557ed2bc80SVladimir Oltean 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
8567ed2bc80SVladimir Oltean 		else if (likely(tx_swbd->dma))
857d4fd0404SClaudiu Manoil 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
858f4a0be84SClaudiu Manoil 
8599d2b68ccSVladimir Oltean 		if (xdp_frame) {
8609d2b68ccSVladimir Oltean 			xdp_return_frame(xdp_frame);
8619d2b68ccSVladimir Oltean 		} else if (skb) {
86252066149SVladimir Oltean 			if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
8637294380cSYangbo Lu 				/* Start work to release lock for next one-step
8647294380cSYangbo Lu 				 * timestamping packet. And send one skb in
8657294380cSYangbo Lu 				 * tx_skbs queue if has.
8667294380cSYangbo Lu 				 */
867b6faf160SYangbo Lu 				schedule_work(&priv->tx_onestep_tstamp);
8687294380cSYangbo Lu 			} else if (unlikely(do_twostep_tstamp)) {
8699d2b68ccSVladimir Oltean 				enetc_tstamp_tx(skb, tstamp);
8707294380cSYangbo Lu 				do_twostep_tstamp = false;
871d3982312SY.b. Lu 			}
8729d2b68ccSVladimir Oltean 			napi_consume_skb(skb, napi_budget);
873d4fd0404SClaudiu Manoil 		}
874d4fd0404SClaudiu Manoil 
875d4fd0404SClaudiu Manoil 		tx_byte_cnt += tx_swbd->len;
8761ee8d6f3SVladimir Oltean 		/* Scrub the swbd here so we don't have to do that
8771ee8d6f3SVladimir Oltean 		 * when we reuse it during xmit
8781ee8d6f3SVladimir Oltean 		 */
8791ee8d6f3SVladimir Oltean 		memset(tx_swbd, 0, sizeof(*tx_swbd));
880d4fd0404SClaudiu Manoil 
881d4fd0404SClaudiu Manoil 		bds_to_clean--;
882d4fd0404SClaudiu Manoil 		tx_swbd++;
883d4fd0404SClaudiu Manoil 		i++;
884d4fd0404SClaudiu Manoil 		if (unlikely(i == tx_ring->bd_count)) {
885d4fd0404SClaudiu Manoil 			i = 0;
886d4fd0404SClaudiu Manoil 			tx_swbd = tx_ring->tx_swbd;
887d4fd0404SClaudiu Manoil 		}
888d4fd0404SClaudiu Manoil 
889d4fd0404SClaudiu Manoil 		/* BD iteration loop end */
890a93580a0SVladimir Oltean 		if (is_eof) {
891d4fd0404SClaudiu Manoil 			tx_frm_cnt++;
892d4fd0404SClaudiu Manoil 			/* re-arm interrupt source */
893fd5736bfSAlex Marginean 			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
894d4fd0404SClaudiu Manoil 					 BIT(16 + tx_ring->index));
895d4fd0404SClaudiu Manoil 		}
896d4fd0404SClaudiu Manoil 
897d4fd0404SClaudiu Manoil 		if (unlikely(!bds_to_clean))
898d4fd0404SClaudiu Manoil 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
899d4fd0404SClaudiu Manoil 	}
900d4fd0404SClaudiu Manoil 
901d4fd0404SClaudiu Manoil 	tx_ring->next_to_clean = i;
902d4fd0404SClaudiu Manoil 	tx_ring->stats.packets += tx_frm_cnt;
903d4fd0404SClaudiu Manoil 	tx_ring->stats.bytes += tx_byte_cnt;
904285e8dedSPo Liu 	tx_ring->stats.win_drop += tx_win_drop;
905d4fd0404SClaudiu Manoil 
906d4fd0404SClaudiu Manoil 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
907d4fd0404SClaudiu Manoil 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
90837184349SWei Fang 		     !test_bit(ENETC_TX_DOWN, &priv->flags) &&
909d4fd0404SClaudiu Manoil 		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
910d4fd0404SClaudiu Manoil 		netif_wake_subqueue(ndev, tx_ring->index);
911d4fd0404SClaudiu Manoil 	}
912d4fd0404SClaudiu Manoil 
913d4fd0404SClaudiu Manoil 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
914d4fd0404SClaudiu Manoil }
915d4fd0404SClaudiu Manoil 
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)916d4fd0404SClaudiu Manoil static bool enetc_new_page(struct enetc_bdr *rx_ring,
917d4fd0404SClaudiu Manoil 			   struct enetc_rx_swbd *rx_swbd)
918d4fd0404SClaudiu Manoil {
9197ed2bc80SVladimir Oltean 	bool xdp = !!(rx_ring->xdp.prog);
920d4fd0404SClaudiu Manoil 	struct page *page;
921d4fd0404SClaudiu Manoil 	dma_addr_t addr;
922d4fd0404SClaudiu Manoil 
923d4fd0404SClaudiu Manoil 	page = dev_alloc_page();
924d4fd0404SClaudiu Manoil 	if (unlikely(!page))
925d4fd0404SClaudiu Manoil 		return false;
926d4fd0404SClaudiu Manoil 
9277ed2bc80SVladimir Oltean 	/* For XDP_TX, we forgo dma_unmap -> dma_map */
9287ed2bc80SVladimir Oltean 	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
9297ed2bc80SVladimir Oltean 
9307ed2bc80SVladimir Oltean 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
931d4fd0404SClaudiu Manoil 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
932d4fd0404SClaudiu Manoil 		__free_page(page);
933d4fd0404SClaudiu Manoil 
934d4fd0404SClaudiu Manoil 		return false;
935d4fd0404SClaudiu Manoil 	}
936d4fd0404SClaudiu Manoil 
937d4fd0404SClaudiu Manoil 	rx_swbd->dma = addr;
938d4fd0404SClaudiu Manoil 	rx_swbd->page = page;
939d1b15102SVladimir Oltean 	rx_swbd->page_offset = rx_ring->buffer_offset;
940d4fd0404SClaudiu Manoil 
941d4fd0404SClaudiu Manoil 	return true;
942d4fd0404SClaudiu Manoil }
943d4fd0404SClaudiu Manoil 
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)944d4fd0404SClaudiu Manoil static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
945d4fd0404SClaudiu Manoil {
946d4fd0404SClaudiu Manoil 	struct enetc_rx_swbd *rx_swbd;
947d4fd0404SClaudiu Manoil 	union enetc_rx_bd *rxbd;
948d4fd0404SClaudiu Manoil 	int i, j;
949d4fd0404SClaudiu Manoil 
950d4fd0404SClaudiu Manoil 	i = rx_ring->next_to_use;
951d4fd0404SClaudiu Manoil 	rx_swbd = &rx_ring->rx_swbd[i];
952714239acSClaudiu Manoil 	rxbd = enetc_rxbd(rx_ring, i);
953d4fd0404SClaudiu Manoil 
954d4fd0404SClaudiu Manoil 	for (j = 0; j < buff_cnt; j++) {
955d4fd0404SClaudiu Manoil 		/* try reuse page */
956d4fd0404SClaudiu Manoil 		if (unlikely(!rx_swbd->page)) {
957d4fd0404SClaudiu Manoil 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
958d4fd0404SClaudiu Manoil 				rx_ring->stats.rx_alloc_errs++;
959d4fd0404SClaudiu Manoil 				break;
960d4fd0404SClaudiu Manoil 			}
961d4fd0404SClaudiu Manoil 		}
962d4fd0404SClaudiu Manoil 
963d4fd0404SClaudiu Manoil 		/* update RxBD */
964d4fd0404SClaudiu Manoil 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
965d4fd0404SClaudiu Manoil 					   rx_swbd->page_offset);
966d4fd0404SClaudiu Manoil 		/* clear 'R" as well */
967d4fd0404SClaudiu Manoil 		rxbd->r.lstatus = 0;
968d4fd0404SClaudiu Manoil 
969c027aa92SVladimir Oltean 		enetc_rxbd_next(rx_ring, &rxbd, &i);
970c027aa92SVladimir Oltean 		rx_swbd = &rx_ring->rx_swbd[i];
971d4fd0404SClaudiu Manoil 	}
972d4fd0404SClaudiu Manoil 
973d4fd0404SClaudiu Manoil 	if (likely(j)) {
974d4fd0404SClaudiu Manoil 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
975d4fd0404SClaudiu Manoil 		rx_ring->next_to_use = i;
9767a5222cbSVladimir Oltean 
9777a5222cbSVladimir Oltean 		/* update ENETC's consumer index */
9787a5222cbSVladimir Oltean 		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
979d4fd0404SClaudiu Manoil 	}
980d4fd0404SClaudiu Manoil 
981d4fd0404SClaudiu Manoil 	return j;
982d4fd0404SClaudiu Manoil }
983d4fd0404SClaudiu Manoil 
984434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)985d3982312SY.b. Lu static void enetc_get_rx_tstamp(struct net_device *ndev,
986d3982312SY.b. Lu 				union enetc_rx_bd *rxbd,
987d3982312SY.b. Lu 				struct sk_buff *skb)
988d3982312SY.b. Lu {
989d3982312SY.b. Lu 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
990d3982312SY.b. Lu 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
991d3982312SY.b. Lu 	struct enetc_hw *hw = &priv->si->hw;
992cec4f328SY.b. Lu 	u32 lo, hi, tstamp_lo;
993d3982312SY.b. Lu 	u64 tstamp;
994d3982312SY.b. Lu 
995cec4f328SY.b. Lu 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
996fd5736bfSAlex Marginean 		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
997fd5736bfSAlex Marginean 		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
998434cebabSClaudiu Manoil 		rxbd = enetc_rxbd_ext(rxbd);
999434cebabSClaudiu Manoil 		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
1000cec4f328SY.b. Lu 		if (lo <= tstamp_lo)
1001d3982312SY.b. Lu 			hi -= 1;
1002d3982312SY.b. Lu 
1003cec4f328SY.b. Lu 		tstamp = (u64)hi << 32 | tstamp_lo;
1004d3982312SY.b. Lu 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1005d3982312SY.b. Lu 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1006d3982312SY.b. Lu 	}
1007d3982312SY.b. Lu }
1008d3982312SY.b. Lu #endif
1009d3982312SY.b. Lu 
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)1010d4fd0404SClaudiu Manoil static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1011d4fd0404SClaudiu Manoil 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
1012d4fd0404SClaudiu Manoil {
1013d3982312SY.b. Lu 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1014827b6fd0SVladimir Oltean 
1015d3982312SY.b. Lu 	/* TODO: hashing */
1016d4fd0404SClaudiu Manoil 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1017d4fd0404SClaudiu Manoil 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1018d4fd0404SClaudiu Manoil 
1019d4fd0404SClaudiu Manoil 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1020d4fd0404SClaudiu Manoil 		skb->ip_summed = CHECKSUM_COMPLETE;
1021d4fd0404SClaudiu Manoil 	}
1022d4fd0404SClaudiu Manoil 
1023827b6fd0SVladimir Oltean 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1024827b6fd0SVladimir Oltean 		__be16 tpid = 0;
1025827b6fd0SVladimir Oltean 
1026827b6fd0SVladimir Oltean 		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1027827b6fd0SVladimir Oltean 		case 0:
1028827b6fd0SVladimir Oltean 			tpid = htons(ETH_P_8021Q);
1029827b6fd0SVladimir Oltean 			break;
1030827b6fd0SVladimir Oltean 		case 1:
1031827b6fd0SVladimir Oltean 			tpid = htons(ETH_P_8021AD);
1032827b6fd0SVladimir Oltean 			break;
1033827b6fd0SVladimir Oltean 		case 2:
1034827b6fd0SVladimir Oltean 			tpid = htons(enetc_port_rd(&priv->si->hw,
1035827b6fd0SVladimir Oltean 						   ENETC_PCVLANR1));
1036827b6fd0SVladimir Oltean 			break;
1037827b6fd0SVladimir Oltean 		case 3:
1038827b6fd0SVladimir Oltean 			tpid = htons(enetc_port_rd(&priv->si->hw,
1039827b6fd0SVladimir Oltean 						   ENETC_PCVLANR2));
1040827b6fd0SVladimir Oltean 			break;
1041827b6fd0SVladimir Oltean 		default:
1042827b6fd0SVladimir Oltean 			break;
1043827b6fd0SVladimir Oltean 		}
1044827b6fd0SVladimir Oltean 
1045827b6fd0SVladimir Oltean 		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1046827b6fd0SVladimir Oltean 	}
1047827b6fd0SVladimir Oltean 
1048434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1049d3982312SY.b. Lu 	if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1050d3982312SY.b. Lu 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1051d3982312SY.b. Lu #endif
1052d4fd0404SClaudiu Manoil }
1053d4fd0404SClaudiu Manoil 
10547ed2bc80SVladimir Oltean /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
10557ed2bc80SVladimir Oltean  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
10567ed2bc80SVladimir Oltean  * mapped buffers.
10577ed2bc80SVladimir Oltean  */
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)1058d4fd0404SClaudiu Manoil static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1059d4fd0404SClaudiu Manoil 					       int i, u16 size)
1060d4fd0404SClaudiu Manoil {
1061d4fd0404SClaudiu Manoil 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1062d4fd0404SClaudiu Manoil 
1063d4fd0404SClaudiu Manoil 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1064d4fd0404SClaudiu Manoil 				      rx_swbd->page_offset,
10657ed2bc80SVladimir Oltean 				      size, rx_swbd->dir);
1066d4fd0404SClaudiu Manoil 	return rx_swbd;
1067d4fd0404SClaudiu Manoil }
1068d4fd0404SClaudiu Manoil 
10696b04830dSVladimir Oltean /* Reuse the current page without performing half-page buffer flipping */
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1070d4fd0404SClaudiu Manoil static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1071d4fd0404SClaudiu Manoil 			      struct enetc_rx_swbd *rx_swbd)
1072d4fd0404SClaudiu Manoil {
1073d1b15102SVladimir Oltean 	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1074d1b15102SVladimir Oltean 
1075d4fd0404SClaudiu Manoil 	enetc_reuse_page(rx_ring, rx_swbd);
1076d4fd0404SClaudiu Manoil 
1077d4fd0404SClaudiu Manoil 	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1078d4fd0404SClaudiu Manoil 					 rx_swbd->page_offset,
10797ed2bc80SVladimir Oltean 					 buffer_size, rx_swbd->dir);
10806b04830dSVladimir Oltean 
10816b04830dSVladimir Oltean 	rx_swbd->page = NULL;
10826b04830dSVladimir Oltean }
10836b04830dSVladimir Oltean 
10846b04830dSVladimir Oltean /* Reuse the current page by performing half-page buffer flipping */
enetc_flip_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)10856b04830dSVladimir Oltean static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
10866b04830dSVladimir Oltean 			       struct enetc_rx_swbd *rx_swbd)
10876b04830dSVladimir Oltean {
10886b04830dSVladimir Oltean 	if (likely(enetc_page_reusable(rx_swbd->page))) {
10896b04830dSVladimir Oltean 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
10906b04830dSVladimir Oltean 		page_ref_inc(rx_swbd->page);
10916b04830dSVladimir Oltean 
10926b04830dSVladimir Oltean 		enetc_put_rx_buff(rx_ring, rx_swbd);
1093d4fd0404SClaudiu Manoil 	} else {
10947ed2bc80SVladimir Oltean 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
10957ed2bc80SVladimir Oltean 			       rx_swbd->dir);
1096d4fd0404SClaudiu Manoil 		rx_swbd->page = NULL;
1097d4fd0404SClaudiu Manoil 	}
10986b04830dSVladimir Oltean }
1099d4fd0404SClaudiu Manoil 
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)1100d4fd0404SClaudiu Manoil static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1101d4fd0404SClaudiu Manoil 						int i, u16 size)
1102d4fd0404SClaudiu Manoil {
1103d4fd0404SClaudiu Manoil 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1104d4fd0404SClaudiu Manoil 	struct sk_buff *skb;
1105d4fd0404SClaudiu Manoil 	void *ba;
1106d4fd0404SClaudiu Manoil 
1107d4fd0404SClaudiu Manoil 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1108d1b15102SVladimir Oltean 	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1109d4fd0404SClaudiu Manoil 	if (unlikely(!skb)) {
1110d4fd0404SClaudiu Manoil 		rx_ring->stats.rx_alloc_errs++;
1111d4fd0404SClaudiu Manoil 		return NULL;
1112d4fd0404SClaudiu Manoil 	}
1113d4fd0404SClaudiu Manoil 
1114d1b15102SVladimir Oltean 	skb_reserve(skb, rx_ring->buffer_offset);
1115d4fd0404SClaudiu Manoil 	__skb_put(skb, size);
1116d4fd0404SClaudiu Manoil 
11176b04830dSVladimir Oltean 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1118d4fd0404SClaudiu Manoil 
1119d4fd0404SClaudiu Manoil 	return skb;
1120d4fd0404SClaudiu Manoil }
1121d4fd0404SClaudiu Manoil 
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)1122d4fd0404SClaudiu Manoil static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1123d4fd0404SClaudiu Manoil 				     u16 size, struct sk_buff *skb)
1124d4fd0404SClaudiu Manoil {
1125d4fd0404SClaudiu Manoil 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1126d4fd0404SClaudiu Manoil 
1127d4fd0404SClaudiu Manoil 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1128d4fd0404SClaudiu Manoil 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1129d4fd0404SClaudiu Manoil 
11306b04830dSVladimir Oltean 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1131d4fd0404SClaudiu Manoil }
1132d4fd0404SClaudiu Manoil 
enetc_check_bd_errors_and_consume(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i)11332fa423f5SVladimir Oltean static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
11342fa423f5SVladimir Oltean 					      u32 bd_status,
11352fa423f5SVladimir Oltean 					      union enetc_rx_bd **rxbd, int *i)
11362fa423f5SVladimir Oltean {
11372fa423f5SVladimir Oltean 	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
11382fa423f5SVladimir Oltean 		return false;
11392fa423f5SVladimir Oltean 
1140672f9a21SVladimir Oltean 	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
11412fa423f5SVladimir Oltean 	enetc_rxbd_next(rx_ring, rxbd, i);
11422fa423f5SVladimir Oltean 
11432fa423f5SVladimir Oltean 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
11442fa423f5SVladimir Oltean 		dma_rmb();
11452fa423f5SVladimir Oltean 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
11462fa423f5SVladimir Oltean 
1147672f9a21SVladimir Oltean 		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
11482fa423f5SVladimir Oltean 		enetc_rxbd_next(rx_ring, rxbd, i);
11492fa423f5SVladimir Oltean 	}
11502fa423f5SVladimir Oltean 
11512fa423f5SVladimir Oltean 	rx_ring->ndev->stats.rx_dropped++;
11522fa423f5SVladimir Oltean 	rx_ring->ndev->stats.rx_errors++;
11532fa423f5SVladimir Oltean 
11542fa423f5SVladimir Oltean 	return true;
11552fa423f5SVladimir Oltean }
11562fa423f5SVladimir Oltean 
enetc_build_skb(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,int buffer_size)1157a800abd3SVladimir Oltean static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1158a800abd3SVladimir Oltean 				       u32 bd_status, union enetc_rx_bd **rxbd,
1159d1b15102SVladimir Oltean 				       int *i, int *cleaned_cnt, int buffer_size)
1160a800abd3SVladimir Oltean {
1161a800abd3SVladimir Oltean 	struct sk_buff *skb;
1162a800abd3SVladimir Oltean 	u16 size;
1163a800abd3SVladimir Oltean 
1164a800abd3SVladimir Oltean 	size = le16_to_cpu((*rxbd)->r.buf_len);
1165a800abd3SVladimir Oltean 	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1166a800abd3SVladimir Oltean 	if (!skb)
1167a800abd3SVladimir Oltean 		return NULL;
1168a800abd3SVladimir Oltean 
1169a800abd3SVladimir Oltean 	enetc_get_offloads(rx_ring, *rxbd, skb);
1170a800abd3SVladimir Oltean 
1171a800abd3SVladimir Oltean 	(*cleaned_cnt)++;
1172a800abd3SVladimir Oltean 
1173a800abd3SVladimir Oltean 	enetc_rxbd_next(rx_ring, rxbd, i);
1174a800abd3SVladimir Oltean 
1175a800abd3SVladimir Oltean 	/* not last BD in frame? */
1176a800abd3SVladimir Oltean 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1177a800abd3SVladimir Oltean 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1178d1b15102SVladimir Oltean 		size = buffer_size;
1179a800abd3SVladimir Oltean 
1180a800abd3SVladimir Oltean 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1181a800abd3SVladimir Oltean 			dma_rmb();
1182a800abd3SVladimir Oltean 			size = le16_to_cpu((*rxbd)->r.buf_len);
1183a800abd3SVladimir Oltean 		}
1184a800abd3SVladimir Oltean 
1185a800abd3SVladimir Oltean 		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1186a800abd3SVladimir Oltean 
1187a800abd3SVladimir Oltean 		(*cleaned_cnt)++;
1188a800abd3SVladimir Oltean 
1189a800abd3SVladimir Oltean 		enetc_rxbd_next(rx_ring, rxbd, i);
1190a800abd3SVladimir Oltean 	}
1191a800abd3SVladimir Oltean 
1192a800abd3SVladimir Oltean 	skb_record_rx_queue(skb, rx_ring->index);
1193a800abd3SVladimir Oltean 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1194a800abd3SVladimir Oltean 
1195a800abd3SVladimir Oltean 	return skb;
1196a800abd3SVladimir Oltean }
1197a800abd3SVladimir Oltean 
1198d4fd0404SClaudiu Manoil #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1199d4fd0404SClaudiu Manoil 
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)1200d4fd0404SClaudiu Manoil static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1201d4fd0404SClaudiu Manoil 			       struct napi_struct *napi, int work_limit)
1202d4fd0404SClaudiu Manoil {
1203d4fd0404SClaudiu Manoil 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1204d4fd0404SClaudiu Manoil 	int cleaned_cnt, i;
1205d4fd0404SClaudiu Manoil 
1206d4fd0404SClaudiu Manoil 	cleaned_cnt = enetc_bd_unused(rx_ring);
1207d4fd0404SClaudiu Manoil 	/* next descriptor to process */
1208d4fd0404SClaudiu Manoil 	i = rx_ring->next_to_clean;
1209d4fd0404SClaudiu Manoil 
1210d4fd0404SClaudiu Manoil 	while (likely(rx_frm_cnt < work_limit)) {
1211d4fd0404SClaudiu Manoil 		union enetc_rx_bd *rxbd;
1212d4fd0404SClaudiu Manoil 		struct sk_buff *skb;
1213d4fd0404SClaudiu Manoil 		u32 bd_status;
1214d4fd0404SClaudiu Manoil 
12157a5222cbSVladimir Oltean 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
12167a5222cbSVladimir Oltean 			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
12177a5222cbSVladimir Oltean 							    cleaned_cnt);
1218d4fd0404SClaudiu Manoil 
1219714239acSClaudiu Manoil 		rxbd = enetc_rxbd(rx_ring, i);
1220d4fd0404SClaudiu Manoil 		bd_status = le32_to_cpu(rxbd->r.lstatus);
12216d36ecdbSVladimir Oltean 		if (!bd_status)
1222d4fd0404SClaudiu Manoil 			break;
1223d4fd0404SClaudiu Manoil 
1224fd5736bfSAlex Marginean 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1225d4fd0404SClaudiu Manoil 		dma_rmb(); /* for reading other rxbd fields */
12262fa423f5SVladimir Oltean 
12272fa423f5SVladimir Oltean 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
12282fa423f5SVladimir Oltean 						      &rxbd, &i))
12292fa423f5SVladimir Oltean 			break;
12302fa423f5SVladimir Oltean 
1231a800abd3SVladimir Oltean 		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1232d1b15102SVladimir Oltean 				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);
12336d36ecdbSVladimir Oltean 		if (!skb)
1234d4fd0404SClaudiu Manoil 			break;
1235d4fd0404SClaudiu Manoil 
12367190d0ffSWei Fang 		/* When set, the outer VLAN header is extracted and reported
12377190d0ffSWei Fang 		 * in the receive buffer descriptor. So rx_byte_cnt should
12387190d0ffSWei Fang 		 * add the length of the extracted VLAN header.
12397190d0ffSWei Fang 		 */
12407190d0ffSWei Fang 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
12417190d0ffSWei Fang 			rx_byte_cnt += VLAN_HLEN;
12427190d0ffSWei Fang 		rx_byte_cnt += skb->len + ETH_HLEN;
1243a800abd3SVladimir Oltean 		rx_frm_cnt++;
1244d4fd0404SClaudiu Manoil 
1245d4fd0404SClaudiu Manoil 		napi_gro_receive(napi, skb);
1246d4fd0404SClaudiu Manoil 	}
1247d4fd0404SClaudiu Manoil 
1248d4fd0404SClaudiu Manoil 	rx_ring->next_to_clean = i;
1249d4fd0404SClaudiu Manoil 
1250d4fd0404SClaudiu Manoil 	rx_ring->stats.packets += rx_frm_cnt;
1251d4fd0404SClaudiu Manoil 	rx_ring->stats.bytes += rx_byte_cnt;
1252d4fd0404SClaudiu Manoil 
1253d4fd0404SClaudiu Manoil 	return rx_frm_cnt;
1254d4fd0404SClaudiu Manoil }
1255d4fd0404SClaudiu Manoil 
enetc_xdp_map_tx_buff(struct enetc_bdr * tx_ring,int i,struct enetc_tx_swbd * tx_swbd,int frm_len)12567ed2bc80SVladimir Oltean static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
12577ed2bc80SVladimir Oltean 				  struct enetc_tx_swbd *tx_swbd,
12587ed2bc80SVladimir Oltean 				  int frm_len)
12597ed2bc80SVladimir Oltean {
12607ed2bc80SVladimir Oltean 	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
12617ed2bc80SVladimir Oltean 
12627ed2bc80SVladimir Oltean 	prefetchw(txbd);
12637ed2bc80SVladimir Oltean 
12647ed2bc80SVladimir Oltean 	enetc_clear_tx_bd(txbd);
12657ed2bc80SVladimir Oltean 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
12667ed2bc80SVladimir Oltean 	txbd->buf_len = cpu_to_le16(tx_swbd->len);
12677ed2bc80SVladimir Oltean 	txbd->frm_len = cpu_to_le16(frm_len);
12687ed2bc80SVladimir Oltean 
12697ed2bc80SVladimir Oltean 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
12707ed2bc80SVladimir Oltean }
12717ed2bc80SVladimir Oltean 
12727ed2bc80SVladimir Oltean /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
12737ed2bc80SVladimir Oltean  * descriptors.
12747ed2bc80SVladimir Oltean  */
enetc_xdp_tx(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,int num_tx_swbd)12757ed2bc80SVladimir Oltean static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
12767ed2bc80SVladimir Oltean 			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
12777ed2bc80SVladimir Oltean {
12787ed2bc80SVladimir Oltean 	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
12797ed2bc80SVladimir Oltean 	int i, k, frm_len = tmp_tx_swbd->len;
12807ed2bc80SVladimir Oltean 
12817ed2bc80SVladimir Oltean 	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
12827ed2bc80SVladimir Oltean 		return false;
12837ed2bc80SVladimir Oltean 
12847ed2bc80SVladimir Oltean 	while (unlikely(!tmp_tx_swbd->is_eof)) {
12857ed2bc80SVladimir Oltean 		tmp_tx_swbd++;
12867ed2bc80SVladimir Oltean 		frm_len += tmp_tx_swbd->len;
12877ed2bc80SVladimir Oltean 	}
12887ed2bc80SVladimir Oltean 
12897ed2bc80SVladimir Oltean 	i = tx_ring->next_to_use;
12907ed2bc80SVladimir Oltean 
12917ed2bc80SVladimir Oltean 	for (k = 0; k < num_tx_swbd; k++) {
12927ed2bc80SVladimir Oltean 		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
12937ed2bc80SVladimir Oltean 
12947ed2bc80SVladimir Oltean 		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
12957ed2bc80SVladimir Oltean 
12967ed2bc80SVladimir Oltean 		/* last BD needs 'F' bit set */
12977ed2bc80SVladimir Oltean 		if (xdp_tx_swbd->is_eof) {
12987ed2bc80SVladimir Oltean 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
12997ed2bc80SVladimir Oltean 
13007ed2bc80SVladimir Oltean 			txbd->flags = ENETC_TXBD_FLAGS_F;
13017ed2bc80SVladimir Oltean 		}
13027ed2bc80SVladimir Oltean 
13037ed2bc80SVladimir Oltean 		enetc_bdr_idx_inc(tx_ring, &i);
13047ed2bc80SVladimir Oltean 	}
13057ed2bc80SVladimir Oltean 
13067ed2bc80SVladimir Oltean 	tx_ring->next_to_use = i;
13077ed2bc80SVladimir Oltean 
13087ed2bc80SVladimir Oltean 	return true;
13097ed2bc80SVladimir Oltean }
13107ed2bc80SVladimir Oltean 
enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,struct xdp_frame * xdp_frame)13119d2b68ccSVladimir Oltean static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
13129d2b68ccSVladimir Oltean 					  struct enetc_tx_swbd *xdp_tx_arr,
13139d2b68ccSVladimir Oltean 					  struct xdp_frame *xdp_frame)
13149d2b68ccSVladimir Oltean {
13159d2b68ccSVladimir Oltean 	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
13169d2b68ccSVladimir Oltean 	struct skb_shared_info *shinfo;
13179d2b68ccSVladimir Oltean 	void *data = xdp_frame->data;
13189d2b68ccSVladimir Oltean 	int len = xdp_frame->len;
13199d2b68ccSVladimir Oltean 	skb_frag_t *frag;
13209d2b68ccSVladimir Oltean 	dma_addr_t dma;
13219d2b68ccSVladimir Oltean 	unsigned int f;
13229d2b68ccSVladimir Oltean 	int n = 0;
13239d2b68ccSVladimir Oltean 
13249d2b68ccSVladimir Oltean 	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
13259d2b68ccSVladimir Oltean 	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
13269d2b68ccSVladimir Oltean 		netdev_err(tx_ring->ndev, "DMA map error\n");
13279d2b68ccSVladimir Oltean 		return -1;
13289d2b68ccSVladimir Oltean 	}
13299d2b68ccSVladimir Oltean 
13309d2b68ccSVladimir Oltean 	xdp_tx_swbd->dma = dma;
13319d2b68ccSVladimir Oltean 	xdp_tx_swbd->dir = DMA_TO_DEVICE;
13329d2b68ccSVladimir Oltean 	xdp_tx_swbd->len = len;
13339d2b68ccSVladimir Oltean 	xdp_tx_swbd->is_xdp_redirect = true;
13349d2b68ccSVladimir Oltean 	xdp_tx_swbd->is_eof = false;
13359d2b68ccSVladimir Oltean 	xdp_tx_swbd->xdp_frame = NULL;
13369d2b68ccSVladimir Oltean 
13379d2b68ccSVladimir Oltean 	n++;
1338c7030d14SLorenzo Bianconi 
1339c7030d14SLorenzo Bianconi 	if (!xdp_frame_has_frags(xdp_frame))
1340c7030d14SLorenzo Bianconi 		goto out;
1341c7030d14SLorenzo Bianconi 
13429d2b68ccSVladimir Oltean 	xdp_tx_swbd = &xdp_tx_arr[n];
13439d2b68ccSVladimir Oltean 
13449d2b68ccSVladimir Oltean 	shinfo = xdp_get_shared_info_from_frame(xdp_frame);
13459d2b68ccSVladimir Oltean 
13469d2b68ccSVladimir Oltean 	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
13479d2b68ccSVladimir Oltean 	     f++, frag++) {
13489d2b68ccSVladimir Oltean 		data = skb_frag_address(frag);
13499d2b68ccSVladimir Oltean 		len = skb_frag_size(frag);
13509d2b68ccSVladimir Oltean 
13519d2b68ccSVladimir Oltean 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
13529d2b68ccSVladimir Oltean 		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
13539d2b68ccSVladimir Oltean 			/* Undo the DMA mapping for all fragments */
1354626b598aSDan Carpenter 			while (--n >= 0)
13559d2b68ccSVladimir Oltean 				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
13569d2b68ccSVladimir Oltean 
13579d2b68ccSVladimir Oltean 			netdev_err(tx_ring->ndev, "DMA map error\n");
13589d2b68ccSVladimir Oltean 			return -1;
13599d2b68ccSVladimir Oltean 		}
13609d2b68ccSVladimir Oltean 
13619d2b68ccSVladimir Oltean 		xdp_tx_swbd->dma = dma;
13629d2b68ccSVladimir Oltean 		xdp_tx_swbd->dir = DMA_TO_DEVICE;
13639d2b68ccSVladimir Oltean 		xdp_tx_swbd->len = len;
13649d2b68ccSVladimir Oltean 		xdp_tx_swbd->is_xdp_redirect = true;
13659d2b68ccSVladimir Oltean 		xdp_tx_swbd->is_eof = false;
13669d2b68ccSVladimir Oltean 		xdp_tx_swbd->xdp_frame = NULL;
13679d2b68ccSVladimir Oltean 
13689d2b68ccSVladimir Oltean 		n++;
13699d2b68ccSVladimir Oltean 		xdp_tx_swbd = &xdp_tx_arr[n];
13709d2b68ccSVladimir Oltean 	}
1371c7030d14SLorenzo Bianconi out:
13729d2b68ccSVladimir Oltean 	xdp_tx_arr[n - 1].is_eof = true;
13739d2b68ccSVladimir Oltean 	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
13749d2b68ccSVladimir Oltean 
13759d2b68ccSVladimir Oltean 	return n;
13769d2b68ccSVladimir Oltean }
13779d2b68ccSVladimir Oltean 
enetc_xdp_xmit(struct net_device * ndev,int num_frames,struct xdp_frame ** frames,u32 flags)13789d2b68ccSVladimir Oltean int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
13799d2b68ccSVladimir Oltean 		   struct xdp_frame **frames, u32 flags)
13809d2b68ccSVladimir Oltean {
13819d2b68ccSVladimir Oltean 	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
13829d2b68ccSVladimir Oltean 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
13839d2b68ccSVladimir Oltean 	struct enetc_bdr *tx_ring;
13849d2b68ccSVladimir Oltean 	int xdp_tx_bd_cnt, i, k;
13859d2b68ccSVladimir Oltean 	int xdp_tx_frm_cnt = 0;
13869d2b68ccSVladimir Oltean 
138737184349SWei Fang 	if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags)))
138837184349SWei Fang 		return -ENETDOWN;
138937184349SWei Fang 
139024e39309SVladimir Oltean 	enetc_lock_mdio();
139124e39309SVladimir Oltean 
13927eab503bSVladimir Oltean 	tx_ring = priv->xdp_tx_ring[smp_processor_id()];
13939d2b68ccSVladimir Oltean 
13949d2b68ccSVladimir Oltean 	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
13959d2b68ccSVladimir Oltean 
13969d2b68ccSVladimir Oltean 	for (k = 0; k < num_frames; k++) {
13979d2b68ccSVladimir Oltean 		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
13989d2b68ccSVladimir Oltean 							       xdp_redirect_arr,
13999d2b68ccSVladimir Oltean 							       frames[k]);
14009d2b68ccSVladimir Oltean 		if (unlikely(xdp_tx_bd_cnt < 0))
14019d2b68ccSVladimir Oltean 			break;
14029d2b68ccSVladimir Oltean 
14039d2b68ccSVladimir Oltean 		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
14049d2b68ccSVladimir Oltean 					   xdp_tx_bd_cnt))) {
14059d2b68ccSVladimir Oltean 			for (i = 0; i < xdp_tx_bd_cnt; i++)
14069d2b68ccSVladimir Oltean 				enetc_unmap_tx_buff(tx_ring,
14079d2b68ccSVladimir Oltean 						    &xdp_redirect_arr[i]);
14089d2b68ccSVladimir Oltean 			tx_ring->stats.xdp_tx_drops++;
14099d2b68ccSVladimir Oltean 			break;
14109d2b68ccSVladimir Oltean 		}
14119d2b68ccSVladimir Oltean 
14129d2b68ccSVladimir Oltean 		xdp_tx_frm_cnt++;
14139d2b68ccSVladimir Oltean 	}
14149d2b68ccSVladimir Oltean 
14159d2b68ccSVladimir Oltean 	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
14169d2b68ccSVladimir Oltean 		enetc_update_tx_ring_tail(tx_ring);
14179d2b68ccSVladimir Oltean 
14189d2b68ccSVladimir Oltean 	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
14199d2b68ccSVladimir Oltean 
142024e39309SVladimir Oltean 	enetc_unlock_mdio();
142124e39309SVladimir Oltean 
14229d2b68ccSVladimir Oltean 	return xdp_tx_frm_cnt;
14239d2b68ccSVladimir Oltean }
1424e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
14259d2b68ccSVladimir Oltean 
enetc_map_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,struct xdp_buff * xdp_buff,u16 size)1426d1b15102SVladimir Oltean static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1427d1b15102SVladimir Oltean 				     struct xdp_buff *xdp_buff, u16 size)
1428d1b15102SVladimir Oltean {
1429d1b15102SVladimir Oltean 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1430d1b15102SVladimir Oltean 	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1431d1b15102SVladimir Oltean 
14327ed2bc80SVladimir Oltean 	/* To be used for XDP_TX */
14337ed2bc80SVladimir Oltean 	rx_swbd->len = size;
14347ed2bc80SVladimir Oltean 
1435d1b15102SVladimir Oltean 	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1436d1b15102SVladimir Oltean 			 rx_ring->buffer_offset, size, false);
1437d1b15102SVladimir Oltean }
1438d1b15102SVladimir Oltean 
enetc_add_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,u16 size,struct xdp_buff * xdp_buff)1439d1b15102SVladimir Oltean static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1440d1b15102SVladimir Oltean 				     u16 size, struct xdp_buff *xdp_buff)
1441d1b15102SVladimir Oltean {
1442d1b15102SVladimir Oltean 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1443d1b15102SVladimir Oltean 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1444c7030d14SLorenzo Bianconi 	skb_frag_t *frag;
1445d1b15102SVladimir Oltean 
14467ed2bc80SVladimir Oltean 	/* To be used for XDP_TX */
14477ed2bc80SVladimir Oltean 	rx_swbd->len = size;
14487ed2bc80SVladimir Oltean 
14498feb020fSLorenzo Bianconi 	if (!xdp_buff_has_frags(xdp_buff)) {
14508feb020fSLorenzo Bianconi 		xdp_buff_set_frags_flag(xdp_buff);
14518feb020fSLorenzo Bianconi 		shinfo->xdp_frags_size = size;
1452c7030d14SLorenzo Bianconi 		shinfo->nr_frags = 0;
14538feb020fSLorenzo Bianconi 	} else {
14548feb020fSLorenzo Bianconi 		shinfo->xdp_frags_size += size;
14558feb020fSLorenzo Bianconi 	}
14568feb020fSLorenzo Bianconi 
14578feb020fSLorenzo Bianconi 	if (page_is_pfmemalloc(rx_swbd->page))
14588feb020fSLorenzo Bianconi 		xdp_buff_set_frag_pfmemalloc(xdp_buff);
14598feb020fSLorenzo Bianconi 
1460c7030d14SLorenzo Bianconi 	frag = &shinfo->frags[shinfo->nr_frags];
1461b51f4113SYunsheng Lin 	skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1462b51f4113SYunsheng Lin 				size);
1463d1b15102SVladimir Oltean 
1464d1b15102SVladimir Oltean 	shinfo->nr_frags++;
1465d1b15102SVladimir Oltean }
1466d1b15102SVladimir Oltean 
enetc_build_xdp_buff(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,struct xdp_buff * xdp_buff)1467d1b15102SVladimir Oltean static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1468d1b15102SVladimir Oltean 				 union enetc_rx_bd **rxbd, int *i,
1469d1b15102SVladimir Oltean 				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1470d1b15102SVladimir Oltean {
1471d1b15102SVladimir Oltean 	u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1472d1b15102SVladimir Oltean 
1473d1b15102SVladimir Oltean 	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1474d1b15102SVladimir Oltean 
1475d1b15102SVladimir Oltean 	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1476d1b15102SVladimir Oltean 	(*cleaned_cnt)++;
1477d1b15102SVladimir Oltean 	enetc_rxbd_next(rx_ring, rxbd, i);
1478d1b15102SVladimir Oltean 
1479d1b15102SVladimir Oltean 	/* not last BD in frame? */
1480d1b15102SVladimir Oltean 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1481d1b15102SVladimir Oltean 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1482d1b15102SVladimir Oltean 		size = ENETC_RXB_DMA_SIZE_XDP;
1483d1b15102SVladimir Oltean 
1484d1b15102SVladimir Oltean 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1485d1b15102SVladimir Oltean 			dma_rmb();
1486d1b15102SVladimir Oltean 			size = le16_to_cpu((*rxbd)->r.buf_len);
1487d1b15102SVladimir Oltean 		}
1488d1b15102SVladimir Oltean 
1489d1b15102SVladimir Oltean 		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1490d1b15102SVladimir Oltean 		(*cleaned_cnt)++;
1491d1b15102SVladimir Oltean 		enetc_rxbd_next(rx_ring, rxbd, i);
1492d1b15102SVladimir Oltean 	}
1493d1b15102SVladimir Oltean }
1494d1b15102SVladimir Oltean 
14957ed2bc80SVladimir Oltean /* Convert RX buffer descriptors to TX buffer descriptors. These will be
149692ff9a6eSVladimir Oltean  * recycled back into the RX ring in enetc_clean_tx_ring.
14977ed2bc80SVladimir Oltean  */
enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd * xdp_tx_arr,struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)14987ed2bc80SVladimir Oltean static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
14997ed2bc80SVladimir Oltean 					struct enetc_bdr *rx_ring,
15007ed2bc80SVladimir Oltean 					int rx_ring_first, int rx_ring_last)
15017ed2bc80SVladimir Oltean {
15027ed2bc80SVladimir Oltean 	int n = 0;
15037ed2bc80SVladimir Oltean 
15047ed2bc80SVladimir Oltean 	for (; rx_ring_first != rx_ring_last;
15057ed2bc80SVladimir Oltean 	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
15067ed2bc80SVladimir Oltean 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
15077ed2bc80SVladimir Oltean 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
15087ed2bc80SVladimir Oltean 
15097ed2bc80SVladimir Oltean 		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */
15107ed2bc80SVladimir Oltean 		tx_swbd->dma = rx_swbd->dma;
15117ed2bc80SVladimir Oltean 		tx_swbd->dir = rx_swbd->dir;
15127ed2bc80SVladimir Oltean 		tx_swbd->page = rx_swbd->page;
15137ed2bc80SVladimir Oltean 		tx_swbd->page_offset = rx_swbd->page_offset;
15147ed2bc80SVladimir Oltean 		tx_swbd->len = rx_swbd->len;
15157ed2bc80SVladimir Oltean 		tx_swbd->is_dma_page = true;
15167ed2bc80SVladimir Oltean 		tx_swbd->is_xdp_tx = true;
15177ed2bc80SVladimir Oltean 		tx_swbd->is_eof = false;
15187ed2bc80SVladimir Oltean 	}
15197ed2bc80SVladimir Oltean 
15207ed2bc80SVladimir Oltean 	/* We rely on caller providing an rx_ring_last > rx_ring_first */
15217ed2bc80SVladimir Oltean 	xdp_tx_arr[n - 1].is_eof = true;
15227ed2bc80SVladimir Oltean 
15237ed2bc80SVladimir Oltean 	return n;
15247ed2bc80SVladimir Oltean }
15257ed2bc80SVladimir Oltean 
enetc_xdp_drop(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1526d1b15102SVladimir Oltean static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1527d1b15102SVladimir Oltean 			   int rx_ring_last)
1528d1b15102SVladimir Oltean {
1529d1b15102SVladimir Oltean 	while (rx_ring_first != rx_ring_last) {
15306b04830dSVladimir Oltean 		enetc_put_rx_buff(rx_ring,
1531d1b15102SVladimir Oltean 				  &rx_ring->rx_swbd[rx_ring_first]);
1532d1b15102SVladimir Oltean 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1533d1b15102SVladimir Oltean 	}
1534d1b15102SVladimir Oltean }
1535d1b15102SVladimir Oltean 
enetc_clean_rx_ring_xdp(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit,struct bpf_prog * prog)1536d1b15102SVladimir Oltean static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1537d1b15102SVladimir Oltean 				   struct napi_struct *napi, int work_limit,
1538d1b15102SVladimir Oltean 				   struct bpf_prog *prog)
1539d1b15102SVladimir Oltean {
15409d2b68ccSVladimir Oltean 	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
15417ed2bc80SVladimir Oltean 	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
15427ed2bc80SVladimir Oltean 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1543d1b15102SVladimir Oltean 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
15447eab503bSVladimir Oltean 	struct enetc_bdr *tx_ring;
1545d1b15102SVladimir Oltean 	int cleaned_cnt, i;
1546d1b15102SVladimir Oltean 	u32 xdp_act;
1547d1b15102SVladimir Oltean 
1548d1b15102SVladimir Oltean 	cleaned_cnt = enetc_bd_unused(rx_ring);
1549d1b15102SVladimir Oltean 	/* next descriptor to process */
1550d1b15102SVladimir Oltean 	i = rx_ring->next_to_clean;
1551d1b15102SVladimir Oltean 
1552d1b15102SVladimir Oltean 	while (likely(rx_frm_cnt < work_limit)) {
1553d1b15102SVladimir Oltean 		union enetc_rx_bd *rxbd, *orig_rxbd;
1554d1b15102SVladimir Oltean 		int orig_i, orig_cleaned_cnt;
1555d1b15102SVladimir Oltean 		struct xdp_buff xdp_buff;
1556d1b15102SVladimir Oltean 		struct sk_buff *skb;
1557d1b15102SVladimir Oltean 		u32 bd_status;
1558628050ecSVladimir Oltean 		int err;
1559d1b15102SVladimir Oltean 
1560d1b15102SVladimir Oltean 		rxbd = enetc_rxbd(rx_ring, i);
1561d1b15102SVladimir Oltean 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1562d1b15102SVladimir Oltean 		if (!bd_status)
1563d1b15102SVladimir Oltean 			break;
1564d1b15102SVladimir Oltean 
1565d1b15102SVladimir Oltean 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1566d1b15102SVladimir Oltean 		dma_rmb(); /* for reading other rxbd fields */
1567d1b15102SVladimir Oltean 
1568d1b15102SVladimir Oltean 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1569d1b15102SVladimir Oltean 						      &rxbd, &i))
1570d1b15102SVladimir Oltean 			break;
1571d1b15102SVladimir Oltean 
1572d1b15102SVladimir Oltean 		orig_rxbd = rxbd;
1573d1b15102SVladimir Oltean 		orig_cleaned_cnt = cleaned_cnt;
1574d1b15102SVladimir Oltean 		orig_i = i;
1575d1b15102SVladimir Oltean 
1576d1b15102SVladimir Oltean 		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1577d1b15102SVladimir Oltean 				     &cleaned_cnt, &xdp_buff);
1578d1b15102SVladimir Oltean 
1579fdebd850SWei Fang 		/* When set, the outer VLAN header is extracted and reported
1580fdebd850SWei Fang 		 * in the receive buffer descriptor. So rx_byte_cnt should
1581fdebd850SWei Fang 		 * add the length of the extracted VLAN header.
1582fdebd850SWei Fang 		 */
1583fdebd850SWei Fang 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1584fdebd850SWei Fang 			rx_byte_cnt += VLAN_HLEN;
1585fdebd850SWei Fang 		rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1586fdebd850SWei Fang 
1587d1b15102SVladimir Oltean 		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1588d1b15102SVladimir Oltean 
1589d1b15102SVladimir Oltean 		switch (xdp_act) {
1590975acc83SVladimir Oltean 		default:
1591c8064e5bSPaolo Abeni 			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1592975acc83SVladimir Oltean 			fallthrough;
1593d1b15102SVladimir Oltean 		case XDP_ABORTED:
1594d1b15102SVladimir Oltean 			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1595d1b15102SVladimir Oltean 			fallthrough;
1596d1b15102SVladimir Oltean 		case XDP_DROP:
1597d1b15102SVladimir Oltean 			enetc_xdp_drop(rx_ring, orig_i, i);
15981c95443eSWei Fang 			rx_ring->stats.xdp_drops++;
1599d1b15102SVladimir Oltean 			break;
1600d1b15102SVladimir Oltean 		case XDP_PASS:
1601d1b15102SVladimir Oltean 			rxbd = orig_rxbd;
1602d1b15102SVladimir Oltean 			cleaned_cnt = orig_cleaned_cnt;
1603d1b15102SVladimir Oltean 			i = orig_i;
1604d1b15102SVladimir Oltean 
1605d1b15102SVladimir Oltean 			skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1606d1b15102SVladimir Oltean 					      &i, &cleaned_cnt,
1607d1b15102SVladimir Oltean 					      ENETC_RXB_DMA_SIZE_XDP);
1608d1b15102SVladimir Oltean 			if (unlikely(!skb))
16098f50d8bbSVladimir Oltean 				goto out;
1610d1b15102SVladimir Oltean 
1611d1b15102SVladimir Oltean 			napi_gro_receive(napi, skb);
1612d1b15102SVladimir Oltean 			break;
16137ed2bc80SVladimir Oltean 		case XDP_TX:
16147eab503bSVladimir Oltean 			tx_ring = priv->xdp_tx_ring[rx_ring->index];
161537184349SWei Fang 			if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) {
161637184349SWei Fang 				enetc_xdp_drop(rx_ring, orig_i, i);
161737184349SWei Fang 				tx_ring->stats.xdp_tx_drops++;
161837184349SWei Fang 				break;
161937184349SWei Fang 			}
162037184349SWei Fang 
16217ed2bc80SVladimir Oltean 			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
16227ed2bc80SVladimir Oltean 								     rx_ring,
16237ed2bc80SVladimir Oltean 								     orig_i, i);
16247ed2bc80SVladimir Oltean 
16257ed2bc80SVladimir Oltean 			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
16267ed2bc80SVladimir Oltean 				enetc_xdp_drop(rx_ring, orig_i, i);
16277ed2bc80SVladimir Oltean 				tx_ring->stats.xdp_tx_drops++;
16287ed2bc80SVladimir Oltean 			} else {
16297ed2bc80SVladimir Oltean 				tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
16307ed2bc80SVladimir Oltean 				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
16317ed2bc80SVladimir Oltean 				xdp_tx_frm_cnt++;
163292ff9a6eSVladimir Oltean 				/* The XDP_TX enqueue was successful, so we
163392ff9a6eSVladimir Oltean 				 * need to scrub the RX software BDs because
163492ff9a6eSVladimir Oltean 				 * the ownership of the buffers no longer
163592ff9a6eSVladimir Oltean 				 * belongs to the RX ring, and we must prevent
163692ff9a6eSVladimir Oltean 				 * enetc_refill_rx_ring() from reusing
163792ff9a6eSVladimir Oltean 				 * rx_swbd->page.
163892ff9a6eSVladimir Oltean 				 */
163992ff9a6eSVladimir Oltean 				while (orig_i != i) {
164092ff9a6eSVladimir Oltean 					rx_ring->rx_swbd[orig_i].page = NULL;
164192ff9a6eSVladimir Oltean 					enetc_bdr_idx_inc(rx_ring, &orig_i);
164292ff9a6eSVladimir Oltean 				}
16437ed2bc80SVladimir Oltean 			}
16447ed2bc80SVladimir Oltean 			break;
16459d2b68ccSVladimir Oltean 		case XDP_REDIRECT:
1646628050ecSVladimir Oltean 			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1647628050ecSVladimir Oltean 			if (unlikely(err)) {
1648628050ecSVladimir Oltean 				enetc_xdp_drop(rx_ring, orig_i, i);
1649628050ecSVladimir Oltean 				rx_ring->stats.xdp_redirect_failures++;
1650628050ecSVladimir Oltean 			} else {
16519d2b68ccSVladimir Oltean 				while (orig_i != i) {
16526b04830dSVladimir Oltean 					enetc_flip_rx_buff(rx_ring,
16539d2b68ccSVladimir Oltean 							   &rx_ring->rx_swbd[orig_i]);
16549d2b68ccSVladimir Oltean 					enetc_bdr_idx_inc(rx_ring, &orig_i);
16559d2b68ccSVladimir Oltean 				}
16569d2b68ccSVladimir Oltean 				xdp_redirect_frm_cnt++;
16579d2b68ccSVladimir Oltean 				rx_ring->stats.xdp_redirect++;
16589d2b68ccSVladimir Oltean 			}
1659d1b15102SVladimir Oltean 		}
1660d1b15102SVladimir Oltean 
1661d1b15102SVladimir Oltean 		rx_frm_cnt++;
1662d1b15102SVladimir Oltean 	}
1663d1b15102SVladimir Oltean 
16648f50d8bbSVladimir Oltean out:
1665d1b15102SVladimir Oltean 	rx_ring->next_to_clean = i;
1666d1b15102SVladimir Oltean 
1667d1b15102SVladimir Oltean 	rx_ring->stats.packets += rx_frm_cnt;
1668d1b15102SVladimir Oltean 	rx_ring->stats.bytes += rx_byte_cnt;
1669d1b15102SVladimir Oltean 
16709d2b68ccSVladimir Oltean 	if (xdp_redirect_frm_cnt)
16719d2b68ccSVladimir Oltean 		xdp_do_flush_map();
16729d2b68ccSVladimir Oltean 
16737ed2bc80SVladimir Oltean 	if (xdp_tx_frm_cnt)
16747ed2bc80SVladimir Oltean 		enetc_update_tx_ring_tail(tx_ring);
16757ed2bc80SVladimir Oltean 
16767ed2bc80SVladimir Oltean 	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
16777ed2bc80SVladimir Oltean 		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
16787ed2bc80SVladimir Oltean 				     rx_ring->xdp.xdp_tx_in_flight);
16797ed2bc80SVladimir Oltean 
1680d1b15102SVladimir Oltean 	return rx_frm_cnt;
1681d1b15102SVladimir Oltean }
1682d1b15102SVladimir Oltean 
enetc_poll(struct napi_struct * napi,int budget)16838580b3c3SVladimir Oltean static int enetc_poll(struct napi_struct *napi, int budget)
16848580b3c3SVladimir Oltean {
16858580b3c3SVladimir Oltean 	struct enetc_int_vector
16868580b3c3SVladimir Oltean 		*v = container_of(napi, struct enetc_int_vector, napi);
1687d1b15102SVladimir Oltean 	struct enetc_bdr *rx_ring = &v->rx_ring;
1688d1b15102SVladimir Oltean 	struct bpf_prog *prog;
16898580b3c3SVladimir Oltean 	bool complete = true;
16908580b3c3SVladimir Oltean 	int work_done;
16918580b3c3SVladimir Oltean 	int i;
16928580b3c3SVladimir Oltean 
16938580b3c3SVladimir Oltean 	enetc_lock_mdio();
16948580b3c3SVladimir Oltean 
16958580b3c3SVladimir Oltean 	for (i = 0; i < v->count_tx_rings; i++)
16968580b3c3SVladimir Oltean 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
16978580b3c3SVladimir Oltean 			complete = false;
16988580b3c3SVladimir Oltean 
1699d1b15102SVladimir Oltean 	prog = rx_ring->xdp.prog;
1700d1b15102SVladimir Oltean 	if (prog)
1701d1b15102SVladimir Oltean 		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1702d1b15102SVladimir Oltean 	else
1703d1b15102SVladimir Oltean 		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
17048580b3c3SVladimir Oltean 	if (work_done == budget)
17058580b3c3SVladimir Oltean 		complete = false;
17068580b3c3SVladimir Oltean 	if (work_done)
17078580b3c3SVladimir Oltean 		v->rx_napi_work = true;
17088580b3c3SVladimir Oltean 
17098580b3c3SVladimir Oltean 	if (!complete) {
17108580b3c3SVladimir Oltean 		enetc_unlock_mdio();
17118580b3c3SVladimir Oltean 		return budget;
17128580b3c3SVladimir Oltean 	}
17138580b3c3SVladimir Oltean 
17148580b3c3SVladimir Oltean 	napi_complete_done(napi, work_done);
17158580b3c3SVladimir Oltean 
17168580b3c3SVladimir Oltean 	if (likely(v->rx_dim_en))
17178580b3c3SVladimir Oltean 		enetc_rx_net_dim(v);
17188580b3c3SVladimir Oltean 
17198580b3c3SVladimir Oltean 	v->rx_napi_work = false;
17208580b3c3SVladimir Oltean 
17218580b3c3SVladimir Oltean 	/* enable interrupts */
17228580b3c3SVladimir Oltean 	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
17238580b3c3SVladimir Oltean 
17248580b3c3SVladimir Oltean 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
17258580b3c3SVladimir Oltean 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
17268580b3c3SVladimir Oltean 				 ENETC_TBIER_TXTIE);
17278580b3c3SVladimir Oltean 
17288580b3c3SVladimir Oltean 	enetc_unlock_mdio();
17298580b3c3SVladimir Oltean 
17308580b3c3SVladimir Oltean 	return work_done;
17318580b3c3SVladimir Oltean }
17328580b3c3SVladimir Oltean 
1733d4fd0404SClaudiu Manoil /* Probing and Init */
1734d382563fSClaudiu Manoil #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)1735d4fd0404SClaudiu Manoil void enetc_get_si_caps(struct enetc_si *si)
1736d4fd0404SClaudiu Manoil {
1737d4fd0404SClaudiu Manoil 	struct enetc_hw *hw = &si->hw;
1738d4fd0404SClaudiu Manoil 	u32 val;
1739d4fd0404SClaudiu Manoil 
1740d4fd0404SClaudiu Manoil 	/* find out how many of various resources we have to work with */
1741d4fd0404SClaudiu Manoil 	val = enetc_rd(hw, ENETC_SICAPR0);
1742d4fd0404SClaudiu Manoil 	si->num_rx_rings = (val >> 16) & 0xff;
1743d4fd0404SClaudiu Manoil 	si->num_tx_rings = val & 0xff;
1744d382563fSClaudiu Manoil 
1745d382563fSClaudiu Manoil 	val = enetc_rd(hw, ENETC_SIRFSCAPR);
1746d382563fSClaudiu Manoil 	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1747d382563fSClaudiu Manoil 	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1748d382563fSClaudiu Manoil 
1749d382563fSClaudiu Manoil 	si->num_rss = 0;
1750d382563fSClaudiu Manoil 	val = enetc_rd(hw, ENETC_SIPCAPR0);
1751d382563fSClaudiu Manoil 	if (val & ENETC_SIPCAPR0_RSS) {
17522e47cb41SPo Liu 		u32 rss;
17532e47cb41SPo Liu 
17542e47cb41SPo Liu 		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
17552e47cb41SPo Liu 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1756d382563fSClaudiu Manoil 	}
17572e47cb41SPo Liu 
17582e47cb41SPo Liu 	if (val & ENETC_SIPCAPR0_QBV)
17592e47cb41SPo Liu 		si->hw_features |= ENETC_SI_F_QBV;
176079e49982SPo Liu 
176194557a9aSVladimir Oltean 	if (val & ENETC_SIPCAPR0_QBU)
176294557a9aSVladimir Oltean 		si->hw_features |= ENETC_SI_F_QBU;
176394557a9aSVladimir Oltean 
176479e49982SPo Liu 	if (val & ENETC_SIPCAPR0_PSFP)
176579e49982SPo Liu 		si->hw_features |= ENETC_SI_F_PSFP;
1766d4fd0404SClaudiu Manoil }
1767e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_get_si_caps);
1768d4fd0404SClaudiu Manoil 
enetc_dma_alloc_bdr(struct enetc_bdr_resource * res)1769f3ce29e1SVladimir Oltean static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
1770d4fd0404SClaudiu Manoil {
1771f3ce29e1SVladimir Oltean 	size_t bd_base_size = res->bd_count * res->bd_size;
1772f3ce29e1SVladimir Oltean 
1773f3ce29e1SVladimir Oltean 	res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
1774f3ce29e1SVladimir Oltean 					  &res->bd_dma_base, GFP_KERNEL);
1775f3ce29e1SVladimir Oltean 	if (!res->bd_base)
1776d4fd0404SClaudiu Manoil 		return -ENOMEM;
1777d4fd0404SClaudiu Manoil 
1778d4fd0404SClaudiu Manoil 	/* h/w requires 128B alignment */
1779f3ce29e1SVladimir Oltean 	if (!IS_ALIGNED(res->bd_dma_base, 128)) {
1780f3ce29e1SVladimir Oltean 		dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1781f3ce29e1SVladimir Oltean 				  res->bd_dma_base);
1782d4fd0404SClaudiu Manoil 		return -EINVAL;
1783d4fd0404SClaudiu Manoil 	}
1784d4fd0404SClaudiu Manoil 
1785d4fd0404SClaudiu Manoil 	return 0;
1786d4fd0404SClaudiu Manoil }
1787d4fd0404SClaudiu Manoil 
enetc_dma_free_bdr(const struct enetc_bdr_resource * res)1788f3ce29e1SVladimir Oltean static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
17890d6cfd0fSVladimir Oltean {
1790f3ce29e1SVladimir Oltean 	size_t bd_base_size = res->bd_count * res->bd_size;
1791f3ce29e1SVladimir Oltean 
1792f3ce29e1SVladimir Oltean 	dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1793f3ce29e1SVladimir Oltean 			  res->bd_dma_base);
17940d6cfd0fSVladimir Oltean }
17950d6cfd0fSVladimir Oltean 
enetc_alloc_tx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count)1796f3ce29e1SVladimir Oltean static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
1797f3ce29e1SVladimir Oltean 				   struct device *dev, size_t bd_count)
1798d4fd0404SClaudiu Manoil {
1799d4fd0404SClaudiu Manoil 	int err;
1800d4fd0404SClaudiu Manoil 
1801f3ce29e1SVladimir Oltean 	res->dev = dev;
1802f3ce29e1SVladimir Oltean 	res->bd_count = bd_count;
1803f3ce29e1SVladimir Oltean 	res->bd_size = sizeof(union enetc_tx_bd);
1804f3ce29e1SVladimir Oltean 
1805fa87c546SJulia Lawall 	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
1806f3ce29e1SVladimir Oltean 	if (!res->tx_swbd)
1807d4fd0404SClaudiu Manoil 		return -ENOMEM;
1808d4fd0404SClaudiu Manoil 
1809f3ce29e1SVladimir Oltean 	err = enetc_dma_alloc_bdr(res);
1810fb8629e2SIoana Ciornei 	if (err)
1811fb8629e2SIoana Ciornei 		goto err_alloc_bdr;
1812fb8629e2SIoana Ciornei 
1813f3ce29e1SVladimir Oltean 	res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
1814f3ce29e1SVladimir Oltean 					      &res->tso_headers_dma,
1815fb8629e2SIoana Ciornei 					      GFP_KERNEL);
1816f3ce29e1SVladimir Oltean 	if (!res->tso_headers) {
1817e79d8264SDan Carpenter 		err = -ENOMEM;
1818fb8629e2SIoana Ciornei 		goto err_alloc_tso;
1819e79d8264SDan Carpenter 	}
1820d4fd0404SClaudiu Manoil 
1821d4fd0404SClaudiu Manoil 	return 0;
1822fb8629e2SIoana Ciornei 
1823fb8629e2SIoana Ciornei err_alloc_tso:
1824f3ce29e1SVladimir Oltean 	enetc_dma_free_bdr(res);
1825fb8629e2SIoana Ciornei err_alloc_bdr:
1826f3ce29e1SVladimir Oltean 	vfree(res->tx_swbd);
1827f3ce29e1SVladimir Oltean 	res->tx_swbd = NULL;
1828fb8629e2SIoana Ciornei 
1829fb8629e2SIoana Ciornei 	return err;
1830d4fd0404SClaudiu Manoil }
1831d4fd0404SClaudiu Manoil 
enetc_free_tx_resource(const struct enetc_bdr_resource * res)1832f3ce29e1SVladimir Oltean static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
1833d4fd0404SClaudiu Manoil {
1834f3ce29e1SVladimir Oltean 	dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
1835f3ce29e1SVladimir Oltean 			  res->tso_headers, res->tso_headers_dma);
1836f3ce29e1SVladimir Oltean 	enetc_dma_free_bdr(res);
1837f3ce29e1SVladimir Oltean 	vfree(res->tx_swbd);
1838d4fd0404SClaudiu Manoil }
1839d4fd0404SClaudiu Manoil 
1840f3ce29e1SVladimir Oltean static struct enetc_bdr_resource *
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)1841f3ce29e1SVladimir Oltean enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1842d4fd0404SClaudiu Manoil {
1843f3ce29e1SVladimir Oltean 	struct enetc_bdr_resource *tx_res;
1844d4fd0404SClaudiu Manoil 	int i, err;
1845d4fd0404SClaudiu Manoil 
1846f3ce29e1SVladimir Oltean 	tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
1847f3ce29e1SVladimir Oltean 	if (!tx_res)
1848f3ce29e1SVladimir Oltean 		return ERR_PTR(-ENOMEM);
1849f3ce29e1SVladimir Oltean 
1850d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++) {
1851f3ce29e1SVladimir Oltean 		struct enetc_bdr *tx_ring = priv->tx_ring[i];
1852d4fd0404SClaudiu Manoil 
1853f3ce29e1SVladimir Oltean 		err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
1854f3ce29e1SVladimir Oltean 					      tx_ring->bd_count);
1855d4fd0404SClaudiu Manoil 		if (err)
1856d4fd0404SClaudiu Manoil 			goto fail;
1857d4fd0404SClaudiu Manoil 	}
1858d4fd0404SClaudiu Manoil 
1859f3ce29e1SVladimir Oltean 	return tx_res;
1860d4fd0404SClaudiu Manoil 
1861d4fd0404SClaudiu Manoil fail:
1862d4fd0404SClaudiu Manoil 	while (i-- > 0)
1863f3ce29e1SVladimir Oltean 		enetc_free_tx_resource(&tx_res[i]);
1864d4fd0404SClaudiu Manoil 
1865f3ce29e1SVladimir Oltean 	kfree(tx_res);
1866f3ce29e1SVladimir Oltean 
1867f3ce29e1SVladimir Oltean 	return ERR_PTR(err);
1868d4fd0404SClaudiu Manoil }
1869d4fd0404SClaudiu Manoil 
enetc_free_tx_resources(const struct enetc_bdr_resource * tx_res,size_t num_resources)1870f3ce29e1SVladimir Oltean static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
1871f3ce29e1SVladimir Oltean 				    size_t num_resources)
1872d4fd0404SClaudiu Manoil {
1873f3ce29e1SVladimir Oltean 	size_t i;
1874d4fd0404SClaudiu Manoil 
1875f3ce29e1SVladimir Oltean 	for (i = 0; i < num_resources; i++)
1876f3ce29e1SVladimir Oltean 		enetc_free_tx_resource(&tx_res[i]);
1877f3ce29e1SVladimir Oltean 
1878f3ce29e1SVladimir Oltean 	kfree(tx_res);
1879d4fd0404SClaudiu Manoil }
1880d4fd0404SClaudiu Manoil 
enetc_alloc_rx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count,bool extended)1881f3ce29e1SVladimir Oltean static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
1882f3ce29e1SVladimir Oltean 				   struct device *dev, size_t bd_count,
1883f3ce29e1SVladimir Oltean 				   bool extended)
1884d4fd0404SClaudiu Manoil {
1885d4fd0404SClaudiu Manoil 	int err;
1886d4fd0404SClaudiu Manoil 
1887f3ce29e1SVladimir Oltean 	res->dev = dev;
1888f3ce29e1SVladimir Oltean 	res->bd_count = bd_count;
1889f3ce29e1SVladimir Oltean 	res->bd_size = sizeof(union enetc_rx_bd);
1890f3ce29e1SVladimir Oltean 	if (extended)
1891f3ce29e1SVladimir Oltean 		res->bd_size *= 2;
1892f3ce29e1SVladimir Oltean 
1893fa87c546SJulia Lawall 	res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
1894f3ce29e1SVladimir Oltean 	if (!res->rx_swbd)
1895d4fd0404SClaudiu Manoil 		return -ENOMEM;
1896d4fd0404SClaudiu Manoil 
1897f3ce29e1SVladimir Oltean 	err = enetc_dma_alloc_bdr(res);
1898d4fd0404SClaudiu Manoil 	if (err) {
1899f3ce29e1SVladimir Oltean 		vfree(res->rx_swbd);
1900d4fd0404SClaudiu Manoil 		return err;
1901d4fd0404SClaudiu Manoil 	}
1902d4fd0404SClaudiu Manoil 
1903d4fd0404SClaudiu Manoil 	return 0;
1904d4fd0404SClaudiu Manoil }
1905d4fd0404SClaudiu Manoil 
enetc_free_rx_resource(const struct enetc_bdr_resource * res)1906f3ce29e1SVladimir Oltean static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
1907d4fd0404SClaudiu Manoil {
1908f3ce29e1SVladimir Oltean 	enetc_dma_free_bdr(res);
1909f3ce29e1SVladimir Oltean 	vfree(res->rx_swbd);
1910d4fd0404SClaudiu Manoil }
1911d4fd0404SClaudiu Manoil 
1912f3ce29e1SVladimir Oltean static struct enetc_bdr_resource *
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv,bool extended)1913f3ce29e1SVladimir Oltean enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
1914d4fd0404SClaudiu Manoil {
1915f3ce29e1SVladimir Oltean 	struct enetc_bdr_resource *rx_res;
1916d4fd0404SClaudiu Manoil 	int i, err;
1917d4fd0404SClaudiu Manoil 
1918f3ce29e1SVladimir Oltean 	rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
1919f3ce29e1SVladimir Oltean 	if (!rx_res)
1920f3ce29e1SVladimir Oltean 		return ERR_PTR(-ENOMEM);
1921d4fd0404SClaudiu Manoil 
1922f3ce29e1SVladimir Oltean 	for (i = 0; i < priv->num_rx_rings; i++) {
1923f3ce29e1SVladimir Oltean 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
1924f3ce29e1SVladimir Oltean 
1925f3ce29e1SVladimir Oltean 		err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
1926f3ce29e1SVladimir Oltean 					      rx_ring->bd_count, extended);
1927d4fd0404SClaudiu Manoil 		if (err)
1928d4fd0404SClaudiu Manoil 			goto fail;
1929d4fd0404SClaudiu Manoil 	}
1930d4fd0404SClaudiu Manoil 
1931f3ce29e1SVladimir Oltean 	return rx_res;
1932d4fd0404SClaudiu Manoil 
1933d4fd0404SClaudiu Manoil fail:
1934d4fd0404SClaudiu Manoil 	while (i-- > 0)
1935f3ce29e1SVladimir Oltean 		enetc_free_rx_resource(&rx_res[i]);
1936d4fd0404SClaudiu Manoil 
1937f3ce29e1SVladimir Oltean 	kfree(rx_res);
1938f3ce29e1SVladimir Oltean 
1939f3ce29e1SVladimir Oltean 	return ERR_PTR(err);
1940d4fd0404SClaudiu Manoil }
1941d4fd0404SClaudiu Manoil 
enetc_free_rx_resources(const struct enetc_bdr_resource * rx_res,size_t num_resources)1942f3ce29e1SVladimir Oltean static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
1943f3ce29e1SVladimir Oltean 				    size_t num_resources)
1944f3ce29e1SVladimir Oltean {
1945f3ce29e1SVladimir Oltean 	size_t i;
1946f3ce29e1SVladimir Oltean 
1947f3ce29e1SVladimir Oltean 	for (i = 0; i < num_resources; i++)
1948f3ce29e1SVladimir Oltean 		enetc_free_rx_resource(&rx_res[i]);
1949f3ce29e1SVladimir Oltean 
1950f3ce29e1SVladimir Oltean 	kfree(rx_res);
1951f3ce29e1SVladimir Oltean }
1952f3ce29e1SVladimir Oltean 
enetc_assign_tx_resource(struct enetc_bdr * tx_ring,const struct enetc_bdr_resource * res)1953f3ce29e1SVladimir Oltean static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
1954f3ce29e1SVladimir Oltean 				     const struct enetc_bdr_resource *res)
1955f3ce29e1SVladimir Oltean {
1956f3ce29e1SVladimir Oltean 	tx_ring->bd_base = res ? res->bd_base : NULL;
1957f3ce29e1SVladimir Oltean 	tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1958f3ce29e1SVladimir Oltean 	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
1959f3ce29e1SVladimir Oltean 	tx_ring->tso_headers = res ? res->tso_headers : NULL;
1960f3ce29e1SVladimir Oltean 	tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
1961f3ce29e1SVladimir Oltean }
1962f3ce29e1SVladimir Oltean 
enetc_assign_rx_resource(struct enetc_bdr * rx_ring,const struct enetc_bdr_resource * res)1963f3ce29e1SVladimir Oltean static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
1964f3ce29e1SVladimir Oltean 				     const struct enetc_bdr_resource *res)
1965f3ce29e1SVladimir Oltean {
1966f3ce29e1SVladimir Oltean 	rx_ring->bd_base = res ? res->bd_base : NULL;
1967f3ce29e1SVladimir Oltean 	rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1968f3ce29e1SVladimir Oltean 	rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
1969f3ce29e1SVladimir Oltean }
1970f3ce29e1SVladimir Oltean 
enetc_assign_tx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)1971f3ce29e1SVladimir Oltean static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
1972f3ce29e1SVladimir Oltean 				      const struct enetc_bdr_resource *res)
1973d4fd0404SClaudiu Manoil {
1974d4fd0404SClaudiu Manoil 	int i;
1975d4fd0404SClaudiu Manoil 
1976f3ce29e1SVladimir Oltean 	if (priv->tx_res)
1977f3ce29e1SVladimir Oltean 		enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
1978f3ce29e1SVladimir Oltean 
1979f3ce29e1SVladimir Oltean 	for (i = 0; i < priv->num_tx_rings; i++) {
1980f3ce29e1SVladimir Oltean 		enetc_assign_tx_resource(priv->tx_ring[i],
1981f3ce29e1SVladimir Oltean 					 res ? &res[i] : NULL);
1982f3ce29e1SVladimir Oltean 	}
1983f3ce29e1SVladimir Oltean 
1984f3ce29e1SVladimir Oltean 	priv->tx_res = res;
1985f3ce29e1SVladimir Oltean }
1986f3ce29e1SVladimir Oltean 
enetc_assign_rx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)1987f3ce29e1SVladimir Oltean static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
1988f3ce29e1SVladimir Oltean 				      const struct enetc_bdr_resource *res)
1989f3ce29e1SVladimir Oltean {
1990f3ce29e1SVladimir Oltean 	int i;
1991f3ce29e1SVladimir Oltean 
1992f3ce29e1SVladimir Oltean 	if (priv->rx_res)
1993f3ce29e1SVladimir Oltean 		enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
1994f3ce29e1SVladimir Oltean 
1995f3ce29e1SVladimir Oltean 	for (i = 0; i < priv->num_rx_rings; i++) {
1996f3ce29e1SVladimir Oltean 		enetc_assign_rx_resource(priv->rx_ring[i],
1997f3ce29e1SVladimir Oltean 					 res ? &res[i] : NULL);
1998f3ce29e1SVladimir Oltean 	}
1999f3ce29e1SVladimir Oltean 
2000f3ce29e1SVladimir Oltean 	priv->rx_res = res;
2001d4fd0404SClaudiu Manoil }
2002d4fd0404SClaudiu Manoil 
enetc_free_tx_ring(struct enetc_bdr * tx_ring)2003d4fd0404SClaudiu Manoil static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
2004d4fd0404SClaudiu Manoil {
2005d4fd0404SClaudiu Manoil 	int i;
2006d4fd0404SClaudiu Manoil 
2007d4fd0404SClaudiu Manoil 	for (i = 0; i < tx_ring->bd_count; i++) {
2008d4fd0404SClaudiu Manoil 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
2009d4fd0404SClaudiu Manoil 
20109d2b68ccSVladimir Oltean 		enetc_free_tx_frame(tx_ring, tx_swbd);
2011d4fd0404SClaudiu Manoil 	}
2012d4fd0404SClaudiu Manoil }
2013d4fd0404SClaudiu Manoil 
enetc_free_rx_ring(struct enetc_bdr * rx_ring)2014d4fd0404SClaudiu Manoil static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2015d4fd0404SClaudiu Manoil {
2016d4fd0404SClaudiu Manoil 	int i;
2017d4fd0404SClaudiu Manoil 
2018d4fd0404SClaudiu Manoil 	for (i = 0; i < rx_ring->bd_count; i++) {
2019d4fd0404SClaudiu Manoil 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2020d4fd0404SClaudiu Manoil 
2021d4fd0404SClaudiu Manoil 		if (!rx_swbd->page)
2022d4fd0404SClaudiu Manoil 			continue;
2023d4fd0404SClaudiu Manoil 
20247ed2bc80SVladimir Oltean 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
20257ed2bc80SVladimir Oltean 			       rx_swbd->dir);
2026d4fd0404SClaudiu Manoil 		__free_page(rx_swbd->page);
2027d4fd0404SClaudiu Manoil 		rx_swbd->page = NULL;
2028d4fd0404SClaudiu Manoil 	}
2029d4fd0404SClaudiu Manoil }
2030d4fd0404SClaudiu Manoil 
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)2031d4fd0404SClaudiu Manoil static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2032d4fd0404SClaudiu Manoil {
2033d4fd0404SClaudiu Manoil 	int i;
2034d4fd0404SClaudiu Manoil 
2035d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_rx_rings; i++)
2036d4fd0404SClaudiu Manoil 		enetc_free_rx_ring(priv->rx_ring[i]);
2037d4fd0404SClaudiu Manoil 
2038d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++)
2039d4fd0404SClaudiu Manoil 		enetc_free_tx_ring(priv->tx_ring[i]);
2040d4fd0404SClaudiu Manoil }
2041d4fd0404SClaudiu Manoil 
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)2042d382563fSClaudiu Manoil static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2043d382563fSClaudiu Manoil {
2044d382563fSClaudiu Manoil 	int *rss_table;
2045d382563fSClaudiu Manoil 	int i;
2046d382563fSClaudiu Manoil 
2047d382563fSClaudiu Manoil 	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2048d382563fSClaudiu Manoil 	if (!rss_table)
2049d382563fSClaudiu Manoil 		return -ENOMEM;
2050d382563fSClaudiu Manoil 
2051d382563fSClaudiu Manoil 	/* Set up RSS table defaults */
2052d382563fSClaudiu Manoil 	for (i = 0; i < si->num_rss; i++)
2053d382563fSClaudiu Manoil 		rss_table[i] = i % num_groups;
2054d382563fSClaudiu Manoil 
2055d382563fSClaudiu Manoil 	enetc_set_rss_table(si, rss_table, si->num_rss);
2056d382563fSClaudiu Manoil 
2057d382563fSClaudiu Manoil 	kfree(rss_table);
2058d382563fSClaudiu Manoil 
2059d382563fSClaudiu Manoil 	return 0;
2060d382563fSClaudiu Manoil }
2061d382563fSClaudiu Manoil 
enetc_configure_si(struct enetc_ndev_priv * priv)2062c646d10dSVladimir Oltean int enetc_configure_si(struct enetc_ndev_priv *priv)
2063d4fd0404SClaudiu Manoil {
2064d4fd0404SClaudiu Manoil 	struct enetc_si *si = priv->si;
2065d4fd0404SClaudiu Manoil 	struct enetc_hw *hw = &si->hw;
2066d382563fSClaudiu Manoil 	int err;
2067d4fd0404SClaudiu Manoil 
2068d4fd0404SClaudiu Manoil 	/* set SI cache attributes */
2069d4fd0404SClaudiu Manoil 	enetc_wr(hw, ENETC_SICAR0,
2070d4fd0404SClaudiu Manoil 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2071d4fd0404SClaudiu Manoil 	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2072d4fd0404SClaudiu Manoil 	/* enable SI */
2073d4fd0404SClaudiu Manoil 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2074d4fd0404SClaudiu Manoil 
2075d382563fSClaudiu Manoil 	if (si->num_rss) {
2076d382563fSClaudiu Manoil 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2077d382563fSClaudiu Manoil 		if (err)
2078d382563fSClaudiu Manoil 			return err;
2079d382563fSClaudiu Manoil 	}
2080d382563fSClaudiu Manoil 
2081d4fd0404SClaudiu Manoil 	return 0;
2082d4fd0404SClaudiu Manoil }
2083e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_configure_si);
2084d4fd0404SClaudiu Manoil 
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)2085d4fd0404SClaudiu Manoil void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2086d4fd0404SClaudiu Manoil {
2087d4fd0404SClaudiu Manoil 	struct enetc_si *si = priv->si;
2088d4fd0404SClaudiu Manoil 	int cpus = num_online_cpus();
2089d4fd0404SClaudiu Manoil 
209002293dd4SClaudiu Manoil 	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
209102293dd4SClaudiu Manoil 	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2092d4fd0404SClaudiu Manoil 
2093d4fd0404SClaudiu Manoil 	/* Enable all available TX rings in order to configure as many
2094d4fd0404SClaudiu Manoil 	 * priorities as possible, when needed.
2095d4fd0404SClaudiu Manoil 	 * TODO: Make # of TX rings run-time configurable
2096d4fd0404SClaudiu Manoil 	 */
2097d4fd0404SClaudiu Manoil 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2098d4fd0404SClaudiu Manoil 	priv->num_tx_rings = si->num_tx_rings;
2099d4fd0404SClaudiu Manoil 	priv->bdr_int_num = cpus;
2100ae0e6a5dSClaudiu Manoil 	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2101ae0e6a5dSClaudiu Manoil 	priv->tx_ictt = ENETC_TXIC_TIMETHR;
2102d4fd0404SClaudiu Manoil }
2103e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2104d4fd0404SClaudiu Manoil 
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)2105d4fd0404SClaudiu Manoil int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2106d4fd0404SClaudiu Manoil {
2107d4fd0404SClaudiu Manoil 	struct enetc_si *si = priv->si;
2108d4fd0404SClaudiu Manoil 
2109d382563fSClaudiu Manoil 	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2110d382563fSClaudiu Manoil 				  GFP_KERNEL);
21114b47c0b8SVladimir Oltean 	if (!priv->cls_rules)
21124b47c0b8SVladimir Oltean 		return -ENOMEM;
2113d382563fSClaudiu Manoil 
2114d4fd0404SClaudiu Manoil 	return 0;
2115d4fd0404SClaudiu Manoil }
2116e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2117d4fd0404SClaudiu Manoil 
enetc_free_si_resources(struct enetc_ndev_priv * priv)2118d4fd0404SClaudiu Manoil void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2119d4fd0404SClaudiu Manoil {
2120d382563fSClaudiu Manoil 	kfree(priv->cls_rules);
2121d4fd0404SClaudiu Manoil }
2122e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2123d4fd0404SClaudiu Manoil 
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2124d4fd0404SClaudiu Manoil static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2125d4fd0404SClaudiu Manoil {
2126d4fd0404SClaudiu Manoil 	int idx = tx_ring->index;
2127d4fd0404SClaudiu Manoil 	u32 tbmr;
2128d4fd0404SClaudiu Manoil 
2129d4fd0404SClaudiu Manoil 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2130d4fd0404SClaudiu Manoil 		       lower_32_bits(tx_ring->bd_dma_base));
2131d4fd0404SClaudiu Manoil 
2132d4fd0404SClaudiu Manoil 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2133d4fd0404SClaudiu Manoil 		       upper_32_bits(tx_ring->bd_dma_base));
2134d4fd0404SClaudiu Manoil 
2135d4fd0404SClaudiu Manoil 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2136d4fd0404SClaudiu Manoil 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2137d4fd0404SClaudiu Manoil 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
2138d4fd0404SClaudiu Manoil 
2139d4fd0404SClaudiu Manoil 	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2140d4fd0404SClaudiu Manoil 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2141d4fd0404SClaudiu Manoil 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2142d4fd0404SClaudiu Manoil 
2143d4fd0404SClaudiu Manoil 	/* enable Tx ints by setting pkt thr to 1 */
214412460a0aSClaudiu Manoil 	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2145d4fd0404SClaudiu Manoil 
2146ff58fda0SVladimir Oltean 	tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2147d4fd0404SClaudiu Manoil 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2148d4fd0404SClaudiu Manoil 		tbmr |= ENETC_TBMR_VIH;
2149d4fd0404SClaudiu Manoil 
2150d4fd0404SClaudiu Manoil 	/* enable ring */
2151d4fd0404SClaudiu Manoil 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2152d4fd0404SClaudiu Manoil 
2153d4fd0404SClaudiu Manoil 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2154d4fd0404SClaudiu Manoil 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2155d4fd0404SClaudiu Manoil 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
2156d4fd0404SClaudiu Manoil }
2157d4fd0404SClaudiu Manoil 
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring,bool extended)2158d075db51SVladimir Oltean static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2159d075db51SVladimir Oltean 			      bool extended)
2160d4fd0404SClaudiu Manoil {
2161d4fd0404SClaudiu Manoil 	int idx = rx_ring->index;
2162ff58fda0SVladimir Oltean 	u32 rbmr = 0;
2163d4fd0404SClaudiu Manoil 
2164d4fd0404SClaudiu Manoil 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2165d4fd0404SClaudiu Manoil 		       lower_32_bits(rx_ring->bd_dma_base));
2166d4fd0404SClaudiu Manoil 
2167d4fd0404SClaudiu Manoil 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2168d4fd0404SClaudiu Manoil 		       upper_32_bits(rx_ring->bd_dma_base));
2169d4fd0404SClaudiu Manoil 
2170d4fd0404SClaudiu Manoil 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2171d4fd0404SClaudiu Manoil 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2172d4fd0404SClaudiu Manoil 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
2173d4fd0404SClaudiu Manoil 
2174d1b15102SVladimir Oltean 	if (rx_ring->xdp.prog)
2175d1b15102SVladimir Oltean 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2176d1b15102SVladimir Oltean 	else
2177d4fd0404SClaudiu Manoil 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2178d4fd0404SClaudiu Manoil 
217984ce1ca3SVladimir Oltean 	/* Also prepare the consumer index in case page allocation never
218084ce1ca3SVladimir Oltean 	 * succeeds. In that case, hardware will never advance producer index
218184ce1ca3SVladimir Oltean 	 * to match consumer index, and will drop all frames.
218284ce1ca3SVladimir Oltean 	 */
2183d4fd0404SClaudiu Manoil 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
218484ce1ca3SVladimir Oltean 	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2185d4fd0404SClaudiu Manoil 
2186d4fd0404SClaudiu Manoil 	/* enable Rx ints by setting pkt thr to 1 */
218712460a0aSClaudiu Manoil 	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2188d4fd0404SClaudiu Manoil 
2189d075db51SVladimir Oltean 	rx_ring->ext_en = extended;
2190434cebabSClaudiu Manoil 	if (rx_ring->ext_en)
2191d3982312SY.b. Lu 		rbmr |= ENETC_RBMR_BDS;
2192434cebabSClaudiu Manoil 
2193d4fd0404SClaudiu Manoil 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2194d4fd0404SClaudiu Manoil 		rbmr |= ENETC_RBMR_VTE;
2195d4fd0404SClaudiu Manoil 
2196d4fd0404SClaudiu Manoil 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2197d4fd0404SClaudiu Manoil 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2198d4fd0404SClaudiu Manoil 
2199fbf1cff9SVladimir Oltean 	rx_ring->next_to_clean = 0;
2200fbf1cff9SVladimir Oltean 	rx_ring->next_to_use = 0;
2201fbf1cff9SVladimir Oltean 	rx_ring->next_to_alloc = 0;
2202fbf1cff9SVladimir Oltean 
22037a5222cbSVladimir Oltean 	enetc_lock_mdio();
2204d4fd0404SClaudiu Manoil 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
22057a5222cbSVladimir Oltean 	enetc_unlock_mdio();
2206d4fd0404SClaudiu Manoil 
2207d4fd0404SClaudiu Manoil 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2208d4fd0404SClaudiu Manoil }
2209d4fd0404SClaudiu Manoil 
enetc_setup_bdrs(struct enetc_ndev_priv * priv,bool extended)2210d075db51SVladimir Oltean static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2211d4fd0404SClaudiu Manoil {
2212715bf261SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
2213d4fd0404SClaudiu Manoil 	int i;
2214d4fd0404SClaudiu Manoil 
2215d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++)
2216715bf261SVladimir Oltean 		enetc_setup_txbdr(hw, priv->tx_ring[i]);
2217d4fd0404SClaudiu Manoil 
2218d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_rx_rings; i++)
2219d075db51SVladimir Oltean 		enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2220d4fd0404SClaudiu Manoil }
2221d4fd0404SClaudiu Manoil 
enetc_enable_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2222ff58fda0SVladimir Oltean static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2223ff58fda0SVladimir Oltean {
2224ff58fda0SVladimir Oltean 	int idx = tx_ring->index;
2225ff58fda0SVladimir Oltean 	u32 tbmr;
2226ff58fda0SVladimir Oltean 
2227ff58fda0SVladimir Oltean 	tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2228ff58fda0SVladimir Oltean 	tbmr |= ENETC_TBMR_EN;
2229ff58fda0SVladimir Oltean 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2230ff58fda0SVladimir Oltean }
2231ff58fda0SVladimir Oltean 
enetc_enable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2232ff58fda0SVladimir Oltean static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2233ff58fda0SVladimir Oltean {
2234ff58fda0SVladimir Oltean 	int idx = rx_ring->index;
2235ff58fda0SVladimir Oltean 	u32 rbmr;
2236ff58fda0SVladimir Oltean 
2237ff58fda0SVladimir Oltean 	rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2238ff58fda0SVladimir Oltean 	rbmr |= ENETC_RBMR_EN;
2239ff58fda0SVladimir Oltean 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2240ff58fda0SVladimir Oltean }
2241ff58fda0SVladimir Oltean 
enetc_enable_rx_bdrs(struct enetc_ndev_priv * priv)2242a419f478SWei Fang static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv)
2243a419f478SWei Fang {
2244a419f478SWei Fang 	struct enetc_hw *hw = &priv->si->hw;
2245a419f478SWei Fang 	int i;
2246a419f478SWei Fang 
2247a419f478SWei Fang 	for (i = 0; i < priv->num_rx_rings; i++)
2248a419f478SWei Fang 		enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2249a419f478SWei Fang }
2250a419f478SWei Fang 
enetc_enable_tx_bdrs(struct enetc_ndev_priv * priv)2251a419f478SWei Fang static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv)
2252ff58fda0SVladimir Oltean {
2253ff58fda0SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
2254ff58fda0SVladimir Oltean 	int i;
2255ff58fda0SVladimir Oltean 
2256ff58fda0SVladimir Oltean 	for (i = 0; i < priv->num_tx_rings; i++)
2257ff58fda0SVladimir Oltean 		enetc_enable_txbdr(hw, priv->tx_ring[i]);
2258ff58fda0SVladimir Oltean }
2259ff58fda0SVladimir Oltean 
enetc_disable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2260ff58fda0SVladimir Oltean static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2261d4fd0404SClaudiu Manoil {
2262d4fd0404SClaudiu Manoil 	int idx = rx_ring->index;
2263d4fd0404SClaudiu Manoil 
2264d4fd0404SClaudiu Manoil 	/* disable EN bit on ring */
2265d4fd0404SClaudiu Manoil 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2266d4fd0404SClaudiu Manoil }
2267d4fd0404SClaudiu Manoil 
enetc_disable_txbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2268ff58fda0SVladimir Oltean static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2269d4fd0404SClaudiu Manoil {
2270ff58fda0SVladimir Oltean 	int idx = rx_ring->index;
2271d4fd0404SClaudiu Manoil 
2272d4fd0404SClaudiu Manoil 	/* disable EN bit on ring */
2273d4fd0404SClaudiu Manoil 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2274ff58fda0SVladimir Oltean }
2275ff58fda0SVladimir Oltean 
enetc_disable_rx_bdrs(struct enetc_ndev_priv * priv)2276a419f478SWei Fang static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv)
2277a419f478SWei Fang {
2278a419f478SWei Fang 	struct enetc_hw *hw = &priv->si->hw;
2279a419f478SWei Fang 	int i;
2280a419f478SWei Fang 
2281a419f478SWei Fang 	for (i = 0; i < priv->num_rx_rings; i++)
2282a419f478SWei Fang 		enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2283a419f478SWei Fang }
2284a419f478SWei Fang 
enetc_disable_tx_bdrs(struct enetc_ndev_priv * priv)2285a419f478SWei Fang static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv)
2286ff58fda0SVladimir Oltean {
2287ff58fda0SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
2288ff58fda0SVladimir Oltean 	int i;
2289ff58fda0SVladimir Oltean 
2290ff58fda0SVladimir Oltean 	for (i = 0; i < priv->num_tx_rings; i++)
2291ff58fda0SVladimir Oltean 		enetc_disable_txbdr(hw, priv->tx_ring[i]);
2292ff58fda0SVladimir Oltean }
2293ff58fda0SVladimir Oltean 
enetc_wait_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2294ff58fda0SVladimir Oltean static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2295ff58fda0SVladimir Oltean {
2296ff58fda0SVladimir Oltean 	int delay = 8, timeout = 100;
2297ff58fda0SVladimir Oltean 	int idx = tx_ring->index;
2298d4fd0404SClaudiu Manoil 
2299d4fd0404SClaudiu Manoil 	/* wait for busy to clear */
2300d4fd0404SClaudiu Manoil 	while (delay < timeout &&
2301d4fd0404SClaudiu Manoil 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2302d4fd0404SClaudiu Manoil 		msleep(delay);
2303d4fd0404SClaudiu Manoil 		delay *= 2;
2304d4fd0404SClaudiu Manoil 	}
2305d4fd0404SClaudiu Manoil 
2306d4fd0404SClaudiu Manoil 	if (delay >= timeout)
2307d4fd0404SClaudiu Manoil 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2308d4fd0404SClaudiu Manoil 			    idx);
2309d4fd0404SClaudiu Manoil }
2310d4fd0404SClaudiu Manoil 
enetc_wait_bdrs(struct enetc_ndev_priv * priv)2311ff58fda0SVladimir Oltean static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2312d4fd0404SClaudiu Manoil {
2313715bf261SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
2314d4fd0404SClaudiu Manoil 	int i;
2315d4fd0404SClaudiu Manoil 
2316d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++)
2317ff58fda0SVladimir Oltean 		enetc_wait_txbdr(hw, priv->tx_ring[i]);
2318d4fd0404SClaudiu Manoil }
2319d4fd0404SClaudiu Manoil 
enetc_setup_irqs(struct enetc_ndev_priv * priv)2320d4fd0404SClaudiu Manoil static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2321d4fd0404SClaudiu Manoil {
2322d4fd0404SClaudiu Manoil 	struct pci_dev *pdev = priv->si->pdev;
2323715bf261SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
2324d4fd0404SClaudiu Manoil 	int i, j, err;
2325d4fd0404SClaudiu Manoil 
2326d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->bdr_int_num; i++) {
2327d4fd0404SClaudiu Manoil 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2328d4fd0404SClaudiu Manoil 		struct enetc_int_vector *v = priv->int_vector[i];
2329d4fd0404SClaudiu Manoil 		int entry = ENETC_BDR_INT_BASE_IDX + i;
2330d4fd0404SClaudiu Manoil 
2331d4fd0404SClaudiu Manoil 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2332d4fd0404SClaudiu Manoil 			 priv->ndev->name, i);
23331e8fc4ffSJinjie Ruan 		err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);
2334d4fd0404SClaudiu Manoil 		if (err) {
2335d4fd0404SClaudiu Manoil 			dev_err(priv->dev, "request_irq() failed!\n");
2336d4fd0404SClaudiu Manoil 			goto irq_err;
2337d4fd0404SClaudiu Manoil 		}
2338d4fd0404SClaudiu Manoil 
2339d4fd0404SClaudiu Manoil 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2340d4fd0404SClaudiu Manoil 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
234191571081SClaudiu Manoil 		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2342d4fd0404SClaudiu Manoil 
2343d4fd0404SClaudiu Manoil 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2344d4fd0404SClaudiu Manoil 
2345d4fd0404SClaudiu Manoil 		for (j = 0; j < v->count_tx_rings; j++) {
2346d4fd0404SClaudiu Manoil 			int idx = v->tx_ring[j].index;
2347d4fd0404SClaudiu Manoil 
2348d4fd0404SClaudiu Manoil 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2349d4fd0404SClaudiu Manoil 		}
23507237a494SClaudiu Manoil 		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2351d4fd0404SClaudiu Manoil 	}
2352d4fd0404SClaudiu Manoil 
2353d4fd0404SClaudiu Manoil 	return 0;
2354d4fd0404SClaudiu Manoil 
2355d4fd0404SClaudiu Manoil irq_err:
2356d4fd0404SClaudiu Manoil 	while (i--) {
2357d4fd0404SClaudiu Manoil 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2358d4fd0404SClaudiu Manoil 
2359d4fd0404SClaudiu Manoil 		irq_set_affinity_hint(irq, NULL);
2360d4fd0404SClaudiu Manoil 		free_irq(irq, priv->int_vector[i]);
2361d4fd0404SClaudiu Manoil 	}
2362d4fd0404SClaudiu Manoil 
2363d4fd0404SClaudiu Manoil 	return err;
2364d4fd0404SClaudiu Manoil }
2365d4fd0404SClaudiu Manoil 
enetc_free_irqs(struct enetc_ndev_priv * priv)2366d4fd0404SClaudiu Manoil static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2367d4fd0404SClaudiu Manoil {
2368d4fd0404SClaudiu Manoil 	struct pci_dev *pdev = priv->si->pdev;
2369d4fd0404SClaudiu Manoil 	int i;
2370d4fd0404SClaudiu Manoil 
2371d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->bdr_int_num; i++) {
2372d4fd0404SClaudiu Manoil 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2373d4fd0404SClaudiu Manoil 
2374d4fd0404SClaudiu Manoil 		irq_set_affinity_hint(irq, NULL);
2375d4fd0404SClaudiu Manoil 		free_irq(irq, priv->int_vector[i]);
2376d4fd0404SClaudiu Manoil 	}
2377d4fd0404SClaudiu Manoil }
2378d4fd0404SClaudiu Manoil 
enetc_setup_interrupts(struct enetc_ndev_priv * priv)2379bbb96dc7SClaudiu Manoil static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2380d4fd0404SClaudiu Manoil {
238191571081SClaudiu Manoil 	struct enetc_hw *hw = &priv->si->hw;
238291571081SClaudiu Manoil 	u32 icpt, ictt;
2383d4fd0404SClaudiu Manoil 	int i;
2384d4fd0404SClaudiu Manoil 
2385d4fd0404SClaudiu Manoil 	/* enable Tx & Rx event indication */
2386ae0e6a5dSClaudiu Manoil 	if (priv->ic_mode &
2387ae0e6a5dSClaudiu Manoil 	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
238891571081SClaudiu Manoil 		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
238991571081SClaudiu Manoil 		/* init to non-0 minimum, will be adjusted later */
239091571081SClaudiu Manoil 		ictt = 0x1;
239191571081SClaudiu Manoil 	} else {
239291571081SClaudiu Manoil 		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
239391571081SClaudiu Manoil 		ictt = 0;
2394d4fd0404SClaudiu Manoil 	}
2395d4fd0404SClaudiu Manoil 
239691571081SClaudiu Manoil 	for (i = 0; i < priv->num_rx_rings; i++) {
239791571081SClaudiu Manoil 		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
239891571081SClaudiu Manoil 		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
239991571081SClaudiu Manoil 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
240091571081SClaudiu Manoil 	}
240191571081SClaudiu Manoil 
240291571081SClaudiu Manoil 	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
240391571081SClaudiu Manoil 		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
240491571081SClaudiu Manoil 	else
240591571081SClaudiu Manoil 		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
240691571081SClaudiu Manoil 
2407d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++) {
240891571081SClaudiu Manoil 		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
240991571081SClaudiu Manoil 		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
241091571081SClaudiu Manoil 		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2411d4fd0404SClaudiu Manoil 	}
2412d4fd0404SClaudiu Manoil }
2413d4fd0404SClaudiu Manoil 
enetc_clear_interrupts(struct enetc_ndev_priv * priv)2414bbb96dc7SClaudiu Manoil static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2415d4fd0404SClaudiu Manoil {
2416715bf261SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
2417d4fd0404SClaudiu Manoil 	int i;
2418d4fd0404SClaudiu Manoil 
2419d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++)
2420715bf261SVladimir Oltean 		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2421d4fd0404SClaudiu Manoil 
2422d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_rx_rings; i++)
2423715bf261SVladimir Oltean 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2424d4fd0404SClaudiu Manoil }
2425d4fd0404SClaudiu Manoil 
enetc_phylink_connect(struct net_device * ndev)242671b77a7aSClaudiu Manoil static int enetc_phylink_connect(struct net_device *ndev)
2427d4fd0404SClaudiu Manoil {
24282e47cb41SPo Liu 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2429a6a10d45SYangbo Lu 	struct ethtool_eee edata;
243071b77a7aSClaudiu Manoil 	int err;
2431d4fd0404SClaudiu Manoil 
2432598ca0d0SVladimir Oltean 	if (!priv->phylink) {
2433598ca0d0SVladimir Oltean 		/* phy-less mode */
2434598ca0d0SVladimir Oltean 		netif_carrier_on(ndev);
2435598ca0d0SVladimir Oltean 		return 0;
2436598ca0d0SVladimir Oltean 	}
2437d4fd0404SClaudiu Manoil 
243871b77a7aSClaudiu Manoil 	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
243971b77a7aSClaudiu Manoil 	if (err) {
2440d4fd0404SClaudiu Manoil 		dev_err(&ndev->dev, "could not attach to PHY\n");
244171b77a7aSClaudiu Manoil 		return err;
2442d4fd0404SClaudiu Manoil 	}
2443d4fd0404SClaudiu Manoil 
2444a6a10d45SYangbo Lu 	/* disable EEE autoneg, until ENETC driver supports it */
2445a6a10d45SYangbo Lu 	memset(&edata, 0, sizeof(struct ethtool_eee));
244671b77a7aSClaudiu Manoil 	phylink_ethtool_set_eee(priv->phylink, &edata);
2447a6a10d45SYangbo Lu 
2448598ca0d0SVladimir Oltean 	phylink_start(priv->phylink);
2449598ca0d0SVladimir Oltean 
2450d4fd0404SClaudiu Manoil 	return 0;
2451d4fd0404SClaudiu Manoil }
2452d4fd0404SClaudiu Manoil 
enetc_tx_onestep_tstamp(struct work_struct * work)24537294380cSYangbo Lu static void enetc_tx_onestep_tstamp(struct work_struct *work)
24547294380cSYangbo Lu {
24557294380cSYangbo Lu 	struct enetc_ndev_priv *priv;
24567294380cSYangbo Lu 	struct sk_buff *skb;
24577294380cSYangbo Lu 
24587294380cSYangbo Lu 	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
24597294380cSYangbo Lu 
24603c463721SVladimir Oltean 	netif_tx_lock_bh(priv->ndev);
24617294380cSYangbo Lu 
24627294380cSYangbo Lu 	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
24637294380cSYangbo Lu 	skb = skb_dequeue(&priv->tx_skbs);
24647294380cSYangbo Lu 	if (skb)
24657294380cSYangbo Lu 		enetc_start_xmit(skb, priv->ndev);
24667294380cSYangbo Lu 
24673c463721SVladimir Oltean 	netif_tx_unlock_bh(priv->ndev);
24687294380cSYangbo Lu }
24697294380cSYangbo Lu 
enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv * priv)24707294380cSYangbo Lu static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
24717294380cSYangbo Lu {
24727294380cSYangbo Lu 	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
24737294380cSYangbo Lu 	skb_queue_head_init(&priv->tx_skbs);
24747294380cSYangbo Lu }
24757294380cSYangbo Lu 
enetc_start(struct net_device * ndev)247691571081SClaudiu Manoil void enetc_start(struct net_device *ndev)
2477bbb96dc7SClaudiu Manoil {
2478bbb96dc7SClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2479bbb96dc7SClaudiu Manoil 	int i;
2480bbb96dc7SClaudiu Manoil 
2481bbb96dc7SClaudiu Manoil 	enetc_setup_interrupts(priv);
2482bbb96dc7SClaudiu Manoil 
2483bbb96dc7SClaudiu Manoil 	for (i = 0; i < priv->bdr_int_num; i++) {
2484bbb96dc7SClaudiu Manoil 		int irq = pci_irq_vector(priv->si->pdev,
2485bbb96dc7SClaudiu Manoil 					 ENETC_BDR_INT_BASE_IDX + i);
2486bbb96dc7SClaudiu Manoil 
2487bbb96dc7SClaudiu Manoil 		napi_enable(&priv->int_vector[i]->napi);
2488bbb96dc7SClaudiu Manoil 		enable_irq(irq);
2489bbb96dc7SClaudiu Manoil 	}
2490bbb96dc7SClaudiu Manoil 
2491760a7c96SWei Fang 	enetc_enable_tx_bdrs(priv);
2492760a7c96SWei Fang 
2493a419f478SWei Fang 	enetc_enable_rx_bdrs(priv);
2494ff58fda0SVladimir Oltean 
2495bbb96dc7SClaudiu Manoil 	netif_tx_start_all_queues(ndev);
249637184349SWei Fang 
249737184349SWei Fang 	clear_bit(ENETC_TX_DOWN, &priv->flags);
2498bbb96dc7SClaudiu Manoil }
2499e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_start);
2500bbb96dc7SClaudiu Manoil 
enetc_open(struct net_device * ndev)2501d4fd0404SClaudiu Manoil int enetc_open(struct net_device *ndev)
2502d4fd0404SClaudiu Manoil {
2503d4fd0404SClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2504f3ce29e1SVladimir Oltean 	struct enetc_bdr_resource *tx_res, *rx_res;
2505d075db51SVladimir Oltean 	bool extended;
2506bbb96dc7SClaudiu Manoil 	int err;
2507d4fd0404SClaudiu Manoil 
2508d075db51SVladimir Oltean 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2509d075db51SVladimir Oltean 
2510d4fd0404SClaudiu Manoil 	err = enetc_setup_irqs(priv);
2511d4fd0404SClaudiu Manoil 	if (err)
2512d4fd0404SClaudiu Manoil 		return err;
2513d4fd0404SClaudiu Manoil 
251471b77a7aSClaudiu Manoil 	err = enetc_phylink_connect(ndev);
2515d4fd0404SClaudiu Manoil 	if (err)
2516d4fd0404SClaudiu Manoil 		goto err_phy_connect;
2517d4fd0404SClaudiu Manoil 
2518f3ce29e1SVladimir Oltean 	tx_res = enetc_alloc_tx_resources(priv);
2519f3ce29e1SVladimir Oltean 	if (IS_ERR(tx_res)) {
2520f3ce29e1SVladimir Oltean 		err = PTR_ERR(tx_res);
2521d4fd0404SClaudiu Manoil 		goto err_alloc_tx;
2522f3ce29e1SVladimir Oltean 	}
2523d4fd0404SClaudiu Manoil 
2524f3ce29e1SVladimir Oltean 	rx_res = enetc_alloc_rx_resources(priv, extended);
2525f3ce29e1SVladimir Oltean 	if (IS_ERR(rx_res)) {
2526f3ce29e1SVladimir Oltean 		err = PTR_ERR(rx_res);
2527d4fd0404SClaudiu Manoil 		goto err_alloc_rx;
2528f3ce29e1SVladimir Oltean 	}
2529d4fd0404SClaudiu Manoil 
25307294380cSYangbo Lu 	enetc_tx_onestep_tstamp_init(priv);
2531f3ce29e1SVladimir Oltean 	enetc_assign_tx_resources(priv, tx_res);
2532f3ce29e1SVladimir Oltean 	enetc_assign_rx_resources(priv, rx_res);
2533d075db51SVladimir Oltean 	enetc_setup_bdrs(priv, extended);
2534bbb96dc7SClaudiu Manoil 	enetc_start(ndev);
2535d4fd0404SClaudiu Manoil 
2536d4fd0404SClaudiu Manoil 	return 0;
2537d4fd0404SClaudiu Manoil 
2538d4fd0404SClaudiu Manoil err_alloc_rx:
2539f3ce29e1SVladimir Oltean 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2540d4fd0404SClaudiu Manoil err_alloc_tx:
254171b77a7aSClaudiu Manoil 	if (priv->phylink)
254271b77a7aSClaudiu Manoil 		phylink_disconnect_phy(priv->phylink);
2543d4fd0404SClaudiu Manoil err_phy_connect:
2544d4fd0404SClaudiu Manoil 	enetc_free_irqs(priv);
2545d4fd0404SClaudiu Manoil 
2546d4fd0404SClaudiu Manoil 	return err;
2547d4fd0404SClaudiu Manoil }
2548e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_open);
2549d4fd0404SClaudiu Manoil 
enetc_stop(struct net_device * ndev)255091571081SClaudiu Manoil void enetc_stop(struct net_device *ndev)
2551d4fd0404SClaudiu Manoil {
2552d4fd0404SClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2553d4fd0404SClaudiu Manoil 	int i;
2554d4fd0404SClaudiu Manoil 
255537184349SWei Fang 	set_bit(ENETC_TX_DOWN, &priv->flags);
255637184349SWei Fang 
2557d4fd0404SClaudiu Manoil 	netif_tx_stop_all_queues(ndev);
2558d4fd0404SClaudiu Manoil 
2559a419f478SWei Fang 	enetc_disable_rx_bdrs(priv);
2560ff58fda0SVladimir Oltean 
2561760a7c96SWei Fang 	enetc_wait_bdrs(priv);
2562760a7c96SWei Fang 
2563760a7c96SWei Fang 	enetc_disable_tx_bdrs(priv);
2564760a7c96SWei Fang 
2565d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->bdr_int_num; i++) {
2566bbb96dc7SClaudiu Manoil 		int irq = pci_irq_vector(priv->si->pdev,
2567bbb96dc7SClaudiu Manoil 					 ENETC_BDR_INT_BASE_IDX + i);
2568bbb96dc7SClaudiu Manoil 
2569bbb96dc7SClaudiu Manoil 		disable_irq(irq);
2570d4fd0404SClaudiu Manoil 		napi_synchronize(&priv->int_vector[i]->napi);
2571d4fd0404SClaudiu Manoil 		napi_disable(&priv->int_vector[i]->napi);
2572d4fd0404SClaudiu Manoil 	}
2573d4fd0404SClaudiu Manoil 
2574bbb96dc7SClaudiu Manoil 	enetc_clear_interrupts(priv);
2575bbb96dc7SClaudiu Manoil }
2576e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_stop);
2577bbb96dc7SClaudiu Manoil 
enetc_close(struct net_device * ndev)2578bbb96dc7SClaudiu Manoil int enetc_close(struct net_device *ndev)
2579bbb96dc7SClaudiu Manoil {
2580bbb96dc7SClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2581bbb96dc7SClaudiu Manoil 
2582bbb96dc7SClaudiu Manoil 	enetc_stop(ndev);
2583d4fd0404SClaudiu Manoil 
2584598ca0d0SVladimir Oltean 	if (priv->phylink) {
2585598ca0d0SVladimir Oltean 		phylink_stop(priv->phylink);
258671b77a7aSClaudiu Manoil 		phylink_disconnect_phy(priv->phylink);
2587598ca0d0SVladimir Oltean 	} else {
2588598ca0d0SVladimir Oltean 		netif_carrier_off(ndev);
2589598ca0d0SVladimir Oltean 	}
2590598ca0d0SVladimir Oltean 
2591d4fd0404SClaudiu Manoil 	enetc_free_rxtx_rings(priv);
2592f3ce29e1SVladimir Oltean 
2593f3ce29e1SVladimir Oltean 	/* Avoids dangling pointers and also frees old resources */
2594f3ce29e1SVladimir Oltean 	enetc_assign_rx_resources(priv, NULL);
2595f3ce29e1SVladimir Oltean 	enetc_assign_tx_resources(priv, NULL);
2596f3ce29e1SVladimir Oltean 
2597d4fd0404SClaudiu Manoil 	enetc_free_irqs(priv);
2598d4fd0404SClaudiu Manoil 
2599d4fd0404SClaudiu Manoil 	return 0;
2600d4fd0404SClaudiu Manoil }
2601e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_close);
2602d4fd0404SClaudiu Manoil 
enetc_reconfigure(struct enetc_ndev_priv * priv,bool extended,int (* cb)(struct enetc_ndev_priv * priv,void * ctx),void * ctx)2603c33bfaf9SVladimir Oltean static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
2604c33bfaf9SVladimir Oltean 			     int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
2605c33bfaf9SVladimir Oltean 			     void *ctx)
26065093406cSVladimir Oltean {
26075093406cSVladimir Oltean 	struct enetc_bdr_resource *tx_res, *rx_res;
26085093406cSVladimir Oltean 	int err;
26095093406cSVladimir Oltean 
26105093406cSVladimir Oltean 	ASSERT_RTNL();
26115093406cSVladimir Oltean 
2612c33bfaf9SVladimir Oltean 	/* If the interface is down, run the callback right away,
2613c33bfaf9SVladimir Oltean 	 * without reconfiguration.
2614c33bfaf9SVladimir Oltean 	 */
2615c33bfaf9SVladimir Oltean 	if (!netif_running(priv->ndev)) {
261646a0ecf9SVladimir Oltean 		if (cb) {
261746a0ecf9SVladimir Oltean 			err = cb(priv, ctx);
261846a0ecf9SVladimir Oltean 			if (err)
261946a0ecf9SVladimir Oltean 				return err;
262046a0ecf9SVladimir Oltean 		}
2621c33bfaf9SVladimir Oltean 
26225093406cSVladimir Oltean 		return 0;
2623c33bfaf9SVladimir Oltean 	}
26245093406cSVladimir Oltean 
26255093406cSVladimir Oltean 	tx_res = enetc_alloc_tx_resources(priv);
26265093406cSVladimir Oltean 	if (IS_ERR(tx_res)) {
26275093406cSVladimir Oltean 		err = PTR_ERR(tx_res);
26285093406cSVladimir Oltean 		goto out;
26295093406cSVladimir Oltean 	}
26305093406cSVladimir Oltean 
26315093406cSVladimir Oltean 	rx_res = enetc_alloc_rx_resources(priv, extended);
26325093406cSVladimir Oltean 	if (IS_ERR(rx_res)) {
26335093406cSVladimir Oltean 		err = PTR_ERR(rx_res);
26345093406cSVladimir Oltean 		goto out_free_tx_res;
26355093406cSVladimir Oltean 	}
26365093406cSVladimir Oltean 
26375093406cSVladimir Oltean 	enetc_stop(priv->ndev);
26385093406cSVladimir Oltean 	enetc_free_rxtx_rings(priv);
26395093406cSVladimir Oltean 
2640c33bfaf9SVladimir Oltean 	/* Interface is down, run optional callback now */
264146a0ecf9SVladimir Oltean 	if (cb) {
264246a0ecf9SVladimir Oltean 		err = cb(priv, ctx);
264346a0ecf9SVladimir Oltean 		if (err)
264446a0ecf9SVladimir Oltean 			goto out_restart;
264546a0ecf9SVladimir Oltean 	}
2646c33bfaf9SVladimir Oltean 
26475093406cSVladimir Oltean 	enetc_assign_tx_resources(priv, tx_res);
26485093406cSVladimir Oltean 	enetc_assign_rx_resources(priv, rx_res);
26495093406cSVladimir Oltean 	enetc_setup_bdrs(priv, extended);
26505093406cSVladimir Oltean 	enetc_start(priv->ndev);
26515093406cSVladimir Oltean 
26525093406cSVladimir Oltean 	return 0;
26535093406cSVladimir Oltean 
265446a0ecf9SVladimir Oltean out_restart:
265546a0ecf9SVladimir Oltean 	enetc_setup_bdrs(priv, extended);
265646a0ecf9SVladimir Oltean 	enetc_start(priv->ndev);
265746a0ecf9SVladimir Oltean 	enetc_free_rx_resources(rx_res, priv->num_rx_rings);
26585093406cSVladimir Oltean out_free_tx_res:
26595093406cSVladimir Oltean 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
26605093406cSVladimir Oltean out:
26615093406cSVladimir Oltean 	return err;
26625093406cSVladimir Oltean }
26635093406cSVladimir Oltean 
enetc_debug_tx_ring_prios(struct enetc_ndev_priv * priv)26641a353111SVladimir Oltean static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
26651a353111SVladimir Oltean {
26661a353111SVladimir Oltean 	int i;
26671a353111SVladimir Oltean 
26681a353111SVladimir Oltean 	for (i = 0; i < priv->num_tx_rings; i++)
26691a353111SVladimir Oltean 		netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
26701a353111SVladimir Oltean 			   priv->tx_ring[i]->prio);
26711a353111SVladimir Oltean }
26721a353111SVladimir Oltean 
enetc_reset_tc_mqprio(struct net_device * ndev)26735353599aSVladimir Oltean void enetc_reset_tc_mqprio(struct net_device *ndev)
2674cbe9e835SCamelia Groza {
2675cbe9e835SCamelia Groza 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2676715bf261SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
2677cbe9e835SCamelia Groza 	struct enetc_bdr *tx_ring;
26787eab503bSVladimir Oltean 	int num_stack_tx_queues;
2679cbe9e835SCamelia Groza 	int i;
2680cbe9e835SCamelia Groza 
26817eab503bSVladimir Oltean 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2682cbe9e835SCamelia Groza 
2683cbe9e835SCamelia Groza 	netdev_reset_tc(ndev);
26847eab503bSVladimir Oltean 	netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2685800db2d1SVladimir Oltean 	priv->min_num_stack_tx_queues = num_possible_cpus();
2686cbe9e835SCamelia Groza 
2687cbe9e835SCamelia Groza 	/* Reset all ring priorities to 0 */
2688cbe9e835SCamelia Groza 	for (i = 0; i < priv->num_tx_rings; i++) {
2689cbe9e835SCamelia Groza 		tx_ring = priv->tx_ring[i];
2690290b5fe0SVladimir Oltean 		tx_ring->prio = 0;
2691290b5fe0SVladimir Oltean 		enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2692cbe9e835SCamelia Groza 	}
2693cbe9e835SCamelia Groza 
26941a353111SVladimir Oltean 	enetc_debug_tx_ring_prios(priv);
269501e23b2bSVladimir Oltean 
269682714539SVladimir Oltean 	enetc_change_preemptible_tcs(priv, 0);
26971a353111SVladimir Oltean }
26985353599aSVladimir Oltean EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
26991a353111SVladimir Oltean 
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)27001a353111SVladimir Oltean int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
27011a353111SVladimir Oltean {
270250764da3SVladimir Oltean 	struct tc_mqprio_qopt_offload *mqprio = type_data;
27031a353111SVladimir Oltean 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
270450764da3SVladimir Oltean 	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
27051a353111SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
27061a353111SVladimir Oltean 	int num_stack_tx_queues = 0;
27071a353111SVladimir Oltean 	struct enetc_bdr *tx_ring;
270850764da3SVladimir Oltean 	u8 num_tc = qopt->num_tc;
27091a353111SVladimir Oltean 	int offset, count;
27101a353111SVladimir Oltean 	int err, tc, q;
27111a353111SVladimir Oltean 
27121a353111SVladimir Oltean 	if (!num_tc) {
27131a353111SVladimir Oltean 		enetc_reset_tc_mqprio(ndev);
2714cbe9e835SCamelia Groza 		return 0;
2715cbe9e835SCamelia Groza 	}
2716cbe9e835SCamelia Groza 
27171a353111SVladimir Oltean 	err = netdev_set_num_tc(ndev, num_tc);
27181a353111SVladimir Oltean 	if (err)
27191a353111SVladimir Oltean 		return err;
27201a353111SVladimir Oltean 
27211a353111SVladimir Oltean 	for (tc = 0; tc < num_tc; tc++) {
272250764da3SVladimir Oltean 		offset = qopt->offset[tc];
272350764da3SVladimir Oltean 		count = qopt->count[tc];
27241a353111SVladimir Oltean 		num_stack_tx_queues += count;
27251a353111SVladimir Oltean 
27261a353111SVladimir Oltean 		err = netdev_set_tc_queue(ndev, tc, count, offset);
27271a353111SVladimir Oltean 		if (err)
27281a353111SVladimir Oltean 			goto err_reset_tc;
27291a353111SVladimir Oltean 
27301a353111SVladimir Oltean 		for (q = offset; q < offset + count; q++) {
27311a353111SVladimir Oltean 			tx_ring = priv->tx_ring[q];
27321a353111SVladimir Oltean 			/* The prio_tc_map is skb_tx_hash()'s way of selecting
27331a353111SVladimir Oltean 			 * between TX queues based on skb->priority. As such,
27341a353111SVladimir Oltean 			 * there's nothing to offload based on it.
27351a353111SVladimir Oltean 			 * Make the mqprio "traffic class" be the priority of
27361a353111SVladimir Oltean 			 * this ring group, and leave the Tx IPV to traffic
27371a353111SVladimir Oltean 			 * class mapping as its default mapping value of 1:1.
2738cbe9e835SCamelia Groza 			 */
27391a353111SVladimir Oltean 			tx_ring->prio = tc;
2740290b5fe0SVladimir Oltean 			enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2741cbe9e835SCamelia Groza 		}
27421a353111SVladimir Oltean 	}
2743cbe9e835SCamelia Groza 
27441a353111SVladimir Oltean 	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
27451a353111SVladimir Oltean 	if (err)
27461a353111SVladimir Oltean 		goto err_reset_tc;
2747cbe9e835SCamelia Groza 
27481a353111SVladimir Oltean 	priv->min_num_stack_tx_queues = num_stack_tx_queues;
2749cbe9e835SCamelia Groza 
27501a353111SVladimir Oltean 	enetc_debug_tx_ring_prios(priv);
2751cbe9e835SCamelia Groza 
275282714539SVladimir Oltean 	enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
275301e23b2bSVladimir Oltean 
2754cbe9e835SCamelia Groza 	return 0;
27551a353111SVladimir Oltean 
27561a353111SVladimir Oltean err_reset_tc:
27571a353111SVladimir Oltean 	enetc_reset_tc_mqprio(ndev);
27581a353111SVladimir Oltean 	return err;
2759cbe9e835SCamelia Groza }
2760e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
2761cbe9e835SCamelia Groza 
enetc_reconfigure_xdp_cb(struct enetc_ndev_priv * priv,void * ctx)2762c33bfaf9SVladimir Oltean static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
2763d1b15102SVladimir Oltean {
2764c33bfaf9SVladimir Oltean 	struct bpf_prog *old_prog, *prog = ctx;
27654ea1dd74SVladimir Oltean 	int num_stack_tx_queues;
27664ea1dd74SVladimir Oltean 	int err, i;
2767d1b15102SVladimir Oltean 
2768d1b15102SVladimir Oltean 	old_prog = xchg(&priv->xdp_prog, prog);
27694ea1dd74SVladimir Oltean 
27704ea1dd74SVladimir Oltean 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
27714ea1dd74SVladimir Oltean 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
27724ea1dd74SVladimir Oltean 	if (err) {
27734ea1dd74SVladimir Oltean 		xchg(&priv->xdp_prog, old_prog);
27744ea1dd74SVladimir Oltean 		return err;
27754ea1dd74SVladimir Oltean 	}
27764ea1dd74SVladimir Oltean 
2777d1b15102SVladimir Oltean 	if (old_prog)
2778d1b15102SVladimir Oltean 		bpf_prog_put(old_prog);
2779d1b15102SVladimir Oltean 
2780d1b15102SVladimir Oltean 	for (i = 0; i < priv->num_rx_rings; i++) {
2781d1b15102SVladimir Oltean 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
2782d1b15102SVladimir Oltean 
2783d1b15102SVladimir Oltean 		rx_ring->xdp.prog = prog;
2784d1b15102SVladimir Oltean 
2785d1b15102SVladimir Oltean 		if (prog)
2786d1b15102SVladimir Oltean 			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2787d1b15102SVladimir Oltean 		else
2788d1b15102SVladimir Oltean 			rx_ring->buffer_offset = ENETC_RXB_PAD;
2789d1b15102SVladimir Oltean 	}
2790d1b15102SVladimir Oltean 
2791d1b15102SVladimir Oltean 	return 0;
2792d1b15102SVladimir Oltean }
2793d1b15102SVladimir Oltean 
enetc_setup_xdp_prog(struct net_device * ndev,struct bpf_prog * prog,struct netlink_ext_ack * extack)2794c33bfaf9SVladimir Oltean static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
2795c33bfaf9SVladimir Oltean 				struct netlink_ext_ack *extack)
2796c33bfaf9SVladimir Oltean {
2797800db2d1SVladimir Oltean 	int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
2798c33bfaf9SVladimir Oltean 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2799c33bfaf9SVladimir Oltean 	bool extended;
2800c33bfaf9SVladimir Oltean 
2801800db2d1SVladimir Oltean 	if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
2802800db2d1SVladimir Oltean 	    priv->num_tx_rings) {
2803800db2d1SVladimir Oltean 		NL_SET_ERR_MSG_FMT_MOD(extack,
2804997a5310SArnd Bergmann 				       "Reserving %d XDP TXQs leaves under %d for stack (total %d)",
2805800db2d1SVladimir Oltean 				       num_xdp_tx_queues,
2806800db2d1SVladimir Oltean 				       priv->min_num_stack_tx_queues,
2807800db2d1SVladimir Oltean 				       priv->num_tx_rings);
2808800db2d1SVladimir Oltean 		return -EBUSY;
2809800db2d1SVladimir Oltean 	}
2810800db2d1SVladimir Oltean 
2811c33bfaf9SVladimir Oltean 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2812c33bfaf9SVladimir Oltean 
2813c33bfaf9SVladimir Oltean 	/* The buffer layout is changing, so we need to drain the old
2814c33bfaf9SVladimir Oltean 	 * RX buffers and seed new ones.
2815c33bfaf9SVladimir Oltean 	 */
2816c33bfaf9SVladimir Oltean 	return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
2817c33bfaf9SVladimir Oltean }
2818c33bfaf9SVladimir Oltean 
enetc_setup_bpf(struct net_device * ndev,struct netdev_bpf * bpf)2819766338c7SVladimir Oltean int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2820d1b15102SVladimir Oltean {
2821766338c7SVladimir Oltean 	switch (bpf->command) {
2822d1b15102SVladimir Oltean 	case XDP_SETUP_PROG:
2823766338c7SVladimir Oltean 		return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
2824d1b15102SVladimir Oltean 	default:
2825d1b15102SVladimir Oltean 		return -EINVAL;
2826d1b15102SVladimir Oltean 	}
2827d1b15102SVladimir Oltean 
2828d1b15102SVladimir Oltean 	return 0;
2829d1b15102SVladimir Oltean }
2830e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_setup_bpf);
2831d1b15102SVladimir Oltean 
enetc_get_stats(struct net_device * ndev)2832d4fd0404SClaudiu Manoil struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2833d4fd0404SClaudiu Manoil {
2834d4fd0404SClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2835d4fd0404SClaudiu Manoil 	struct net_device_stats *stats = &ndev->stats;
2836d4fd0404SClaudiu Manoil 	unsigned long packets = 0, bytes = 0;
2837285e8dedSPo Liu 	unsigned long tx_dropped = 0;
2838d4fd0404SClaudiu Manoil 	int i;
2839d4fd0404SClaudiu Manoil 
2840d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_rx_rings; i++) {
2841d4fd0404SClaudiu Manoil 		packets += priv->rx_ring[i]->stats.packets;
2842d4fd0404SClaudiu Manoil 		bytes	+= priv->rx_ring[i]->stats.bytes;
2843d4fd0404SClaudiu Manoil 	}
2844d4fd0404SClaudiu Manoil 
2845d4fd0404SClaudiu Manoil 	stats->rx_packets = packets;
2846d4fd0404SClaudiu Manoil 	stats->rx_bytes = bytes;
2847d4fd0404SClaudiu Manoil 	bytes = 0;
2848d4fd0404SClaudiu Manoil 	packets = 0;
2849d4fd0404SClaudiu Manoil 
2850d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++) {
2851d4fd0404SClaudiu Manoil 		packets += priv->tx_ring[i]->stats.packets;
2852d4fd0404SClaudiu Manoil 		bytes	+= priv->tx_ring[i]->stats.bytes;
2853285e8dedSPo Liu 		tx_dropped += priv->tx_ring[i]->stats.win_drop;
2854d4fd0404SClaudiu Manoil 	}
2855d4fd0404SClaudiu Manoil 
2856d4fd0404SClaudiu Manoil 	stats->tx_packets = packets;
2857d4fd0404SClaudiu Manoil 	stats->tx_bytes = bytes;
2858285e8dedSPo Liu 	stats->tx_dropped = tx_dropped;
2859d4fd0404SClaudiu Manoil 
2860d4fd0404SClaudiu Manoil 	return stats;
2861d4fd0404SClaudiu Manoil }
2862e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_get_stats);
2863d4fd0404SClaudiu Manoil 
enetc_set_rss(struct net_device * ndev,int en)2864d382563fSClaudiu Manoil static int enetc_set_rss(struct net_device *ndev, int en)
2865d382563fSClaudiu Manoil {
2866d382563fSClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2867d382563fSClaudiu Manoil 	struct enetc_hw *hw = &priv->si->hw;
2868d382563fSClaudiu Manoil 	u32 reg;
2869d382563fSClaudiu Manoil 
2870d382563fSClaudiu Manoil 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2871d382563fSClaudiu Manoil 
2872d382563fSClaudiu Manoil 	reg = enetc_rd(hw, ENETC_SIMR);
2873d382563fSClaudiu Manoil 	reg &= ~ENETC_SIMR_RSSE;
2874d382563fSClaudiu Manoil 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
2875d382563fSClaudiu Manoil 	enetc_wr(hw, ENETC_SIMR, reg);
2876d382563fSClaudiu Manoil 
2877d382563fSClaudiu Manoil 	return 0;
2878d382563fSClaudiu Manoil }
2879d382563fSClaudiu Manoil 
enetc_enable_rxvlan(struct net_device * ndev,bool en)28809deba33fSClaudiu Manoil static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
28819deba33fSClaudiu Manoil {
28829deba33fSClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2883715bf261SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
28849deba33fSClaudiu Manoil 	int i;
28859deba33fSClaudiu Manoil 
28869deba33fSClaudiu Manoil 	for (i = 0; i < priv->num_rx_rings; i++)
2887715bf261SVladimir Oltean 		enetc_bdr_enable_rxvlan(hw, i, en);
28889deba33fSClaudiu Manoil }
28899deba33fSClaudiu Manoil 
enetc_enable_txvlan(struct net_device * ndev,bool en)28909deba33fSClaudiu Manoil static void enetc_enable_txvlan(struct net_device *ndev, bool en)
28919deba33fSClaudiu Manoil {
28929deba33fSClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2893715bf261SVladimir Oltean 	struct enetc_hw *hw = &priv->si->hw;
28949deba33fSClaudiu Manoil 	int i;
28959deba33fSClaudiu Manoil 
28969deba33fSClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++)
2897715bf261SVladimir Oltean 		enetc_bdr_enable_txvlan(hw, i, en);
28989deba33fSClaudiu Manoil }
28999deba33fSClaudiu Manoil 
enetc_set_features(struct net_device * ndev,netdev_features_t features)2900fed38e64SVladimir Oltean void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2901d382563fSClaudiu Manoil {
2902d382563fSClaudiu Manoil 	netdev_features_t changed = ndev->features ^ features;
2903d382563fSClaudiu Manoil 
2904d382563fSClaudiu Manoil 	if (changed & NETIF_F_RXHASH)
2905d382563fSClaudiu Manoil 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2906d382563fSClaudiu Manoil 
29079deba33fSClaudiu Manoil 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
29089deba33fSClaudiu Manoil 		enetc_enable_rxvlan(ndev,
29099deba33fSClaudiu Manoil 				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
29109deba33fSClaudiu Manoil 
29119deba33fSClaudiu Manoil 	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
29129deba33fSClaudiu Manoil 		enetc_enable_txvlan(ndev,
29139deba33fSClaudiu Manoil 				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2914d382563fSClaudiu Manoil }
2915e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_set_features);
2916d382563fSClaudiu Manoil 
2917434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)2918d3982312SY.b. Lu static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2919d3982312SY.b. Lu {
2920d3982312SY.b. Lu 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
29215093406cSVladimir Oltean 	int err, new_offloads = priv->active_offloads;
2922d3982312SY.b. Lu 	struct hwtstamp_config config;
2923d3982312SY.b. Lu 
2924d3982312SY.b. Lu 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2925d3982312SY.b. Lu 		return -EFAULT;
2926d3982312SY.b. Lu 
2927d3982312SY.b. Lu 	switch (config.tx_type) {
2928d3982312SY.b. Lu 	case HWTSTAMP_TX_OFF:
29295093406cSVladimir Oltean 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2930d3982312SY.b. Lu 		break;
2931d3982312SY.b. Lu 	case HWTSTAMP_TX_ON:
29325093406cSVladimir Oltean 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
29335093406cSVladimir Oltean 		new_offloads |= ENETC_F_TX_TSTAMP;
2934d3982312SY.b. Lu 		break;
29357294380cSYangbo Lu 	case HWTSTAMP_TX_ONESTEP_SYNC:
29365093406cSVladimir Oltean 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
29375093406cSVladimir Oltean 		new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
29387294380cSYangbo Lu 		break;
2939d3982312SY.b. Lu 	default:
2940d3982312SY.b. Lu 		return -ERANGE;
2941d3982312SY.b. Lu 	}
2942d3982312SY.b. Lu 
2943d3982312SY.b. Lu 	switch (config.rx_filter) {
2944d3982312SY.b. Lu 	case HWTSTAMP_FILTER_NONE:
29455093406cSVladimir Oltean 		new_offloads &= ~ENETC_F_RX_TSTAMP;
2946d3982312SY.b. Lu 		break;
2947d3982312SY.b. Lu 	default:
29485093406cSVladimir Oltean 		new_offloads |= ENETC_F_RX_TSTAMP;
2949d3982312SY.b. Lu 		config.rx_filter = HWTSTAMP_FILTER_ALL;
2950d3982312SY.b. Lu 	}
2951d3982312SY.b. Lu 
29525093406cSVladimir Oltean 	if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
29535093406cSVladimir Oltean 		bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
29545093406cSVladimir Oltean 
2955c33bfaf9SVladimir Oltean 		err = enetc_reconfigure(priv, extended, NULL, NULL);
29565093406cSVladimir Oltean 		if (err)
29575093406cSVladimir Oltean 			return err;
2958434cebabSClaudiu Manoil 	}
2959434cebabSClaudiu Manoil 
29605093406cSVladimir Oltean 	priv->active_offloads = new_offloads;
29615093406cSVladimir Oltean 
2962d3982312SY.b. Lu 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2963d3982312SY.b. Lu 	       -EFAULT : 0;
2964d3982312SY.b. Lu }
2965d3982312SY.b. Lu 
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)2966d3982312SY.b. Lu static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2967d3982312SY.b. Lu {
2968d3982312SY.b. Lu 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2969d3982312SY.b. Lu 	struct hwtstamp_config config;
2970d3982312SY.b. Lu 
2971d3982312SY.b. Lu 	config.flags = 0;
2972d3982312SY.b. Lu 
29737294380cSYangbo Lu 	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
29747294380cSYangbo Lu 		config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
29757294380cSYangbo Lu 	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2976d3982312SY.b. Lu 		config.tx_type = HWTSTAMP_TX_ON;
2977d3982312SY.b. Lu 	else
2978d3982312SY.b. Lu 		config.tx_type = HWTSTAMP_TX_OFF;
2979d3982312SY.b. Lu 
2980d3982312SY.b. Lu 	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2981d3982312SY.b. Lu 			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2982d3982312SY.b. Lu 
2983d3982312SY.b. Lu 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2984d3982312SY.b. Lu 	       -EFAULT : 0;
2985d3982312SY.b. Lu }
2986d3982312SY.b. Lu #endif
2987d3982312SY.b. Lu 
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)2988d3982312SY.b. Lu int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2989d3982312SY.b. Lu {
299071b77a7aSClaudiu Manoil 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2991434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2992d3982312SY.b. Lu 	if (cmd == SIOCSHWTSTAMP)
2993d3982312SY.b. Lu 		return enetc_hwtstamp_set(ndev, rq);
2994d3982312SY.b. Lu 	if (cmd == SIOCGHWTSTAMP)
2995d3982312SY.b. Lu 		return enetc_hwtstamp_get(ndev, rq);
2996d3982312SY.b. Lu #endif
2997a613bafeSMichael Walle 
299871b77a7aSClaudiu Manoil 	if (!priv->phylink)
2999c55b810aSMichael Walle 		return -EOPNOTSUPP;
300071b77a7aSClaudiu Manoil 
300171b77a7aSClaudiu Manoil 	return phylink_mii_ioctl(priv->phylink, rq, cmd);
3002d3982312SY.b. Lu }
3003e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_ioctl);
3004d3982312SY.b. Lu 
enetc_alloc_msix(struct enetc_ndev_priv * priv)3005d4fd0404SClaudiu Manoil int enetc_alloc_msix(struct enetc_ndev_priv *priv)
3006d4fd0404SClaudiu Manoil {
3007d4fd0404SClaudiu Manoil 	struct pci_dev *pdev = priv->si->pdev;
30084ea1dd74SVladimir Oltean 	int num_stack_tx_queues;
30097eab503bSVladimir Oltean 	int first_xdp_tx_ring;
3010d4fd0404SClaudiu Manoil 	int i, n, err, nvec;
30117eab503bSVladimir Oltean 	int v_tx_rings;
3012d4fd0404SClaudiu Manoil 
3013d4fd0404SClaudiu Manoil 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
3014d4fd0404SClaudiu Manoil 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
3015d4fd0404SClaudiu Manoil 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
3016d4fd0404SClaudiu Manoil 
3017d4fd0404SClaudiu Manoil 	if (n < 0)
3018d4fd0404SClaudiu Manoil 		return n;
3019d4fd0404SClaudiu Manoil 
3020d4fd0404SClaudiu Manoil 	if (n != nvec)
3021d4fd0404SClaudiu Manoil 		return -EPERM;
3022d4fd0404SClaudiu Manoil 
3023d4fd0404SClaudiu Manoil 	/* # of tx rings per int vector */
3024d4fd0404SClaudiu Manoil 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
3025d4fd0404SClaudiu Manoil 
3026d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->bdr_int_num; i++) {
3027d4fd0404SClaudiu Manoil 		struct enetc_int_vector *v;
3028d4fd0404SClaudiu Manoil 		struct enetc_bdr *bdr;
3029d4fd0404SClaudiu Manoil 		int j;
3030d4fd0404SClaudiu Manoil 
30311260e772SGustavo A. R. Silva 		v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
3032d4fd0404SClaudiu Manoil 		if (!v) {
3033d4fd0404SClaudiu Manoil 			err = -ENOMEM;
3034d4fd0404SClaudiu Manoil 			goto fail;
3035d4fd0404SClaudiu Manoil 		}
3036d4fd0404SClaudiu Manoil 
3037d4fd0404SClaudiu Manoil 		priv->int_vector[i] = v;
3038d4fd0404SClaudiu Manoil 
3039d1b15102SVladimir Oltean 		bdr = &v->rx_ring;
3040d1b15102SVladimir Oltean 		bdr->index = i;
3041d1b15102SVladimir Oltean 		bdr->ndev = priv->ndev;
3042d1b15102SVladimir Oltean 		bdr->dev = priv->dev;
3043d1b15102SVladimir Oltean 		bdr->bd_count = priv->rx_bd_count;
3044d1b15102SVladimir Oltean 		bdr->buffer_offset = ENETC_RXB_PAD;
3045d1b15102SVladimir Oltean 		priv->rx_ring[i] = bdr;
3046d1b15102SVladimir Oltean 
3047d1b15102SVladimir Oltean 		err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
3048d1b15102SVladimir Oltean 		if (err) {
3049d1b15102SVladimir Oltean 			kfree(v);
3050d1b15102SVladimir Oltean 			goto fail;
3051d1b15102SVladimir Oltean 		}
3052d1b15102SVladimir Oltean 
3053d1b15102SVladimir Oltean 		err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
3054d1b15102SVladimir Oltean 						 MEM_TYPE_PAGE_SHARED, NULL);
3055d1b15102SVladimir Oltean 		if (err) {
3056d1b15102SVladimir Oltean 			xdp_rxq_info_unreg(&bdr->xdp.rxq);
3057d1b15102SVladimir Oltean 			kfree(v);
3058d1b15102SVladimir Oltean 			goto fail;
3059d1b15102SVladimir Oltean 		}
3060d1b15102SVladimir Oltean 
3061ae0e6a5dSClaudiu Manoil 		/* init defaults for adaptive IC */
3062ae0e6a5dSClaudiu Manoil 		if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3063ae0e6a5dSClaudiu Manoil 			v->rx_ictt = 0x1;
3064ae0e6a5dSClaudiu Manoil 			v->rx_dim_en = true;
3065ae0e6a5dSClaudiu Manoil 		}
3066ae0e6a5dSClaudiu Manoil 		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3067b48b89f9SJakub Kicinski 		netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3068d4fd0404SClaudiu Manoil 		v->count_tx_rings = v_tx_rings;
3069d4fd0404SClaudiu Manoil 
3070d4fd0404SClaudiu Manoil 		for (j = 0; j < v_tx_rings; j++) {
3071d4fd0404SClaudiu Manoil 			int idx;
3072d4fd0404SClaudiu Manoil 
3073d4fd0404SClaudiu Manoil 			/* default tx ring mapping policy */
30746c5e6b4cSClaudiu Manoil 			idx = priv->bdr_int_num * j + i;
3075d4fd0404SClaudiu Manoil 			__set_bit(idx, &v->tx_rings_map);
3076d4fd0404SClaudiu Manoil 			bdr = &v->tx_ring[j];
3077d4fd0404SClaudiu Manoil 			bdr->index = idx;
3078d4fd0404SClaudiu Manoil 			bdr->ndev = priv->ndev;
3079d4fd0404SClaudiu Manoil 			bdr->dev = priv->dev;
3080d4fd0404SClaudiu Manoil 			bdr->bd_count = priv->tx_bd_count;
3081d4fd0404SClaudiu Manoil 			priv->tx_ring[idx] = bdr;
3082d4fd0404SClaudiu Manoil 		}
3083d4fd0404SClaudiu Manoil 	}
3084d4fd0404SClaudiu Manoil 
30854ea1dd74SVladimir Oltean 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
30864ea1dd74SVladimir Oltean 
30874ea1dd74SVladimir Oltean 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
30884ea1dd74SVladimir Oltean 	if (err)
30894ea1dd74SVladimir Oltean 		goto fail;
30904ea1dd74SVladimir Oltean 
30914ea1dd74SVladimir Oltean 	err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
30924ea1dd74SVladimir Oltean 	if (err)
30934ea1dd74SVladimir Oltean 		goto fail;
30944ea1dd74SVladimir Oltean 
3095800db2d1SVladimir Oltean 	priv->min_num_stack_tx_queues = num_possible_cpus();
30967eab503bSVladimir Oltean 	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
30977eab503bSVladimir Oltean 	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
30987eab503bSVladimir Oltean 
3099d4fd0404SClaudiu Manoil 	return 0;
3100d4fd0404SClaudiu Manoil 
3101d4fd0404SClaudiu Manoil fail:
3102d4fd0404SClaudiu Manoil 	while (i--) {
3103d1b15102SVladimir Oltean 		struct enetc_int_vector *v = priv->int_vector[i];
3104d1b15102SVladimir Oltean 		struct enetc_bdr *rx_ring = &v->rx_ring;
3105d1b15102SVladimir Oltean 
3106d1b15102SVladimir Oltean 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3107d1b15102SVladimir Oltean 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3108d1b15102SVladimir Oltean 		netif_napi_del(&v->napi);
3109d1b15102SVladimir Oltean 		cancel_work_sync(&v->rx_dim.work);
3110d1b15102SVladimir Oltean 		kfree(v);
3111d4fd0404SClaudiu Manoil 	}
3112d4fd0404SClaudiu Manoil 
3113d4fd0404SClaudiu Manoil 	pci_free_irq_vectors(pdev);
3114d4fd0404SClaudiu Manoil 
3115d4fd0404SClaudiu Manoil 	return err;
3116d4fd0404SClaudiu Manoil }
3117e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3118d4fd0404SClaudiu Manoil 
enetc_free_msix(struct enetc_ndev_priv * priv)3119d4fd0404SClaudiu Manoil void enetc_free_msix(struct enetc_ndev_priv *priv)
3120d4fd0404SClaudiu Manoil {
3121d4fd0404SClaudiu Manoil 	int i;
3122d4fd0404SClaudiu Manoil 
3123d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->bdr_int_num; i++) {
3124d4fd0404SClaudiu Manoil 		struct enetc_int_vector *v = priv->int_vector[i];
3125d1b15102SVladimir Oltean 		struct enetc_bdr *rx_ring = &v->rx_ring;
3126d4fd0404SClaudiu Manoil 
3127d1b15102SVladimir Oltean 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3128d1b15102SVladimir Oltean 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3129d4fd0404SClaudiu Manoil 		netif_napi_del(&v->napi);
3130ae0e6a5dSClaudiu Manoil 		cancel_work_sync(&v->rx_dim.work);
3131d4fd0404SClaudiu Manoil 	}
3132d4fd0404SClaudiu Manoil 
3133d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_rx_rings; i++)
3134d4fd0404SClaudiu Manoil 		priv->rx_ring[i] = NULL;
3135d4fd0404SClaudiu Manoil 
3136d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->num_tx_rings; i++)
3137d4fd0404SClaudiu Manoil 		priv->tx_ring[i] = NULL;
3138d4fd0404SClaudiu Manoil 
3139d4fd0404SClaudiu Manoil 	for (i = 0; i < priv->bdr_int_num; i++) {
3140d4fd0404SClaudiu Manoil 		kfree(priv->int_vector[i]);
3141d4fd0404SClaudiu Manoil 		priv->int_vector[i] = NULL;
3142d4fd0404SClaudiu Manoil 	}
3143d4fd0404SClaudiu Manoil 
3144d4fd0404SClaudiu Manoil 	/* disable all MSIX for this device */
3145d4fd0404SClaudiu Manoil 	pci_free_irq_vectors(priv->si->pdev);
3146d4fd0404SClaudiu Manoil }
3147e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_free_msix);
3148d4fd0404SClaudiu Manoil 
enetc_kfree_si(struct enetc_si * si)3149d4fd0404SClaudiu Manoil static void enetc_kfree_si(struct enetc_si *si)
3150d4fd0404SClaudiu Manoil {
3151d4fd0404SClaudiu Manoil 	char *p = (char *)si - si->pad;
3152d4fd0404SClaudiu Manoil 
3153d4fd0404SClaudiu Manoil 	kfree(p);
3154d4fd0404SClaudiu Manoil }
3155d4fd0404SClaudiu Manoil 
enetc_detect_errata(struct enetc_si * si)3156d4fd0404SClaudiu Manoil static void enetc_detect_errata(struct enetc_si *si)
3157d4fd0404SClaudiu Manoil {
3158d4fd0404SClaudiu Manoil 	if (si->pdev->revision == ENETC_REV1)
315982728b91SClaudiu Manoil 		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3160d4fd0404SClaudiu Manoil }
3161d4fd0404SClaudiu Manoil 
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)3162d4fd0404SClaudiu Manoil int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3163d4fd0404SClaudiu Manoil {
3164d4fd0404SClaudiu Manoil 	struct enetc_si *si, *p;
3165d4fd0404SClaudiu Manoil 	struct enetc_hw *hw;
3166d4fd0404SClaudiu Manoil 	size_t alloc_size;
3167d4fd0404SClaudiu Manoil 	int err, len;
3168d4fd0404SClaudiu Manoil 
3169d4fd0404SClaudiu Manoil 	pcie_flr(pdev);
3170d4fd0404SClaudiu Manoil 	err = pci_enable_device_mem(pdev);
3171a72691eeSCai Huoqing 	if (err)
3172a72691eeSCai Huoqing 		return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3173d4fd0404SClaudiu Manoil 
3174d4fd0404SClaudiu Manoil 	/* set up for high or low dma */
3175d4fd0404SClaudiu Manoil 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3176d4fd0404SClaudiu Manoil 	if (err) {
3177cfcfc8f5SChristophe JAILLET 		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3178d4fd0404SClaudiu Manoil 		goto err_dma;
3179d4fd0404SClaudiu Manoil 	}
3180d4fd0404SClaudiu Manoil 
3181d4fd0404SClaudiu Manoil 	err = pci_request_mem_regions(pdev, name);
3182d4fd0404SClaudiu Manoil 	if (err) {
3183d4fd0404SClaudiu Manoil 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3184d4fd0404SClaudiu Manoil 		goto err_pci_mem_reg;
3185d4fd0404SClaudiu Manoil 	}
3186d4fd0404SClaudiu Manoil 
3187d4fd0404SClaudiu Manoil 	pci_set_master(pdev);
3188d4fd0404SClaudiu Manoil 
3189d4fd0404SClaudiu Manoil 	alloc_size = sizeof(struct enetc_si);
3190d4fd0404SClaudiu Manoil 	if (sizeof_priv) {
3191d4fd0404SClaudiu Manoil 		/* align priv to 32B */
3192d4fd0404SClaudiu Manoil 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3193d4fd0404SClaudiu Manoil 		alloc_size += sizeof_priv;
3194d4fd0404SClaudiu Manoil 	}
3195d4fd0404SClaudiu Manoil 	/* force 32B alignment for enetc_si */
3196d4fd0404SClaudiu Manoil 	alloc_size += ENETC_SI_ALIGN - 1;
3197d4fd0404SClaudiu Manoil 
3198d4fd0404SClaudiu Manoil 	p = kzalloc(alloc_size, GFP_KERNEL);
3199d4fd0404SClaudiu Manoil 	if (!p) {
3200d4fd0404SClaudiu Manoil 		err = -ENOMEM;
3201d4fd0404SClaudiu Manoil 		goto err_alloc_si;
3202d4fd0404SClaudiu Manoil 	}
3203d4fd0404SClaudiu Manoil 
3204d4fd0404SClaudiu Manoil 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3205d4fd0404SClaudiu Manoil 	si->pad = (char *)si - (char *)p;
3206d4fd0404SClaudiu Manoil 
3207d4fd0404SClaudiu Manoil 	pci_set_drvdata(pdev, si);
3208d4fd0404SClaudiu Manoil 	si->pdev = pdev;
3209d4fd0404SClaudiu Manoil 	hw = &si->hw;
3210d4fd0404SClaudiu Manoil 
3211d4fd0404SClaudiu Manoil 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
3212d4fd0404SClaudiu Manoil 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3213d4fd0404SClaudiu Manoil 	if (!hw->reg) {
3214d4fd0404SClaudiu Manoil 		err = -ENXIO;
3215d4fd0404SClaudiu Manoil 		dev_err(&pdev->dev, "ioremap() failed\n");
3216d4fd0404SClaudiu Manoil 		goto err_ioremap;
3217d4fd0404SClaudiu Manoil 	}
3218d4fd0404SClaudiu Manoil 	if (len > ENETC_PORT_BASE)
3219d4fd0404SClaudiu Manoil 		hw->port = hw->reg + ENETC_PORT_BASE;
3220d4fd0404SClaudiu Manoil 	if (len > ENETC_GLOBAL_BASE)
3221d4fd0404SClaudiu Manoil 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
3222d4fd0404SClaudiu Manoil 
3223d4fd0404SClaudiu Manoil 	enetc_detect_errata(si);
3224d4fd0404SClaudiu Manoil 
3225d4fd0404SClaudiu Manoil 	return 0;
3226d4fd0404SClaudiu Manoil 
3227d4fd0404SClaudiu Manoil err_ioremap:
3228d4fd0404SClaudiu Manoil 	enetc_kfree_si(si);
3229d4fd0404SClaudiu Manoil err_alloc_si:
3230d4fd0404SClaudiu Manoil 	pci_release_mem_regions(pdev);
3231d4fd0404SClaudiu Manoil err_pci_mem_reg:
3232d4fd0404SClaudiu Manoil err_dma:
3233d4fd0404SClaudiu Manoil 	pci_disable_device(pdev);
3234d4fd0404SClaudiu Manoil 
3235d4fd0404SClaudiu Manoil 	return err;
3236d4fd0404SClaudiu Manoil }
3237e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_pci_probe);
3238d4fd0404SClaudiu Manoil 
enetc_pci_remove(struct pci_dev * pdev)3239d4fd0404SClaudiu Manoil void enetc_pci_remove(struct pci_dev *pdev)
3240d4fd0404SClaudiu Manoil {
3241d4fd0404SClaudiu Manoil 	struct enetc_si *si = pci_get_drvdata(pdev);
3242d4fd0404SClaudiu Manoil 	struct enetc_hw *hw = &si->hw;
3243d4fd0404SClaudiu Manoil 
3244d4fd0404SClaudiu Manoil 	iounmap(hw->reg);
3245d4fd0404SClaudiu Manoil 	enetc_kfree_si(si);
3246d4fd0404SClaudiu Manoil 	pci_release_mem_regions(pdev);
3247d4fd0404SClaudiu Manoil 	pci_disable_device(pdev);
3248d4fd0404SClaudiu Manoil }
3249e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_pci_remove);
3250e3972399SVladimir Oltean 
3251e3972399SVladimir Oltean MODULE_LICENSE("Dual BSD/GPL");
3252