1d4fd0404SClaudiu Manoil // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2d4fd0404SClaudiu Manoil /* Copyright 2017-2019 NXP */
3d4fd0404SClaudiu Manoil
4d4fd0404SClaudiu Manoil #include "enetc.h"
5d1b15102SVladimir Oltean #include <linux/bpf_trace.h>
6d4fd0404SClaudiu Manoil #include <linux/tcp.h>
7d4fd0404SClaudiu Manoil #include <linux/udp.h>
8bbcbf2eeSStephen Rothwell #include <linux/vmalloc.h>
97294380cSYangbo Lu #include <linux/ptp_classify.h>
10edce2a93SIoana Ciornei #include <net/ip6_checksum.h>
11847cbfc0SVladimir Oltean #include <net/pkt_sched.h>
12fb8629e2SIoana Ciornei #include <net/tso.h>
13d4fd0404SClaudiu Manoil
enetc_port_mac_rd(struct enetc_si * si,u32 reg)1412717decSVladimir Oltean u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
1512717decSVladimir Oltean {
1612717decSVladimir Oltean return enetc_port_rd(&si->hw, reg);
1712717decSVladimir Oltean }
1812717decSVladimir Oltean EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
1912717decSVladimir Oltean
enetc_port_mac_wr(struct enetc_si * si,u32 reg,u32 val)2012717decSVladimir Oltean void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
2112717decSVladimir Oltean {
2212717decSVladimir Oltean enetc_port_wr(&si->hw, reg, val);
2312717decSVladimir Oltean if (si->hw_features & ENETC_SI_F_QBU)
2412717decSVladimir Oltean enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val);
2512717decSVladimir Oltean }
2612717decSVladimir Oltean EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
2712717decSVladimir Oltean
enetc_change_preemptible_tcs(struct enetc_ndev_priv * priv,u8 preemptible_tcs)2882714539SVladimir Oltean static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
2982714539SVladimir Oltean u8 preemptible_tcs)
3001e23b2bSVladimir Oltean {
31*66127f0dSWei Fang if (!(priv->si->hw_features & ENETC_SI_F_QBU))
32*66127f0dSWei Fang return;
33*66127f0dSWei Fang
3482714539SVladimir Oltean priv->preemptible_tcs = preemptible_tcs;
3582714539SVladimir Oltean enetc_mm_commit_preemptible_tcs(priv);
3601e23b2bSVladimir Oltean }
3701e23b2bSVladimir Oltean
enetc_num_stack_tx_queues(struct enetc_ndev_priv * priv)387eab503bSVladimir Oltean static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
397eab503bSVladimir Oltean {
407eab503bSVladimir Oltean int num_tx_rings = priv->num_tx_rings;
417eab503bSVladimir Oltean
421c81a9b3SVladimir Oltean if (priv->xdp_prog)
437eab503bSVladimir Oltean return num_tx_rings - num_possible_cpus();
447eab503bSVladimir Oltean
457eab503bSVladimir Oltean return num_tx_rings;
467eab503bSVladimir Oltean }
477eab503bSVladimir Oltean
enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv * priv,struct enetc_bdr * tx_ring)487eab503bSVladimir Oltean static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
497eab503bSVladimir Oltean struct enetc_bdr *tx_ring)
507eab503bSVladimir Oltean {
517eab503bSVladimir Oltean int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
527eab503bSVladimir Oltean
537eab503bSVladimir Oltean return priv->rx_ring[index];
547eab503bSVladimir Oltean }
557eab503bSVladimir Oltean
enetc_tx_swbd_get_skb(struct enetc_tx_swbd * tx_swbd)569d2b68ccSVladimir Oltean static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
579d2b68ccSVladimir Oltean {
589d2b68ccSVladimir Oltean if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
599d2b68ccSVladimir Oltean return NULL;
609d2b68ccSVladimir Oltean
619d2b68ccSVladimir Oltean return tx_swbd->skb;
629d2b68ccSVladimir Oltean }
639d2b68ccSVladimir Oltean
649d2b68ccSVladimir Oltean static struct xdp_frame *
enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd * tx_swbd)659d2b68ccSVladimir Oltean enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
669d2b68ccSVladimir Oltean {
679d2b68ccSVladimir Oltean if (tx_swbd->is_xdp_redirect)
689d2b68ccSVladimir Oltean return tx_swbd->xdp_frame;
699d2b68ccSVladimir Oltean
709d2b68ccSVladimir Oltean return NULL;
719d2b68ccSVladimir Oltean }
729d2b68ccSVladimir Oltean
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)73d4fd0404SClaudiu Manoil static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
74d4fd0404SClaudiu Manoil struct enetc_tx_swbd *tx_swbd)
75d4fd0404SClaudiu Manoil {
767ed2bc80SVladimir Oltean /* For XDP_TX, pages come from RX, whereas for the other contexts where
777ed2bc80SVladimir Oltean * we have is_dma_page_set, those come from skb_frag_dma_map. We need
787ed2bc80SVladimir Oltean * to match the DMA mapping length, so we need to differentiate those.
797ed2bc80SVladimir Oltean */
80d4fd0404SClaudiu Manoil if (tx_swbd->is_dma_page)
81d4fd0404SClaudiu Manoil dma_unmap_page(tx_ring->dev, tx_swbd->dma,
827ed2bc80SVladimir Oltean tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
837ed2bc80SVladimir Oltean tx_swbd->dir);
84d4fd0404SClaudiu Manoil else
85d4fd0404SClaudiu Manoil dma_unmap_single(tx_ring->dev, tx_swbd->dma,
867ed2bc80SVladimir Oltean tx_swbd->len, tx_swbd->dir);
87d4fd0404SClaudiu Manoil tx_swbd->dma = 0;
88d4fd0404SClaudiu Manoil }
89d4fd0404SClaudiu Manoil
enetc_free_tx_frame(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)909d2b68ccSVladimir Oltean static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
91d4fd0404SClaudiu Manoil struct enetc_tx_swbd *tx_swbd)
92d4fd0404SClaudiu Manoil {
939d2b68ccSVladimir Oltean struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
949d2b68ccSVladimir Oltean struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
959d2b68ccSVladimir Oltean
96d4fd0404SClaudiu Manoil if (tx_swbd->dma)
97d4fd0404SClaudiu Manoil enetc_unmap_tx_buff(tx_ring, tx_swbd);
98d4fd0404SClaudiu Manoil
999d2b68ccSVladimir Oltean if (xdp_frame) {
1009d2b68ccSVladimir Oltean xdp_return_frame(tx_swbd->xdp_frame);
1019d2b68ccSVladimir Oltean tx_swbd->xdp_frame = NULL;
1029d2b68ccSVladimir Oltean } else if (skb) {
1039d2b68ccSVladimir Oltean dev_kfree_skb_any(skb);
104d4fd0404SClaudiu Manoil tx_swbd->skb = NULL;
105d4fd0404SClaudiu Manoil }
106d4fd0404SClaudiu Manoil }
107d4fd0404SClaudiu Manoil
1087ed2bc80SVladimir Oltean /* Let H/W know BD ring has been updated */
enetc_update_tx_ring_tail(struct enetc_bdr * tx_ring)1097ed2bc80SVladimir Oltean static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
1107ed2bc80SVladimir Oltean {
1117ed2bc80SVladimir Oltean /* includes wmb() */
1127ed2bc80SVladimir Oltean enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
1137ed2bc80SVladimir Oltean }
1147ed2bc80SVladimir Oltean
enetc_ptp_parse(struct sk_buff * skb,u8 * udp,u8 * msgtype,u8 * twostep,u16 * correction_offset,u16 * body_offset)1157294380cSYangbo Lu static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
1167294380cSYangbo Lu u8 *msgtype, u8 *twostep,
1177294380cSYangbo Lu u16 *correction_offset, u16 *body_offset)
1187294380cSYangbo Lu {
1197294380cSYangbo Lu unsigned int ptp_class;
1207294380cSYangbo Lu struct ptp_header *hdr;
1217294380cSYangbo Lu unsigned int type;
1227294380cSYangbo Lu u8 *base;
1237294380cSYangbo Lu
1247294380cSYangbo Lu ptp_class = ptp_classify_raw(skb);
1257294380cSYangbo Lu if (ptp_class == PTP_CLASS_NONE)
1267294380cSYangbo Lu return -EINVAL;
1277294380cSYangbo Lu
1287294380cSYangbo Lu hdr = ptp_parse_header(skb, ptp_class);
1297294380cSYangbo Lu if (!hdr)
1307294380cSYangbo Lu return -EINVAL;
1317294380cSYangbo Lu
1327294380cSYangbo Lu type = ptp_class & PTP_CLASS_PMASK;
1337294380cSYangbo Lu if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
1347294380cSYangbo Lu *udp = 1;
1357294380cSYangbo Lu else
1367294380cSYangbo Lu *udp = 0;
1377294380cSYangbo Lu
1387294380cSYangbo Lu *msgtype = ptp_get_msgtype(hdr, ptp_class);
1397294380cSYangbo Lu *twostep = hdr->flag_field[0] & 0x2;
1407294380cSYangbo Lu
1417294380cSYangbo Lu base = skb_mac_header(skb);
1427294380cSYangbo Lu *correction_offset = (u8 *)&hdr->correction - base;
1437294380cSYangbo Lu *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
1447294380cSYangbo Lu
1457294380cSYangbo Lu return 0;
1467294380cSYangbo Lu }
1477294380cSYangbo Lu
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)148f768e751SYangbo Lu static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
149d4fd0404SClaudiu Manoil {
1507294380cSYangbo Lu bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
1517294380cSYangbo Lu struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
1527294380cSYangbo Lu struct enetc_hw *hw = &priv->si->hw;
153d4fd0404SClaudiu Manoil struct enetc_tx_swbd *tx_swbd;
154d4fd0404SClaudiu Manoil int len = skb_headlen(skb);
155d4fd0404SClaudiu Manoil union enetc_tx_bd temp_bd;
1567294380cSYangbo Lu u8 msgtype, twostep, udp;
157d4fd0404SClaudiu Manoil union enetc_tx_bd *txbd;
1587294380cSYangbo Lu u16 offset1, offset2;
159d4fd0404SClaudiu Manoil int i, count = 0;
1607294380cSYangbo Lu skb_frag_t *frag;
161d4fd0404SClaudiu Manoil unsigned int f;
162d4fd0404SClaudiu Manoil dma_addr_t dma;
163d4fd0404SClaudiu Manoil u8 flags = 0;
164d4fd0404SClaudiu Manoil
165d4fd0404SClaudiu Manoil i = tx_ring->next_to_use;
166d4fd0404SClaudiu Manoil txbd = ENETC_TXBD(*tx_ring, i);
167d4fd0404SClaudiu Manoil prefetchw(txbd);
168d4fd0404SClaudiu Manoil
169d4fd0404SClaudiu Manoil dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
170d4fd0404SClaudiu Manoil if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
171d4fd0404SClaudiu Manoil goto dma_err;
172d4fd0404SClaudiu Manoil
173d4fd0404SClaudiu Manoil temp_bd.addr = cpu_to_le64(dma);
174d4fd0404SClaudiu Manoil temp_bd.buf_len = cpu_to_le16(len);
175d4fd0404SClaudiu Manoil temp_bd.lstatus = 0;
176d4fd0404SClaudiu Manoil
177d4fd0404SClaudiu Manoil tx_swbd = &tx_ring->tx_swbd[i];
178d4fd0404SClaudiu Manoil tx_swbd->dma = dma;
179d4fd0404SClaudiu Manoil tx_swbd->len = len;
180d4fd0404SClaudiu Manoil tx_swbd->is_dma_page = 0;
1817ed2bc80SVladimir Oltean tx_swbd->dir = DMA_TO_DEVICE;
182d4fd0404SClaudiu Manoil count++;
183d4fd0404SClaudiu Manoil
184d4fd0404SClaudiu Manoil do_vlan = skb_vlan_tag_present(skb);
1857294380cSYangbo Lu if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
1867294380cSYangbo Lu if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
1877294380cSYangbo Lu &offset2) ||
1887294380cSYangbo Lu msgtype != PTP_MSGTYPE_SYNC || twostep)
1897294380cSYangbo Lu WARN_ONCE(1, "Bad packet for one-step timestamping\n");
1907294380cSYangbo Lu else
1917294380cSYangbo Lu do_onestep_tstamp = true;
1927294380cSYangbo Lu } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
1937294380cSYangbo Lu do_twostep_tstamp = true;
1947294380cSYangbo Lu }
195d4fd0404SClaudiu Manoil
1967294380cSYangbo Lu tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
197285e8dedSPo Liu tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
198285e8dedSPo Liu tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
1997294380cSYangbo Lu
2007294380cSYangbo Lu if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
201d4fd0404SClaudiu Manoil flags |= ENETC_TXBD_FLAGS_EX;
202d4fd0404SClaudiu Manoil
20382728b91SClaudiu Manoil if (tx_ring->tsd_enable)
2040d08c9ecSPo Liu flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
205d4fd0404SClaudiu Manoil
206d4fd0404SClaudiu Manoil /* first BD needs frm_len and offload flags set */
207d4fd0404SClaudiu Manoil temp_bd.frm_len = cpu_to_le16(skb->len);
208d4fd0404SClaudiu Manoil temp_bd.flags = flags;
209d4fd0404SClaudiu Manoil
21082728b91SClaudiu Manoil if (flags & ENETC_TXBD_FLAGS_TSE)
21182728b91SClaudiu Manoil temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
21282728b91SClaudiu Manoil flags);
2130d08c9ecSPo Liu
214d4fd0404SClaudiu Manoil if (flags & ENETC_TXBD_FLAGS_EX) {
215d4fd0404SClaudiu Manoil u8 e_flags = 0;
216d4fd0404SClaudiu Manoil *txbd = temp_bd;
217d4fd0404SClaudiu Manoil enetc_clear_tx_bd(&temp_bd);
218d4fd0404SClaudiu Manoil
219d4fd0404SClaudiu Manoil /* add extension BD for VLAN and/or timestamping */
220d4fd0404SClaudiu Manoil flags = 0;
221d4fd0404SClaudiu Manoil tx_swbd++;
222d4fd0404SClaudiu Manoil txbd++;
223d4fd0404SClaudiu Manoil i++;
224d4fd0404SClaudiu Manoil if (unlikely(i == tx_ring->bd_count)) {
225d4fd0404SClaudiu Manoil i = 0;
226d4fd0404SClaudiu Manoil tx_swbd = tx_ring->tx_swbd;
227d4fd0404SClaudiu Manoil txbd = ENETC_TXBD(*tx_ring, 0);
228d4fd0404SClaudiu Manoil }
229d4fd0404SClaudiu Manoil prefetchw(txbd);
230d4fd0404SClaudiu Manoil
231d4fd0404SClaudiu Manoil if (do_vlan) {
232d4fd0404SClaudiu Manoil temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
233d4fd0404SClaudiu Manoil temp_bd.ext.tpid = 0; /* < C-TAG */
234d4fd0404SClaudiu Manoil e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
235d4fd0404SClaudiu Manoil }
236d4fd0404SClaudiu Manoil
2377294380cSYangbo Lu if (do_onestep_tstamp) {
2387294380cSYangbo Lu u32 lo, hi, val;
2397294380cSYangbo Lu u64 sec, nsec;
2407294380cSYangbo Lu u8 *data;
2417294380cSYangbo Lu
2427294380cSYangbo Lu lo = enetc_rd_hot(hw, ENETC_SICTR0);
2437294380cSYangbo Lu hi = enetc_rd_hot(hw, ENETC_SICTR1);
2447294380cSYangbo Lu sec = (u64)hi << 32 | lo;
2457294380cSYangbo Lu nsec = do_div(sec, 1000000000);
2467294380cSYangbo Lu
2477294380cSYangbo Lu /* Configure extension BD */
2487294380cSYangbo Lu temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
2497294380cSYangbo Lu e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
2507294380cSYangbo Lu
2517294380cSYangbo Lu /* Update originTimestamp field of Sync packet
2527294380cSYangbo Lu * - 48 bits seconds field
2537294380cSYangbo Lu * - 32 bits nanseconds field
2547294380cSYangbo Lu */
2557294380cSYangbo Lu data = skb_mac_header(skb);
2567294380cSYangbo Lu *(__be16 *)(data + offset2) =
2577294380cSYangbo Lu htons((sec >> 32) & 0xffff);
2587294380cSYangbo Lu *(__be32 *)(data + offset2 + 2) =
2597294380cSYangbo Lu htonl(sec & 0xffffffff);
2607294380cSYangbo Lu *(__be32 *)(data + offset2 + 6) = htonl(nsec);
2617294380cSYangbo Lu
2627294380cSYangbo Lu /* Configure single-step register */
2637294380cSYangbo Lu val = ENETC_PM0_SINGLE_STEP_EN;
2647294380cSYangbo Lu val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
2657294380cSYangbo Lu if (udp)
2667294380cSYangbo Lu val |= ENETC_PM0_SINGLE_STEP_CH;
2677294380cSYangbo Lu
26812717decSVladimir Oltean enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
26912717decSVladimir Oltean val);
2707294380cSYangbo Lu } else if (do_twostep_tstamp) {
271d4fd0404SClaudiu Manoil skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
272d4fd0404SClaudiu Manoil e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
273d4fd0404SClaudiu Manoil }
274d4fd0404SClaudiu Manoil
275d4fd0404SClaudiu Manoil temp_bd.ext.e_flags = e_flags;
276d4fd0404SClaudiu Manoil count++;
277d4fd0404SClaudiu Manoil }
278d4fd0404SClaudiu Manoil
279d4fd0404SClaudiu Manoil frag = &skb_shinfo(skb)->frags[0];
280d4fd0404SClaudiu Manoil for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
281d4fd0404SClaudiu Manoil len = skb_frag_size(frag);
282d4fd0404SClaudiu Manoil dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
283d4fd0404SClaudiu Manoil DMA_TO_DEVICE);
284d4fd0404SClaudiu Manoil if (dma_mapping_error(tx_ring->dev, dma))
285d4fd0404SClaudiu Manoil goto dma_err;
286d4fd0404SClaudiu Manoil
287d4fd0404SClaudiu Manoil *txbd = temp_bd;
288d4fd0404SClaudiu Manoil enetc_clear_tx_bd(&temp_bd);
289d4fd0404SClaudiu Manoil
290d4fd0404SClaudiu Manoil flags = 0;
291d4fd0404SClaudiu Manoil tx_swbd++;
292d4fd0404SClaudiu Manoil txbd++;
293d4fd0404SClaudiu Manoil i++;
294d4fd0404SClaudiu Manoil if (unlikely(i == tx_ring->bd_count)) {
295d4fd0404SClaudiu Manoil i = 0;
296d4fd0404SClaudiu Manoil tx_swbd = tx_ring->tx_swbd;
297d4fd0404SClaudiu Manoil txbd = ENETC_TXBD(*tx_ring, 0);
298d4fd0404SClaudiu Manoil }
299d4fd0404SClaudiu Manoil prefetchw(txbd);
300d4fd0404SClaudiu Manoil
301d4fd0404SClaudiu Manoil temp_bd.addr = cpu_to_le64(dma);
302d4fd0404SClaudiu Manoil temp_bd.buf_len = cpu_to_le16(len);
303d4fd0404SClaudiu Manoil
304d4fd0404SClaudiu Manoil tx_swbd->dma = dma;
305d4fd0404SClaudiu Manoil tx_swbd->len = len;
306d4fd0404SClaudiu Manoil tx_swbd->is_dma_page = 1;
3077ed2bc80SVladimir Oltean tx_swbd->dir = DMA_TO_DEVICE;
308d4fd0404SClaudiu Manoil count++;
309d4fd0404SClaudiu Manoil }
310d4fd0404SClaudiu Manoil
311d4fd0404SClaudiu Manoil /* last BD needs 'F' bit set */
312d4fd0404SClaudiu Manoil flags |= ENETC_TXBD_FLAGS_F;
313d4fd0404SClaudiu Manoil temp_bd.flags = flags;
314d4fd0404SClaudiu Manoil *txbd = temp_bd;
315d4fd0404SClaudiu Manoil
316d504498dSVladimir Oltean tx_ring->tx_swbd[i].is_eof = true;
317d4fd0404SClaudiu Manoil tx_ring->tx_swbd[i].skb = skb;
318d4fd0404SClaudiu Manoil
319d4fd0404SClaudiu Manoil enetc_bdr_idx_inc(tx_ring, &i);
320d4fd0404SClaudiu Manoil tx_ring->next_to_use = i;
321d4fd0404SClaudiu Manoil
3224caefbceSMichael Walle skb_tx_timestamp(skb);
3234caefbceSMichael Walle
3247ed2bc80SVladimir Oltean enetc_update_tx_ring_tail(tx_ring);
325d4fd0404SClaudiu Manoil
326d4fd0404SClaudiu Manoil return count;
327d4fd0404SClaudiu Manoil
328d4fd0404SClaudiu Manoil dma_err:
329d4fd0404SClaudiu Manoil dev_err(tx_ring->dev, "DMA map error");
330d4fd0404SClaudiu Manoil
331d4fd0404SClaudiu Manoil do {
332d4fd0404SClaudiu Manoil tx_swbd = &tx_ring->tx_swbd[i];
3339d2b68ccSVladimir Oltean enetc_free_tx_frame(tx_ring, tx_swbd);
334d4fd0404SClaudiu Manoil if (i == 0)
335d4fd0404SClaudiu Manoil i = tx_ring->bd_count;
336d4fd0404SClaudiu Manoil i--;
337d4fd0404SClaudiu Manoil } while (count--);
338d4fd0404SClaudiu Manoil
339d4fd0404SClaudiu Manoil return 0;
340d4fd0404SClaudiu Manoil }
341d4fd0404SClaudiu Manoil
enetc_map_tx_tso_hdr(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,int * i,int hdr_len,int data_len)342fb8629e2SIoana Ciornei static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
343fb8629e2SIoana Ciornei struct enetc_tx_swbd *tx_swbd,
344fb8629e2SIoana Ciornei union enetc_tx_bd *txbd, int *i, int hdr_len,
345fb8629e2SIoana Ciornei int data_len)
346fb8629e2SIoana Ciornei {
347fb8629e2SIoana Ciornei union enetc_tx_bd txbd_tmp;
348fb8629e2SIoana Ciornei u8 flags = 0, e_flags = 0;
349fb8629e2SIoana Ciornei dma_addr_t addr;
350fb8629e2SIoana Ciornei
351fb8629e2SIoana Ciornei enetc_clear_tx_bd(&txbd_tmp);
352fb8629e2SIoana Ciornei addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
353fb8629e2SIoana Ciornei
354fb8629e2SIoana Ciornei if (skb_vlan_tag_present(skb))
355fb8629e2SIoana Ciornei flags |= ENETC_TXBD_FLAGS_EX;
356fb8629e2SIoana Ciornei
357fb8629e2SIoana Ciornei txbd_tmp.addr = cpu_to_le64(addr);
358fb8629e2SIoana Ciornei txbd_tmp.buf_len = cpu_to_le16(hdr_len);
359fb8629e2SIoana Ciornei
360fb8629e2SIoana Ciornei /* first BD needs frm_len and offload flags set */
361fb8629e2SIoana Ciornei txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
362fb8629e2SIoana Ciornei txbd_tmp.flags = flags;
363fb8629e2SIoana Ciornei
364fb8629e2SIoana Ciornei /* For the TSO header we do not set the dma address since we do not
365fb8629e2SIoana Ciornei * want it unmapped when we do cleanup. We still set len so that we
366fb8629e2SIoana Ciornei * count the bytes sent.
367fb8629e2SIoana Ciornei */
368fb8629e2SIoana Ciornei tx_swbd->len = hdr_len;
369fb8629e2SIoana Ciornei tx_swbd->do_twostep_tstamp = false;
370fb8629e2SIoana Ciornei tx_swbd->check_wb = false;
371fb8629e2SIoana Ciornei
372fb8629e2SIoana Ciornei /* Actually write the header in the BD */
373fb8629e2SIoana Ciornei *txbd = txbd_tmp;
374fb8629e2SIoana Ciornei
375fb8629e2SIoana Ciornei /* Add extension BD for VLAN */
376fb8629e2SIoana Ciornei if (flags & ENETC_TXBD_FLAGS_EX) {
377fb8629e2SIoana Ciornei /* Get the next BD */
378fb8629e2SIoana Ciornei enetc_bdr_idx_inc(tx_ring, i);
379fb8629e2SIoana Ciornei txbd = ENETC_TXBD(*tx_ring, *i);
380fb8629e2SIoana Ciornei tx_swbd = &tx_ring->tx_swbd[*i];
381fb8629e2SIoana Ciornei prefetchw(txbd);
382fb8629e2SIoana Ciornei
383fb8629e2SIoana Ciornei /* Setup the VLAN fields */
384fb8629e2SIoana Ciornei enetc_clear_tx_bd(&txbd_tmp);
385fb8629e2SIoana Ciornei txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
386fb8629e2SIoana Ciornei txbd_tmp.ext.tpid = 0; /* < C-TAG */
387fb8629e2SIoana Ciornei e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
388fb8629e2SIoana Ciornei
389fb8629e2SIoana Ciornei /* Write the BD */
390fb8629e2SIoana Ciornei txbd_tmp.ext.e_flags = e_flags;
391fb8629e2SIoana Ciornei *txbd = txbd_tmp;
392fb8629e2SIoana Ciornei }
393fb8629e2SIoana Ciornei }
394fb8629e2SIoana Ciornei
enetc_map_tx_tso_data(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,char * data,int size,bool last_bd)395fb8629e2SIoana Ciornei static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
396fb8629e2SIoana Ciornei struct enetc_tx_swbd *tx_swbd,
397fb8629e2SIoana Ciornei union enetc_tx_bd *txbd, char *data,
398fb8629e2SIoana Ciornei int size, bool last_bd)
399fb8629e2SIoana Ciornei {
400fb8629e2SIoana Ciornei union enetc_tx_bd txbd_tmp;
401fb8629e2SIoana Ciornei dma_addr_t addr;
402fb8629e2SIoana Ciornei u8 flags = 0;
403fb8629e2SIoana Ciornei
404fb8629e2SIoana Ciornei enetc_clear_tx_bd(&txbd_tmp);
405fb8629e2SIoana Ciornei
406fb8629e2SIoana Ciornei addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
407fb8629e2SIoana Ciornei if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
408fb8629e2SIoana Ciornei netdev_err(tx_ring->ndev, "DMA map error\n");
409fb8629e2SIoana Ciornei return -ENOMEM;
410fb8629e2SIoana Ciornei }
411fb8629e2SIoana Ciornei
412fb8629e2SIoana Ciornei if (last_bd) {
413fb8629e2SIoana Ciornei flags |= ENETC_TXBD_FLAGS_F;
414fb8629e2SIoana Ciornei tx_swbd->is_eof = 1;
415fb8629e2SIoana Ciornei }
416fb8629e2SIoana Ciornei
417fb8629e2SIoana Ciornei txbd_tmp.addr = cpu_to_le64(addr);
418fb8629e2SIoana Ciornei txbd_tmp.buf_len = cpu_to_le16(size);
419fb8629e2SIoana Ciornei txbd_tmp.flags = flags;
420fb8629e2SIoana Ciornei
421fb8629e2SIoana Ciornei tx_swbd->dma = addr;
422fb8629e2SIoana Ciornei tx_swbd->len = size;
423fb8629e2SIoana Ciornei tx_swbd->dir = DMA_TO_DEVICE;
424fb8629e2SIoana Ciornei
425fb8629e2SIoana Ciornei *txbd = txbd_tmp;
426fb8629e2SIoana Ciornei
427fb8629e2SIoana Ciornei return 0;
428fb8629e2SIoana Ciornei }
429fb8629e2SIoana Ciornei
enetc_tso_hdr_csum(struct tso_t * tso,struct sk_buff * skb,char * hdr,int hdr_len,int * l4_hdr_len)430fb8629e2SIoana Ciornei static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
431fb8629e2SIoana Ciornei char *hdr, int hdr_len, int *l4_hdr_len)
432fb8629e2SIoana Ciornei {
433fb8629e2SIoana Ciornei char *l4_hdr = hdr + skb_transport_offset(skb);
434fb8629e2SIoana Ciornei int mac_hdr_len = skb_network_offset(skb);
435fb8629e2SIoana Ciornei
436fb8629e2SIoana Ciornei if (tso->tlen != sizeof(struct udphdr)) {
437fb8629e2SIoana Ciornei struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
438fb8629e2SIoana Ciornei
439fb8629e2SIoana Ciornei tcph->check = 0;
440fb8629e2SIoana Ciornei } else {
441fb8629e2SIoana Ciornei struct udphdr *udph = (struct udphdr *)(l4_hdr);
442fb8629e2SIoana Ciornei
443fb8629e2SIoana Ciornei udph->check = 0;
444fb8629e2SIoana Ciornei }
445fb8629e2SIoana Ciornei
446fb8629e2SIoana Ciornei /* Compute the IP checksum. This is necessary since tso_build_hdr()
447fb8629e2SIoana Ciornei * already incremented the IP ID field.
448fb8629e2SIoana Ciornei */
449fb8629e2SIoana Ciornei if (!tso->ipv6) {
450fb8629e2SIoana Ciornei struct iphdr *iph = (void *)(hdr + mac_hdr_len);
451fb8629e2SIoana Ciornei
452fb8629e2SIoana Ciornei iph->check = 0;
453fb8629e2SIoana Ciornei iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
454fb8629e2SIoana Ciornei }
455fb8629e2SIoana Ciornei
456fb8629e2SIoana Ciornei /* Compute the checksum over the L4 header. */
457fb8629e2SIoana Ciornei *l4_hdr_len = hdr_len - skb_transport_offset(skb);
458fb8629e2SIoana Ciornei return csum_partial(l4_hdr, *l4_hdr_len, 0);
459fb8629e2SIoana Ciornei }
460fb8629e2SIoana Ciornei
enetc_tso_complete_csum(struct enetc_bdr * tx_ring,struct tso_t * tso,struct sk_buff * skb,char * hdr,int len,__wsum sum)461fb8629e2SIoana Ciornei static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
462fb8629e2SIoana Ciornei struct sk_buff *skb, char *hdr, int len,
463fb8629e2SIoana Ciornei __wsum sum)
464fb8629e2SIoana Ciornei {
465fb8629e2SIoana Ciornei char *l4_hdr = hdr + skb_transport_offset(skb);
466fb8629e2SIoana Ciornei __sum16 csum_final;
467fb8629e2SIoana Ciornei
468fb8629e2SIoana Ciornei /* Complete the L4 checksum by appending the pseudo-header to the
469fb8629e2SIoana Ciornei * already computed checksum.
470fb8629e2SIoana Ciornei */
471fb8629e2SIoana Ciornei if (!tso->ipv6)
472fb8629e2SIoana Ciornei csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
473fb8629e2SIoana Ciornei ip_hdr(skb)->daddr,
474fb8629e2SIoana Ciornei len, ip_hdr(skb)->protocol, sum);
475fb8629e2SIoana Ciornei else
476fb8629e2SIoana Ciornei csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
477fb8629e2SIoana Ciornei &ipv6_hdr(skb)->daddr,
478fb8629e2SIoana Ciornei len, ipv6_hdr(skb)->nexthdr, sum);
479fb8629e2SIoana Ciornei
480fb8629e2SIoana Ciornei if (tso->tlen != sizeof(struct udphdr)) {
481fb8629e2SIoana Ciornei struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
482fb8629e2SIoana Ciornei
483fb8629e2SIoana Ciornei tcph->check = csum_final;
484fb8629e2SIoana Ciornei } else {
485fb8629e2SIoana Ciornei struct udphdr *udph = (struct udphdr *)(l4_hdr);
486fb8629e2SIoana Ciornei
487fb8629e2SIoana Ciornei udph->check = csum_final;
488fb8629e2SIoana Ciornei }
489fb8629e2SIoana Ciornei }
490fb8629e2SIoana Ciornei
enetc_map_tx_tso_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)491fb8629e2SIoana Ciornei static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
492fb8629e2SIoana Ciornei {
493fb8629e2SIoana Ciornei int hdr_len, total_len, data_len;
494fb8629e2SIoana Ciornei struct enetc_tx_swbd *tx_swbd;
495fb8629e2SIoana Ciornei union enetc_tx_bd *txbd;
496fb8629e2SIoana Ciornei struct tso_t tso;
497fb8629e2SIoana Ciornei __wsum csum, csum2;
498fb8629e2SIoana Ciornei int count = 0, pos;
499fb8629e2SIoana Ciornei int err, i, bd_data_num;
500fb8629e2SIoana Ciornei
501fb8629e2SIoana Ciornei /* Initialize the TSO handler, and prepare the first payload */
502fb8629e2SIoana Ciornei hdr_len = tso_start(skb, &tso);
503fb8629e2SIoana Ciornei total_len = skb->len - hdr_len;
504fb8629e2SIoana Ciornei i = tx_ring->next_to_use;
505fb8629e2SIoana Ciornei
506fb8629e2SIoana Ciornei while (total_len > 0) {
507fb8629e2SIoana Ciornei char *hdr;
508fb8629e2SIoana Ciornei
509fb8629e2SIoana Ciornei /* Get the BD */
510fb8629e2SIoana Ciornei txbd = ENETC_TXBD(*tx_ring, i);
511fb8629e2SIoana Ciornei tx_swbd = &tx_ring->tx_swbd[i];
512fb8629e2SIoana Ciornei prefetchw(txbd);
513fb8629e2SIoana Ciornei
514fb8629e2SIoana Ciornei /* Determine the length of this packet */
515fb8629e2SIoana Ciornei data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
516fb8629e2SIoana Ciornei total_len -= data_len;
517fb8629e2SIoana Ciornei
518fb8629e2SIoana Ciornei /* prepare packet headers: MAC + IP + TCP */
519fb8629e2SIoana Ciornei hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
520fb8629e2SIoana Ciornei tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
521fb8629e2SIoana Ciornei
522fb8629e2SIoana Ciornei /* compute the csum over the L4 header */
523fb8629e2SIoana Ciornei csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
524fb8629e2SIoana Ciornei enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
525fb8629e2SIoana Ciornei bd_data_num = 0;
526fb8629e2SIoana Ciornei count++;
527fb8629e2SIoana Ciornei
528fb8629e2SIoana Ciornei while (data_len > 0) {
529fb8629e2SIoana Ciornei int size;
530fb8629e2SIoana Ciornei
531fb8629e2SIoana Ciornei size = min_t(int, tso.size, data_len);
532fb8629e2SIoana Ciornei
533fb8629e2SIoana Ciornei /* Advance the index in the BDR */
534fb8629e2SIoana Ciornei enetc_bdr_idx_inc(tx_ring, &i);
535fb8629e2SIoana Ciornei txbd = ENETC_TXBD(*tx_ring, i);
536fb8629e2SIoana Ciornei tx_swbd = &tx_ring->tx_swbd[i];
537fb8629e2SIoana Ciornei prefetchw(txbd);
538fb8629e2SIoana Ciornei
539fb8629e2SIoana Ciornei /* Compute the checksum over this segment of data and
540fb8629e2SIoana Ciornei * add it to the csum already computed (over the L4
541fb8629e2SIoana Ciornei * header and possible other data segments).
542fb8629e2SIoana Ciornei */
543fb8629e2SIoana Ciornei csum2 = csum_partial(tso.data, size, 0);
544fb8629e2SIoana Ciornei csum = csum_block_add(csum, csum2, pos);
545fb8629e2SIoana Ciornei pos += size;
546fb8629e2SIoana Ciornei
547fb8629e2SIoana Ciornei err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
548fb8629e2SIoana Ciornei tso.data, size,
549fb8629e2SIoana Ciornei size == data_len);
550fb8629e2SIoana Ciornei if (err)
551fb8629e2SIoana Ciornei goto err_map_data;
552fb8629e2SIoana Ciornei
553fb8629e2SIoana Ciornei data_len -= size;
554fb8629e2SIoana Ciornei count++;
555fb8629e2SIoana Ciornei bd_data_num++;
556fb8629e2SIoana Ciornei tso_build_data(skb, &tso, size);
557fb8629e2SIoana Ciornei
558fb8629e2SIoana Ciornei if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
559fb8629e2SIoana Ciornei goto err_chained_bd;
560fb8629e2SIoana Ciornei }
561fb8629e2SIoana Ciornei
562fb8629e2SIoana Ciornei enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
563fb8629e2SIoana Ciornei
564fb8629e2SIoana Ciornei if (total_len == 0)
565fb8629e2SIoana Ciornei tx_swbd->skb = skb;
566fb8629e2SIoana Ciornei
567fb8629e2SIoana Ciornei /* Go to the next BD */
568fb8629e2SIoana Ciornei enetc_bdr_idx_inc(tx_ring, &i);
569fb8629e2SIoana Ciornei }
570fb8629e2SIoana Ciornei
571fb8629e2SIoana Ciornei tx_ring->next_to_use = i;
572fb8629e2SIoana Ciornei enetc_update_tx_ring_tail(tx_ring);
573fb8629e2SIoana Ciornei
574fb8629e2SIoana Ciornei return count;
575fb8629e2SIoana Ciornei
576fb8629e2SIoana Ciornei err_map_data:
577fb8629e2SIoana Ciornei dev_err(tx_ring->dev, "DMA map error");
578fb8629e2SIoana Ciornei
579fb8629e2SIoana Ciornei err_chained_bd:
580fb8629e2SIoana Ciornei do {
581fb8629e2SIoana Ciornei tx_swbd = &tx_ring->tx_swbd[i];
582fb8629e2SIoana Ciornei enetc_free_tx_frame(tx_ring, tx_swbd);
583fb8629e2SIoana Ciornei if (i == 0)
584fb8629e2SIoana Ciornei i = tx_ring->bd_count;
585fb8629e2SIoana Ciornei i--;
586fb8629e2SIoana Ciornei } while (count--);
587fb8629e2SIoana Ciornei
588fb8629e2SIoana Ciornei return 0;
589fb8629e2SIoana Ciornei }
590fb8629e2SIoana Ciornei
enetc_start_xmit(struct sk_buff * skb,struct net_device * ndev)5917294380cSYangbo Lu static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
5927294380cSYangbo Lu struct net_device *ndev)
5930486185eSVladimir Oltean {
5940486185eSVladimir Oltean struct enetc_ndev_priv *priv = netdev_priv(ndev);
5950486185eSVladimir Oltean struct enetc_bdr *tx_ring;
596acede3c5SIoana Ciornei int count, err;
5970486185eSVladimir Oltean
5987ce9c3d3SYangbo Lu /* Queue one-step Sync packet if already locked */
5997ce9c3d3SYangbo Lu if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
6007ce9c3d3SYangbo Lu if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
6017ce9c3d3SYangbo Lu &priv->flags)) {
6027ce9c3d3SYangbo Lu skb_queue_tail(&priv->tx_skbs, skb);
6037ce9c3d3SYangbo Lu return NETDEV_TX_OK;
6047ce9c3d3SYangbo Lu }
6057ce9c3d3SYangbo Lu }
6067ce9c3d3SYangbo Lu
6070486185eSVladimir Oltean tx_ring = priv->tx_ring[skb->queue_mapping];
6080486185eSVladimir Oltean
609fb8629e2SIoana Ciornei if (skb_is_gso(skb)) {
610fb8629e2SIoana Ciornei if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
611fb8629e2SIoana Ciornei netif_stop_subqueue(ndev, tx_ring->index);
612fb8629e2SIoana Ciornei return NETDEV_TX_BUSY;
613fb8629e2SIoana Ciornei }
614fb8629e2SIoana Ciornei
615fb8629e2SIoana Ciornei enetc_lock_mdio();
616fb8629e2SIoana Ciornei count = enetc_map_tx_tso_buffs(tx_ring, skb);
617fb8629e2SIoana Ciornei enetc_unlock_mdio();
618fb8629e2SIoana Ciornei } else {
6190486185eSVladimir Oltean if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
6200486185eSVladimir Oltean if (unlikely(skb_linearize(skb)))
6210486185eSVladimir Oltean goto drop_packet_err;
6220486185eSVladimir Oltean
6230486185eSVladimir Oltean count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
6240486185eSVladimir Oltean if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
6250486185eSVladimir Oltean netif_stop_subqueue(ndev, tx_ring->index);
6260486185eSVladimir Oltean return NETDEV_TX_BUSY;
6270486185eSVladimir Oltean }
6280486185eSVladimir Oltean
629acede3c5SIoana Ciornei if (skb->ip_summed == CHECKSUM_PARTIAL) {
630acede3c5SIoana Ciornei err = skb_checksum_help(skb);
631acede3c5SIoana Ciornei if (err)
632acede3c5SIoana Ciornei goto drop_packet_err;
633acede3c5SIoana Ciornei }
6340486185eSVladimir Oltean enetc_lock_mdio();
635f768e751SYangbo Lu count = enetc_map_tx_buffs(tx_ring, skb);
6360486185eSVladimir Oltean enetc_unlock_mdio();
637fb8629e2SIoana Ciornei }
6380486185eSVladimir Oltean
6390486185eSVladimir Oltean if (unlikely(!count))
6400486185eSVladimir Oltean goto drop_packet_err;
6410486185eSVladimir Oltean
6420486185eSVladimir Oltean if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
6430486185eSVladimir Oltean netif_stop_subqueue(ndev, tx_ring->index);
6440486185eSVladimir Oltean
6450486185eSVladimir Oltean return NETDEV_TX_OK;
6460486185eSVladimir Oltean
6470486185eSVladimir Oltean drop_packet_err:
6480486185eSVladimir Oltean dev_kfree_skb_any(skb);
6490486185eSVladimir Oltean return NETDEV_TX_OK;
6500486185eSVladimir Oltean }
6510486185eSVladimir Oltean
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)6527294380cSYangbo Lu netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
6537294380cSYangbo Lu {
6547294380cSYangbo Lu struct enetc_ndev_priv *priv = netdev_priv(ndev);
6557294380cSYangbo Lu u8 udp, msgtype, twostep;
6567294380cSYangbo Lu u16 offset1, offset2;
6577294380cSYangbo Lu
6587294380cSYangbo Lu /* Mark tx timestamp type on skb->cb[0] if requires */
6597294380cSYangbo Lu if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
6607294380cSYangbo Lu (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
6617294380cSYangbo Lu skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
6627294380cSYangbo Lu } else {
6637294380cSYangbo Lu skb->cb[0] = 0;
6647294380cSYangbo Lu }
6657294380cSYangbo Lu
6667294380cSYangbo Lu /* Fall back to two-step timestamp if not one-step Sync packet */
6677294380cSYangbo Lu if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
6687294380cSYangbo Lu if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
6697294380cSYangbo Lu &offset1, &offset2) ||
6707294380cSYangbo Lu msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
6717294380cSYangbo Lu skb->cb[0] = ENETC_F_TX_TSTAMP;
6727294380cSYangbo Lu }
6737294380cSYangbo Lu
6747294380cSYangbo Lu return enetc_start_xmit(skb, ndev);
6757294380cSYangbo Lu }
676e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_xmit);
6777294380cSYangbo Lu
enetc_msix(int irq,void * data)678d4fd0404SClaudiu Manoil static irqreturn_t enetc_msix(int irq, void *data)
679d4fd0404SClaudiu Manoil {
680d4fd0404SClaudiu Manoil struct enetc_int_vector *v = data;
681d4fd0404SClaudiu Manoil int i;
682d4fd0404SClaudiu Manoil
683fd5736bfSAlex Marginean enetc_lock_mdio();
684fd5736bfSAlex Marginean
685d4fd0404SClaudiu Manoil /* disable interrupts */
686fd5736bfSAlex Marginean enetc_wr_reg_hot(v->rbier, 0);
687fd5736bfSAlex Marginean enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
688d4fd0404SClaudiu Manoil
6890574e200SClaudiu Manoil for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
690fd5736bfSAlex Marginean enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
691fd5736bfSAlex Marginean
692fd5736bfSAlex Marginean enetc_unlock_mdio();
693d4fd0404SClaudiu Manoil
694215602a8SJiafei Pan napi_schedule(&v->napi);
695d4fd0404SClaudiu Manoil
696d4fd0404SClaudiu Manoil return IRQ_HANDLED;
697d4fd0404SClaudiu Manoil }
698d4fd0404SClaudiu Manoil
enetc_rx_dim_work(struct work_struct * w)699ae0e6a5dSClaudiu Manoil static void enetc_rx_dim_work(struct work_struct *w)
700ae0e6a5dSClaudiu Manoil {
701ae0e6a5dSClaudiu Manoil struct dim *dim = container_of(w, struct dim, work);
702ae0e6a5dSClaudiu Manoil struct dim_cq_moder moder =
703ae0e6a5dSClaudiu Manoil net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
704ae0e6a5dSClaudiu Manoil struct enetc_int_vector *v =
705ae0e6a5dSClaudiu Manoil container_of(dim, struct enetc_int_vector, rx_dim);
706ae0e6a5dSClaudiu Manoil
707ae0e6a5dSClaudiu Manoil v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
708ae0e6a5dSClaudiu Manoil dim->state = DIM_START_MEASURE;
709ae0e6a5dSClaudiu Manoil }
710ae0e6a5dSClaudiu Manoil
enetc_rx_net_dim(struct enetc_int_vector * v)711ae0e6a5dSClaudiu Manoil static void enetc_rx_net_dim(struct enetc_int_vector *v)
712ae0e6a5dSClaudiu Manoil {
7139f7afa05SClaudiu Manoil struct dim_sample dim_sample = {};
714ae0e6a5dSClaudiu Manoil
715ae0e6a5dSClaudiu Manoil v->comp_cnt++;
716ae0e6a5dSClaudiu Manoil
717ae0e6a5dSClaudiu Manoil if (!v->rx_napi_work)
718ae0e6a5dSClaudiu Manoil return;
719ae0e6a5dSClaudiu Manoil
720ae0e6a5dSClaudiu Manoil dim_update_sample(v->comp_cnt,
721ae0e6a5dSClaudiu Manoil v->rx_ring.stats.packets,
722ae0e6a5dSClaudiu Manoil v->rx_ring.stats.bytes,
723ae0e6a5dSClaudiu Manoil &dim_sample);
724ae0e6a5dSClaudiu Manoil net_dim(&v->rx_dim, dim_sample);
725ae0e6a5dSClaudiu Manoil }
726ae0e6a5dSClaudiu Manoil
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)727d4fd0404SClaudiu Manoil static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
728d4fd0404SClaudiu Manoil {
729fd5736bfSAlex Marginean int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
730d4fd0404SClaudiu Manoil
731d4fd0404SClaudiu Manoil return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
732d4fd0404SClaudiu Manoil }
733d4fd0404SClaudiu Manoil
enetc_page_reusable(struct page * page)73465d0cbb4SVladimir Oltean static bool enetc_page_reusable(struct page *page)
73565d0cbb4SVladimir Oltean {
73665d0cbb4SVladimir Oltean return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
73765d0cbb4SVladimir Oltean }
73865d0cbb4SVladimir Oltean
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)73965d0cbb4SVladimir Oltean static void enetc_reuse_page(struct enetc_bdr *rx_ring,
74065d0cbb4SVladimir Oltean struct enetc_rx_swbd *old)
74165d0cbb4SVladimir Oltean {
74265d0cbb4SVladimir Oltean struct enetc_rx_swbd *new;
74365d0cbb4SVladimir Oltean
74465d0cbb4SVladimir Oltean new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
74565d0cbb4SVladimir Oltean
74665d0cbb4SVladimir Oltean /* next buf that may reuse a page */
74765d0cbb4SVladimir Oltean enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
74865d0cbb4SVladimir Oltean
74965d0cbb4SVladimir Oltean /* copy page reference */
75065d0cbb4SVladimir Oltean *new = *old;
75165d0cbb4SVladimir Oltean }
75265d0cbb4SVladimir Oltean
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)753d3982312SY.b. Lu static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
754d3982312SY.b. Lu u64 *tstamp)
755d3982312SY.b. Lu {
756cec4f328SY.b. Lu u32 lo, hi, tstamp_lo;
757d3982312SY.b. Lu
7586d36ecdbSVladimir Oltean lo = enetc_rd_hot(hw, ENETC_SICTR0);
7596d36ecdbSVladimir Oltean hi = enetc_rd_hot(hw, ENETC_SICTR1);
760cec4f328SY.b. Lu tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
761cec4f328SY.b. Lu if (lo <= tstamp_lo)
762d3982312SY.b. Lu hi -= 1;
763cec4f328SY.b. Lu *tstamp = (u64)hi << 32 | tstamp_lo;
764d3982312SY.b. Lu }
765d3982312SY.b. Lu
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)766d3982312SY.b. Lu static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
767d3982312SY.b. Lu {
768d3982312SY.b. Lu struct skb_shared_hwtstamps shhwtstamps;
769d3982312SY.b. Lu
770d3982312SY.b. Lu if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
771d3982312SY.b. Lu memset(&shhwtstamps, 0, sizeof(shhwtstamps));
772d3982312SY.b. Lu shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
773847cbfc0SVladimir Oltean skb_txtime_consumed(skb);
774d3982312SY.b. Lu skb_tstamp_tx(skb, &shhwtstamps);
775d3982312SY.b. Lu }
776d3982312SY.b. Lu }
777d3982312SY.b. Lu
enetc_recycle_xdp_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)7787ed2bc80SVladimir Oltean static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
7797ed2bc80SVladimir Oltean struct enetc_tx_swbd *tx_swbd)
7807ed2bc80SVladimir Oltean {
7817ed2bc80SVladimir Oltean struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
7827ed2bc80SVladimir Oltean struct enetc_rx_swbd rx_swbd = {
7837ed2bc80SVladimir Oltean .dma = tx_swbd->dma,
7847ed2bc80SVladimir Oltean .page = tx_swbd->page,
7857ed2bc80SVladimir Oltean .page_offset = tx_swbd->page_offset,
7867ed2bc80SVladimir Oltean .dir = tx_swbd->dir,
7877ed2bc80SVladimir Oltean .len = tx_swbd->len,
7887ed2bc80SVladimir Oltean };
7897eab503bSVladimir Oltean struct enetc_bdr *rx_ring;
7907eab503bSVladimir Oltean
7917eab503bSVladimir Oltean rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
7927ed2bc80SVladimir Oltean
7937ed2bc80SVladimir Oltean if (likely(enetc_swbd_unused(rx_ring))) {
7947ed2bc80SVladimir Oltean enetc_reuse_page(rx_ring, &rx_swbd);
7957ed2bc80SVladimir Oltean
7967ed2bc80SVladimir Oltean /* sync for use by the device */
7977ed2bc80SVladimir Oltean dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
7987ed2bc80SVladimir Oltean rx_swbd.page_offset,
7997ed2bc80SVladimir Oltean ENETC_RXB_DMA_SIZE_XDP,
8007ed2bc80SVladimir Oltean rx_swbd.dir);
8017ed2bc80SVladimir Oltean
8027ed2bc80SVladimir Oltean rx_ring->stats.recycles++;
8037ed2bc80SVladimir Oltean } else {
8047ed2bc80SVladimir Oltean /* RX ring is already full, we need to unmap and free the
8057ed2bc80SVladimir Oltean * page, since there's nothing useful we can do with it.
8067ed2bc80SVladimir Oltean */
8077ed2bc80SVladimir Oltean rx_ring->stats.recycle_failures++;
8087ed2bc80SVladimir Oltean
8097ed2bc80SVladimir Oltean dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
8107ed2bc80SVladimir Oltean rx_swbd.dir);
8117ed2bc80SVladimir Oltean __free_page(rx_swbd.page);
8127ed2bc80SVladimir Oltean }
8137ed2bc80SVladimir Oltean
8147ed2bc80SVladimir Oltean rx_ring->xdp.xdp_tx_in_flight--;
8157ed2bc80SVladimir Oltean }
8167ed2bc80SVladimir Oltean
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)817d4fd0404SClaudiu Manoil static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
818d4fd0404SClaudiu Manoil {
819285e8dedSPo Liu int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
820d4fd0404SClaudiu Manoil struct net_device *ndev = tx_ring->ndev;
8217294380cSYangbo Lu struct enetc_ndev_priv *priv = netdev_priv(ndev);
822d4fd0404SClaudiu Manoil struct enetc_tx_swbd *tx_swbd;
823d4fd0404SClaudiu Manoil int i, bds_to_clean;
8247294380cSYangbo Lu bool do_twostep_tstamp;
825d3982312SY.b. Lu u64 tstamp = 0;
826d4fd0404SClaudiu Manoil
827d4fd0404SClaudiu Manoil i = tx_ring->next_to_clean;
828d4fd0404SClaudiu Manoil tx_swbd = &tx_ring->tx_swbd[i];
829fd5736bfSAlex Marginean
830d4fd0404SClaudiu Manoil bds_to_clean = enetc_bd_ready_count(tx_ring, i);
831d4fd0404SClaudiu Manoil
8327294380cSYangbo Lu do_twostep_tstamp = false;
833d3982312SY.b. Lu
834d4fd0404SClaudiu Manoil while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
8359d2b68ccSVladimir Oltean struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
8369d2b68ccSVladimir Oltean struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
837a93580a0SVladimir Oltean bool is_eof = tx_swbd->is_eof;
8389d2b68ccSVladimir Oltean
839d3982312SY.b. Lu if (unlikely(tx_swbd->check_wb)) {
840ae77bdbcSVladimir Oltean union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
841d3982312SY.b. Lu
842d3982312SY.b. Lu if (txbd->flags & ENETC_TXBD_FLAGS_W &&
8437294380cSYangbo Lu tx_swbd->do_twostep_tstamp) {
844d3982312SY.b. Lu enetc_get_tx_tstamp(&priv->si->hw, txbd,
845d3982312SY.b. Lu &tstamp);
8467294380cSYangbo Lu do_twostep_tstamp = true;
847d3982312SY.b. Lu }
848285e8dedSPo Liu
849285e8dedSPo Liu if (tx_swbd->qbv_en &&
850285e8dedSPo Liu txbd->wb.status & ENETC_TXBD_STATS_WIN)
851285e8dedSPo Liu tx_win_drop++;
852d3982312SY.b. Lu }
853d3982312SY.b. Lu
8547ed2bc80SVladimir Oltean if (tx_swbd->is_xdp_tx)
8557ed2bc80SVladimir Oltean enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
8567ed2bc80SVladimir Oltean else if (likely(tx_swbd->dma))
857d4fd0404SClaudiu Manoil enetc_unmap_tx_buff(tx_ring, tx_swbd);
858f4a0be84SClaudiu Manoil
8599d2b68ccSVladimir Oltean if (xdp_frame) {
8609d2b68ccSVladimir Oltean xdp_return_frame(xdp_frame);
8619d2b68ccSVladimir Oltean } else if (skb) {
86252066149SVladimir Oltean if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
8637294380cSYangbo Lu /* Start work to release lock for next one-step
8647294380cSYangbo Lu * timestamping packet. And send one skb in
8657294380cSYangbo Lu * tx_skbs queue if has.
8667294380cSYangbo Lu */
867b6faf160SYangbo Lu schedule_work(&priv->tx_onestep_tstamp);
8687294380cSYangbo Lu } else if (unlikely(do_twostep_tstamp)) {
8699d2b68ccSVladimir Oltean enetc_tstamp_tx(skb, tstamp);
8707294380cSYangbo Lu do_twostep_tstamp = false;
871d3982312SY.b. Lu }
8729d2b68ccSVladimir Oltean napi_consume_skb(skb, napi_budget);
873d4fd0404SClaudiu Manoil }
874d4fd0404SClaudiu Manoil
875d4fd0404SClaudiu Manoil tx_byte_cnt += tx_swbd->len;
8761ee8d6f3SVladimir Oltean /* Scrub the swbd here so we don't have to do that
8771ee8d6f3SVladimir Oltean * when we reuse it during xmit
8781ee8d6f3SVladimir Oltean */
8791ee8d6f3SVladimir Oltean memset(tx_swbd, 0, sizeof(*tx_swbd));
880d4fd0404SClaudiu Manoil
881d4fd0404SClaudiu Manoil bds_to_clean--;
882d4fd0404SClaudiu Manoil tx_swbd++;
883d4fd0404SClaudiu Manoil i++;
884d4fd0404SClaudiu Manoil if (unlikely(i == tx_ring->bd_count)) {
885d4fd0404SClaudiu Manoil i = 0;
886d4fd0404SClaudiu Manoil tx_swbd = tx_ring->tx_swbd;
887d4fd0404SClaudiu Manoil }
888d4fd0404SClaudiu Manoil
889d4fd0404SClaudiu Manoil /* BD iteration loop end */
890a93580a0SVladimir Oltean if (is_eof) {
891d4fd0404SClaudiu Manoil tx_frm_cnt++;
892d4fd0404SClaudiu Manoil /* re-arm interrupt source */
893fd5736bfSAlex Marginean enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
894d4fd0404SClaudiu Manoil BIT(16 + tx_ring->index));
895d4fd0404SClaudiu Manoil }
896d4fd0404SClaudiu Manoil
897d4fd0404SClaudiu Manoil if (unlikely(!bds_to_clean))
898d4fd0404SClaudiu Manoil bds_to_clean = enetc_bd_ready_count(tx_ring, i);
899d4fd0404SClaudiu Manoil }
900d4fd0404SClaudiu Manoil
901d4fd0404SClaudiu Manoil tx_ring->next_to_clean = i;
902d4fd0404SClaudiu Manoil tx_ring->stats.packets += tx_frm_cnt;
903d4fd0404SClaudiu Manoil tx_ring->stats.bytes += tx_byte_cnt;
904285e8dedSPo Liu tx_ring->stats.win_drop += tx_win_drop;
905d4fd0404SClaudiu Manoil
906d4fd0404SClaudiu Manoil if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
907d4fd0404SClaudiu Manoil __netif_subqueue_stopped(ndev, tx_ring->index) &&
90837184349SWei Fang !test_bit(ENETC_TX_DOWN, &priv->flags) &&
909d4fd0404SClaudiu Manoil (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
910d4fd0404SClaudiu Manoil netif_wake_subqueue(ndev, tx_ring->index);
911d4fd0404SClaudiu Manoil }
912d4fd0404SClaudiu Manoil
913d4fd0404SClaudiu Manoil return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
914d4fd0404SClaudiu Manoil }
915d4fd0404SClaudiu Manoil
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)916d4fd0404SClaudiu Manoil static bool enetc_new_page(struct enetc_bdr *rx_ring,
917d4fd0404SClaudiu Manoil struct enetc_rx_swbd *rx_swbd)
918d4fd0404SClaudiu Manoil {
9197ed2bc80SVladimir Oltean bool xdp = !!(rx_ring->xdp.prog);
920d4fd0404SClaudiu Manoil struct page *page;
921d4fd0404SClaudiu Manoil dma_addr_t addr;
922d4fd0404SClaudiu Manoil
923d4fd0404SClaudiu Manoil page = dev_alloc_page();
924d4fd0404SClaudiu Manoil if (unlikely(!page))
925d4fd0404SClaudiu Manoil return false;
926d4fd0404SClaudiu Manoil
9277ed2bc80SVladimir Oltean /* For XDP_TX, we forgo dma_unmap -> dma_map */
9287ed2bc80SVladimir Oltean rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
9297ed2bc80SVladimir Oltean
9307ed2bc80SVladimir Oltean addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
931d4fd0404SClaudiu Manoil if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
932d4fd0404SClaudiu Manoil __free_page(page);
933d4fd0404SClaudiu Manoil
934d4fd0404SClaudiu Manoil return false;
935d4fd0404SClaudiu Manoil }
936d4fd0404SClaudiu Manoil
937d4fd0404SClaudiu Manoil rx_swbd->dma = addr;
938d4fd0404SClaudiu Manoil rx_swbd->page = page;
939d1b15102SVladimir Oltean rx_swbd->page_offset = rx_ring->buffer_offset;
940d4fd0404SClaudiu Manoil
941d4fd0404SClaudiu Manoil return true;
942d4fd0404SClaudiu Manoil }
943d4fd0404SClaudiu Manoil
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)944d4fd0404SClaudiu Manoil static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
945d4fd0404SClaudiu Manoil {
946d4fd0404SClaudiu Manoil struct enetc_rx_swbd *rx_swbd;
947d4fd0404SClaudiu Manoil union enetc_rx_bd *rxbd;
948d4fd0404SClaudiu Manoil int i, j;
949d4fd0404SClaudiu Manoil
950d4fd0404SClaudiu Manoil i = rx_ring->next_to_use;
951d4fd0404SClaudiu Manoil rx_swbd = &rx_ring->rx_swbd[i];
952714239acSClaudiu Manoil rxbd = enetc_rxbd(rx_ring, i);
953d4fd0404SClaudiu Manoil
954d4fd0404SClaudiu Manoil for (j = 0; j < buff_cnt; j++) {
955d4fd0404SClaudiu Manoil /* try reuse page */
956d4fd0404SClaudiu Manoil if (unlikely(!rx_swbd->page)) {
957d4fd0404SClaudiu Manoil if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
958d4fd0404SClaudiu Manoil rx_ring->stats.rx_alloc_errs++;
959d4fd0404SClaudiu Manoil break;
960d4fd0404SClaudiu Manoil }
961d4fd0404SClaudiu Manoil }
962d4fd0404SClaudiu Manoil
963d4fd0404SClaudiu Manoil /* update RxBD */
964d4fd0404SClaudiu Manoil rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
965d4fd0404SClaudiu Manoil rx_swbd->page_offset);
966d4fd0404SClaudiu Manoil /* clear 'R" as well */
967d4fd0404SClaudiu Manoil rxbd->r.lstatus = 0;
968d4fd0404SClaudiu Manoil
969c027aa92SVladimir Oltean enetc_rxbd_next(rx_ring, &rxbd, &i);
970c027aa92SVladimir Oltean rx_swbd = &rx_ring->rx_swbd[i];
971d4fd0404SClaudiu Manoil }
972d4fd0404SClaudiu Manoil
973d4fd0404SClaudiu Manoil if (likely(j)) {
974d4fd0404SClaudiu Manoil rx_ring->next_to_alloc = i; /* keep track from page reuse */
975d4fd0404SClaudiu Manoil rx_ring->next_to_use = i;
9767a5222cbSVladimir Oltean
9777a5222cbSVladimir Oltean /* update ENETC's consumer index */
9787a5222cbSVladimir Oltean enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
979d4fd0404SClaudiu Manoil }
980d4fd0404SClaudiu Manoil
981d4fd0404SClaudiu Manoil return j;
982d4fd0404SClaudiu Manoil }
983d4fd0404SClaudiu Manoil
984434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)985d3982312SY.b. Lu static void enetc_get_rx_tstamp(struct net_device *ndev,
986d3982312SY.b. Lu union enetc_rx_bd *rxbd,
987d3982312SY.b. Lu struct sk_buff *skb)
988d3982312SY.b. Lu {
989d3982312SY.b. Lu struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
990d3982312SY.b. Lu struct enetc_ndev_priv *priv = netdev_priv(ndev);
991d3982312SY.b. Lu struct enetc_hw *hw = &priv->si->hw;
992cec4f328SY.b. Lu u32 lo, hi, tstamp_lo;
993d3982312SY.b. Lu u64 tstamp;
994d3982312SY.b. Lu
995cec4f328SY.b. Lu if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
996fd5736bfSAlex Marginean lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
997fd5736bfSAlex Marginean hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
998434cebabSClaudiu Manoil rxbd = enetc_rxbd_ext(rxbd);
999434cebabSClaudiu Manoil tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
1000cec4f328SY.b. Lu if (lo <= tstamp_lo)
1001d3982312SY.b. Lu hi -= 1;
1002d3982312SY.b. Lu
1003cec4f328SY.b. Lu tstamp = (u64)hi << 32 | tstamp_lo;
1004d3982312SY.b. Lu memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1005d3982312SY.b. Lu shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1006d3982312SY.b. Lu }
1007d3982312SY.b. Lu }
1008d3982312SY.b. Lu #endif
1009d3982312SY.b. Lu
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)1010d4fd0404SClaudiu Manoil static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1011d4fd0404SClaudiu Manoil union enetc_rx_bd *rxbd, struct sk_buff *skb)
1012d4fd0404SClaudiu Manoil {
1013d3982312SY.b. Lu struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1014827b6fd0SVladimir Oltean
1015d3982312SY.b. Lu /* TODO: hashing */
1016d4fd0404SClaudiu Manoil if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1017d4fd0404SClaudiu Manoil u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1018d4fd0404SClaudiu Manoil
1019d4fd0404SClaudiu Manoil skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1020d4fd0404SClaudiu Manoil skb->ip_summed = CHECKSUM_COMPLETE;
1021d4fd0404SClaudiu Manoil }
1022d4fd0404SClaudiu Manoil
1023827b6fd0SVladimir Oltean if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1024827b6fd0SVladimir Oltean __be16 tpid = 0;
1025827b6fd0SVladimir Oltean
1026827b6fd0SVladimir Oltean switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1027827b6fd0SVladimir Oltean case 0:
1028827b6fd0SVladimir Oltean tpid = htons(ETH_P_8021Q);
1029827b6fd0SVladimir Oltean break;
1030827b6fd0SVladimir Oltean case 1:
1031827b6fd0SVladimir Oltean tpid = htons(ETH_P_8021AD);
1032827b6fd0SVladimir Oltean break;
1033827b6fd0SVladimir Oltean case 2:
1034827b6fd0SVladimir Oltean tpid = htons(enetc_port_rd(&priv->si->hw,
1035827b6fd0SVladimir Oltean ENETC_PCVLANR1));
1036827b6fd0SVladimir Oltean break;
1037827b6fd0SVladimir Oltean case 3:
1038827b6fd0SVladimir Oltean tpid = htons(enetc_port_rd(&priv->si->hw,
1039827b6fd0SVladimir Oltean ENETC_PCVLANR2));
1040827b6fd0SVladimir Oltean break;
1041827b6fd0SVladimir Oltean default:
1042827b6fd0SVladimir Oltean break;
1043827b6fd0SVladimir Oltean }
1044827b6fd0SVladimir Oltean
1045827b6fd0SVladimir Oltean __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1046827b6fd0SVladimir Oltean }
1047827b6fd0SVladimir Oltean
1048434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1049d3982312SY.b. Lu if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1050d3982312SY.b. Lu enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1051d3982312SY.b. Lu #endif
1052d4fd0404SClaudiu Manoil }
1053d4fd0404SClaudiu Manoil
10547ed2bc80SVladimir Oltean /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
10557ed2bc80SVladimir Oltean * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
10567ed2bc80SVladimir Oltean * mapped buffers.
10577ed2bc80SVladimir Oltean */
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)1058d4fd0404SClaudiu Manoil static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1059d4fd0404SClaudiu Manoil int i, u16 size)
1060d4fd0404SClaudiu Manoil {
1061d4fd0404SClaudiu Manoil struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1062d4fd0404SClaudiu Manoil
1063d4fd0404SClaudiu Manoil dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1064d4fd0404SClaudiu Manoil rx_swbd->page_offset,
10657ed2bc80SVladimir Oltean size, rx_swbd->dir);
1066d4fd0404SClaudiu Manoil return rx_swbd;
1067d4fd0404SClaudiu Manoil }
1068d4fd0404SClaudiu Manoil
10696b04830dSVladimir Oltean /* Reuse the current page without performing half-page buffer flipping */
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1070d4fd0404SClaudiu Manoil static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1071d4fd0404SClaudiu Manoil struct enetc_rx_swbd *rx_swbd)
1072d4fd0404SClaudiu Manoil {
1073d1b15102SVladimir Oltean size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1074d1b15102SVladimir Oltean
1075d4fd0404SClaudiu Manoil enetc_reuse_page(rx_ring, rx_swbd);
1076d4fd0404SClaudiu Manoil
1077d4fd0404SClaudiu Manoil dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1078d4fd0404SClaudiu Manoil rx_swbd->page_offset,
10797ed2bc80SVladimir Oltean buffer_size, rx_swbd->dir);
10806b04830dSVladimir Oltean
10816b04830dSVladimir Oltean rx_swbd->page = NULL;
10826b04830dSVladimir Oltean }
10836b04830dSVladimir Oltean
10846b04830dSVladimir Oltean /* Reuse the current page by performing half-page buffer flipping */
enetc_flip_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)10856b04830dSVladimir Oltean static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
10866b04830dSVladimir Oltean struct enetc_rx_swbd *rx_swbd)
10876b04830dSVladimir Oltean {
10886b04830dSVladimir Oltean if (likely(enetc_page_reusable(rx_swbd->page))) {
10896b04830dSVladimir Oltean rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
10906b04830dSVladimir Oltean page_ref_inc(rx_swbd->page);
10916b04830dSVladimir Oltean
10926b04830dSVladimir Oltean enetc_put_rx_buff(rx_ring, rx_swbd);
1093d4fd0404SClaudiu Manoil } else {
10947ed2bc80SVladimir Oltean dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
10957ed2bc80SVladimir Oltean rx_swbd->dir);
1096d4fd0404SClaudiu Manoil rx_swbd->page = NULL;
1097d4fd0404SClaudiu Manoil }
10986b04830dSVladimir Oltean }
1099d4fd0404SClaudiu Manoil
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)1100d4fd0404SClaudiu Manoil static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1101d4fd0404SClaudiu Manoil int i, u16 size)
1102d4fd0404SClaudiu Manoil {
1103d4fd0404SClaudiu Manoil struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1104d4fd0404SClaudiu Manoil struct sk_buff *skb;
1105d4fd0404SClaudiu Manoil void *ba;
1106d4fd0404SClaudiu Manoil
1107d4fd0404SClaudiu Manoil ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1108d1b15102SVladimir Oltean skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1109d4fd0404SClaudiu Manoil if (unlikely(!skb)) {
1110d4fd0404SClaudiu Manoil rx_ring->stats.rx_alloc_errs++;
1111d4fd0404SClaudiu Manoil return NULL;
1112d4fd0404SClaudiu Manoil }
1113d4fd0404SClaudiu Manoil
1114d1b15102SVladimir Oltean skb_reserve(skb, rx_ring->buffer_offset);
1115d4fd0404SClaudiu Manoil __skb_put(skb, size);
1116d4fd0404SClaudiu Manoil
11176b04830dSVladimir Oltean enetc_flip_rx_buff(rx_ring, rx_swbd);
1118d4fd0404SClaudiu Manoil
1119d4fd0404SClaudiu Manoil return skb;
1120d4fd0404SClaudiu Manoil }
1121d4fd0404SClaudiu Manoil
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)1122d4fd0404SClaudiu Manoil static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1123d4fd0404SClaudiu Manoil u16 size, struct sk_buff *skb)
1124d4fd0404SClaudiu Manoil {
1125d4fd0404SClaudiu Manoil struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1126d4fd0404SClaudiu Manoil
1127d4fd0404SClaudiu Manoil skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1128d4fd0404SClaudiu Manoil rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1129d4fd0404SClaudiu Manoil
11306b04830dSVladimir Oltean enetc_flip_rx_buff(rx_ring, rx_swbd);
1131d4fd0404SClaudiu Manoil }
1132d4fd0404SClaudiu Manoil
enetc_check_bd_errors_and_consume(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i)11332fa423f5SVladimir Oltean static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
11342fa423f5SVladimir Oltean u32 bd_status,
11352fa423f5SVladimir Oltean union enetc_rx_bd **rxbd, int *i)
11362fa423f5SVladimir Oltean {
11372fa423f5SVladimir Oltean if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
11382fa423f5SVladimir Oltean return false;
11392fa423f5SVladimir Oltean
1140672f9a21SVladimir Oltean enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
11412fa423f5SVladimir Oltean enetc_rxbd_next(rx_ring, rxbd, i);
11422fa423f5SVladimir Oltean
11432fa423f5SVladimir Oltean while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
11442fa423f5SVladimir Oltean dma_rmb();
11452fa423f5SVladimir Oltean bd_status = le32_to_cpu((*rxbd)->r.lstatus);
11462fa423f5SVladimir Oltean
1147672f9a21SVladimir Oltean enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
11482fa423f5SVladimir Oltean enetc_rxbd_next(rx_ring, rxbd, i);
11492fa423f5SVladimir Oltean }
11502fa423f5SVladimir Oltean
11512fa423f5SVladimir Oltean rx_ring->ndev->stats.rx_dropped++;
11522fa423f5SVladimir Oltean rx_ring->ndev->stats.rx_errors++;
11532fa423f5SVladimir Oltean
11542fa423f5SVladimir Oltean return true;
11552fa423f5SVladimir Oltean }
11562fa423f5SVladimir Oltean
enetc_build_skb(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,int buffer_size)1157a800abd3SVladimir Oltean static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1158a800abd3SVladimir Oltean u32 bd_status, union enetc_rx_bd **rxbd,
1159d1b15102SVladimir Oltean int *i, int *cleaned_cnt, int buffer_size)
1160a800abd3SVladimir Oltean {
1161a800abd3SVladimir Oltean struct sk_buff *skb;
1162a800abd3SVladimir Oltean u16 size;
1163a800abd3SVladimir Oltean
1164a800abd3SVladimir Oltean size = le16_to_cpu((*rxbd)->r.buf_len);
1165a800abd3SVladimir Oltean skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1166a800abd3SVladimir Oltean if (!skb)
1167a800abd3SVladimir Oltean return NULL;
1168a800abd3SVladimir Oltean
1169a800abd3SVladimir Oltean enetc_get_offloads(rx_ring, *rxbd, skb);
1170a800abd3SVladimir Oltean
1171a800abd3SVladimir Oltean (*cleaned_cnt)++;
1172a800abd3SVladimir Oltean
1173a800abd3SVladimir Oltean enetc_rxbd_next(rx_ring, rxbd, i);
1174a800abd3SVladimir Oltean
1175a800abd3SVladimir Oltean /* not last BD in frame? */
1176a800abd3SVladimir Oltean while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1177a800abd3SVladimir Oltean bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1178d1b15102SVladimir Oltean size = buffer_size;
1179a800abd3SVladimir Oltean
1180a800abd3SVladimir Oltean if (bd_status & ENETC_RXBD_LSTATUS_F) {
1181a800abd3SVladimir Oltean dma_rmb();
1182a800abd3SVladimir Oltean size = le16_to_cpu((*rxbd)->r.buf_len);
1183a800abd3SVladimir Oltean }
1184a800abd3SVladimir Oltean
1185a800abd3SVladimir Oltean enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1186a800abd3SVladimir Oltean
1187a800abd3SVladimir Oltean (*cleaned_cnt)++;
1188a800abd3SVladimir Oltean
1189a800abd3SVladimir Oltean enetc_rxbd_next(rx_ring, rxbd, i);
1190a800abd3SVladimir Oltean }
1191a800abd3SVladimir Oltean
1192a800abd3SVladimir Oltean skb_record_rx_queue(skb, rx_ring->index);
1193a800abd3SVladimir Oltean skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1194a800abd3SVladimir Oltean
1195a800abd3SVladimir Oltean return skb;
1196a800abd3SVladimir Oltean }
1197a800abd3SVladimir Oltean
1198d4fd0404SClaudiu Manoil #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1199d4fd0404SClaudiu Manoil
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)1200d4fd0404SClaudiu Manoil static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1201d4fd0404SClaudiu Manoil struct napi_struct *napi, int work_limit)
1202d4fd0404SClaudiu Manoil {
1203d4fd0404SClaudiu Manoil int rx_frm_cnt = 0, rx_byte_cnt = 0;
1204d4fd0404SClaudiu Manoil int cleaned_cnt, i;
1205d4fd0404SClaudiu Manoil
1206d4fd0404SClaudiu Manoil cleaned_cnt = enetc_bd_unused(rx_ring);
1207d4fd0404SClaudiu Manoil /* next descriptor to process */
1208d4fd0404SClaudiu Manoil i = rx_ring->next_to_clean;
1209d4fd0404SClaudiu Manoil
1210d4fd0404SClaudiu Manoil while (likely(rx_frm_cnt < work_limit)) {
1211d4fd0404SClaudiu Manoil union enetc_rx_bd *rxbd;
1212d4fd0404SClaudiu Manoil struct sk_buff *skb;
1213d4fd0404SClaudiu Manoil u32 bd_status;
1214d4fd0404SClaudiu Manoil
12157a5222cbSVladimir Oltean if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
12167a5222cbSVladimir Oltean cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
12177a5222cbSVladimir Oltean cleaned_cnt);
1218d4fd0404SClaudiu Manoil
1219714239acSClaudiu Manoil rxbd = enetc_rxbd(rx_ring, i);
1220d4fd0404SClaudiu Manoil bd_status = le32_to_cpu(rxbd->r.lstatus);
12216d36ecdbSVladimir Oltean if (!bd_status)
1222d4fd0404SClaudiu Manoil break;
1223d4fd0404SClaudiu Manoil
1224fd5736bfSAlex Marginean enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1225d4fd0404SClaudiu Manoil dma_rmb(); /* for reading other rxbd fields */
12262fa423f5SVladimir Oltean
12272fa423f5SVladimir Oltean if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
12282fa423f5SVladimir Oltean &rxbd, &i))
12292fa423f5SVladimir Oltean break;
12302fa423f5SVladimir Oltean
1231a800abd3SVladimir Oltean skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1232d1b15102SVladimir Oltean &cleaned_cnt, ENETC_RXB_DMA_SIZE);
12336d36ecdbSVladimir Oltean if (!skb)
1234d4fd0404SClaudiu Manoil break;
1235d4fd0404SClaudiu Manoil
12367190d0ffSWei Fang /* When set, the outer VLAN header is extracted and reported
12377190d0ffSWei Fang * in the receive buffer descriptor. So rx_byte_cnt should
12387190d0ffSWei Fang * add the length of the extracted VLAN header.
12397190d0ffSWei Fang */
12407190d0ffSWei Fang if (bd_status & ENETC_RXBD_FLAG_VLAN)
12417190d0ffSWei Fang rx_byte_cnt += VLAN_HLEN;
12427190d0ffSWei Fang rx_byte_cnt += skb->len + ETH_HLEN;
1243a800abd3SVladimir Oltean rx_frm_cnt++;
1244d4fd0404SClaudiu Manoil
1245d4fd0404SClaudiu Manoil napi_gro_receive(napi, skb);
1246d4fd0404SClaudiu Manoil }
1247d4fd0404SClaudiu Manoil
1248d4fd0404SClaudiu Manoil rx_ring->next_to_clean = i;
1249d4fd0404SClaudiu Manoil
1250d4fd0404SClaudiu Manoil rx_ring->stats.packets += rx_frm_cnt;
1251d4fd0404SClaudiu Manoil rx_ring->stats.bytes += rx_byte_cnt;
1252d4fd0404SClaudiu Manoil
1253d4fd0404SClaudiu Manoil return rx_frm_cnt;
1254d4fd0404SClaudiu Manoil }
1255d4fd0404SClaudiu Manoil
enetc_xdp_map_tx_buff(struct enetc_bdr * tx_ring,int i,struct enetc_tx_swbd * tx_swbd,int frm_len)12567ed2bc80SVladimir Oltean static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
12577ed2bc80SVladimir Oltean struct enetc_tx_swbd *tx_swbd,
12587ed2bc80SVladimir Oltean int frm_len)
12597ed2bc80SVladimir Oltean {
12607ed2bc80SVladimir Oltean union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
12617ed2bc80SVladimir Oltean
12627ed2bc80SVladimir Oltean prefetchw(txbd);
12637ed2bc80SVladimir Oltean
12647ed2bc80SVladimir Oltean enetc_clear_tx_bd(txbd);
12657ed2bc80SVladimir Oltean txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
12667ed2bc80SVladimir Oltean txbd->buf_len = cpu_to_le16(tx_swbd->len);
12677ed2bc80SVladimir Oltean txbd->frm_len = cpu_to_le16(frm_len);
12687ed2bc80SVladimir Oltean
12697ed2bc80SVladimir Oltean memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
12707ed2bc80SVladimir Oltean }
12717ed2bc80SVladimir Oltean
12727ed2bc80SVladimir Oltean /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
12737ed2bc80SVladimir Oltean * descriptors.
12747ed2bc80SVladimir Oltean */
enetc_xdp_tx(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,int num_tx_swbd)12757ed2bc80SVladimir Oltean static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
12767ed2bc80SVladimir Oltean struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
12777ed2bc80SVladimir Oltean {
12787ed2bc80SVladimir Oltean struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
12797ed2bc80SVladimir Oltean int i, k, frm_len = tmp_tx_swbd->len;
12807ed2bc80SVladimir Oltean
12817ed2bc80SVladimir Oltean if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
12827ed2bc80SVladimir Oltean return false;
12837ed2bc80SVladimir Oltean
12847ed2bc80SVladimir Oltean while (unlikely(!tmp_tx_swbd->is_eof)) {
12857ed2bc80SVladimir Oltean tmp_tx_swbd++;
12867ed2bc80SVladimir Oltean frm_len += tmp_tx_swbd->len;
12877ed2bc80SVladimir Oltean }
12887ed2bc80SVladimir Oltean
12897ed2bc80SVladimir Oltean i = tx_ring->next_to_use;
12907ed2bc80SVladimir Oltean
12917ed2bc80SVladimir Oltean for (k = 0; k < num_tx_swbd; k++) {
12927ed2bc80SVladimir Oltean struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
12937ed2bc80SVladimir Oltean
12947ed2bc80SVladimir Oltean enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
12957ed2bc80SVladimir Oltean
12967ed2bc80SVladimir Oltean /* last BD needs 'F' bit set */
12977ed2bc80SVladimir Oltean if (xdp_tx_swbd->is_eof) {
12987ed2bc80SVladimir Oltean union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
12997ed2bc80SVladimir Oltean
13007ed2bc80SVladimir Oltean txbd->flags = ENETC_TXBD_FLAGS_F;
13017ed2bc80SVladimir Oltean }
13027ed2bc80SVladimir Oltean
13037ed2bc80SVladimir Oltean enetc_bdr_idx_inc(tx_ring, &i);
13047ed2bc80SVladimir Oltean }
13057ed2bc80SVladimir Oltean
13067ed2bc80SVladimir Oltean tx_ring->next_to_use = i;
13077ed2bc80SVladimir Oltean
13087ed2bc80SVladimir Oltean return true;
13097ed2bc80SVladimir Oltean }
13107ed2bc80SVladimir Oltean
enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,struct xdp_frame * xdp_frame)13119d2b68ccSVladimir Oltean static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
13129d2b68ccSVladimir Oltean struct enetc_tx_swbd *xdp_tx_arr,
13139d2b68ccSVladimir Oltean struct xdp_frame *xdp_frame)
13149d2b68ccSVladimir Oltean {
13159d2b68ccSVladimir Oltean struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
13169d2b68ccSVladimir Oltean struct skb_shared_info *shinfo;
13179d2b68ccSVladimir Oltean void *data = xdp_frame->data;
13189d2b68ccSVladimir Oltean int len = xdp_frame->len;
13199d2b68ccSVladimir Oltean skb_frag_t *frag;
13209d2b68ccSVladimir Oltean dma_addr_t dma;
13219d2b68ccSVladimir Oltean unsigned int f;
13229d2b68ccSVladimir Oltean int n = 0;
13239d2b68ccSVladimir Oltean
13249d2b68ccSVladimir Oltean dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
13259d2b68ccSVladimir Oltean if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
13269d2b68ccSVladimir Oltean netdev_err(tx_ring->ndev, "DMA map error\n");
13279d2b68ccSVladimir Oltean return -1;
13289d2b68ccSVladimir Oltean }
13299d2b68ccSVladimir Oltean
13309d2b68ccSVladimir Oltean xdp_tx_swbd->dma = dma;
13319d2b68ccSVladimir Oltean xdp_tx_swbd->dir = DMA_TO_DEVICE;
13329d2b68ccSVladimir Oltean xdp_tx_swbd->len = len;
13339d2b68ccSVladimir Oltean xdp_tx_swbd->is_xdp_redirect = true;
13349d2b68ccSVladimir Oltean xdp_tx_swbd->is_eof = false;
13359d2b68ccSVladimir Oltean xdp_tx_swbd->xdp_frame = NULL;
13369d2b68ccSVladimir Oltean
13379d2b68ccSVladimir Oltean n++;
1338c7030d14SLorenzo Bianconi
1339c7030d14SLorenzo Bianconi if (!xdp_frame_has_frags(xdp_frame))
1340c7030d14SLorenzo Bianconi goto out;
1341c7030d14SLorenzo Bianconi
13429d2b68ccSVladimir Oltean xdp_tx_swbd = &xdp_tx_arr[n];
13439d2b68ccSVladimir Oltean
13449d2b68ccSVladimir Oltean shinfo = xdp_get_shared_info_from_frame(xdp_frame);
13459d2b68ccSVladimir Oltean
13469d2b68ccSVladimir Oltean for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
13479d2b68ccSVladimir Oltean f++, frag++) {
13489d2b68ccSVladimir Oltean data = skb_frag_address(frag);
13499d2b68ccSVladimir Oltean len = skb_frag_size(frag);
13509d2b68ccSVladimir Oltean
13519d2b68ccSVladimir Oltean dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
13529d2b68ccSVladimir Oltean if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
13539d2b68ccSVladimir Oltean /* Undo the DMA mapping for all fragments */
1354626b598aSDan Carpenter while (--n >= 0)
13559d2b68ccSVladimir Oltean enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
13569d2b68ccSVladimir Oltean
13579d2b68ccSVladimir Oltean netdev_err(tx_ring->ndev, "DMA map error\n");
13589d2b68ccSVladimir Oltean return -1;
13599d2b68ccSVladimir Oltean }
13609d2b68ccSVladimir Oltean
13619d2b68ccSVladimir Oltean xdp_tx_swbd->dma = dma;
13629d2b68ccSVladimir Oltean xdp_tx_swbd->dir = DMA_TO_DEVICE;
13639d2b68ccSVladimir Oltean xdp_tx_swbd->len = len;
13649d2b68ccSVladimir Oltean xdp_tx_swbd->is_xdp_redirect = true;
13659d2b68ccSVladimir Oltean xdp_tx_swbd->is_eof = false;
13669d2b68ccSVladimir Oltean xdp_tx_swbd->xdp_frame = NULL;
13679d2b68ccSVladimir Oltean
13689d2b68ccSVladimir Oltean n++;
13699d2b68ccSVladimir Oltean xdp_tx_swbd = &xdp_tx_arr[n];
13709d2b68ccSVladimir Oltean }
1371c7030d14SLorenzo Bianconi out:
13729d2b68ccSVladimir Oltean xdp_tx_arr[n - 1].is_eof = true;
13739d2b68ccSVladimir Oltean xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
13749d2b68ccSVladimir Oltean
13759d2b68ccSVladimir Oltean return n;
13769d2b68ccSVladimir Oltean }
13779d2b68ccSVladimir Oltean
enetc_xdp_xmit(struct net_device * ndev,int num_frames,struct xdp_frame ** frames,u32 flags)13789d2b68ccSVladimir Oltean int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
13799d2b68ccSVladimir Oltean struct xdp_frame **frames, u32 flags)
13809d2b68ccSVladimir Oltean {
13819d2b68ccSVladimir Oltean struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
13829d2b68ccSVladimir Oltean struct enetc_ndev_priv *priv = netdev_priv(ndev);
13839d2b68ccSVladimir Oltean struct enetc_bdr *tx_ring;
13849d2b68ccSVladimir Oltean int xdp_tx_bd_cnt, i, k;
13859d2b68ccSVladimir Oltean int xdp_tx_frm_cnt = 0;
13869d2b68ccSVladimir Oltean
138737184349SWei Fang if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags)))
138837184349SWei Fang return -ENETDOWN;
138937184349SWei Fang
139024e39309SVladimir Oltean enetc_lock_mdio();
139124e39309SVladimir Oltean
13927eab503bSVladimir Oltean tx_ring = priv->xdp_tx_ring[smp_processor_id()];
13939d2b68ccSVladimir Oltean
13949d2b68ccSVladimir Oltean prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
13959d2b68ccSVladimir Oltean
13969d2b68ccSVladimir Oltean for (k = 0; k < num_frames; k++) {
13979d2b68ccSVladimir Oltean xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
13989d2b68ccSVladimir Oltean xdp_redirect_arr,
13999d2b68ccSVladimir Oltean frames[k]);
14009d2b68ccSVladimir Oltean if (unlikely(xdp_tx_bd_cnt < 0))
14019d2b68ccSVladimir Oltean break;
14029d2b68ccSVladimir Oltean
14039d2b68ccSVladimir Oltean if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
14049d2b68ccSVladimir Oltean xdp_tx_bd_cnt))) {
14059d2b68ccSVladimir Oltean for (i = 0; i < xdp_tx_bd_cnt; i++)
14069d2b68ccSVladimir Oltean enetc_unmap_tx_buff(tx_ring,
14079d2b68ccSVladimir Oltean &xdp_redirect_arr[i]);
14089d2b68ccSVladimir Oltean tx_ring->stats.xdp_tx_drops++;
14099d2b68ccSVladimir Oltean break;
14109d2b68ccSVladimir Oltean }
14119d2b68ccSVladimir Oltean
14129d2b68ccSVladimir Oltean xdp_tx_frm_cnt++;
14139d2b68ccSVladimir Oltean }
14149d2b68ccSVladimir Oltean
14159d2b68ccSVladimir Oltean if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
14169d2b68ccSVladimir Oltean enetc_update_tx_ring_tail(tx_ring);
14179d2b68ccSVladimir Oltean
14189d2b68ccSVladimir Oltean tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
14199d2b68ccSVladimir Oltean
142024e39309SVladimir Oltean enetc_unlock_mdio();
142124e39309SVladimir Oltean
14229d2b68ccSVladimir Oltean return xdp_tx_frm_cnt;
14239d2b68ccSVladimir Oltean }
1424e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
14259d2b68ccSVladimir Oltean
enetc_map_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,struct xdp_buff * xdp_buff,u16 size)1426d1b15102SVladimir Oltean static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1427d1b15102SVladimir Oltean struct xdp_buff *xdp_buff, u16 size)
1428d1b15102SVladimir Oltean {
1429d1b15102SVladimir Oltean struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1430d1b15102SVladimir Oltean void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1431d1b15102SVladimir Oltean
14327ed2bc80SVladimir Oltean /* To be used for XDP_TX */
14337ed2bc80SVladimir Oltean rx_swbd->len = size;
14347ed2bc80SVladimir Oltean
1435d1b15102SVladimir Oltean xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1436d1b15102SVladimir Oltean rx_ring->buffer_offset, size, false);
1437d1b15102SVladimir Oltean }
1438d1b15102SVladimir Oltean
enetc_add_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,u16 size,struct xdp_buff * xdp_buff)1439d1b15102SVladimir Oltean static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1440d1b15102SVladimir Oltean u16 size, struct xdp_buff *xdp_buff)
1441d1b15102SVladimir Oltean {
1442d1b15102SVladimir Oltean struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1443d1b15102SVladimir Oltean struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1444c7030d14SLorenzo Bianconi skb_frag_t *frag;
1445d1b15102SVladimir Oltean
14467ed2bc80SVladimir Oltean /* To be used for XDP_TX */
14477ed2bc80SVladimir Oltean rx_swbd->len = size;
14487ed2bc80SVladimir Oltean
14498feb020fSLorenzo Bianconi if (!xdp_buff_has_frags(xdp_buff)) {
14508feb020fSLorenzo Bianconi xdp_buff_set_frags_flag(xdp_buff);
14518feb020fSLorenzo Bianconi shinfo->xdp_frags_size = size;
1452c7030d14SLorenzo Bianconi shinfo->nr_frags = 0;
14538feb020fSLorenzo Bianconi } else {
14548feb020fSLorenzo Bianconi shinfo->xdp_frags_size += size;
14558feb020fSLorenzo Bianconi }
14568feb020fSLorenzo Bianconi
14578feb020fSLorenzo Bianconi if (page_is_pfmemalloc(rx_swbd->page))
14588feb020fSLorenzo Bianconi xdp_buff_set_frag_pfmemalloc(xdp_buff);
14598feb020fSLorenzo Bianconi
1460c7030d14SLorenzo Bianconi frag = &shinfo->frags[shinfo->nr_frags];
1461b51f4113SYunsheng Lin skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1462b51f4113SYunsheng Lin size);
1463d1b15102SVladimir Oltean
1464d1b15102SVladimir Oltean shinfo->nr_frags++;
1465d1b15102SVladimir Oltean }
1466d1b15102SVladimir Oltean
enetc_build_xdp_buff(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,struct xdp_buff * xdp_buff)1467d1b15102SVladimir Oltean static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1468d1b15102SVladimir Oltean union enetc_rx_bd **rxbd, int *i,
1469d1b15102SVladimir Oltean int *cleaned_cnt, struct xdp_buff *xdp_buff)
1470d1b15102SVladimir Oltean {
1471d1b15102SVladimir Oltean u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1472d1b15102SVladimir Oltean
1473d1b15102SVladimir Oltean xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1474d1b15102SVladimir Oltean
1475d1b15102SVladimir Oltean enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1476d1b15102SVladimir Oltean (*cleaned_cnt)++;
1477d1b15102SVladimir Oltean enetc_rxbd_next(rx_ring, rxbd, i);
1478d1b15102SVladimir Oltean
1479d1b15102SVladimir Oltean /* not last BD in frame? */
1480d1b15102SVladimir Oltean while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1481d1b15102SVladimir Oltean bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1482d1b15102SVladimir Oltean size = ENETC_RXB_DMA_SIZE_XDP;
1483d1b15102SVladimir Oltean
1484d1b15102SVladimir Oltean if (bd_status & ENETC_RXBD_LSTATUS_F) {
1485d1b15102SVladimir Oltean dma_rmb();
1486d1b15102SVladimir Oltean size = le16_to_cpu((*rxbd)->r.buf_len);
1487d1b15102SVladimir Oltean }
1488d1b15102SVladimir Oltean
1489d1b15102SVladimir Oltean enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1490d1b15102SVladimir Oltean (*cleaned_cnt)++;
1491d1b15102SVladimir Oltean enetc_rxbd_next(rx_ring, rxbd, i);
1492d1b15102SVladimir Oltean }
1493d1b15102SVladimir Oltean }
1494d1b15102SVladimir Oltean
14957ed2bc80SVladimir Oltean /* Convert RX buffer descriptors to TX buffer descriptors. These will be
149692ff9a6eSVladimir Oltean * recycled back into the RX ring in enetc_clean_tx_ring.
14977ed2bc80SVladimir Oltean */
enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd * xdp_tx_arr,struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)14987ed2bc80SVladimir Oltean static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
14997ed2bc80SVladimir Oltean struct enetc_bdr *rx_ring,
15007ed2bc80SVladimir Oltean int rx_ring_first, int rx_ring_last)
15017ed2bc80SVladimir Oltean {
15027ed2bc80SVladimir Oltean int n = 0;
15037ed2bc80SVladimir Oltean
15047ed2bc80SVladimir Oltean for (; rx_ring_first != rx_ring_last;
15057ed2bc80SVladimir Oltean n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
15067ed2bc80SVladimir Oltean struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
15077ed2bc80SVladimir Oltean struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
15087ed2bc80SVladimir Oltean
15097ed2bc80SVladimir Oltean /* No need to dma_map, we already have DMA_BIDIRECTIONAL */
15107ed2bc80SVladimir Oltean tx_swbd->dma = rx_swbd->dma;
15117ed2bc80SVladimir Oltean tx_swbd->dir = rx_swbd->dir;
15127ed2bc80SVladimir Oltean tx_swbd->page = rx_swbd->page;
15137ed2bc80SVladimir Oltean tx_swbd->page_offset = rx_swbd->page_offset;
15147ed2bc80SVladimir Oltean tx_swbd->len = rx_swbd->len;
15157ed2bc80SVladimir Oltean tx_swbd->is_dma_page = true;
15167ed2bc80SVladimir Oltean tx_swbd->is_xdp_tx = true;
15177ed2bc80SVladimir Oltean tx_swbd->is_eof = false;
15187ed2bc80SVladimir Oltean }
15197ed2bc80SVladimir Oltean
15207ed2bc80SVladimir Oltean /* We rely on caller providing an rx_ring_last > rx_ring_first */
15217ed2bc80SVladimir Oltean xdp_tx_arr[n - 1].is_eof = true;
15227ed2bc80SVladimir Oltean
15237ed2bc80SVladimir Oltean return n;
15247ed2bc80SVladimir Oltean }
15257ed2bc80SVladimir Oltean
enetc_xdp_drop(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1526d1b15102SVladimir Oltean static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1527d1b15102SVladimir Oltean int rx_ring_last)
1528d1b15102SVladimir Oltean {
1529d1b15102SVladimir Oltean while (rx_ring_first != rx_ring_last) {
15306b04830dSVladimir Oltean enetc_put_rx_buff(rx_ring,
1531d1b15102SVladimir Oltean &rx_ring->rx_swbd[rx_ring_first]);
1532d1b15102SVladimir Oltean enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1533d1b15102SVladimir Oltean }
1534d1b15102SVladimir Oltean }
1535d1b15102SVladimir Oltean
enetc_clean_rx_ring_xdp(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit,struct bpf_prog * prog)1536d1b15102SVladimir Oltean static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1537d1b15102SVladimir Oltean struct napi_struct *napi, int work_limit,
1538d1b15102SVladimir Oltean struct bpf_prog *prog)
1539d1b15102SVladimir Oltean {
15409d2b68ccSVladimir Oltean int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
15417ed2bc80SVladimir Oltean struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
15427ed2bc80SVladimir Oltean struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1543d1b15102SVladimir Oltean int rx_frm_cnt = 0, rx_byte_cnt = 0;
15447eab503bSVladimir Oltean struct enetc_bdr *tx_ring;
1545d1b15102SVladimir Oltean int cleaned_cnt, i;
1546d1b15102SVladimir Oltean u32 xdp_act;
1547d1b15102SVladimir Oltean
1548d1b15102SVladimir Oltean cleaned_cnt = enetc_bd_unused(rx_ring);
1549d1b15102SVladimir Oltean /* next descriptor to process */
1550d1b15102SVladimir Oltean i = rx_ring->next_to_clean;
1551d1b15102SVladimir Oltean
1552d1b15102SVladimir Oltean while (likely(rx_frm_cnt < work_limit)) {
1553d1b15102SVladimir Oltean union enetc_rx_bd *rxbd, *orig_rxbd;
1554d1b15102SVladimir Oltean int orig_i, orig_cleaned_cnt;
1555d1b15102SVladimir Oltean struct xdp_buff xdp_buff;
1556d1b15102SVladimir Oltean struct sk_buff *skb;
1557d1b15102SVladimir Oltean u32 bd_status;
1558628050ecSVladimir Oltean int err;
1559d1b15102SVladimir Oltean
1560d1b15102SVladimir Oltean rxbd = enetc_rxbd(rx_ring, i);
1561d1b15102SVladimir Oltean bd_status = le32_to_cpu(rxbd->r.lstatus);
1562d1b15102SVladimir Oltean if (!bd_status)
1563d1b15102SVladimir Oltean break;
1564d1b15102SVladimir Oltean
1565d1b15102SVladimir Oltean enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1566d1b15102SVladimir Oltean dma_rmb(); /* for reading other rxbd fields */
1567d1b15102SVladimir Oltean
1568d1b15102SVladimir Oltean if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1569d1b15102SVladimir Oltean &rxbd, &i))
1570d1b15102SVladimir Oltean break;
1571d1b15102SVladimir Oltean
1572d1b15102SVladimir Oltean orig_rxbd = rxbd;
1573d1b15102SVladimir Oltean orig_cleaned_cnt = cleaned_cnt;
1574d1b15102SVladimir Oltean orig_i = i;
1575d1b15102SVladimir Oltean
1576d1b15102SVladimir Oltean enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1577d1b15102SVladimir Oltean &cleaned_cnt, &xdp_buff);
1578d1b15102SVladimir Oltean
1579fdebd850SWei Fang /* When set, the outer VLAN header is extracted and reported
1580fdebd850SWei Fang * in the receive buffer descriptor. So rx_byte_cnt should
1581fdebd850SWei Fang * add the length of the extracted VLAN header.
1582fdebd850SWei Fang */
1583fdebd850SWei Fang if (bd_status & ENETC_RXBD_FLAG_VLAN)
1584fdebd850SWei Fang rx_byte_cnt += VLAN_HLEN;
1585fdebd850SWei Fang rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1586fdebd850SWei Fang
1587d1b15102SVladimir Oltean xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1588d1b15102SVladimir Oltean
1589d1b15102SVladimir Oltean switch (xdp_act) {
1590975acc83SVladimir Oltean default:
1591c8064e5bSPaolo Abeni bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1592975acc83SVladimir Oltean fallthrough;
1593d1b15102SVladimir Oltean case XDP_ABORTED:
1594d1b15102SVladimir Oltean trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1595d1b15102SVladimir Oltean fallthrough;
1596d1b15102SVladimir Oltean case XDP_DROP:
1597d1b15102SVladimir Oltean enetc_xdp_drop(rx_ring, orig_i, i);
15981c95443eSWei Fang rx_ring->stats.xdp_drops++;
1599d1b15102SVladimir Oltean break;
1600d1b15102SVladimir Oltean case XDP_PASS:
1601d1b15102SVladimir Oltean rxbd = orig_rxbd;
1602d1b15102SVladimir Oltean cleaned_cnt = orig_cleaned_cnt;
1603d1b15102SVladimir Oltean i = orig_i;
1604d1b15102SVladimir Oltean
1605d1b15102SVladimir Oltean skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1606d1b15102SVladimir Oltean &i, &cleaned_cnt,
1607d1b15102SVladimir Oltean ENETC_RXB_DMA_SIZE_XDP);
1608d1b15102SVladimir Oltean if (unlikely(!skb))
16098f50d8bbSVladimir Oltean goto out;
1610d1b15102SVladimir Oltean
1611d1b15102SVladimir Oltean napi_gro_receive(napi, skb);
1612d1b15102SVladimir Oltean break;
16137ed2bc80SVladimir Oltean case XDP_TX:
16147eab503bSVladimir Oltean tx_ring = priv->xdp_tx_ring[rx_ring->index];
161537184349SWei Fang if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) {
161637184349SWei Fang enetc_xdp_drop(rx_ring, orig_i, i);
161737184349SWei Fang tx_ring->stats.xdp_tx_drops++;
161837184349SWei Fang break;
161937184349SWei Fang }
162037184349SWei Fang
16217ed2bc80SVladimir Oltean xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
16227ed2bc80SVladimir Oltean rx_ring,
16237ed2bc80SVladimir Oltean orig_i, i);
16247ed2bc80SVladimir Oltean
16257ed2bc80SVladimir Oltean if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
16267ed2bc80SVladimir Oltean enetc_xdp_drop(rx_ring, orig_i, i);
16277ed2bc80SVladimir Oltean tx_ring->stats.xdp_tx_drops++;
16287ed2bc80SVladimir Oltean } else {
16297ed2bc80SVladimir Oltean tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
16307ed2bc80SVladimir Oltean rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
16317ed2bc80SVladimir Oltean xdp_tx_frm_cnt++;
163292ff9a6eSVladimir Oltean /* The XDP_TX enqueue was successful, so we
163392ff9a6eSVladimir Oltean * need to scrub the RX software BDs because
163492ff9a6eSVladimir Oltean * the ownership of the buffers no longer
163592ff9a6eSVladimir Oltean * belongs to the RX ring, and we must prevent
163692ff9a6eSVladimir Oltean * enetc_refill_rx_ring() from reusing
163792ff9a6eSVladimir Oltean * rx_swbd->page.
163892ff9a6eSVladimir Oltean */
163992ff9a6eSVladimir Oltean while (orig_i != i) {
164092ff9a6eSVladimir Oltean rx_ring->rx_swbd[orig_i].page = NULL;
164192ff9a6eSVladimir Oltean enetc_bdr_idx_inc(rx_ring, &orig_i);
164292ff9a6eSVladimir Oltean }
16437ed2bc80SVladimir Oltean }
16447ed2bc80SVladimir Oltean break;
16459d2b68ccSVladimir Oltean case XDP_REDIRECT:
1646628050ecSVladimir Oltean err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1647628050ecSVladimir Oltean if (unlikely(err)) {
1648628050ecSVladimir Oltean enetc_xdp_drop(rx_ring, orig_i, i);
1649628050ecSVladimir Oltean rx_ring->stats.xdp_redirect_failures++;
1650628050ecSVladimir Oltean } else {
16519d2b68ccSVladimir Oltean while (orig_i != i) {
16526b04830dSVladimir Oltean enetc_flip_rx_buff(rx_ring,
16539d2b68ccSVladimir Oltean &rx_ring->rx_swbd[orig_i]);
16549d2b68ccSVladimir Oltean enetc_bdr_idx_inc(rx_ring, &orig_i);
16559d2b68ccSVladimir Oltean }
16569d2b68ccSVladimir Oltean xdp_redirect_frm_cnt++;
16579d2b68ccSVladimir Oltean rx_ring->stats.xdp_redirect++;
16589d2b68ccSVladimir Oltean }
1659d1b15102SVladimir Oltean }
1660d1b15102SVladimir Oltean
1661d1b15102SVladimir Oltean rx_frm_cnt++;
1662d1b15102SVladimir Oltean }
1663d1b15102SVladimir Oltean
16648f50d8bbSVladimir Oltean out:
1665d1b15102SVladimir Oltean rx_ring->next_to_clean = i;
1666d1b15102SVladimir Oltean
1667d1b15102SVladimir Oltean rx_ring->stats.packets += rx_frm_cnt;
1668d1b15102SVladimir Oltean rx_ring->stats.bytes += rx_byte_cnt;
1669d1b15102SVladimir Oltean
16709d2b68ccSVladimir Oltean if (xdp_redirect_frm_cnt)
16719d2b68ccSVladimir Oltean xdp_do_flush_map();
16729d2b68ccSVladimir Oltean
16737ed2bc80SVladimir Oltean if (xdp_tx_frm_cnt)
16747ed2bc80SVladimir Oltean enetc_update_tx_ring_tail(tx_ring);
16757ed2bc80SVladimir Oltean
16767ed2bc80SVladimir Oltean if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
16777ed2bc80SVladimir Oltean enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
16787ed2bc80SVladimir Oltean rx_ring->xdp.xdp_tx_in_flight);
16797ed2bc80SVladimir Oltean
1680d1b15102SVladimir Oltean return rx_frm_cnt;
1681d1b15102SVladimir Oltean }
1682d1b15102SVladimir Oltean
enetc_poll(struct napi_struct * napi,int budget)16838580b3c3SVladimir Oltean static int enetc_poll(struct napi_struct *napi, int budget)
16848580b3c3SVladimir Oltean {
16858580b3c3SVladimir Oltean struct enetc_int_vector
16868580b3c3SVladimir Oltean *v = container_of(napi, struct enetc_int_vector, napi);
1687d1b15102SVladimir Oltean struct enetc_bdr *rx_ring = &v->rx_ring;
1688d1b15102SVladimir Oltean struct bpf_prog *prog;
16898580b3c3SVladimir Oltean bool complete = true;
16908580b3c3SVladimir Oltean int work_done;
16918580b3c3SVladimir Oltean int i;
16928580b3c3SVladimir Oltean
16938580b3c3SVladimir Oltean enetc_lock_mdio();
16948580b3c3SVladimir Oltean
16958580b3c3SVladimir Oltean for (i = 0; i < v->count_tx_rings; i++)
16968580b3c3SVladimir Oltean if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
16978580b3c3SVladimir Oltean complete = false;
16988580b3c3SVladimir Oltean
1699d1b15102SVladimir Oltean prog = rx_ring->xdp.prog;
1700d1b15102SVladimir Oltean if (prog)
1701d1b15102SVladimir Oltean work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1702d1b15102SVladimir Oltean else
1703d1b15102SVladimir Oltean work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
17048580b3c3SVladimir Oltean if (work_done == budget)
17058580b3c3SVladimir Oltean complete = false;
17068580b3c3SVladimir Oltean if (work_done)
17078580b3c3SVladimir Oltean v->rx_napi_work = true;
17088580b3c3SVladimir Oltean
17098580b3c3SVladimir Oltean if (!complete) {
17108580b3c3SVladimir Oltean enetc_unlock_mdio();
17118580b3c3SVladimir Oltean return budget;
17128580b3c3SVladimir Oltean }
17138580b3c3SVladimir Oltean
17148580b3c3SVladimir Oltean napi_complete_done(napi, work_done);
17158580b3c3SVladimir Oltean
17168580b3c3SVladimir Oltean if (likely(v->rx_dim_en))
17178580b3c3SVladimir Oltean enetc_rx_net_dim(v);
17188580b3c3SVladimir Oltean
17198580b3c3SVladimir Oltean v->rx_napi_work = false;
17208580b3c3SVladimir Oltean
17218580b3c3SVladimir Oltean /* enable interrupts */
17228580b3c3SVladimir Oltean enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
17238580b3c3SVladimir Oltean
17248580b3c3SVladimir Oltean for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
17258580b3c3SVladimir Oltean enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
17268580b3c3SVladimir Oltean ENETC_TBIER_TXTIE);
17278580b3c3SVladimir Oltean
17288580b3c3SVladimir Oltean enetc_unlock_mdio();
17298580b3c3SVladimir Oltean
17308580b3c3SVladimir Oltean return work_done;
17318580b3c3SVladimir Oltean }
17328580b3c3SVladimir Oltean
1733d4fd0404SClaudiu Manoil /* Probing and Init */
1734d382563fSClaudiu Manoil #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)1735d4fd0404SClaudiu Manoil void enetc_get_si_caps(struct enetc_si *si)
1736d4fd0404SClaudiu Manoil {
1737d4fd0404SClaudiu Manoil struct enetc_hw *hw = &si->hw;
1738d4fd0404SClaudiu Manoil u32 val;
1739d4fd0404SClaudiu Manoil
1740d4fd0404SClaudiu Manoil /* find out how many of various resources we have to work with */
1741d4fd0404SClaudiu Manoil val = enetc_rd(hw, ENETC_SICAPR0);
1742d4fd0404SClaudiu Manoil si->num_rx_rings = (val >> 16) & 0xff;
1743d4fd0404SClaudiu Manoil si->num_tx_rings = val & 0xff;
1744d382563fSClaudiu Manoil
1745d382563fSClaudiu Manoil val = enetc_rd(hw, ENETC_SIRFSCAPR);
1746d382563fSClaudiu Manoil si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1747d382563fSClaudiu Manoil si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1748d382563fSClaudiu Manoil
1749d382563fSClaudiu Manoil si->num_rss = 0;
1750d382563fSClaudiu Manoil val = enetc_rd(hw, ENETC_SIPCAPR0);
1751d382563fSClaudiu Manoil if (val & ENETC_SIPCAPR0_RSS) {
17522e47cb41SPo Liu u32 rss;
17532e47cb41SPo Liu
17542e47cb41SPo Liu rss = enetc_rd(hw, ENETC_SIRSSCAPR);
17552e47cb41SPo Liu si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1756d382563fSClaudiu Manoil }
17572e47cb41SPo Liu
17582e47cb41SPo Liu if (val & ENETC_SIPCAPR0_QBV)
17592e47cb41SPo Liu si->hw_features |= ENETC_SI_F_QBV;
176079e49982SPo Liu
176194557a9aSVladimir Oltean if (val & ENETC_SIPCAPR0_QBU)
176294557a9aSVladimir Oltean si->hw_features |= ENETC_SI_F_QBU;
176394557a9aSVladimir Oltean
176479e49982SPo Liu if (val & ENETC_SIPCAPR0_PSFP)
176579e49982SPo Liu si->hw_features |= ENETC_SI_F_PSFP;
1766d4fd0404SClaudiu Manoil }
1767e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_get_si_caps);
1768d4fd0404SClaudiu Manoil
enetc_dma_alloc_bdr(struct enetc_bdr_resource * res)1769f3ce29e1SVladimir Oltean static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
1770d4fd0404SClaudiu Manoil {
1771f3ce29e1SVladimir Oltean size_t bd_base_size = res->bd_count * res->bd_size;
1772f3ce29e1SVladimir Oltean
1773f3ce29e1SVladimir Oltean res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
1774f3ce29e1SVladimir Oltean &res->bd_dma_base, GFP_KERNEL);
1775f3ce29e1SVladimir Oltean if (!res->bd_base)
1776d4fd0404SClaudiu Manoil return -ENOMEM;
1777d4fd0404SClaudiu Manoil
1778d4fd0404SClaudiu Manoil /* h/w requires 128B alignment */
1779f3ce29e1SVladimir Oltean if (!IS_ALIGNED(res->bd_dma_base, 128)) {
1780f3ce29e1SVladimir Oltean dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1781f3ce29e1SVladimir Oltean res->bd_dma_base);
1782d4fd0404SClaudiu Manoil return -EINVAL;
1783d4fd0404SClaudiu Manoil }
1784d4fd0404SClaudiu Manoil
1785d4fd0404SClaudiu Manoil return 0;
1786d4fd0404SClaudiu Manoil }
1787d4fd0404SClaudiu Manoil
enetc_dma_free_bdr(const struct enetc_bdr_resource * res)1788f3ce29e1SVladimir Oltean static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
17890d6cfd0fSVladimir Oltean {
1790f3ce29e1SVladimir Oltean size_t bd_base_size = res->bd_count * res->bd_size;
1791f3ce29e1SVladimir Oltean
1792f3ce29e1SVladimir Oltean dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1793f3ce29e1SVladimir Oltean res->bd_dma_base);
17940d6cfd0fSVladimir Oltean }
17950d6cfd0fSVladimir Oltean
enetc_alloc_tx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count)1796f3ce29e1SVladimir Oltean static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
1797f3ce29e1SVladimir Oltean struct device *dev, size_t bd_count)
1798d4fd0404SClaudiu Manoil {
1799d4fd0404SClaudiu Manoil int err;
1800d4fd0404SClaudiu Manoil
1801f3ce29e1SVladimir Oltean res->dev = dev;
1802f3ce29e1SVladimir Oltean res->bd_count = bd_count;
1803f3ce29e1SVladimir Oltean res->bd_size = sizeof(union enetc_tx_bd);
1804f3ce29e1SVladimir Oltean
1805fa87c546SJulia Lawall res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
1806f3ce29e1SVladimir Oltean if (!res->tx_swbd)
1807d4fd0404SClaudiu Manoil return -ENOMEM;
1808d4fd0404SClaudiu Manoil
1809f3ce29e1SVladimir Oltean err = enetc_dma_alloc_bdr(res);
1810fb8629e2SIoana Ciornei if (err)
1811fb8629e2SIoana Ciornei goto err_alloc_bdr;
1812fb8629e2SIoana Ciornei
1813f3ce29e1SVladimir Oltean res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
1814f3ce29e1SVladimir Oltean &res->tso_headers_dma,
1815fb8629e2SIoana Ciornei GFP_KERNEL);
1816f3ce29e1SVladimir Oltean if (!res->tso_headers) {
1817e79d8264SDan Carpenter err = -ENOMEM;
1818fb8629e2SIoana Ciornei goto err_alloc_tso;
1819e79d8264SDan Carpenter }
1820d4fd0404SClaudiu Manoil
1821d4fd0404SClaudiu Manoil return 0;
1822fb8629e2SIoana Ciornei
1823fb8629e2SIoana Ciornei err_alloc_tso:
1824f3ce29e1SVladimir Oltean enetc_dma_free_bdr(res);
1825fb8629e2SIoana Ciornei err_alloc_bdr:
1826f3ce29e1SVladimir Oltean vfree(res->tx_swbd);
1827f3ce29e1SVladimir Oltean res->tx_swbd = NULL;
1828fb8629e2SIoana Ciornei
1829fb8629e2SIoana Ciornei return err;
1830d4fd0404SClaudiu Manoil }
1831d4fd0404SClaudiu Manoil
enetc_free_tx_resource(const struct enetc_bdr_resource * res)1832f3ce29e1SVladimir Oltean static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
1833d4fd0404SClaudiu Manoil {
1834f3ce29e1SVladimir Oltean dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
1835f3ce29e1SVladimir Oltean res->tso_headers, res->tso_headers_dma);
1836f3ce29e1SVladimir Oltean enetc_dma_free_bdr(res);
1837f3ce29e1SVladimir Oltean vfree(res->tx_swbd);
1838d4fd0404SClaudiu Manoil }
1839d4fd0404SClaudiu Manoil
1840f3ce29e1SVladimir Oltean static struct enetc_bdr_resource *
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)1841f3ce29e1SVladimir Oltean enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1842d4fd0404SClaudiu Manoil {
1843f3ce29e1SVladimir Oltean struct enetc_bdr_resource *tx_res;
1844d4fd0404SClaudiu Manoil int i, err;
1845d4fd0404SClaudiu Manoil
1846f3ce29e1SVladimir Oltean tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
1847f3ce29e1SVladimir Oltean if (!tx_res)
1848f3ce29e1SVladimir Oltean return ERR_PTR(-ENOMEM);
1849f3ce29e1SVladimir Oltean
1850d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++) {
1851f3ce29e1SVladimir Oltean struct enetc_bdr *tx_ring = priv->tx_ring[i];
1852d4fd0404SClaudiu Manoil
1853f3ce29e1SVladimir Oltean err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
1854f3ce29e1SVladimir Oltean tx_ring->bd_count);
1855d4fd0404SClaudiu Manoil if (err)
1856d4fd0404SClaudiu Manoil goto fail;
1857d4fd0404SClaudiu Manoil }
1858d4fd0404SClaudiu Manoil
1859f3ce29e1SVladimir Oltean return tx_res;
1860d4fd0404SClaudiu Manoil
1861d4fd0404SClaudiu Manoil fail:
1862d4fd0404SClaudiu Manoil while (i-- > 0)
1863f3ce29e1SVladimir Oltean enetc_free_tx_resource(&tx_res[i]);
1864d4fd0404SClaudiu Manoil
1865f3ce29e1SVladimir Oltean kfree(tx_res);
1866f3ce29e1SVladimir Oltean
1867f3ce29e1SVladimir Oltean return ERR_PTR(err);
1868d4fd0404SClaudiu Manoil }
1869d4fd0404SClaudiu Manoil
enetc_free_tx_resources(const struct enetc_bdr_resource * tx_res,size_t num_resources)1870f3ce29e1SVladimir Oltean static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
1871f3ce29e1SVladimir Oltean size_t num_resources)
1872d4fd0404SClaudiu Manoil {
1873f3ce29e1SVladimir Oltean size_t i;
1874d4fd0404SClaudiu Manoil
1875f3ce29e1SVladimir Oltean for (i = 0; i < num_resources; i++)
1876f3ce29e1SVladimir Oltean enetc_free_tx_resource(&tx_res[i]);
1877f3ce29e1SVladimir Oltean
1878f3ce29e1SVladimir Oltean kfree(tx_res);
1879d4fd0404SClaudiu Manoil }
1880d4fd0404SClaudiu Manoil
enetc_alloc_rx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count,bool extended)1881f3ce29e1SVladimir Oltean static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
1882f3ce29e1SVladimir Oltean struct device *dev, size_t bd_count,
1883f3ce29e1SVladimir Oltean bool extended)
1884d4fd0404SClaudiu Manoil {
1885d4fd0404SClaudiu Manoil int err;
1886d4fd0404SClaudiu Manoil
1887f3ce29e1SVladimir Oltean res->dev = dev;
1888f3ce29e1SVladimir Oltean res->bd_count = bd_count;
1889f3ce29e1SVladimir Oltean res->bd_size = sizeof(union enetc_rx_bd);
1890f3ce29e1SVladimir Oltean if (extended)
1891f3ce29e1SVladimir Oltean res->bd_size *= 2;
1892f3ce29e1SVladimir Oltean
1893fa87c546SJulia Lawall res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
1894f3ce29e1SVladimir Oltean if (!res->rx_swbd)
1895d4fd0404SClaudiu Manoil return -ENOMEM;
1896d4fd0404SClaudiu Manoil
1897f3ce29e1SVladimir Oltean err = enetc_dma_alloc_bdr(res);
1898d4fd0404SClaudiu Manoil if (err) {
1899f3ce29e1SVladimir Oltean vfree(res->rx_swbd);
1900d4fd0404SClaudiu Manoil return err;
1901d4fd0404SClaudiu Manoil }
1902d4fd0404SClaudiu Manoil
1903d4fd0404SClaudiu Manoil return 0;
1904d4fd0404SClaudiu Manoil }
1905d4fd0404SClaudiu Manoil
enetc_free_rx_resource(const struct enetc_bdr_resource * res)1906f3ce29e1SVladimir Oltean static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
1907d4fd0404SClaudiu Manoil {
1908f3ce29e1SVladimir Oltean enetc_dma_free_bdr(res);
1909f3ce29e1SVladimir Oltean vfree(res->rx_swbd);
1910d4fd0404SClaudiu Manoil }
1911d4fd0404SClaudiu Manoil
1912f3ce29e1SVladimir Oltean static struct enetc_bdr_resource *
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv,bool extended)1913f3ce29e1SVladimir Oltean enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
1914d4fd0404SClaudiu Manoil {
1915f3ce29e1SVladimir Oltean struct enetc_bdr_resource *rx_res;
1916d4fd0404SClaudiu Manoil int i, err;
1917d4fd0404SClaudiu Manoil
1918f3ce29e1SVladimir Oltean rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
1919f3ce29e1SVladimir Oltean if (!rx_res)
1920f3ce29e1SVladimir Oltean return ERR_PTR(-ENOMEM);
1921d4fd0404SClaudiu Manoil
1922f3ce29e1SVladimir Oltean for (i = 0; i < priv->num_rx_rings; i++) {
1923f3ce29e1SVladimir Oltean struct enetc_bdr *rx_ring = priv->rx_ring[i];
1924f3ce29e1SVladimir Oltean
1925f3ce29e1SVladimir Oltean err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
1926f3ce29e1SVladimir Oltean rx_ring->bd_count, extended);
1927d4fd0404SClaudiu Manoil if (err)
1928d4fd0404SClaudiu Manoil goto fail;
1929d4fd0404SClaudiu Manoil }
1930d4fd0404SClaudiu Manoil
1931f3ce29e1SVladimir Oltean return rx_res;
1932d4fd0404SClaudiu Manoil
1933d4fd0404SClaudiu Manoil fail:
1934d4fd0404SClaudiu Manoil while (i-- > 0)
1935f3ce29e1SVladimir Oltean enetc_free_rx_resource(&rx_res[i]);
1936d4fd0404SClaudiu Manoil
1937f3ce29e1SVladimir Oltean kfree(rx_res);
1938f3ce29e1SVladimir Oltean
1939f3ce29e1SVladimir Oltean return ERR_PTR(err);
1940d4fd0404SClaudiu Manoil }
1941d4fd0404SClaudiu Manoil
enetc_free_rx_resources(const struct enetc_bdr_resource * rx_res,size_t num_resources)1942f3ce29e1SVladimir Oltean static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
1943f3ce29e1SVladimir Oltean size_t num_resources)
1944f3ce29e1SVladimir Oltean {
1945f3ce29e1SVladimir Oltean size_t i;
1946f3ce29e1SVladimir Oltean
1947f3ce29e1SVladimir Oltean for (i = 0; i < num_resources; i++)
1948f3ce29e1SVladimir Oltean enetc_free_rx_resource(&rx_res[i]);
1949f3ce29e1SVladimir Oltean
1950f3ce29e1SVladimir Oltean kfree(rx_res);
1951f3ce29e1SVladimir Oltean }
1952f3ce29e1SVladimir Oltean
enetc_assign_tx_resource(struct enetc_bdr * tx_ring,const struct enetc_bdr_resource * res)1953f3ce29e1SVladimir Oltean static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
1954f3ce29e1SVladimir Oltean const struct enetc_bdr_resource *res)
1955f3ce29e1SVladimir Oltean {
1956f3ce29e1SVladimir Oltean tx_ring->bd_base = res ? res->bd_base : NULL;
1957f3ce29e1SVladimir Oltean tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1958f3ce29e1SVladimir Oltean tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
1959f3ce29e1SVladimir Oltean tx_ring->tso_headers = res ? res->tso_headers : NULL;
1960f3ce29e1SVladimir Oltean tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
1961f3ce29e1SVladimir Oltean }
1962f3ce29e1SVladimir Oltean
enetc_assign_rx_resource(struct enetc_bdr * rx_ring,const struct enetc_bdr_resource * res)1963f3ce29e1SVladimir Oltean static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
1964f3ce29e1SVladimir Oltean const struct enetc_bdr_resource *res)
1965f3ce29e1SVladimir Oltean {
1966f3ce29e1SVladimir Oltean rx_ring->bd_base = res ? res->bd_base : NULL;
1967f3ce29e1SVladimir Oltean rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1968f3ce29e1SVladimir Oltean rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
1969f3ce29e1SVladimir Oltean }
1970f3ce29e1SVladimir Oltean
enetc_assign_tx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)1971f3ce29e1SVladimir Oltean static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
1972f3ce29e1SVladimir Oltean const struct enetc_bdr_resource *res)
1973d4fd0404SClaudiu Manoil {
1974d4fd0404SClaudiu Manoil int i;
1975d4fd0404SClaudiu Manoil
1976f3ce29e1SVladimir Oltean if (priv->tx_res)
1977f3ce29e1SVladimir Oltean enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
1978f3ce29e1SVladimir Oltean
1979f3ce29e1SVladimir Oltean for (i = 0; i < priv->num_tx_rings; i++) {
1980f3ce29e1SVladimir Oltean enetc_assign_tx_resource(priv->tx_ring[i],
1981f3ce29e1SVladimir Oltean res ? &res[i] : NULL);
1982f3ce29e1SVladimir Oltean }
1983f3ce29e1SVladimir Oltean
1984f3ce29e1SVladimir Oltean priv->tx_res = res;
1985f3ce29e1SVladimir Oltean }
1986f3ce29e1SVladimir Oltean
enetc_assign_rx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)1987f3ce29e1SVladimir Oltean static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
1988f3ce29e1SVladimir Oltean const struct enetc_bdr_resource *res)
1989f3ce29e1SVladimir Oltean {
1990f3ce29e1SVladimir Oltean int i;
1991f3ce29e1SVladimir Oltean
1992f3ce29e1SVladimir Oltean if (priv->rx_res)
1993f3ce29e1SVladimir Oltean enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
1994f3ce29e1SVladimir Oltean
1995f3ce29e1SVladimir Oltean for (i = 0; i < priv->num_rx_rings; i++) {
1996f3ce29e1SVladimir Oltean enetc_assign_rx_resource(priv->rx_ring[i],
1997f3ce29e1SVladimir Oltean res ? &res[i] : NULL);
1998f3ce29e1SVladimir Oltean }
1999f3ce29e1SVladimir Oltean
2000f3ce29e1SVladimir Oltean priv->rx_res = res;
2001d4fd0404SClaudiu Manoil }
2002d4fd0404SClaudiu Manoil
enetc_free_tx_ring(struct enetc_bdr * tx_ring)2003d4fd0404SClaudiu Manoil static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
2004d4fd0404SClaudiu Manoil {
2005d4fd0404SClaudiu Manoil int i;
2006d4fd0404SClaudiu Manoil
2007d4fd0404SClaudiu Manoil for (i = 0; i < tx_ring->bd_count; i++) {
2008d4fd0404SClaudiu Manoil struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
2009d4fd0404SClaudiu Manoil
20109d2b68ccSVladimir Oltean enetc_free_tx_frame(tx_ring, tx_swbd);
2011d4fd0404SClaudiu Manoil }
2012d4fd0404SClaudiu Manoil }
2013d4fd0404SClaudiu Manoil
enetc_free_rx_ring(struct enetc_bdr * rx_ring)2014d4fd0404SClaudiu Manoil static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2015d4fd0404SClaudiu Manoil {
2016d4fd0404SClaudiu Manoil int i;
2017d4fd0404SClaudiu Manoil
2018d4fd0404SClaudiu Manoil for (i = 0; i < rx_ring->bd_count; i++) {
2019d4fd0404SClaudiu Manoil struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2020d4fd0404SClaudiu Manoil
2021d4fd0404SClaudiu Manoil if (!rx_swbd->page)
2022d4fd0404SClaudiu Manoil continue;
2023d4fd0404SClaudiu Manoil
20247ed2bc80SVladimir Oltean dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
20257ed2bc80SVladimir Oltean rx_swbd->dir);
2026d4fd0404SClaudiu Manoil __free_page(rx_swbd->page);
2027d4fd0404SClaudiu Manoil rx_swbd->page = NULL;
2028d4fd0404SClaudiu Manoil }
2029d4fd0404SClaudiu Manoil }
2030d4fd0404SClaudiu Manoil
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)2031d4fd0404SClaudiu Manoil static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2032d4fd0404SClaudiu Manoil {
2033d4fd0404SClaudiu Manoil int i;
2034d4fd0404SClaudiu Manoil
2035d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_rx_rings; i++)
2036d4fd0404SClaudiu Manoil enetc_free_rx_ring(priv->rx_ring[i]);
2037d4fd0404SClaudiu Manoil
2038d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++)
2039d4fd0404SClaudiu Manoil enetc_free_tx_ring(priv->tx_ring[i]);
2040d4fd0404SClaudiu Manoil }
2041d4fd0404SClaudiu Manoil
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)2042d382563fSClaudiu Manoil static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2043d382563fSClaudiu Manoil {
2044d382563fSClaudiu Manoil int *rss_table;
2045d382563fSClaudiu Manoil int i;
2046d382563fSClaudiu Manoil
2047d382563fSClaudiu Manoil rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2048d382563fSClaudiu Manoil if (!rss_table)
2049d382563fSClaudiu Manoil return -ENOMEM;
2050d382563fSClaudiu Manoil
2051d382563fSClaudiu Manoil /* Set up RSS table defaults */
2052d382563fSClaudiu Manoil for (i = 0; i < si->num_rss; i++)
2053d382563fSClaudiu Manoil rss_table[i] = i % num_groups;
2054d382563fSClaudiu Manoil
2055d382563fSClaudiu Manoil enetc_set_rss_table(si, rss_table, si->num_rss);
2056d382563fSClaudiu Manoil
2057d382563fSClaudiu Manoil kfree(rss_table);
2058d382563fSClaudiu Manoil
2059d382563fSClaudiu Manoil return 0;
2060d382563fSClaudiu Manoil }
2061d382563fSClaudiu Manoil
enetc_configure_si(struct enetc_ndev_priv * priv)2062c646d10dSVladimir Oltean int enetc_configure_si(struct enetc_ndev_priv *priv)
2063d4fd0404SClaudiu Manoil {
2064d4fd0404SClaudiu Manoil struct enetc_si *si = priv->si;
2065d4fd0404SClaudiu Manoil struct enetc_hw *hw = &si->hw;
2066d382563fSClaudiu Manoil int err;
2067d4fd0404SClaudiu Manoil
2068d4fd0404SClaudiu Manoil /* set SI cache attributes */
2069d4fd0404SClaudiu Manoil enetc_wr(hw, ENETC_SICAR0,
2070d4fd0404SClaudiu Manoil ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2071d4fd0404SClaudiu Manoil enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2072d4fd0404SClaudiu Manoil /* enable SI */
2073d4fd0404SClaudiu Manoil enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2074d4fd0404SClaudiu Manoil
2075d382563fSClaudiu Manoil if (si->num_rss) {
2076d382563fSClaudiu Manoil err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2077d382563fSClaudiu Manoil if (err)
2078d382563fSClaudiu Manoil return err;
2079d382563fSClaudiu Manoil }
2080d382563fSClaudiu Manoil
2081d4fd0404SClaudiu Manoil return 0;
2082d4fd0404SClaudiu Manoil }
2083e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_configure_si);
2084d4fd0404SClaudiu Manoil
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)2085d4fd0404SClaudiu Manoil void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2086d4fd0404SClaudiu Manoil {
2087d4fd0404SClaudiu Manoil struct enetc_si *si = priv->si;
2088d4fd0404SClaudiu Manoil int cpus = num_online_cpus();
2089d4fd0404SClaudiu Manoil
209002293dd4SClaudiu Manoil priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
209102293dd4SClaudiu Manoil priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2092d4fd0404SClaudiu Manoil
2093d4fd0404SClaudiu Manoil /* Enable all available TX rings in order to configure as many
2094d4fd0404SClaudiu Manoil * priorities as possible, when needed.
2095d4fd0404SClaudiu Manoil * TODO: Make # of TX rings run-time configurable
2096d4fd0404SClaudiu Manoil */
2097d4fd0404SClaudiu Manoil priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2098d4fd0404SClaudiu Manoil priv->num_tx_rings = si->num_tx_rings;
2099d4fd0404SClaudiu Manoil priv->bdr_int_num = cpus;
2100ae0e6a5dSClaudiu Manoil priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2101ae0e6a5dSClaudiu Manoil priv->tx_ictt = ENETC_TXIC_TIMETHR;
2102d4fd0404SClaudiu Manoil }
2103e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2104d4fd0404SClaudiu Manoil
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)2105d4fd0404SClaudiu Manoil int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2106d4fd0404SClaudiu Manoil {
2107d4fd0404SClaudiu Manoil struct enetc_si *si = priv->si;
2108d4fd0404SClaudiu Manoil
2109d382563fSClaudiu Manoil priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2110d382563fSClaudiu Manoil GFP_KERNEL);
21114b47c0b8SVladimir Oltean if (!priv->cls_rules)
21124b47c0b8SVladimir Oltean return -ENOMEM;
2113d382563fSClaudiu Manoil
2114d4fd0404SClaudiu Manoil return 0;
2115d4fd0404SClaudiu Manoil }
2116e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2117d4fd0404SClaudiu Manoil
enetc_free_si_resources(struct enetc_ndev_priv * priv)2118d4fd0404SClaudiu Manoil void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2119d4fd0404SClaudiu Manoil {
2120d382563fSClaudiu Manoil kfree(priv->cls_rules);
2121d4fd0404SClaudiu Manoil }
2122e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2123d4fd0404SClaudiu Manoil
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2124d4fd0404SClaudiu Manoil static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2125d4fd0404SClaudiu Manoil {
2126d4fd0404SClaudiu Manoil int idx = tx_ring->index;
2127d4fd0404SClaudiu Manoil u32 tbmr;
2128d4fd0404SClaudiu Manoil
2129d4fd0404SClaudiu Manoil enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2130d4fd0404SClaudiu Manoil lower_32_bits(tx_ring->bd_dma_base));
2131d4fd0404SClaudiu Manoil
2132d4fd0404SClaudiu Manoil enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2133d4fd0404SClaudiu Manoil upper_32_bits(tx_ring->bd_dma_base));
2134d4fd0404SClaudiu Manoil
2135d4fd0404SClaudiu Manoil WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2136d4fd0404SClaudiu Manoil enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2137d4fd0404SClaudiu Manoil ENETC_RTBLENR_LEN(tx_ring->bd_count));
2138d4fd0404SClaudiu Manoil
2139d4fd0404SClaudiu Manoil /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2140d4fd0404SClaudiu Manoil tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2141d4fd0404SClaudiu Manoil tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2142d4fd0404SClaudiu Manoil
2143d4fd0404SClaudiu Manoil /* enable Tx ints by setting pkt thr to 1 */
214412460a0aSClaudiu Manoil enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2145d4fd0404SClaudiu Manoil
2146ff58fda0SVladimir Oltean tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2147d4fd0404SClaudiu Manoil if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2148d4fd0404SClaudiu Manoil tbmr |= ENETC_TBMR_VIH;
2149d4fd0404SClaudiu Manoil
2150d4fd0404SClaudiu Manoil /* enable ring */
2151d4fd0404SClaudiu Manoil enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2152d4fd0404SClaudiu Manoil
2153d4fd0404SClaudiu Manoil tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2154d4fd0404SClaudiu Manoil tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2155d4fd0404SClaudiu Manoil tx_ring->idr = hw->reg + ENETC_SITXIDR;
2156d4fd0404SClaudiu Manoil }
2157d4fd0404SClaudiu Manoil
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring,bool extended)2158d075db51SVladimir Oltean static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2159d075db51SVladimir Oltean bool extended)
2160d4fd0404SClaudiu Manoil {
2161d4fd0404SClaudiu Manoil int idx = rx_ring->index;
2162ff58fda0SVladimir Oltean u32 rbmr = 0;
2163d4fd0404SClaudiu Manoil
2164d4fd0404SClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2165d4fd0404SClaudiu Manoil lower_32_bits(rx_ring->bd_dma_base));
2166d4fd0404SClaudiu Manoil
2167d4fd0404SClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2168d4fd0404SClaudiu Manoil upper_32_bits(rx_ring->bd_dma_base));
2169d4fd0404SClaudiu Manoil
2170d4fd0404SClaudiu Manoil WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2171d4fd0404SClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2172d4fd0404SClaudiu Manoil ENETC_RTBLENR_LEN(rx_ring->bd_count));
2173d4fd0404SClaudiu Manoil
2174d1b15102SVladimir Oltean if (rx_ring->xdp.prog)
2175d1b15102SVladimir Oltean enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2176d1b15102SVladimir Oltean else
2177d4fd0404SClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2178d4fd0404SClaudiu Manoil
217984ce1ca3SVladimir Oltean /* Also prepare the consumer index in case page allocation never
218084ce1ca3SVladimir Oltean * succeeds. In that case, hardware will never advance producer index
218184ce1ca3SVladimir Oltean * to match consumer index, and will drop all frames.
218284ce1ca3SVladimir Oltean */
2183d4fd0404SClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
218484ce1ca3SVladimir Oltean enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2185d4fd0404SClaudiu Manoil
2186d4fd0404SClaudiu Manoil /* enable Rx ints by setting pkt thr to 1 */
218712460a0aSClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2188d4fd0404SClaudiu Manoil
2189d075db51SVladimir Oltean rx_ring->ext_en = extended;
2190434cebabSClaudiu Manoil if (rx_ring->ext_en)
2191d3982312SY.b. Lu rbmr |= ENETC_RBMR_BDS;
2192434cebabSClaudiu Manoil
2193d4fd0404SClaudiu Manoil if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2194d4fd0404SClaudiu Manoil rbmr |= ENETC_RBMR_VTE;
2195d4fd0404SClaudiu Manoil
2196d4fd0404SClaudiu Manoil rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2197d4fd0404SClaudiu Manoil rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2198d4fd0404SClaudiu Manoil
2199fbf1cff9SVladimir Oltean rx_ring->next_to_clean = 0;
2200fbf1cff9SVladimir Oltean rx_ring->next_to_use = 0;
2201fbf1cff9SVladimir Oltean rx_ring->next_to_alloc = 0;
2202fbf1cff9SVladimir Oltean
22037a5222cbSVladimir Oltean enetc_lock_mdio();
2204d4fd0404SClaudiu Manoil enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
22057a5222cbSVladimir Oltean enetc_unlock_mdio();
2206d4fd0404SClaudiu Manoil
2207d4fd0404SClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2208d4fd0404SClaudiu Manoil }
2209d4fd0404SClaudiu Manoil
enetc_setup_bdrs(struct enetc_ndev_priv * priv,bool extended)2210d075db51SVladimir Oltean static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2211d4fd0404SClaudiu Manoil {
2212715bf261SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
2213d4fd0404SClaudiu Manoil int i;
2214d4fd0404SClaudiu Manoil
2215d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++)
2216715bf261SVladimir Oltean enetc_setup_txbdr(hw, priv->tx_ring[i]);
2217d4fd0404SClaudiu Manoil
2218d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_rx_rings; i++)
2219d075db51SVladimir Oltean enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2220d4fd0404SClaudiu Manoil }
2221d4fd0404SClaudiu Manoil
enetc_enable_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2222ff58fda0SVladimir Oltean static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2223ff58fda0SVladimir Oltean {
2224ff58fda0SVladimir Oltean int idx = tx_ring->index;
2225ff58fda0SVladimir Oltean u32 tbmr;
2226ff58fda0SVladimir Oltean
2227ff58fda0SVladimir Oltean tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2228ff58fda0SVladimir Oltean tbmr |= ENETC_TBMR_EN;
2229ff58fda0SVladimir Oltean enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2230ff58fda0SVladimir Oltean }
2231ff58fda0SVladimir Oltean
enetc_enable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2232ff58fda0SVladimir Oltean static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2233ff58fda0SVladimir Oltean {
2234ff58fda0SVladimir Oltean int idx = rx_ring->index;
2235ff58fda0SVladimir Oltean u32 rbmr;
2236ff58fda0SVladimir Oltean
2237ff58fda0SVladimir Oltean rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2238ff58fda0SVladimir Oltean rbmr |= ENETC_RBMR_EN;
2239ff58fda0SVladimir Oltean enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2240ff58fda0SVladimir Oltean }
2241ff58fda0SVladimir Oltean
enetc_enable_rx_bdrs(struct enetc_ndev_priv * priv)2242a419f478SWei Fang static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv)
2243a419f478SWei Fang {
2244a419f478SWei Fang struct enetc_hw *hw = &priv->si->hw;
2245a419f478SWei Fang int i;
2246a419f478SWei Fang
2247a419f478SWei Fang for (i = 0; i < priv->num_rx_rings; i++)
2248a419f478SWei Fang enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2249a419f478SWei Fang }
2250a419f478SWei Fang
enetc_enable_tx_bdrs(struct enetc_ndev_priv * priv)2251a419f478SWei Fang static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv)
2252ff58fda0SVladimir Oltean {
2253ff58fda0SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
2254ff58fda0SVladimir Oltean int i;
2255ff58fda0SVladimir Oltean
2256ff58fda0SVladimir Oltean for (i = 0; i < priv->num_tx_rings; i++)
2257ff58fda0SVladimir Oltean enetc_enable_txbdr(hw, priv->tx_ring[i]);
2258ff58fda0SVladimir Oltean }
2259ff58fda0SVladimir Oltean
enetc_disable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2260ff58fda0SVladimir Oltean static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2261d4fd0404SClaudiu Manoil {
2262d4fd0404SClaudiu Manoil int idx = rx_ring->index;
2263d4fd0404SClaudiu Manoil
2264d4fd0404SClaudiu Manoil /* disable EN bit on ring */
2265d4fd0404SClaudiu Manoil enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2266d4fd0404SClaudiu Manoil }
2267d4fd0404SClaudiu Manoil
enetc_disable_txbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2268ff58fda0SVladimir Oltean static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2269d4fd0404SClaudiu Manoil {
2270ff58fda0SVladimir Oltean int idx = rx_ring->index;
2271d4fd0404SClaudiu Manoil
2272d4fd0404SClaudiu Manoil /* disable EN bit on ring */
2273d4fd0404SClaudiu Manoil enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2274ff58fda0SVladimir Oltean }
2275ff58fda0SVladimir Oltean
enetc_disable_rx_bdrs(struct enetc_ndev_priv * priv)2276a419f478SWei Fang static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv)
2277a419f478SWei Fang {
2278a419f478SWei Fang struct enetc_hw *hw = &priv->si->hw;
2279a419f478SWei Fang int i;
2280a419f478SWei Fang
2281a419f478SWei Fang for (i = 0; i < priv->num_rx_rings; i++)
2282a419f478SWei Fang enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2283a419f478SWei Fang }
2284a419f478SWei Fang
enetc_disable_tx_bdrs(struct enetc_ndev_priv * priv)2285a419f478SWei Fang static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv)
2286ff58fda0SVladimir Oltean {
2287ff58fda0SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
2288ff58fda0SVladimir Oltean int i;
2289ff58fda0SVladimir Oltean
2290ff58fda0SVladimir Oltean for (i = 0; i < priv->num_tx_rings; i++)
2291ff58fda0SVladimir Oltean enetc_disable_txbdr(hw, priv->tx_ring[i]);
2292ff58fda0SVladimir Oltean }
2293ff58fda0SVladimir Oltean
enetc_wait_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2294ff58fda0SVladimir Oltean static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2295ff58fda0SVladimir Oltean {
2296ff58fda0SVladimir Oltean int delay = 8, timeout = 100;
2297ff58fda0SVladimir Oltean int idx = tx_ring->index;
2298d4fd0404SClaudiu Manoil
2299d4fd0404SClaudiu Manoil /* wait for busy to clear */
2300d4fd0404SClaudiu Manoil while (delay < timeout &&
2301d4fd0404SClaudiu Manoil enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2302d4fd0404SClaudiu Manoil msleep(delay);
2303d4fd0404SClaudiu Manoil delay *= 2;
2304d4fd0404SClaudiu Manoil }
2305d4fd0404SClaudiu Manoil
2306d4fd0404SClaudiu Manoil if (delay >= timeout)
2307d4fd0404SClaudiu Manoil netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2308d4fd0404SClaudiu Manoil idx);
2309d4fd0404SClaudiu Manoil }
2310d4fd0404SClaudiu Manoil
enetc_wait_bdrs(struct enetc_ndev_priv * priv)2311ff58fda0SVladimir Oltean static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2312d4fd0404SClaudiu Manoil {
2313715bf261SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
2314d4fd0404SClaudiu Manoil int i;
2315d4fd0404SClaudiu Manoil
2316d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++)
2317ff58fda0SVladimir Oltean enetc_wait_txbdr(hw, priv->tx_ring[i]);
2318d4fd0404SClaudiu Manoil }
2319d4fd0404SClaudiu Manoil
enetc_setup_irqs(struct enetc_ndev_priv * priv)2320d4fd0404SClaudiu Manoil static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2321d4fd0404SClaudiu Manoil {
2322d4fd0404SClaudiu Manoil struct pci_dev *pdev = priv->si->pdev;
2323715bf261SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
2324d4fd0404SClaudiu Manoil int i, j, err;
2325d4fd0404SClaudiu Manoil
2326d4fd0404SClaudiu Manoil for (i = 0; i < priv->bdr_int_num; i++) {
2327d4fd0404SClaudiu Manoil int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2328d4fd0404SClaudiu Manoil struct enetc_int_vector *v = priv->int_vector[i];
2329d4fd0404SClaudiu Manoil int entry = ENETC_BDR_INT_BASE_IDX + i;
2330d4fd0404SClaudiu Manoil
2331d4fd0404SClaudiu Manoil snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2332d4fd0404SClaudiu Manoil priv->ndev->name, i);
23331e8fc4ffSJinjie Ruan err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);
2334d4fd0404SClaudiu Manoil if (err) {
2335d4fd0404SClaudiu Manoil dev_err(priv->dev, "request_irq() failed!\n");
2336d4fd0404SClaudiu Manoil goto irq_err;
2337d4fd0404SClaudiu Manoil }
2338d4fd0404SClaudiu Manoil
2339d4fd0404SClaudiu Manoil v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2340d4fd0404SClaudiu Manoil v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
234191571081SClaudiu Manoil v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2342d4fd0404SClaudiu Manoil
2343d4fd0404SClaudiu Manoil enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2344d4fd0404SClaudiu Manoil
2345d4fd0404SClaudiu Manoil for (j = 0; j < v->count_tx_rings; j++) {
2346d4fd0404SClaudiu Manoil int idx = v->tx_ring[j].index;
2347d4fd0404SClaudiu Manoil
2348d4fd0404SClaudiu Manoil enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2349d4fd0404SClaudiu Manoil }
23507237a494SClaudiu Manoil irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2351d4fd0404SClaudiu Manoil }
2352d4fd0404SClaudiu Manoil
2353d4fd0404SClaudiu Manoil return 0;
2354d4fd0404SClaudiu Manoil
2355d4fd0404SClaudiu Manoil irq_err:
2356d4fd0404SClaudiu Manoil while (i--) {
2357d4fd0404SClaudiu Manoil int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2358d4fd0404SClaudiu Manoil
2359d4fd0404SClaudiu Manoil irq_set_affinity_hint(irq, NULL);
2360d4fd0404SClaudiu Manoil free_irq(irq, priv->int_vector[i]);
2361d4fd0404SClaudiu Manoil }
2362d4fd0404SClaudiu Manoil
2363d4fd0404SClaudiu Manoil return err;
2364d4fd0404SClaudiu Manoil }
2365d4fd0404SClaudiu Manoil
enetc_free_irqs(struct enetc_ndev_priv * priv)2366d4fd0404SClaudiu Manoil static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2367d4fd0404SClaudiu Manoil {
2368d4fd0404SClaudiu Manoil struct pci_dev *pdev = priv->si->pdev;
2369d4fd0404SClaudiu Manoil int i;
2370d4fd0404SClaudiu Manoil
2371d4fd0404SClaudiu Manoil for (i = 0; i < priv->bdr_int_num; i++) {
2372d4fd0404SClaudiu Manoil int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2373d4fd0404SClaudiu Manoil
2374d4fd0404SClaudiu Manoil irq_set_affinity_hint(irq, NULL);
2375d4fd0404SClaudiu Manoil free_irq(irq, priv->int_vector[i]);
2376d4fd0404SClaudiu Manoil }
2377d4fd0404SClaudiu Manoil }
2378d4fd0404SClaudiu Manoil
enetc_setup_interrupts(struct enetc_ndev_priv * priv)2379bbb96dc7SClaudiu Manoil static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2380d4fd0404SClaudiu Manoil {
238191571081SClaudiu Manoil struct enetc_hw *hw = &priv->si->hw;
238291571081SClaudiu Manoil u32 icpt, ictt;
2383d4fd0404SClaudiu Manoil int i;
2384d4fd0404SClaudiu Manoil
2385d4fd0404SClaudiu Manoil /* enable Tx & Rx event indication */
2386ae0e6a5dSClaudiu Manoil if (priv->ic_mode &
2387ae0e6a5dSClaudiu Manoil (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
238891571081SClaudiu Manoil icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
238991571081SClaudiu Manoil /* init to non-0 minimum, will be adjusted later */
239091571081SClaudiu Manoil ictt = 0x1;
239191571081SClaudiu Manoil } else {
239291571081SClaudiu Manoil icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
239391571081SClaudiu Manoil ictt = 0;
2394d4fd0404SClaudiu Manoil }
2395d4fd0404SClaudiu Manoil
239691571081SClaudiu Manoil for (i = 0; i < priv->num_rx_rings; i++) {
239791571081SClaudiu Manoil enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
239891571081SClaudiu Manoil enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
239991571081SClaudiu Manoil enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
240091571081SClaudiu Manoil }
240191571081SClaudiu Manoil
240291571081SClaudiu Manoil if (priv->ic_mode & ENETC_IC_TX_MANUAL)
240391571081SClaudiu Manoil icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
240491571081SClaudiu Manoil else
240591571081SClaudiu Manoil icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
240691571081SClaudiu Manoil
2407d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++) {
240891571081SClaudiu Manoil enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
240991571081SClaudiu Manoil enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
241091571081SClaudiu Manoil enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2411d4fd0404SClaudiu Manoil }
2412d4fd0404SClaudiu Manoil }
2413d4fd0404SClaudiu Manoil
enetc_clear_interrupts(struct enetc_ndev_priv * priv)2414bbb96dc7SClaudiu Manoil static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2415d4fd0404SClaudiu Manoil {
2416715bf261SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
2417d4fd0404SClaudiu Manoil int i;
2418d4fd0404SClaudiu Manoil
2419d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++)
2420715bf261SVladimir Oltean enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2421d4fd0404SClaudiu Manoil
2422d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_rx_rings; i++)
2423715bf261SVladimir Oltean enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2424d4fd0404SClaudiu Manoil }
2425d4fd0404SClaudiu Manoil
enetc_phylink_connect(struct net_device * ndev)242671b77a7aSClaudiu Manoil static int enetc_phylink_connect(struct net_device *ndev)
2427d4fd0404SClaudiu Manoil {
24282e47cb41SPo Liu struct enetc_ndev_priv *priv = netdev_priv(ndev);
2429a6a10d45SYangbo Lu struct ethtool_eee edata;
243071b77a7aSClaudiu Manoil int err;
2431d4fd0404SClaudiu Manoil
2432598ca0d0SVladimir Oltean if (!priv->phylink) {
2433598ca0d0SVladimir Oltean /* phy-less mode */
2434598ca0d0SVladimir Oltean netif_carrier_on(ndev);
2435598ca0d0SVladimir Oltean return 0;
2436598ca0d0SVladimir Oltean }
2437d4fd0404SClaudiu Manoil
243871b77a7aSClaudiu Manoil err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
243971b77a7aSClaudiu Manoil if (err) {
2440d4fd0404SClaudiu Manoil dev_err(&ndev->dev, "could not attach to PHY\n");
244171b77a7aSClaudiu Manoil return err;
2442d4fd0404SClaudiu Manoil }
2443d4fd0404SClaudiu Manoil
2444a6a10d45SYangbo Lu /* disable EEE autoneg, until ENETC driver supports it */
2445a6a10d45SYangbo Lu memset(&edata, 0, sizeof(struct ethtool_eee));
244671b77a7aSClaudiu Manoil phylink_ethtool_set_eee(priv->phylink, &edata);
2447a6a10d45SYangbo Lu
2448598ca0d0SVladimir Oltean phylink_start(priv->phylink);
2449598ca0d0SVladimir Oltean
2450d4fd0404SClaudiu Manoil return 0;
2451d4fd0404SClaudiu Manoil }
2452d4fd0404SClaudiu Manoil
enetc_tx_onestep_tstamp(struct work_struct * work)24537294380cSYangbo Lu static void enetc_tx_onestep_tstamp(struct work_struct *work)
24547294380cSYangbo Lu {
24557294380cSYangbo Lu struct enetc_ndev_priv *priv;
24567294380cSYangbo Lu struct sk_buff *skb;
24577294380cSYangbo Lu
24587294380cSYangbo Lu priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
24597294380cSYangbo Lu
24603c463721SVladimir Oltean netif_tx_lock_bh(priv->ndev);
24617294380cSYangbo Lu
24627294380cSYangbo Lu clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
24637294380cSYangbo Lu skb = skb_dequeue(&priv->tx_skbs);
24647294380cSYangbo Lu if (skb)
24657294380cSYangbo Lu enetc_start_xmit(skb, priv->ndev);
24667294380cSYangbo Lu
24673c463721SVladimir Oltean netif_tx_unlock_bh(priv->ndev);
24687294380cSYangbo Lu }
24697294380cSYangbo Lu
enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv * priv)24707294380cSYangbo Lu static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
24717294380cSYangbo Lu {
24727294380cSYangbo Lu INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
24737294380cSYangbo Lu skb_queue_head_init(&priv->tx_skbs);
24747294380cSYangbo Lu }
24757294380cSYangbo Lu
enetc_start(struct net_device * ndev)247691571081SClaudiu Manoil void enetc_start(struct net_device *ndev)
2477bbb96dc7SClaudiu Manoil {
2478bbb96dc7SClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2479bbb96dc7SClaudiu Manoil int i;
2480bbb96dc7SClaudiu Manoil
2481bbb96dc7SClaudiu Manoil enetc_setup_interrupts(priv);
2482bbb96dc7SClaudiu Manoil
2483bbb96dc7SClaudiu Manoil for (i = 0; i < priv->bdr_int_num; i++) {
2484bbb96dc7SClaudiu Manoil int irq = pci_irq_vector(priv->si->pdev,
2485bbb96dc7SClaudiu Manoil ENETC_BDR_INT_BASE_IDX + i);
2486bbb96dc7SClaudiu Manoil
2487bbb96dc7SClaudiu Manoil napi_enable(&priv->int_vector[i]->napi);
2488bbb96dc7SClaudiu Manoil enable_irq(irq);
2489bbb96dc7SClaudiu Manoil }
2490bbb96dc7SClaudiu Manoil
2491760a7c96SWei Fang enetc_enable_tx_bdrs(priv);
2492760a7c96SWei Fang
2493a419f478SWei Fang enetc_enable_rx_bdrs(priv);
2494ff58fda0SVladimir Oltean
2495bbb96dc7SClaudiu Manoil netif_tx_start_all_queues(ndev);
249637184349SWei Fang
249737184349SWei Fang clear_bit(ENETC_TX_DOWN, &priv->flags);
2498bbb96dc7SClaudiu Manoil }
2499e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_start);
2500bbb96dc7SClaudiu Manoil
enetc_open(struct net_device * ndev)2501d4fd0404SClaudiu Manoil int enetc_open(struct net_device *ndev)
2502d4fd0404SClaudiu Manoil {
2503d4fd0404SClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2504f3ce29e1SVladimir Oltean struct enetc_bdr_resource *tx_res, *rx_res;
2505d075db51SVladimir Oltean bool extended;
2506bbb96dc7SClaudiu Manoil int err;
2507d4fd0404SClaudiu Manoil
2508d075db51SVladimir Oltean extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2509d075db51SVladimir Oltean
2510d4fd0404SClaudiu Manoil err = enetc_setup_irqs(priv);
2511d4fd0404SClaudiu Manoil if (err)
2512d4fd0404SClaudiu Manoil return err;
2513d4fd0404SClaudiu Manoil
251471b77a7aSClaudiu Manoil err = enetc_phylink_connect(ndev);
2515d4fd0404SClaudiu Manoil if (err)
2516d4fd0404SClaudiu Manoil goto err_phy_connect;
2517d4fd0404SClaudiu Manoil
2518f3ce29e1SVladimir Oltean tx_res = enetc_alloc_tx_resources(priv);
2519f3ce29e1SVladimir Oltean if (IS_ERR(tx_res)) {
2520f3ce29e1SVladimir Oltean err = PTR_ERR(tx_res);
2521d4fd0404SClaudiu Manoil goto err_alloc_tx;
2522f3ce29e1SVladimir Oltean }
2523d4fd0404SClaudiu Manoil
2524f3ce29e1SVladimir Oltean rx_res = enetc_alloc_rx_resources(priv, extended);
2525f3ce29e1SVladimir Oltean if (IS_ERR(rx_res)) {
2526f3ce29e1SVladimir Oltean err = PTR_ERR(rx_res);
2527d4fd0404SClaudiu Manoil goto err_alloc_rx;
2528f3ce29e1SVladimir Oltean }
2529d4fd0404SClaudiu Manoil
25307294380cSYangbo Lu enetc_tx_onestep_tstamp_init(priv);
2531f3ce29e1SVladimir Oltean enetc_assign_tx_resources(priv, tx_res);
2532f3ce29e1SVladimir Oltean enetc_assign_rx_resources(priv, rx_res);
2533d075db51SVladimir Oltean enetc_setup_bdrs(priv, extended);
2534bbb96dc7SClaudiu Manoil enetc_start(ndev);
2535d4fd0404SClaudiu Manoil
2536d4fd0404SClaudiu Manoil return 0;
2537d4fd0404SClaudiu Manoil
2538d4fd0404SClaudiu Manoil err_alloc_rx:
2539f3ce29e1SVladimir Oltean enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2540d4fd0404SClaudiu Manoil err_alloc_tx:
254171b77a7aSClaudiu Manoil if (priv->phylink)
254271b77a7aSClaudiu Manoil phylink_disconnect_phy(priv->phylink);
2543d4fd0404SClaudiu Manoil err_phy_connect:
2544d4fd0404SClaudiu Manoil enetc_free_irqs(priv);
2545d4fd0404SClaudiu Manoil
2546d4fd0404SClaudiu Manoil return err;
2547d4fd0404SClaudiu Manoil }
2548e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_open);
2549d4fd0404SClaudiu Manoil
enetc_stop(struct net_device * ndev)255091571081SClaudiu Manoil void enetc_stop(struct net_device *ndev)
2551d4fd0404SClaudiu Manoil {
2552d4fd0404SClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2553d4fd0404SClaudiu Manoil int i;
2554d4fd0404SClaudiu Manoil
255537184349SWei Fang set_bit(ENETC_TX_DOWN, &priv->flags);
255637184349SWei Fang
2557d4fd0404SClaudiu Manoil netif_tx_stop_all_queues(ndev);
2558d4fd0404SClaudiu Manoil
2559a419f478SWei Fang enetc_disable_rx_bdrs(priv);
2560ff58fda0SVladimir Oltean
2561760a7c96SWei Fang enetc_wait_bdrs(priv);
2562760a7c96SWei Fang
2563760a7c96SWei Fang enetc_disable_tx_bdrs(priv);
2564760a7c96SWei Fang
2565d4fd0404SClaudiu Manoil for (i = 0; i < priv->bdr_int_num; i++) {
2566bbb96dc7SClaudiu Manoil int irq = pci_irq_vector(priv->si->pdev,
2567bbb96dc7SClaudiu Manoil ENETC_BDR_INT_BASE_IDX + i);
2568bbb96dc7SClaudiu Manoil
2569bbb96dc7SClaudiu Manoil disable_irq(irq);
2570d4fd0404SClaudiu Manoil napi_synchronize(&priv->int_vector[i]->napi);
2571d4fd0404SClaudiu Manoil napi_disable(&priv->int_vector[i]->napi);
2572d4fd0404SClaudiu Manoil }
2573d4fd0404SClaudiu Manoil
2574bbb96dc7SClaudiu Manoil enetc_clear_interrupts(priv);
2575bbb96dc7SClaudiu Manoil }
2576e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_stop);
2577bbb96dc7SClaudiu Manoil
enetc_close(struct net_device * ndev)2578bbb96dc7SClaudiu Manoil int enetc_close(struct net_device *ndev)
2579bbb96dc7SClaudiu Manoil {
2580bbb96dc7SClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2581bbb96dc7SClaudiu Manoil
2582bbb96dc7SClaudiu Manoil enetc_stop(ndev);
2583d4fd0404SClaudiu Manoil
2584598ca0d0SVladimir Oltean if (priv->phylink) {
2585598ca0d0SVladimir Oltean phylink_stop(priv->phylink);
258671b77a7aSClaudiu Manoil phylink_disconnect_phy(priv->phylink);
2587598ca0d0SVladimir Oltean } else {
2588598ca0d0SVladimir Oltean netif_carrier_off(ndev);
2589598ca0d0SVladimir Oltean }
2590598ca0d0SVladimir Oltean
2591d4fd0404SClaudiu Manoil enetc_free_rxtx_rings(priv);
2592f3ce29e1SVladimir Oltean
2593f3ce29e1SVladimir Oltean /* Avoids dangling pointers and also frees old resources */
2594f3ce29e1SVladimir Oltean enetc_assign_rx_resources(priv, NULL);
2595f3ce29e1SVladimir Oltean enetc_assign_tx_resources(priv, NULL);
2596f3ce29e1SVladimir Oltean
2597d4fd0404SClaudiu Manoil enetc_free_irqs(priv);
2598d4fd0404SClaudiu Manoil
2599d4fd0404SClaudiu Manoil return 0;
2600d4fd0404SClaudiu Manoil }
2601e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_close);
2602d4fd0404SClaudiu Manoil
enetc_reconfigure(struct enetc_ndev_priv * priv,bool extended,int (* cb)(struct enetc_ndev_priv * priv,void * ctx),void * ctx)2603c33bfaf9SVladimir Oltean static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
2604c33bfaf9SVladimir Oltean int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
2605c33bfaf9SVladimir Oltean void *ctx)
26065093406cSVladimir Oltean {
26075093406cSVladimir Oltean struct enetc_bdr_resource *tx_res, *rx_res;
26085093406cSVladimir Oltean int err;
26095093406cSVladimir Oltean
26105093406cSVladimir Oltean ASSERT_RTNL();
26115093406cSVladimir Oltean
2612c33bfaf9SVladimir Oltean /* If the interface is down, run the callback right away,
2613c33bfaf9SVladimir Oltean * without reconfiguration.
2614c33bfaf9SVladimir Oltean */
2615c33bfaf9SVladimir Oltean if (!netif_running(priv->ndev)) {
261646a0ecf9SVladimir Oltean if (cb) {
261746a0ecf9SVladimir Oltean err = cb(priv, ctx);
261846a0ecf9SVladimir Oltean if (err)
261946a0ecf9SVladimir Oltean return err;
262046a0ecf9SVladimir Oltean }
2621c33bfaf9SVladimir Oltean
26225093406cSVladimir Oltean return 0;
2623c33bfaf9SVladimir Oltean }
26245093406cSVladimir Oltean
26255093406cSVladimir Oltean tx_res = enetc_alloc_tx_resources(priv);
26265093406cSVladimir Oltean if (IS_ERR(tx_res)) {
26275093406cSVladimir Oltean err = PTR_ERR(tx_res);
26285093406cSVladimir Oltean goto out;
26295093406cSVladimir Oltean }
26305093406cSVladimir Oltean
26315093406cSVladimir Oltean rx_res = enetc_alloc_rx_resources(priv, extended);
26325093406cSVladimir Oltean if (IS_ERR(rx_res)) {
26335093406cSVladimir Oltean err = PTR_ERR(rx_res);
26345093406cSVladimir Oltean goto out_free_tx_res;
26355093406cSVladimir Oltean }
26365093406cSVladimir Oltean
26375093406cSVladimir Oltean enetc_stop(priv->ndev);
26385093406cSVladimir Oltean enetc_free_rxtx_rings(priv);
26395093406cSVladimir Oltean
2640c33bfaf9SVladimir Oltean /* Interface is down, run optional callback now */
264146a0ecf9SVladimir Oltean if (cb) {
264246a0ecf9SVladimir Oltean err = cb(priv, ctx);
264346a0ecf9SVladimir Oltean if (err)
264446a0ecf9SVladimir Oltean goto out_restart;
264546a0ecf9SVladimir Oltean }
2646c33bfaf9SVladimir Oltean
26475093406cSVladimir Oltean enetc_assign_tx_resources(priv, tx_res);
26485093406cSVladimir Oltean enetc_assign_rx_resources(priv, rx_res);
26495093406cSVladimir Oltean enetc_setup_bdrs(priv, extended);
26505093406cSVladimir Oltean enetc_start(priv->ndev);
26515093406cSVladimir Oltean
26525093406cSVladimir Oltean return 0;
26535093406cSVladimir Oltean
265446a0ecf9SVladimir Oltean out_restart:
265546a0ecf9SVladimir Oltean enetc_setup_bdrs(priv, extended);
265646a0ecf9SVladimir Oltean enetc_start(priv->ndev);
265746a0ecf9SVladimir Oltean enetc_free_rx_resources(rx_res, priv->num_rx_rings);
26585093406cSVladimir Oltean out_free_tx_res:
26595093406cSVladimir Oltean enetc_free_tx_resources(tx_res, priv->num_tx_rings);
26605093406cSVladimir Oltean out:
26615093406cSVladimir Oltean return err;
26625093406cSVladimir Oltean }
26635093406cSVladimir Oltean
enetc_debug_tx_ring_prios(struct enetc_ndev_priv * priv)26641a353111SVladimir Oltean static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
26651a353111SVladimir Oltean {
26661a353111SVladimir Oltean int i;
26671a353111SVladimir Oltean
26681a353111SVladimir Oltean for (i = 0; i < priv->num_tx_rings; i++)
26691a353111SVladimir Oltean netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
26701a353111SVladimir Oltean priv->tx_ring[i]->prio);
26711a353111SVladimir Oltean }
26721a353111SVladimir Oltean
enetc_reset_tc_mqprio(struct net_device * ndev)26735353599aSVladimir Oltean void enetc_reset_tc_mqprio(struct net_device *ndev)
2674cbe9e835SCamelia Groza {
2675cbe9e835SCamelia Groza struct enetc_ndev_priv *priv = netdev_priv(ndev);
2676715bf261SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
2677cbe9e835SCamelia Groza struct enetc_bdr *tx_ring;
26787eab503bSVladimir Oltean int num_stack_tx_queues;
2679cbe9e835SCamelia Groza int i;
2680cbe9e835SCamelia Groza
26817eab503bSVladimir Oltean num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2682cbe9e835SCamelia Groza
2683cbe9e835SCamelia Groza netdev_reset_tc(ndev);
26847eab503bSVladimir Oltean netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2685800db2d1SVladimir Oltean priv->min_num_stack_tx_queues = num_possible_cpus();
2686cbe9e835SCamelia Groza
2687cbe9e835SCamelia Groza /* Reset all ring priorities to 0 */
2688cbe9e835SCamelia Groza for (i = 0; i < priv->num_tx_rings; i++) {
2689cbe9e835SCamelia Groza tx_ring = priv->tx_ring[i];
2690290b5fe0SVladimir Oltean tx_ring->prio = 0;
2691290b5fe0SVladimir Oltean enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2692cbe9e835SCamelia Groza }
2693cbe9e835SCamelia Groza
26941a353111SVladimir Oltean enetc_debug_tx_ring_prios(priv);
269501e23b2bSVladimir Oltean
269682714539SVladimir Oltean enetc_change_preemptible_tcs(priv, 0);
26971a353111SVladimir Oltean }
26985353599aSVladimir Oltean EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
26991a353111SVladimir Oltean
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)27001a353111SVladimir Oltean int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
27011a353111SVladimir Oltean {
270250764da3SVladimir Oltean struct tc_mqprio_qopt_offload *mqprio = type_data;
27031a353111SVladimir Oltean struct enetc_ndev_priv *priv = netdev_priv(ndev);
270450764da3SVladimir Oltean struct tc_mqprio_qopt *qopt = &mqprio->qopt;
27051a353111SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
27061a353111SVladimir Oltean int num_stack_tx_queues = 0;
27071a353111SVladimir Oltean struct enetc_bdr *tx_ring;
270850764da3SVladimir Oltean u8 num_tc = qopt->num_tc;
27091a353111SVladimir Oltean int offset, count;
27101a353111SVladimir Oltean int err, tc, q;
27111a353111SVladimir Oltean
27121a353111SVladimir Oltean if (!num_tc) {
27131a353111SVladimir Oltean enetc_reset_tc_mqprio(ndev);
2714cbe9e835SCamelia Groza return 0;
2715cbe9e835SCamelia Groza }
2716cbe9e835SCamelia Groza
27171a353111SVladimir Oltean err = netdev_set_num_tc(ndev, num_tc);
27181a353111SVladimir Oltean if (err)
27191a353111SVladimir Oltean return err;
27201a353111SVladimir Oltean
27211a353111SVladimir Oltean for (tc = 0; tc < num_tc; tc++) {
272250764da3SVladimir Oltean offset = qopt->offset[tc];
272350764da3SVladimir Oltean count = qopt->count[tc];
27241a353111SVladimir Oltean num_stack_tx_queues += count;
27251a353111SVladimir Oltean
27261a353111SVladimir Oltean err = netdev_set_tc_queue(ndev, tc, count, offset);
27271a353111SVladimir Oltean if (err)
27281a353111SVladimir Oltean goto err_reset_tc;
27291a353111SVladimir Oltean
27301a353111SVladimir Oltean for (q = offset; q < offset + count; q++) {
27311a353111SVladimir Oltean tx_ring = priv->tx_ring[q];
27321a353111SVladimir Oltean /* The prio_tc_map is skb_tx_hash()'s way of selecting
27331a353111SVladimir Oltean * between TX queues based on skb->priority. As such,
27341a353111SVladimir Oltean * there's nothing to offload based on it.
27351a353111SVladimir Oltean * Make the mqprio "traffic class" be the priority of
27361a353111SVladimir Oltean * this ring group, and leave the Tx IPV to traffic
27371a353111SVladimir Oltean * class mapping as its default mapping value of 1:1.
2738cbe9e835SCamelia Groza */
27391a353111SVladimir Oltean tx_ring->prio = tc;
2740290b5fe0SVladimir Oltean enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2741cbe9e835SCamelia Groza }
27421a353111SVladimir Oltean }
2743cbe9e835SCamelia Groza
27441a353111SVladimir Oltean err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
27451a353111SVladimir Oltean if (err)
27461a353111SVladimir Oltean goto err_reset_tc;
2747cbe9e835SCamelia Groza
27481a353111SVladimir Oltean priv->min_num_stack_tx_queues = num_stack_tx_queues;
2749cbe9e835SCamelia Groza
27501a353111SVladimir Oltean enetc_debug_tx_ring_prios(priv);
2751cbe9e835SCamelia Groza
275282714539SVladimir Oltean enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
275301e23b2bSVladimir Oltean
2754cbe9e835SCamelia Groza return 0;
27551a353111SVladimir Oltean
27561a353111SVladimir Oltean err_reset_tc:
27571a353111SVladimir Oltean enetc_reset_tc_mqprio(ndev);
27581a353111SVladimir Oltean return err;
2759cbe9e835SCamelia Groza }
2760e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
2761cbe9e835SCamelia Groza
enetc_reconfigure_xdp_cb(struct enetc_ndev_priv * priv,void * ctx)2762c33bfaf9SVladimir Oltean static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
2763d1b15102SVladimir Oltean {
2764c33bfaf9SVladimir Oltean struct bpf_prog *old_prog, *prog = ctx;
27654ea1dd74SVladimir Oltean int num_stack_tx_queues;
27664ea1dd74SVladimir Oltean int err, i;
2767d1b15102SVladimir Oltean
2768d1b15102SVladimir Oltean old_prog = xchg(&priv->xdp_prog, prog);
27694ea1dd74SVladimir Oltean
27704ea1dd74SVladimir Oltean num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
27714ea1dd74SVladimir Oltean err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
27724ea1dd74SVladimir Oltean if (err) {
27734ea1dd74SVladimir Oltean xchg(&priv->xdp_prog, old_prog);
27744ea1dd74SVladimir Oltean return err;
27754ea1dd74SVladimir Oltean }
27764ea1dd74SVladimir Oltean
2777d1b15102SVladimir Oltean if (old_prog)
2778d1b15102SVladimir Oltean bpf_prog_put(old_prog);
2779d1b15102SVladimir Oltean
2780d1b15102SVladimir Oltean for (i = 0; i < priv->num_rx_rings; i++) {
2781d1b15102SVladimir Oltean struct enetc_bdr *rx_ring = priv->rx_ring[i];
2782d1b15102SVladimir Oltean
2783d1b15102SVladimir Oltean rx_ring->xdp.prog = prog;
2784d1b15102SVladimir Oltean
2785d1b15102SVladimir Oltean if (prog)
2786d1b15102SVladimir Oltean rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2787d1b15102SVladimir Oltean else
2788d1b15102SVladimir Oltean rx_ring->buffer_offset = ENETC_RXB_PAD;
2789d1b15102SVladimir Oltean }
2790d1b15102SVladimir Oltean
2791d1b15102SVladimir Oltean return 0;
2792d1b15102SVladimir Oltean }
2793d1b15102SVladimir Oltean
enetc_setup_xdp_prog(struct net_device * ndev,struct bpf_prog * prog,struct netlink_ext_ack * extack)2794c33bfaf9SVladimir Oltean static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
2795c33bfaf9SVladimir Oltean struct netlink_ext_ack *extack)
2796c33bfaf9SVladimir Oltean {
2797800db2d1SVladimir Oltean int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
2798c33bfaf9SVladimir Oltean struct enetc_ndev_priv *priv = netdev_priv(ndev);
2799c33bfaf9SVladimir Oltean bool extended;
2800c33bfaf9SVladimir Oltean
2801800db2d1SVladimir Oltean if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
2802800db2d1SVladimir Oltean priv->num_tx_rings) {
2803800db2d1SVladimir Oltean NL_SET_ERR_MSG_FMT_MOD(extack,
2804997a5310SArnd Bergmann "Reserving %d XDP TXQs leaves under %d for stack (total %d)",
2805800db2d1SVladimir Oltean num_xdp_tx_queues,
2806800db2d1SVladimir Oltean priv->min_num_stack_tx_queues,
2807800db2d1SVladimir Oltean priv->num_tx_rings);
2808800db2d1SVladimir Oltean return -EBUSY;
2809800db2d1SVladimir Oltean }
2810800db2d1SVladimir Oltean
2811c33bfaf9SVladimir Oltean extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2812c33bfaf9SVladimir Oltean
2813c33bfaf9SVladimir Oltean /* The buffer layout is changing, so we need to drain the old
2814c33bfaf9SVladimir Oltean * RX buffers and seed new ones.
2815c33bfaf9SVladimir Oltean */
2816c33bfaf9SVladimir Oltean return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
2817c33bfaf9SVladimir Oltean }
2818c33bfaf9SVladimir Oltean
enetc_setup_bpf(struct net_device * ndev,struct netdev_bpf * bpf)2819766338c7SVladimir Oltean int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2820d1b15102SVladimir Oltean {
2821766338c7SVladimir Oltean switch (bpf->command) {
2822d1b15102SVladimir Oltean case XDP_SETUP_PROG:
2823766338c7SVladimir Oltean return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
2824d1b15102SVladimir Oltean default:
2825d1b15102SVladimir Oltean return -EINVAL;
2826d1b15102SVladimir Oltean }
2827d1b15102SVladimir Oltean
2828d1b15102SVladimir Oltean return 0;
2829d1b15102SVladimir Oltean }
2830e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_setup_bpf);
2831d1b15102SVladimir Oltean
enetc_get_stats(struct net_device * ndev)2832d4fd0404SClaudiu Manoil struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2833d4fd0404SClaudiu Manoil {
2834d4fd0404SClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2835d4fd0404SClaudiu Manoil struct net_device_stats *stats = &ndev->stats;
2836d4fd0404SClaudiu Manoil unsigned long packets = 0, bytes = 0;
2837285e8dedSPo Liu unsigned long tx_dropped = 0;
2838d4fd0404SClaudiu Manoil int i;
2839d4fd0404SClaudiu Manoil
2840d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_rx_rings; i++) {
2841d4fd0404SClaudiu Manoil packets += priv->rx_ring[i]->stats.packets;
2842d4fd0404SClaudiu Manoil bytes += priv->rx_ring[i]->stats.bytes;
2843d4fd0404SClaudiu Manoil }
2844d4fd0404SClaudiu Manoil
2845d4fd0404SClaudiu Manoil stats->rx_packets = packets;
2846d4fd0404SClaudiu Manoil stats->rx_bytes = bytes;
2847d4fd0404SClaudiu Manoil bytes = 0;
2848d4fd0404SClaudiu Manoil packets = 0;
2849d4fd0404SClaudiu Manoil
2850d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++) {
2851d4fd0404SClaudiu Manoil packets += priv->tx_ring[i]->stats.packets;
2852d4fd0404SClaudiu Manoil bytes += priv->tx_ring[i]->stats.bytes;
2853285e8dedSPo Liu tx_dropped += priv->tx_ring[i]->stats.win_drop;
2854d4fd0404SClaudiu Manoil }
2855d4fd0404SClaudiu Manoil
2856d4fd0404SClaudiu Manoil stats->tx_packets = packets;
2857d4fd0404SClaudiu Manoil stats->tx_bytes = bytes;
2858285e8dedSPo Liu stats->tx_dropped = tx_dropped;
2859d4fd0404SClaudiu Manoil
2860d4fd0404SClaudiu Manoil return stats;
2861d4fd0404SClaudiu Manoil }
2862e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_get_stats);
2863d4fd0404SClaudiu Manoil
enetc_set_rss(struct net_device * ndev,int en)2864d382563fSClaudiu Manoil static int enetc_set_rss(struct net_device *ndev, int en)
2865d382563fSClaudiu Manoil {
2866d382563fSClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2867d382563fSClaudiu Manoil struct enetc_hw *hw = &priv->si->hw;
2868d382563fSClaudiu Manoil u32 reg;
2869d382563fSClaudiu Manoil
2870d382563fSClaudiu Manoil enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2871d382563fSClaudiu Manoil
2872d382563fSClaudiu Manoil reg = enetc_rd(hw, ENETC_SIMR);
2873d382563fSClaudiu Manoil reg &= ~ENETC_SIMR_RSSE;
2874d382563fSClaudiu Manoil reg |= (en) ? ENETC_SIMR_RSSE : 0;
2875d382563fSClaudiu Manoil enetc_wr(hw, ENETC_SIMR, reg);
2876d382563fSClaudiu Manoil
2877d382563fSClaudiu Manoil return 0;
2878d382563fSClaudiu Manoil }
2879d382563fSClaudiu Manoil
enetc_enable_rxvlan(struct net_device * ndev,bool en)28809deba33fSClaudiu Manoil static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
28819deba33fSClaudiu Manoil {
28829deba33fSClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2883715bf261SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
28849deba33fSClaudiu Manoil int i;
28859deba33fSClaudiu Manoil
28869deba33fSClaudiu Manoil for (i = 0; i < priv->num_rx_rings; i++)
2887715bf261SVladimir Oltean enetc_bdr_enable_rxvlan(hw, i, en);
28889deba33fSClaudiu Manoil }
28899deba33fSClaudiu Manoil
enetc_enable_txvlan(struct net_device * ndev,bool en)28909deba33fSClaudiu Manoil static void enetc_enable_txvlan(struct net_device *ndev, bool en)
28919deba33fSClaudiu Manoil {
28929deba33fSClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2893715bf261SVladimir Oltean struct enetc_hw *hw = &priv->si->hw;
28949deba33fSClaudiu Manoil int i;
28959deba33fSClaudiu Manoil
28969deba33fSClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++)
2897715bf261SVladimir Oltean enetc_bdr_enable_txvlan(hw, i, en);
28989deba33fSClaudiu Manoil }
28999deba33fSClaudiu Manoil
enetc_set_features(struct net_device * ndev,netdev_features_t features)2900fed38e64SVladimir Oltean void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2901d382563fSClaudiu Manoil {
2902d382563fSClaudiu Manoil netdev_features_t changed = ndev->features ^ features;
2903d382563fSClaudiu Manoil
2904d382563fSClaudiu Manoil if (changed & NETIF_F_RXHASH)
2905d382563fSClaudiu Manoil enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2906d382563fSClaudiu Manoil
29079deba33fSClaudiu Manoil if (changed & NETIF_F_HW_VLAN_CTAG_RX)
29089deba33fSClaudiu Manoil enetc_enable_rxvlan(ndev,
29099deba33fSClaudiu Manoil !!(features & NETIF_F_HW_VLAN_CTAG_RX));
29109deba33fSClaudiu Manoil
29119deba33fSClaudiu Manoil if (changed & NETIF_F_HW_VLAN_CTAG_TX)
29129deba33fSClaudiu Manoil enetc_enable_txvlan(ndev,
29139deba33fSClaudiu Manoil !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2914d382563fSClaudiu Manoil }
2915e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_set_features);
2916d382563fSClaudiu Manoil
2917434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)2918d3982312SY.b. Lu static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2919d3982312SY.b. Lu {
2920d3982312SY.b. Lu struct enetc_ndev_priv *priv = netdev_priv(ndev);
29215093406cSVladimir Oltean int err, new_offloads = priv->active_offloads;
2922d3982312SY.b. Lu struct hwtstamp_config config;
2923d3982312SY.b. Lu
2924d3982312SY.b. Lu if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2925d3982312SY.b. Lu return -EFAULT;
2926d3982312SY.b. Lu
2927d3982312SY.b. Lu switch (config.tx_type) {
2928d3982312SY.b. Lu case HWTSTAMP_TX_OFF:
29295093406cSVladimir Oltean new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2930d3982312SY.b. Lu break;
2931d3982312SY.b. Lu case HWTSTAMP_TX_ON:
29325093406cSVladimir Oltean new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
29335093406cSVladimir Oltean new_offloads |= ENETC_F_TX_TSTAMP;
2934d3982312SY.b. Lu break;
29357294380cSYangbo Lu case HWTSTAMP_TX_ONESTEP_SYNC:
29365093406cSVladimir Oltean new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
29375093406cSVladimir Oltean new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
29387294380cSYangbo Lu break;
2939d3982312SY.b. Lu default:
2940d3982312SY.b. Lu return -ERANGE;
2941d3982312SY.b. Lu }
2942d3982312SY.b. Lu
2943d3982312SY.b. Lu switch (config.rx_filter) {
2944d3982312SY.b. Lu case HWTSTAMP_FILTER_NONE:
29455093406cSVladimir Oltean new_offloads &= ~ENETC_F_RX_TSTAMP;
2946d3982312SY.b. Lu break;
2947d3982312SY.b. Lu default:
29485093406cSVladimir Oltean new_offloads |= ENETC_F_RX_TSTAMP;
2949d3982312SY.b. Lu config.rx_filter = HWTSTAMP_FILTER_ALL;
2950d3982312SY.b. Lu }
2951d3982312SY.b. Lu
29525093406cSVladimir Oltean if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
29535093406cSVladimir Oltean bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
29545093406cSVladimir Oltean
2955c33bfaf9SVladimir Oltean err = enetc_reconfigure(priv, extended, NULL, NULL);
29565093406cSVladimir Oltean if (err)
29575093406cSVladimir Oltean return err;
2958434cebabSClaudiu Manoil }
2959434cebabSClaudiu Manoil
29605093406cSVladimir Oltean priv->active_offloads = new_offloads;
29615093406cSVladimir Oltean
2962d3982312SY.b. Lu return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2963d3982312SY.b. Lu -EFAULT : 0;
2964d3982312SY.b. Lu }
2965d3982312SY.b. Lu
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)2966d3982312SY.b. Lu static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2967d3982312SY.b. Lu {
2968d3982312SY.b. Lu struct enetc_ndev_priv *priv = netdev_priv(ndev);
2969d3982312SY.b. Lu struct hwtstamp_config config;
2970d3982312SY.b. Lu
2971d3982312SY.b. Lu config.flags = 0;
2972d3982312SY.b. Lu
29737294380cSYangbo Lu if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
29747294380cSYangbo Lu config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
29757294380cSYangbo Lu else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2976d3982312SY.b. Lu config.tx_type = HWTSTAMP_TX_ON;
2977d3982312SY.b. Lu else
2978d3982312SY.b. Lu config.tx_type = HWTSTAMP_TX_OFF;
2979d3982312SY.b. Lu
2980d3982312SY.b. Lu config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2981d3982312SY.b. Lu HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2982d3982312SY.b. Lu
2983d3982312SY.b. Lu return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2984d3982312SY.b. Lu -EFAULT : 0;
2985d3982312SY.b. Lu }
2986d3982312SY.b. Lu #endif
2987d3982312SY.b. Lu
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)2988d3982312SY.b. Lu int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2989d3982312SY.b. Lu {
299071b77a7aSClaudiu Manoil struct enetc_ndev_priv *priv = netdev_priv(ndev);
2991434cebabSClaudiu Manoil #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2992d3982312SY.b. Lu if (cmd == SIOCSHWTSTAMP)
2993d3982312SY.b. Lu return enetc_hwtstamp_set(ndev, rq);
2994d3982312SY.b. Lu if (cmd == SIOCGHWTSTAMP)
2995d3982312SY.b. Lu return enetc_hwtstamp_get(ndev, rq);
2996d3982312SY.b. Lu #endif
2997a613bafeSMichael Walle
299871b77a7aSClaudiu Manoil if (!priv->phylink)
2999c55b810aSMichael Walle return -EOPNOTSUPP;
300071b77a7aSClaudiu Manoil
300171b77a7aSClaudiu Manoil return phylink_mii_ioctl(priv->phylink, rq, cmd);
3002d3982312SY.b. Lu }
3003e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_ioctl);
3004d3982312SY.b. Lu
enetc_alloc_msix(struct enetc_ndev_priv * priv)3005d4fd0404SClaudiu Manoil int enetc_alloc_msix(struct enetc_ndev_priv *priv)
3006d4fd0404SClaudiu Manoil {
3007d4fd0404SClaudiu Manoil struct pci_dev *pdev = priv->si->pdev;
30084ea1dd74SVladimir Oltean int num_stack_tx_queues;
30097eab503bSVladimir Oltean int first_xdp_tx_ring;
3010d4fd0404SClaudiu Manoil int i, n, err, nvec;
30117eab503bSVladimir Oltean int v_tx_rings;
3012d4fd0404SClaudiu Manoil
3013d4fd0404SClaudiu Manoil nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
3014d4fd0404SClaudiu Manoil /* allocate MSIX for both messaging and Rx/Tx interrupts */
3015d4fd0404SClaudiu Manoil n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
3016d4fd0404SClaudiu Manoil
3017d4fd0404SClaudiu Manoil if (n < 0)
3018d4fd0404SClaudiu Manoil return n;
3019d4fd0404SClaudiu Manoil
3020d4fd0404SClaudiu Manoil if (n != nvec)
3021d4fd0404SClaudiu Manoil return -EPERM;
3022d4fd0404SClaudiu Manoil
3023d4fd0404SClaudiu Manoil /* # of tx rings per int vector */
3024d4fd0404SClaudiu Manoil v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
3025d4fd0404SClaudiu Manoil
3026d4fd0404SClaudiu Manoil for (i = 0; i < priv->bdr_int_num; i++) {
3027d4fd0404SClaudiu Manoil struct enetc_int_vector *v;
3028d4fd0404SClaudiu Manoil struct enetc_bdr *bdr;
3029d4fd0404SClaudiu Manoil int j;
3030d4fd0404SClaudiu Manoil
30311260e772SGustavo A. R. Silva v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
3032d4fd0404SClaudiu Manoil if (!v) {
3033d4fd0404SClaudiu Manoil err = -ENOMEM;
3034d4fd0404SClaudiu Manoil goto fail;
3035d4fd0404SClaudiu Manoil }
3036d4fd0404SClaudiu Manoil
3037d4fd0404SClaudiu Manoil priv->int_vector[i] = v;
3038d4fd0404SClaudiu Manoil
3039d1b15102SVladimir Oltean bdr = &v->rx_ring;
3040d1b15102SVladimir Oltean bdr->index = i;
3041d1b15102SVladimir Oltean bdr->ndev = priv->ndev;
3042d1b15102SVladimir Oltean bdr->dev = priv->dev;
3043d1b15102SVladimir Oltean bdr->bd_count = priv->rx_bd_count;
3044d1b15102SVladimir Oltean bdr->buffer_offset = ENETC_RXB_PAD;
3045d1b15102SVladimir Oltean priv->rx_ring[i] = bdr;
3046d1b15102SVladimir Oltean
3047d1b15102SVladimir Oltean err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
3048d1b15102SVladimir Oltean if (err) {
3049d1b15102SVladimir Oltean kfree(v);
3050d1b15102SVladimir Oltean goto fail;
3051d1b15102SVladimir Oltean }
3052d1b15102SVladimir Oltean
3053d1b15102SVladimir Oltean err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
3054d1b15102SVladimir Oltean MEM_TYPE_PAGE_SHARED, NULL);
3055d1b15102SVladimir Oltean if (err) {
3056d1b15102SVladimir Oltean xdp_rxq_info_unreg(&bdr->xdp.rxq);
3057d1b15102SVladimir Oltean kfree(v);
3058d1b15102SVladimir Oltean goto fail;
3059d1b15102SVladimir Oltean }
3060d1b15102SVladimir Oltean
3061ae0e6a5dSClaudiu Manoil /* init defaults for adaptive IC */
3062ae0e6a5dSClaudiu Manoil if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3063ae0e6a5dSClaudiu Manoil v->rx_ictt = 0x1;
3064ae0e6a5dSClaudiu Manoil v->rx_dim_en = true;
3065ae0e6a5dSClaudiu Manoil }
3066ae0e6a5dSClaudiu Manoil INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3067b48b89f9SJakub Kicinski netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3068d4fd0404SClaudiu Manoil v->count_tx_rings = v_tx_rings;
3069d4fd0404SClaudiu Manoil
3070d4fd0404SClaudiu Manoil for (j = 0; j < v_tx_rings; j++) {
3071d4fd0404SClaudiu Manoil int idx;
3072d4fd0404SClaudiu Manoil
3073d4fd0404SClaudiu Manoil /* default tx ring mapping policy */
30746c5e6b4cSClaudiu Manoil idx = priv->bdr_int_num * j + i;
3075d4fd0404SClaudiu Manoil __set_bit(idx, &v->tx_rings_map);
3076d4fd0404SClaudiu Manoil bdr = &v->tx_ring[j];
3077d4fd0404SClaudiu Manoil bdr->index = idx;
3078d4fd0404SClaudiu Manoil bdr->ndev = priv->ndev;
3079d4fd0404SClaudiu Manoil bdr->dev = priv->dev;
3080d4fd0404SClaudiu Manoil bdr->bd_count = priv->tx_bd_count;
3081d4fd0404SClaudiu Manoil priv->tx_ring[idx] = bdr;
3082d4fd0404SClaudiu Manoil }
3083d4fd0404SClaudiu Manoil }
3084d4fd0404SClaudiu Manoil
30854ea1dd74SVladimir Oltean num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
30864ea1dd74SVladimir Oltean
30874ea1dd74SVladimir Oltean err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
30884ea1dd74SVladimir Oltean if (err)
30894ea1dd74SVladimir Oltean goto fail;
30904ea1dd74SVladimir Oltean
30914ea1dd74SVladimir Oltean err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
30924ea1dd74SVladimir Oltean if (err)
30934ea1dd74SVladimir Oltean goto fail;
30944ea1dd74SVladimir Oltean
3095800db2d1SVladimir Oltean priv->min_num_stack_tx_queues = num_possible_cpus();
30967eab503bSVladimir Oltean first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
30977eab503bSVladimir Oltean priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
30987eab503bSVladimir Oltean
3099d4fd0404SClaudiu Manoil return 0;
3100d4fd0404SClaudiu Manoil
3101d4fd0404SClaudiu Manoil fail:
3102d4fd0404SClaudiu Manoil while (i--) {
3103d1b15102SVladimir Oltean struct enetc_int_vector *v = priv->int_vector[i];
3104d1b15102SVladimir Oltean struct enetc_bdr *rx_ring = &v->rx_ring;
3105d1b15102SVladimir Oltean
3106d1b15102SVladimir Oltean xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3107d1b15102SVladimir Oltean xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3108d1b15102SVladimir Oltean netif_napi_del(&v->napi);
3109d1b15102SVladimir Oltean cancel_work_sync(&v->rx_dim.work);
3110d1b15102SVladimir Oltean kfree(v);
3111d4fd0404SClaudiu Manoil }
3112d4fd0404SClaudiu Manoil
3113d4fd0404SClaudiu Manoil pci_free_irq_vectors(pdev);
3114d4fd0404SClaudiu Manoil
3115d4fd0404SClaudiu Manoil return err;
3116d4fd0404SClaudiu Manoil }
3117e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3118d4fd0404SClaudiu Manoil
enetc_free_msix(struct enetc_ndev_priv * priv)3119d4fd0404SClaudiu Manoil void enetc_free_msix(struct enetc_ndev_priv *priv)
3120d4fd0404SClaudiu Manoil {
3121d4fd0404SClaudiu Manoil int i;
3122d4fd0404SClaudiu Manoil
3123d4fd0404SClaudiu Manoil for (i = 0; i < priv->bdr_int_num; i++) {
3124d4fd0404SClaudiu Manoil struct enetc_int_vector *v = priv->int_vector[i];
3125d1b15102SVladimir Oltean struct enetc_bdr *rx_ring = &v->rx_ring;
3126d4fd0404SClaudiu Manoil
3127d1b15102SVladimir Oltean xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3128d1b15102SVladimir Oltean xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3129d4fd0404SClaudiu Manoil netif_napi_del(&v->napi);
3130ae0e6a5dSClaudiu Manoil cancel_work_sync(&v->rx_dim.work);
3131d4fd0404SClaudiu Manoil }
3132d4fd0404SClaudiu Manoil
3133d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_rx_rings; i++)
3134d4fd0404SClaudiu Manoil priv->rx_ring[i] = NULL;
3135d4fd0404SClaudiu Manoil
3136d4fd0404SClaudiu Manoil for (i = 0; i < priv->num_tx_rings; i++)
3137d4fd0404SClaudiu Manoil priv->tx_ring[i] = NULL;
3138d4fd0404SClaudiu Manoil
3139d4fd0404SClaudiu Manoil for (i = 0; i < priv->bdr_int_num; i++) {
3140d4fd0404SClaudiu Manoil kfree(priv->int_vector[i]);
3141d4fd0404SClaudiu Manoil priv->int_vector[i] = NULL;
3142d4fd0404SClaudiu Manoil }
3143d4fd0404SClaudiu Manoil
3144d4fd0404SClaudiu Manoil /* disable all MSIX for this device */
3145d4fd0404SClaudiu Manoil pci_free_irq_vectors(priv->si->pdev);
3146d4fd0404SClaudiu Manoil }
3147e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_free_msix);
3148d4fd0404SClaudiu Manoil
enetc_kfree_si(struct enetc_si * si)3149d4fd0404SClaudiu Manoil static void enetc_kfree_si(struct enetc_si *si)
3150d4fd0404SClaudiu Manoil {
3151d4fd0404SClaudiu Manoil char *p = (char *)si - si->pad;
3152d4fd0404SClaudiu Manoil
3153d4fd0404SClaudiu Manoil kfree(p);
3154d4fd0404SClaudiu Manoil }
3155d4fd0404SClaudiu Manoil
enetc_detect_errata(struct enetc_si * si)3156d4fd0404SClaudiu Manoil static void enetc_detect_errata(struct enetc_si *si)
3157d4fd0404SClaudiu Manoil {
3158d4fd0404SClaudiu Manoil if (si->pdev->revision == ENETC_REV1)
315982728b91SClaudiu Manoil si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3160d4fd0404SClaudiu Manoil }
3161d4fd0404SClaudiu Manoil
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)3162d4fd0404SClaudiu Manoil int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3163d4fd0404SClaudiu Manoil {
3164d4fd0404SClaudiu Manoil struct enetc_si *si, *p;
3165d4fd0404SClaudiu Manoil struct enetc_hw *hw;
3166d4fd0404SClaudiu Manoil size_t alloc_size;
3167d4fd0404SClaudiu Manoil int err, len;
3168d4fd0404SClaudiu Manoil
3169d4fd0404SClaudiu Manoil pcie_flr(pdev);
3170d4fd0404SClaudiu Manoil err = pci_enable_device_mem(pdev);
3171a72691eeSCai Huoqing if (err)
3172a72691eeSCai Huoqing return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3173d4fd0404SClaudiu Manoil
3174d4fd0404SClaudiu Manoil /* set up for high or low dma */
3175d4fd0404SClaudiu Manoil err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3176d4fd0404SClaudiu Manoil if (err) {
3177cfcfc8f5SChristophe JAILLET dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3178d4fd0404SClaudiu Manoil goto err_dma;
3179d4fd0404SClaudiu Manoil }
3180d4fd0404SClaudiu Manoil
3181d4fd0404SClaudiu Manoil err = pci_request_mem_regions(pdev, name);
3182d4fd0404SClaudiu Manoil if (err) {
3183d4fd0404SClaudiu Manoil dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3184d4fd0404SClaudiu Manoil goto err_pci_mem_reg;
3185d4fd0404SClaudiu Manoil }
3186d4fd0404SClaudiu Manoil
3187d4fd0404SClaudiu Manoil pci_set_master(pdev);
3188d4fd0404SClaudiu Manoil
3189d4fd0404SClaudiu Manoil alloc_size = sizeof(struct enetc_si);
3190d4fd0404SClaudiu Manoil if (sizeof_priv) {
3191d4fd0404SClaudiu Manoil /* align priv to 32B */
3192d4fd0404SClaudiu Manoil alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3193d4fd0404SClaudiu Manoil alloc_size += sizeof_priv;
3194d4fd0404SClaudiu Manoil }
3195d4fd0404SClaudiu Manoil /* force 32B alignment for enetc_si */
3196d4fd0404SClaudiu Manoil alloc_size += ENETC_SI_ALIGN - 1;
3197d4fd0404SClaudiu Manoil
3198d4fd0404SClaudiu Manoil p = kzalloc(alloc_size, GFP_KERNEL);
3199d4fd0404SClaudiu Manoil if (!p) {
3200d4fd0404SClaudiu Manoil err = -ENOMEM;
3201d4fd0404SClaudiu Manoil goto err_alloc_si;
3202d4fd0404SClaudiu Manoil }
3203d4fd0404SClaudiu Manoil
3204d4fd0404SClaudiu Manoil si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3205d4fd0404SClaudiu Manoil si->pad = (char *)si - (char *)p;
3206d4fd0404SClaudiu Manoil
3207d4fd0404SClaudiu Manoil pci_set_drvdata(pdev, si);
3208d4fd0404SClaudiu Manoil si->pdev = pdev;
3209d4fd0404SClaudiu Manoil hw = &si->hw;
3210d4fd0404SClaudiu Manoil
3211d4fd0404SClaudiu Manoil len = pci_resource_len(pdev, ENETC_BAR_REGS);
3212d4fd0404SClaudiu Manoil hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3213d4fd0404SClaudiu Manoil if (!hw->reg) {
3214d4fd0404SClaudiu Manoil err = -ENXIO;
3215d4fd0404SClaudiu Manoil dev_err(&pdev->dev, "ioremap() failed\n");
3216d4fd0404SClaudiu Manoil goto err_ioremap;
3217d4fd0404SClaudiu Manoil }
3218d4fd0404SClaudiu Manoil if (len > ENETC_PORT_BASE)
3219d4fd0404SClaudiu Manoil hw->port = hw->reg + ENETC_PORT_BASE;
3220d4fd0404SClaudiu Manoil if (len > ENETC_GLOBAL_BASE)
3221d4fd0404SClaudiu Manoil hw->global = hw->reg + ENETC_GLOBAL_BASE;
3222d4fd0404SClaudiu Manoil
3223d4fd0404SClaudiu Manoil enetc_detect_errata(si);
3224d4fd0404SClaudiu Manoil
3225d4fd0404SClaudiu Manoil return 0;
3226d4fd0404SClaudiu Manoil
3227d4fd0404SClaudiu Manoil err_ioremap:
3228d4fd0404SClaudiu Manoil enetc_kfree_si(si);
3229d4fd0404SClaudiu Manoil err_alloc_si:
3230d4fd0404SClaudiu Manoil pci_release_mem_regions(pdev);
3231d4fd0404SClaudiu Manoil err_pci_mem_reg:
3232d4fd0404SClaudiu Manoil err_dma:
3233d4fd0404SClaudiu Manoil pci_disable_device(pdev);
3234d4fd0404SClaudiu Manoil
3235d4fd0404SClaudiu Manoil return err;
3236d4fd0404SClaudiu Manoil }
3237e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_pci_probe);
3238d4fd0404SClaudiu Manoil
enetc_pci_remove(struct pci_dev * pdev)3239d4fd0404SClaudiu Manoil void enetc_pci_remove(struct pci_dev *pdev)
3240d4fd0404SClaudiu Manoil {
3241d4fd0404SClaudiu Manoil struct enetc_si *si = pci_get_drvdata(pdev);
3242d4fd0404SClaudiu Manoil struct enetc_hw *hw = &si->hw;
3243d4fd0404SClaudiu Manoil
3244d4fd0404SClaudiu Manoil iounmap(hw->reg);
3245d4fd0404SClaudiu Manoil enetc_kfree_si(si);
3246d4fd0404SClaudiu Manoil pci_release_mem_regions(pdev);
3247d4fd0404SClaudiu Manoil pci_disable_device(pdev);
3248d4fd0404SClaudiu Manoil }
3249e3972399SVladimir Oltean EXPORT_SYMBOL_GPL(enetc_pci_remove);
3250e3972399SVladimir Oltean
3251e3972399SVladimir Oltean MODULE_LICENSE("Dual BSD/GPL");
3252