1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/tcp.h>
7 #include <linux/udp.h>
8 #include <linux/vmalloc.h>
9 #include <linux/ptp_classify.h>
10 #include <net/ip6_checksum.h>
11 #include <net/pkt_sched.h>
12 #include <net/tso.h>
13
enetc_port_mac_rd(struct enetc_si * si,u32 reg)14 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
15 {
16 return enetc_port_rd(&si->hw, reg);
17 }
18 EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
19
enetc_port_mac_wr(struct enetc_si * si,u32 reg,u32 val)20 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
21 {
22 enetc_port_wr(&si->hw, reg, val);
23 if (si->hw_features & ENETC_SI_F_QBU)
24 enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val);
25 }
26 EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
27
enetc_change_preemptible_tcs(struct enetc_ndev_priv * priv,u8 preemptible_tcs)28 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
29 u8 preemptible_tcs)
30 {
31 priv->preemptible_tcs = preemptible_tcs;
32 enetc_mm_commit_preemptible_tcs(priv);
33 }
34
enetc_num_stack_tx_queues(struct enetc_ndev_priv * priv)35 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
36 {
37 int num_tx_rings = priv->num_tx_rings;
38
39 if (priv->xdp_prog)
40 return num_tx_rings - num_possible_cpus();
41
42 return num_tx_rings;
43 }
44
enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv * priv,struct enetc_bdr * tx_ring)45 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
46 struct enetc_bdr *tx_ring)
47 {
48 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
49
50 return priv->rx_ring[index];
51 }
52
enetc_tx_swbd_get_skb(struct enetc_tx_swbd * tx_swbd)53 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
54 {
55 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
56 return NULL;
57
58 return tx_swbd->skb;
59 }
60
61 static struct xdp_frame *
enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd * tx_swbd)62 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
63 {
64 if (tx_swbd->is_xdp_redirect)
65 return tx_swbd->xdp_frame;
66
67 return NULL;
68 }
69
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)70 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
71 struct enetc_tx_swbd *tx_swbd)
72 {
73 /* For XDP_TX, pages come from RX, whereas for the other contexts where
74 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
75 * to match the DMA mapping length, so we need to differentiate those.
76 */
77 if (tx_swbd->is_dma_page)
78 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
79 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
80 tx_swbd->dir);
81 else
82 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
83 tx_swbd->len, tx_swbd->dir);
84 tx_swbd->dma = 0;
85 }
86
enetc_free_tx_frame(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)87 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
88 struct enetc_tx_swbd *tx_swbd)
89 {
90 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
91 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
92
93 if (tx_swbd->dma)
94 enetc_unmap_tx_buff(tx_ring, tx_swbd);
95
96 if (xdp_frame) {
97 xdp_return_frame(tx_swbd->xdp_frame);
98 tx_swbd->xdp_frame = NULL;
99 } else if (skb) {
100 dev_kfree_skb_any(skb);
101 tx_swbd->skb = NULL;
102 }
103 }
104
105 /* Let H/W know BD ring has been updated */
enetc_update_tx_ring_tail(struct enetc_bdr * tx_ring)106 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
107 {
108 /* includes wmb() */
109 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
110 }
111
enetc_ptp_parse(struct sk_buff * skb,u8 * udp,u8 * msgtype,u8 * twostep,u16 * correction_offset,u16 * body_offset)112 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
113 u8 *msgtype, u8 *twostep,
114 u16 *correction_offset, u16 *body_offset)
115 {
116 unsigned int ptp_class;
117 struct ptp_header *hdr;
118 unsigned int type;
119 u8 *base;
120
121 ptp_class = ptp_classify_raw(skb);
122 if (ptp_class == PTP_CLASS_NONE)
123 return -EINVAL;
124
125 hdr = ptp_parse_header(skb, ptp_class);
126 if (!hdr)
127 return -EINVAL;
128
129 type = ptp_class & PTP_CLASS_PMASK;
130 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
131 *udp = 1;
132 else
133 *udp = 0;
134
135 *msgtype = ptp_get_msgtype(hdr, ptp_class);
136 *twostep = hdr->flag_field[0] & 0x2;
137
138 base = skb_mac_header(skb);
139 *correction_offset = (u8 *)&hdr->correction - base;
140 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
141
142 return 0;
143 }
144
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)145 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
146 {
147 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
148 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
149 struct enetc_hw *hw = &priv->si->hw;
150 struct enetc_tx_swbd *tx_swbd;
151 int len = skb_headlen(skb);
152 union enetc_tx_bd temp_bd;
153 u8 msgtype, twostep, udp;
154 union enetc_tx_bd *txbd;
155 u16 offset1, offset2;
156 int i, count = 0;
157 skb_frag_t *frag;
158 unsigned int f;
159 dma_addr_t dma;
160 u8 flags = 0;
161
162 i = tx_ring->next_to_use;
163 txbd = ENETC_TXBD(*tx_ring, i);
164 prefetchw(txbd);
165
166 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
167 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
168 goto dma_err;
169
170 temp_bd.addr = cpu_to_le64(dma);
171 temp_bd.buf_len = cpu_to_le16(len);
172 temp_bd.lstatus = 0;
173
174 tx_swbd = &tx_ring->tx_swbd[i];
175 tx_swbd->dma = dma;
176 tx_swbd->len = len;
177 tx_swbd->is_dma_page = 0;
178 tx_swbd->dir = DMA_TO_DEVICE;
179 count++;
180
181 do_vlan = skb_vlan_tag_present(skb);
182 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
183 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
184 &offset2) ||
185 msgtype != PTP_MSGTYPE_SYNC || twostep)
186 WARN_ONCE(1, "Bad packet for one-step timestamping\n");
187 else
188 do_onestep_tstamp = true;
189 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
190 do_twostep_tstamp = true;
191 }
192
193 tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
194 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
195 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
196
197 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
198 flags |= ENETC_TXBD_FLAGS_EX;
199
200 if (tx_ring->tsd_enable)
201 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
202
203 /* first BD needs frm_len and offload flags set */
204 temp_bd.frm_len = cpu_to_le16(skb->len);
205 temp_bd.flags = flags;
206
207 if (flags & ENETC_TXBD_FLAGS_TSE)
208 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
209 flags);
210
211 if (flags & ENETC_TXBD_FLAGS_EX) {
212 u8 e_flags = 0;
213 *txbd = temp_bd;
214 enetc_clear_tx_bd(&temp_bd);
215
216 /* add extension BD for VLAN and/or timestamping */
217 flags = 0;
218 tx_swbd++;
219 txbd++;
220 i++;
221 if (unlikely(i == tx_ring->bd_count)) {
222 i = 0;
223 tx_swbd = tx_ring->tx_swbd;
224 txbd = ENETC_TXBD(*tx_ring, 0);
225 }
226 prefetchw(txbd);
227
228 if (do_vlan) {
229 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
230 temp_bd.ext.tpid = 0; /* < C-TAG */
231 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
232 }
233
234 if (do_onestep_tstamp) {
235 u32 lo, hi, val;
236 u64 sec, nsec;
237 u8 *data;
238
239 lo = enetc_rd_hot(hw, ENETC_SICTR0);
240 hi = enetc_rd_hot(hw, ENETC_SICTR1);
241 sec = (u64)hi << 32 | lo;
242 nsec = do_div(sec, 1000000000);
243
244 /* Configure extension BD */
245 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
246 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
247
248 /* Update originTimestamp field of Sync packet
249 * - 48 bits seconds field
250 * - 32 bits nanseconds field
251 */
252 data = skb_mac_header(skb);
253 *(__be16 *)(data + offset2) =
254 htons((sec >> 32) & 0xffff);
255 *(__be32 *)(data + offset2 + 2) =
256 htonl(sec & 0xffffffff);
257 *(__be32 *)(data + offset2 + 6) = htonl(nsec);
258
259 /* Configure single-step register */
260 val = ENETC_PM0_SINGLE_STEP_EN;
261 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
262 if (udp)
263 val |= ENETC_PM0_SINGLE_STEP_CH;
264
265 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
266 val);
267 } else if (do_twostep_tstamp) {
268 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
269 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
270 }
271
272 temp_bd.ext.e_flags = e_flags;
273 count++;
274 }
275
276 frag = &skb_shinfo(skb)->frags[0];
277 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
278 len = skb_frag_size(frag);
279 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
280 DMA_TO_DEVICE);
281 if (dma_mapping_error(tx_ring->dev, dma))
282 goto dma_err;
283
284 *txbd = temp_bd;
285 enetc_clear_tx_bd(&temp_bd);
286
287 flags = 0;
288 tx_swbd++;
289 txbd++;
290 i++;
291 if (unlikely(i == tx_ring->bd_count)) {
292 i = 0;
293 tx_swbd = tx_ring->tx_swbd;
294 txbd = ENETC_TXBD(*tx_ring, 0);
295 }
296 prefetchw(txbd);
297
298 temp_bd.addr = cpu_to_le64(dma);
299 temp_bd.buf_len = cpu_to_le16(len);
300
301 tx_swbd->dma = dma;
302 tx_swbd->len = len;
303 tx_swbd->is_dma_page = 1;
304 tx_swbd->dir = DMA_TO_DEVICE;
305 count++;
306 }
307
308 /* last BD needs 'F' bit set */
309 flags |= ENETC_TXBD_FLAGS_F;
310 temp_bd.flags = flags;
311 *txbd = temp_bd;
312
313 tx_ring->tx_swbd[i].is_eof = true;
314 tx_ring->tx_swbd[i].skb = skb;
315
316 enetc_bdr_idx_inc(tx_ring, &i);
317 tx_ring->next_to_use = i;
318
319 skb_tx_timestamp(skb);
320
321 enetc_update_tx_ring_tail(tx_ring);
322
323 return count;
324
325 dma_err:
326 dev_err(tx_ring->dev, "DMA map error");
327
328 do {
329 tx_swbd = &tx_ring->tx_swbd[i];
330 enetc_free_tx_frame(tx_ring, tx_swbd);
331 if (i == 0)
332 i = tx_ring->bd_count;
333 i--;
334 } while (count--);
335
336 return 0;
337 }
338
enetc_map_tx_tso_hdr(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,int * i,int hdr_len,int data_len)339 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
340 struct enetc_tx_swbd *tx_swbd,
341 union enetc_tx_bd *txbd, int *i, int hdr_len,
342 int data_len)
343 {
344 union enetc_tx_bd txbd_tmp;
345 u8 flags = 0, e_flags = 0;
346 dma_addr_t addr;
347
348 enetc_clear_tx_bd(&txbd_tmp);
349 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
350
351 if (skb_vlan_tag_present(skb))
352 flags |= ENETC_TXBD_FLAGS_EX;
353
354 txbd_tmp.addr = cpu_to_le64(addr);
355 txbd_tmp.buf_len = cpu_to_le16(hdr_len);
356
357 /* first BD needs frm_len and offload flags set */
358 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
359 txbd_tmp.flags = flags;
360
361 /* For the TSO header we do not set the dma address since we do not
362 * want it unmapped when we do cleanup. We still set len so that we
363 * count the bytes sent.
364 */
365 tx_swbd->len = hdr_len;
366 tx_swbd->do_twostep_tstamp = false;
367 tx_swbd->check_wb = false;
368
369 /* Actually write the header in the BD */
370 *txbd = txbd_tmp;
371
372 /* Add extension BD for VLAN */
373 if (flags & ENETC_TXBD_FLAGS_EX) {
374 /* Get the next BD */
375 enetc_bdr_idx_inc(tx_ring, i);
376 txbd = ENETC_TXBD(*tx_ring, *i);
377 tx_swbd = &tx_ring->tx_swbd[*i];
378 prefetchw(txbd);
379
380 /* Setup the VLAN fields */
381 enetc_clear_tx_bd(&txbd_tmp);
382 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
383 txbd_tmp.ext.tpid = 0; /* < C-TAG */
384 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
385
386 /* Write the BD */
387 txbd_tmp.ext.e_flags = e_flags;
388 *txbd = txbd_tmp;
389 }
390 }
391
enetc_map_tx_tso_data(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,char * data,int size,bool last_bd)392 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
393 struct enetc_tx_swbd *tx_swbd,
394 union enetc_tx_bd *txbd, char *data,
395 int size, bool last_bd)
396 {
397 union enetc_tx_bd txbd_tmp;
398 dma_addr_t addr;
399 u8 flags = 0;
400
401 enetc_clear_tx_bd(&txbd_tmp);
402
403 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
404 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
405 netdev_err(tx_ring->ndev, "DMA map error\n");
406 return -ENOMEM;
407 }
408
409 if (last_bd) {
410 flags |= ENETC_TXBD_FLAGS_F;
411 tx_swbd->is_eof = 1;
412 }
413
414 txbd_tmp.addr = cpu_to_le64(addr);
415 txbd_tmp.buf_len = cpu_to_le16(size);
416 txbd_tmp.flags = flags;
417
418 tx_swbd->dma = addr;
419 tx_swbd->len = size;
420 tx_swbd->dir = DMA_TO_DEVICE;
421
422 *txbd = txbd_tmp;
423
424 return 0;
425 }
426
enetc_tso_hdr_csum(struct tso_t * tso,struct sk_buff * skb,char * hdr,int hdr_len,int * l4_hdr_len)427 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
428 char *hdr, int hdr_len, int *l4_hdr_len)
429 {
430 char *l4_hdr = hdr + skb_transport_offset(skb);
431 int mac_hdr_len = skb_network_offset(skb);
432
433 if (tso->tlen != sizeof(struct udphdr)) {
434 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
435
436 tcph->check = 0;
437 } else {
438 struct udphdr *udph = (struct udphdr *)(l4_hdr);
439
440 udph->check = 0;
441 }
442
443 /* Compute the IP checksum. This is necessary since tso_build_hdr()
444 * already incremented the IP ID field.
445 */
446 if (!tso->ipv6) {
447 struct iphdr *iph = (void *)(hdr + mac_hdr_len);
448
449 iph->check = 0;
450 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
451 }
452
453 /* Compute the checksum over the L4 header. */
454 *l4_hdr_len = hdr_len - skb_transport_offset(skb);
455 return csum_partial(l4_hdr, *l4_hdr_len, 0);
456 }
457
enetc_tso_complete_csum(struct enetc_bdr * tx_ring,struct tso_t * tso,struct sk_buff * skb,char * hdr,int len,__wsum sum)458 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
459 struct sk_buff *skb, char *hdr, int len,
460 __wsum sum)
461 {
462 char *l4_hdr = hdr + skb_transport_offset(skb);
463 __sum16 csum_final;
464
465 /* Complete the L4 checksum by appending the pseudo-header to the
466 * already computed checksum.
467 */
468 if (!tso->ipv6)
469 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
470 ip_hdr(skb)->daddr,
471 len, ip_hdr(skb)->protocol, sum);
472 else
473 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
474 &ipv6_hdr(skb)->daddr,
475 len, ipv6_hdr(skb)->nexthdr, sum);
476
477 if (tso->tlen != sizeof(struct udphdr)) {
478 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
479
480 tcph->check = csum_final;
481 } else {
482 struct udphdr *udph = (struct udphdr *)(l4_hdr);
483
484 udph->check = csum_final;
485 }
486 }
487
enetc_map_tx_tso_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)488 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
489 {
490 int hdr_len, total_len, data_len;
491 struct enetc_tx_swbd *tx_swbd;
492 union enetc_tx_bd *txbd;
493 struct tso_t tso;
494 __wsum csum, csum2;
495 int count = 0, pos;
496 int err, i, bd_data_num;
497
498 /* Initialize the TSO handler, and prepare the first payload */
499 hdr_len = tso_start(skb, &tso);
500 total_len = skb->len - hdr_len;
501 i = tx_ring->next_to_use;
502
503 while (total_len > 0) {
504 char *hdr;
505
506 /* Get the BD */
507 txbd = ENETC_TXBD(*tx_ring, i);
508 tx_swbd = &tx_ring->tx_swbd[i];
509 prefetchw(txbd);
510
511 /* Determine the length of this packet */
512 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
513 total_len -= data_len;
514
515 /* prepare packet headers: MAC + IP + TCP */
516 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
517 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
518
519 /* compute the csum over the L4 header */
520 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
521 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
522 bd_data_num = 0;
523 count++;
524
525 while (data_len > 0) {
526 int size;
527
528 size = min_t(int, tso.size, data_len);
529
530 /* Advance the index in the BDR */
531 enetc_bdr_idx_inc(tx_ring, &i);
532 txbd = ENETC_TXBD(*tx_ring, i);
533 tx_swbd = &tx_ring->tx_swbd[i];
534 prefetchw(txbd);
535
536 /* Compute the checksum over this segment of data and
537 * add it to the csum already computed (over the L4
538 * header and possible other data segments).
539 */
540 csum2 = csum_partial(tso.data, size, 0);
541 csum = csum_block_add(csum, csum2, pos);
542 pos += size;
543
544 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
545 tso.data, size,
546 size == data_len);
547 if (err)
548 goto err_map_data;
549
550 data_len -= size;
551 count++;
552 bd_data_num++;
553 tso_build_data(skb, &tso, size);
554
555 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
556 goto err_chained_bd;
557 }
558
559 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
560
561 if (total_len == 0)
562 tx_swbd->skb = skb;
563
564 /* Go to the next BD */
565 enetc_bdr_idx_inc(tx_ring, &i);
566 }
567
568 tx_ring->next_to_use = i;
569 enetc_update_tx_ring_tail(tx_ring);
570
571 return count;
572
573 err_map_data:
574 dev_err(tx_ring->dev, "DMA map error");
575
576 err_chained_bd:
577 do {
578 tx_swbd = &tx_ring->tx_swbd[i];
579 enetc_free_tx_frame(tx_ring, tx_swbd);
580 if (i == 0)
581 i = tx_ring->bd_count;
582 i--;
583 } while (count--);
584
585 return 0;
586 }
587
enetc_start_xmit(struct sk_buff * skb,struct net_device * ndev)588 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
589 struct net_device *ndev)
590 {
591 struct enetc_ndev_priv *priv = netdev_priv(ndev);
592 struct enetc_bdr *tx_ring;
593 int count, err;
594
595 /* Queue one-step Sync packet if already locked */
596 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
597 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
598 &priv->flags)) {
599 skb_queue_tail(&priv->tx_skbs, skb);
600 return NETDEV_TX_OK;
601 }
602 }
603
604 tx_ring = priv->tx_ring[skb->queue_mapping];
605
606 if (skb_is_gso(skb)) {
607 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
608 netif_stop_subqueue(ndev, tx_ring->index);
609 return NETDEV_TX_BUSY;
610 }
611
612 enetc_lock_mdio();
613 count = enetc_map_tx_tso_buffs(tx_ring, skb);
614 enetc_unlock_mdio();
615 } else {
616 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
617 if (unlikely(skb_linearize(skb)))
618 goto drop_packet_err;
619
620 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
621 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
622 netif_stop_subqueue(ndev, tx_ring->index);
623 return NETDEV_TX_BUSY;
624 }
625
626 if (skb->ip_summed == CHECKSUM_PARTIAL) {
627 err = skb_checksum_help(skb);
628 if (err)
629 goto drop_packet_err;
630 }
631 enetc_lock_mdio();
632 count = enetc_map_tx_buffs(tx_ring, skb);
633 enetc_unlock_mdio();
634 }
635
636 if (unlikely(!count))
637 goto drop_packet_err;
638
639 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
640 netif_stop_subqueue(ndev, tx_ring->index);
641
642 return NETDEV_TX_OK;
643
644 drop_packet_err:
645 dev_kfree_skb_any(skb);
646 return NETDEV_TX_OK;
647 }
648
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)649 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
650 {
651 struct enetc_ndev_priv *priv = netdev_priv(ndev);
652 u8 udp, msgtype, twostep;
653 u16 offset1, offset2;
654
655 /* Mark tx timestamp type on skb->cb[0] if requires */
656 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
657 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
658 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
659 } else {
660 skb->cb[0] = 0;
661 }
662
663 /* Fall back to two-step timestamp if not one-step Sync packet */
664 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
665 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
666 &offset1, &offset2) ||
667 msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
668 skb->cb[0] = ENETC_F_TX_TSTAMP;
669 }
670
671 return enetc_start_xmit(skb, ndev);
672 }
673 EXPORT_SYMBOL_GPL(enetc_xmit);
674
enetc_msix(int irq,void * data)675 static irqreturn_t enetc_msix(int irq, void *data)
676 {
677 struct enetc_int_vector *v = data;
678 int i;
679
680 enetc_lock_mdio();
681
682 /* disable interrupts */
683 enetc_wr_reg_hot(v->rbier, 0);
684 enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
685
686 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
687 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
688
689 enetc_unlock_mdio();
690
691 napi_schedule(&v->napi);
692
693 return IRQ_HANDLED;
694 }
695
enetc_rx_dim_work(struct work_struct * w)696 static void enetc_rx_dim_work(struct work_struct *w)
697 {
698 struct dim *dim = container_of(w, struct dim, work);
699 struct dim_cq_moder moder =
700 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
701 struct enetc_int_vector *v =
702 container_of(dim, struct enetc_int_vector, rx_dim);
703
704 v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
705 dim->state = DIM_START_MEASURE;
706 }
707
enetc_rx_net_dim(struct enetc_int_vector * v)708 static void enetc_rx_net_dim(struct enetc_int_vector *v)
709 {
710 struct dim_sample dim_sample = {};
711
712 v->comp_cnt++;
713
714 if (!v->rx_napi_work)
715 return;
716
717 dim_update_sample(v->comp_cnt,
718 v->rx_ring.stats.packets,
719 v->rx_ring.stats.bytes,
720 &dim_sample);
721 net_dim(&v->rx_dim, dim_sample);
722 }
723
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)724 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
725 {
726 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
727
728 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
729 }
730
enetc_page_reusable(struct page * page)731 static bool enetc_page_reusable(struct page *page)
732 {
733 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
734 }
735
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)736 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
737 struct enetc_rx_swbd *old)
738 {
739 struct enetc_rx_swbd *new;
740
741 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
742
743 /* next buf that may reuse a page */
744 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
745
746 /* copy page reference */
747 *new = *old;
748 }
749
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)750 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
751 u64 *tstamp)
752 {
753 u32 lo, hi, tstamp_lo;
754
755 lo = enetc_rd_hot(hw, ENETC_SICTR0);
756 hi = enetc_rd_hot(hw, ENETC_SICTR1);
757 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
758 if (lo <= tstamp_lo)
759 hi -= 1;
760 *tstamp = (u64)hi << 32 | tstamp_lo;
761 }
762
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)763 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
764 {
765 struct skb_shared_hwtstamps shhwtstamps;
766
767 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
768 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
769 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
770 skb_txtime_consumed(skb);
771 skb_tstamp_tx(skb, &shhwtstamps);
772 }
773 }
774
enetc_recycle_xdp_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)775 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
776 struct enetc_tx_swbd *tx_swbd)
777 {
778 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
779 struct enetc_rx_swbd rx_swbd = {
780 .dma = tx_swbd->dma,
781 .page = tx_swbd->page,
782 .page_offset = tx_swbd->page_offset,
783 .dir = tx_swbd->dir,
784 .len = tx_swbd->len,
785 };
786 struct enetc_bdr *rx_ring;
787
788 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
789
790 if (likely(enetc_swbd_unused(rx_ring))) {
791 enetc_reuse_page(rx_ring, &rx_swbd);
792
793 /* sync for use by the device */
794 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
795 rx_swbd.page_offset,
796 ENETC_RXB_DMA_SIZE_XDP,
797 rx_swbd.dir);
798
799 rx_ring->stats.recycles++;
800 } else {
801 /* RX ring is already full, we need to unmap and free the
802 * page, since there's nothing useful we can do with it.
803 */
804 rx_ring->stats.recycle_failures++;
805
806 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
807 rx_swbd.dir);
808 __free_page(rx_swbd.page);
809 }
810
811 rx_ring->xdp.xdp_tx_in_flight--;
812 }
813
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)814 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
815 {
816 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
817 struct net_device *ndev = tx_ring->ndev;
818 struct enetc_ndev_priv *priv = netdev_priv(ndev);
819 struct enetc_tx_swbd *tx_swbd;
820 int i, bds_to_clean;
821 bool do_twostep_tstamp;
822 u64 tstamp = 0;
823
824 i = tx_ring->next_to_clean;
825 tx_swbd = &tx_ring->tx_swbd[i];
826
827 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
828
829 do_twostep_tstamp = false;
830
831 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
832 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
833 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
834 bool is_eof = tx_swbd->is_eof;
835
836 if (unlikely(tx_swbd->check_wb)) {
837 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
838
839 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
840 tx_swbd->do_twostep_tstamp) {
841 enetc_get_tx_tstamp(&priv->si->hw, txbd,
842 &tstamp);
843 do_twostep_tstamp = true;
844 }
845
846 if (tx_swbd->qbv_en &&
847 txbd->wb.status & ENETC_TXBD_STATS_WIN)
848 tx_win_drop++;
849 }
850
851 if (tx_swbd->is_xdp_tx)
852 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
853 else if (likely(tx_swbd->dma))
854 enetc_unmap_tx_buff(tx_ring, tx_swbd);
855
856 if (xdp_frame) {
857 xdp_return_frame(xdp_frame);
858 } else if (skb) {
859 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
860 /* Start work to release lock for next one-step
861 * timestamping packet. And send one skb in
862 * tx_skbs queue if has.
863 */
864 schedule_work(&priv->tx_onestep_tstamp);
865 } else if (unlikely(do_twostep_tstamp)) {
866 enetc_tstamp_tx(skb, tstamp);
867 do_twostep_tstamp = false;
868 }
869 napi_consume_skb(skb, napi_budget);
870 }
871
872 tx_byte_cnt += tx_swbd->len;
873 /* Scrub the swbd here so we don't have to do that
874 * when we reuse it during xmit
875 */
876 memset(tx_swbd, 0, sizeof(*tx_swbd));
877
878 bds_to_clean--;
879 tx_swbd++;
880 i++;
881 if (unlikely(i == tx_ring->bd_count)) {
882 i = 0;
883 tx_swbd = tx_ring->tx_swbd;
884 }
885
886 /* BD iteration loop end */
887 if (is_eof) {
888 tx_frm_cnt++;
889 /* re-arm interrupt source */
890 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
891 BIT(16 + tx_ring->index));
892 }
893
894 if (unlikely(!bds_to_clean))
895 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
896 }
897
898 tx_ring->next_to_clean = i;
899 tx_ring->stats.packets += tx_frm_cnt;
900 tx_ring->stats.bytes += tx_byte_cnt;
901 tx_ring->stats.win_drop += tx_win_drop;
902
903 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
904 __netif_subqueue_stopped(ndev, tx_ring->index) &&
905 !test_bit(ENETC_TX_DOWN, &priv->flags) &&
906 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
907 netif_wake_subqueue(ndev, tx_ring->index);
908 }
909
910 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
911 }
912
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)913 static bool enetc_new_page(struct enetc_bdr *rx_ring,
914 struct enetc_rx_swbd *rx_swbd)
915 {
916 bool xdp = !!(rx_ring->xdp.prog);
917 struct page *page;
918 dma_addr_t addr;
919
920 page = dev_alloc_page();
921 if (unlikely(!page))
922 return false;
923
924 /* For XDP_TX, we forgo dma_unmap -> dma_map */
925 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
926
927 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
928 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
929 __free_page(page);
930
931 return false;
932 }
933
934 rx_swbd->dma = addr;
935 rx_swbd->page = page;
936 rx_swbd->page_offset = rx_ring->buffer_offset;
937
938 return true;
939 }
940
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)941 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
942 {
943 struct enetc_rx_swbd *rx_swbd;
944 union enetc_rx_bd *rxbd;
945 int i, j;
946
947 i = rx_ring->next_to_use;
948 rx_swbd = &rx_ring->rx_swbd[i];
949 rxbd = enetc_rxbd(rx_ring, i);
950
951 for (j = 0; j < buff_cnt; j++) {
952 /* try reuse page */
953 if (unlikely(!rx_swbd->page)) {
954 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
955 rx_ring->stats.rx_alloc_errs++;
956 break;
957 }
958 }
959
960 /* update RxBD */
961 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
962 rx_swbd->page_offset);
963 /* clear 'R" as well */
964 rxbd->r.lstatus = 0;
965
966 enetc_rxbd_next(rx_ring, &rxbd, &i);
967 rx_swbd = &rx_ring->rx_swbd[i];
968 }
969
970 if (likely(j)) {
971 rx_ring->next_to_alloc = i; /* keep track from page reuse */
972 rx_ring->next_to_use = i;
973
974 /* update ENETC's consumer index */
975 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
976 }
977
978 return j;
979 }
980
981 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)982 static void enetc_get_rx_tstamp(struct net_device *ndev,
983 union enetc_rx_bd *rxbd,
984 struct sk_buff *skb)
985 {
986 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
987 struct enetc_ndev_priv *priv = netdev_priv(ndev);
988 struct enetc_hw *hw = &priv->si->hw;
989 u32 lo, hi, tstamp_lo;
990 u64 tstamp;
991
992 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
993 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
994 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
995 rxbd = enetc_rxbd_ext(rxbd);
996 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
997 if (lo <= tstamp_lo)
998 hi -= 1;
999
1000 tstamp = (u64)hi << 32 | tstamp_lo;
1001 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1002 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1003 }
1004 }
1005 #endif
1006
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)1007 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1008 union enetc_rx_bd *rxbd, struct sk_buff *skb)
1009 {
1010 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1011
1012 /* TODO: hashing */
1013 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1014 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1015
1016 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1017 skb->ip_summed = CHECKSUM_COMPLETE;
1018 }
1019
1020 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1021 __be16 tpid = 0;
1022
1023 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1024 case 0:
1025 tpid = htons(ETH_P_8021Q);
1026 break;
1027 case 1:
1028 tpid = htons(ETH_P_8021AD);
1029 break;
1030 case 2:
1031 tpid = htons(enetc_port_rd(&priv->si->hw,
1032 ENETC_PCVLANR1));
1033 break;
1034 case 3:
1035 tpid = htons(enetc_port_rd(&priv->si->hw,
1036 ENETC_PCVLANR2));
1037 break;
1038 default:
1039 break;
1040 }
1041
1042 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1043 }
1044
1045 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1046 if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1047 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1048 #endif
1049 }
1050
1051 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1052 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1053 * mapped buffers.
1054 */
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)1055 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1056 int i, u16 size)
1057 {
1058 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1059
1060 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1061 rx_swbd->page_offset,
1062 size, rx_swbd->dir);
1063 return rx_swbd;
1064 }
1065
1066 /* Reuse the current page without performing half-page buffer flipping */
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1067 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1068 struct enetc_rx_swbd *rx_swbd)
1069 {
1070 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1071
1072 enetc_reuse_page(rx_ring, rx_swbd);
1073
1074 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1075 rx_swbd->page_offset,
1076 buffer_size, rx_swbd->dir);
1077
1078 rx_swbd->page = NULL;
1079 }
1080
1081 /* Reuse the current page by performing half-page buffer flipping */
enetc_flip_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1082 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1083 struct enetc_rx_swbd *rx_swbd)
1084 {
1085 if (likely(enetc_page_reusable(rx_swbd->page))) {
1086 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1087 page_ref_inc(rx_swbd->page);
1088
1089 enetc_put_rx_buff(rx_ring, rx_swbd);
1090 } else {
1091 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1092 rx_swbd->dir);
1093 rx_swbd->page = NULL;
1094 }
1095 }
1096
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)1097 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1098 int i, u16 size)
1099 {
1100 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1101 struct sk_buff *skb;
1102 void *ba;
1103
1104 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1105 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1106 if (unlikely(!skb)) {
1107 rx_ring->stats.rx_alloc_errs++;
1108 return NULL;
1109 }
1110
1111 skb_reserve(skb, rx_ring->buffer_offset);
1112 __skb_put(skb, size);
1113
1114 enetc_flip_rx_buff(rx_ring, rx_swbd);
1115
1116 return skb;
1117 }
1118
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)1119 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1120 u16 size, struct sk_buff *skb)
1121 {
1122 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1123
1124 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1125 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1126
1127 enetc_flip_rx_buff(rx_ring, rx_swbd);
1128 }
1129
enetc_check_bd_errors_and_consume(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i)1130 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1131 u32 bd_status,
1132 union enetc_rx_bd **rxbd, int *i)
1133 {
1134 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1135 return false;
1136
1137 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1138 enetc_rxbd_next(rx_ring, rxbd, i);
1139
1140 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1141 dma_rmb();
1142 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1143
1144 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1145 enetc_rxbd_next(rx_ring, rxbd, i);
1146 }
1147
1148 rx_ring->ndev->stats.rx_dropped++;
1149 rx_ring->ndev->stats.rx_errors++;
1150
1151 return true;
1152 }
1153
enetc_build_skb(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,int buffer_size)1154 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1155 u32 bd_status, union enetc_rx_bd **rxbd,
1156 int *i, int *cleaned_cnt, int buffer_size)
1157 {
1158 struct sk_buff *skb;
1159 u16 size;
1160
1161 size = le16_to_cpu((*rxbd)->r.buf_len);
1162 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1163 if (!skb)
1164 return NULL;
1165
1166 enetc_get_offloads(rx_ring, *rxbd, skb);
1167
1168 (*cleaned_cnt)++;
1169
1170 enetc_rxbd_next(rx_ring, rxbd, i);
1171
1172 /* not last BD in frame? */
1173 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1174 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1175 size = buffer_size;
1176
1177 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1178 dma_rmb();
1179 size = le16_to_cpu((*rxbd)->r.buf_len);
1180 }
1181
1182 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1183
1184 (*cleaned_cnt)++;
1185
1186 enetc_rxbd_next(rx_ring, rxbd, i);
1187 }
1188
1189 skb_record_rx_queue(skb, rx_ring->index);
1190 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1191
1192 return skb;
1193 }
1194
1195 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1196
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)1197 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1198 struct napi_struct *napi, int work_limit)
1199 {
1200 int rx_frm_cnt = 0, rx_byte_cnt = 0;
1201 int cleaned_cnt, i;
1202
1203 cleaned_cnt = enetc_bd_unused(rx_ring);
1204 /* next descriptor to process */
1205 i = rx_ring->next_to_clean;
1206
1207 while (likely(rx_frm_cnt < work_limit)) {
1208 union enetc_rx_bd *rxbd;
1209 struct sk_buff *skb;
1210 u32 bd_status;
1211
1212 if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1213 cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1214 cleaned_cnt);
1215
1216 rxbd = enetc_rxbd(rx_ring, i);
1217 bd_status = le32_to_cpu(rxbd->r.lstatus);
1218 if (!bd_status)
1219 break;
1220
1221 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1222 dma_rmb(); /* for reading other rxbd fields */
1223
1224 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1225 &rxbd, &i))
1226 break;
1227
1228 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1229 &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1230 if (!skb)
1231 break;
1232
1233 /* When set, the outer VLAN header is extracted and reported
1234 * in the receive buffer descriptor. So rx_byte_cnt should
1235 * add the length of the extracted VLAN header.
1236 */
1237 if (bd_status & ENETC_RXBD_FLAG_VLAN)
1238 rx_byte_cnt += VLAN_HLEN;
1239 rx_byte_cnt += skb->len + ETH_HLEN;
1240 rx_frm_cnt++;
1241
1242 napi_gro_receive(napi, skb);
1243 }
1244
1245 rx_ring->next_to_clean = i;
1246
1247 rx_ring->stats.packets += rx_frm_cnt;
1248 rx_ring->stats.bytes += rx_byte_cnt;
1249
1250 return rx_frm_cnt;
1251 }
1252
enetc_xdp_map_tx_buff(struct enetc_bdr * tx_ring,int i,struct enetc_tx_swbd * tx_swbd,int frm_len)1253 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1254 struct enetc_tx_swbd *tx_swbd,
1255 int frm_len)
1256 {
1257 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1258
1259 prefetchw(txbd);
1260
1261 enetc_clear_tx_bd(txbd);
1262 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1263 txbd->buf_len = cpu_to_le16(tx_swbd->len);
1264 txbd->frm_len = cpu_to_le16(frm_len);
1265
1266 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1267 }
1268
1269 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1270 * descriptors.
1271 */
enetc_xdp_tx(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,int num_tx_swbd)1272 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1273 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1274 {
1275 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1276 int i, k, frm_len = tmp_tx_swbd->len;
1277
1278 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1279 return false;
1280
1281 while (unlikely(!tmp_tx_swbd->is_eof)) {
1282 tmp_tx_swbd++;
1283 frm_len += tmp_tx_swbd->len;
1284 }
1285
1286 i = tx_ring->next_to_use;
1287
1288 for (k = 0; k < num_tx_swbd; k++) {
1289 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1290
1291 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1292
1293 /* last BD needs 'F' bit set */
1294 if (xdp_tx_swbd->is_eof) {
1295 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1296
1297 txbd->flags = ENETC_TXBD_FLAGS_F;
1298 }
1299
1300 enetc_bdr_idx_inc(tx_ring, &i);
1301 }
1302
1303 tx_ring->next_to_use = i;
1304
1305 return true;
1306 }
1307
enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,struct xdp_frame * xdp_frame)1308 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1309 struct enetc_tx_swbd *xdp_tx_arr,
1310 struct xdp_frame *xdp_frame)
1311 {
1312 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1313 struct skb_shared_info *shinfo;
1314 void *data = xdp_frame->data;
1315 int len = xdp_frame->len;
1316 skb_frag_t *frag;
1317 dma_addr_t dma;
1318 unsigned int f;
1319 int n = 0;
1320
1321 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1322 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1323 netdev_err(tx_ring->ndev, "DMA map error\n");
1324 return -1;
1325 }
1326
1327 xdp_tx_swbd->dma = dma;
1328 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1329 xdp_tx_swbd->len = len;
1330 xdp_tx_swbd->is_xdp_redirect = true;
1331 xdp_tx_swbd->is_eof = false;
1332 xdp_tx_swbd->xdp_frame = NULL;
1333
1334 n++;
1335
1336 if (!xdp_frame_has_frags(xdp_frame))
1337 goto out;
1338
1339 xdp_tx_swbd = &xdp_tx_arr[n];
1340
1341 shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1342
1343 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1344 f++, frag++) {
1345 data = skb_frag_address(frag);
1346 len = skb_frag_size(frag);
1347
1348 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1349 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1350 /* Undo the DMA mapping for all fragments */
1351 while (--n >= 0)
1352 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1353
1354 netdev_err(tx_ring->ndev, "DMA map error\n");
1355 return -1;
1356 }
1357
1358 xdp_tx_swbd->dma = dma;
1359 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1360 xdp_tx_swbd->len = len;
1361 xdp_tx_swbd->is_xdp_redirect = true;
1362 xdp_tx_swbd->is_eof = false;
1363 xdp_tx_swbd->xdp_frame = NULL;
1364
1365 n++;
1366 xdp_tx_swbd = &xdp_tx_arr[n];
1367 }
1368 out:
1369 xdp_tx_arr[n - 1].is_eof = true;
1370 xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1371
1372 return n;
1373 }
1374
enetc_xdp_xmit(struct net_device * ndev,int num_frames,struct xdp_frame ** frames,u32 flags)1375 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1376 struct xdp_frame **frames, u32 flags)
1377 {
1378 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1379 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1380 struct enetc_bdr *tx_ring;
1381 int xdp_tx_bd_cnt, i, k;
1382 int xdp_tx_frm_cnt = 0;
1383
1384 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags)))
1385 return -ENETDOWN;
1386
1387 enetc_lock_mdio();
1388
1389 tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1390
1391 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1392
1393 for (k = 0; k < num_frames; k++) {
1394 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1395 xdp_redirect_arr,
1396 frames[k]);
1397 if (unlikely(xdp_tx_bd_cnt < 0))
1398 break;
1399
1400 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1401 xdp_tx_bd_cnt))) {
1402 for (i = 0; i < xdp_tx_bd_cnt; i++)
1403 enetc_unmap_tx_buff(tx_ring,
1404 &xdp_redirect_arr[i]);
1405 tx_ring->stats.xdp_tx_drops++;
1406 break;
1407 }
1408
1409 xdp_tx_frm_cnt++;
1410 }
1411
1412 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1413 enetc_update_tx_ring_tail(tx_ring);
1414
1415 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1416
1417 enetc_unlock_mdio();
1418
1419 return xdp_tx_frm_cnt;
1420 }
1421 EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
1422
enetc_map_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,struct xdp_buff * xdp_buff,u16 size)1423 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1424 struct xdp_buff *xdp_buff, u16 size)
1425 {
1426 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1427 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1428
1429 /* To be used for XDP_TX */
1430 rx_swbd->len = size;
1431
1432 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1433 rx_ring->buffer_offset, size, false);
1434 }
1435
enetc_add_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,u16 size,struct xdp_buff * xdp_buff)1436 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1437 u16 size, struct xdp_buff *xdp_buff)
1438 {
1439 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1440 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1441 skb_frag_t *frag;
1442
1443 /* To be used for XDP_TX */
1444 rx_swbd->len = size;
1445
1446 if (!xdp_buff_has_frags(xdp_buff)) {
1447 xdp_buff_set_frags_flag(xdp_buff);
1448 shinfo->xdp_frags_size = size;
1449 shinfo->nr_frags = 0;
1450 } else {
1451 shinfo->xdp_frags_size += size;
1452 }
1453
1454 if (page_is_pfmemalloc(rx_swbd->page))
1455 xdp_buff_set_frag_pfmemalloc(xdp_buff);
1456
1457 frag = &shinfo->frags[shinfo->nr_frags];
1458 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1459 size);
1460
1461 shinfo->nr_frags++;
1462 }
1463
enetc_build_xdp_buff(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,struct xdp_buff * xdp_buff)1464 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1465 union enetc_rx_bd **rxbd, int *i,
1466 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1467 {
1468 u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1469
1470 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1471
1472 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1473 (*cleaned_cnt)++;
1474 enetc_rxbd_next(rx_ring, rxbd, i);
1475
1476 /* not last BD in frame? */
1477 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1478 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1479 size = ENETC_RXB_DMA_SIZE_XDP;
1480
1481 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1482 dma_rmb();
1483 size = le16_to_cpu((*rxbd)->r.buf_len);
1484 }
1485
1486 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1487 (*cleaned_cnt)++;
1488 enetc_rxbd_next(rx_ring, rxbd, i);
1489 }
1490 }
1491
1492 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1493 * recycled back into the RX ring in enetc_clean_tx_ring.
1494 */
enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd * xdp_tx_arr,struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1495 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1496 struct enetc_bdr *rx_ring,
1497 int rx_ring_first, int rx_ring_last)
1498 {
1499 int n = 0;
1500
1501 for (; rx_ring_first != rx_ring_last;
1502 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1503 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1504 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1505
1506 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1507 tx_swbd->dma = rx_swbd->dma;
1508 tx_swbd->dir = rx_swbd->dir;
1509 tx_swbd->page = rx_swbd->page;
1510 tx_swbd->page_offset = rx_swbd->page_offset;
1511 tx_swbd->len = rx_swbd->len;
1512 tx_swbd->is_dma_page = true;
1513 tx_swbd->is_xdp_tx = true;
1514 tx_swbd->is_eof = false;
1515 }
1516
1517 /* We rely on caller providing an rx_ring_last > rx_ring_first */
1518 xdp_tx_arr[n - 1].is_eof = true;
1519
1520 return n;
1521 }
1522
enetc_xdp_drop(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1523 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1524 int rx_ring_last)
1525 {
1526 while (rx_ring_first != rx_ring_last) {
1527 enetc_put_rx_buff(rx_ring,
1528 &rx_ring->rx_swbd[rx_ring_first]);
1529 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1530 }
1531 }
1532
enetc_clean_rx_ring_xdp(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit,struct bpf_prog * prog)1533 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1534 struct napi_struct *napi, int work_limit,
1535 struct bpf_prog *prog)
1536 {
1537 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1538 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1539 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1540 int rx_frm_cnt = 0, rx_byte_cnt = 0;
1541 struct enetc_bdr *tx_ring;
1542 int cleaned_cnt, i;
1543 u32 xdp_act;
1544
1545 cleaned_cnt = enetc_bd_unused(rx_ring);
1546 /* next descriptor to process */
1547 i = rx_ring->next_to_clean;
1548
1549 while (likely(rx_frm_cnt < work_limit)) {
1550 union enetc_rx_bd *rxbd, *orig_rxbd;
1551 int orig_i, orig_cleaned_cnt;
1552 struct xdp_buff xdp_buff;
1553 struct sk_buff *skb;
1554 u32 bd_status;
1555 int err;
1556
1557 rxbd = enetc_rxbd(rx_ring, i);
1558 bd_status = le32_to_cpu(rxbd->r.lstatus);
1559 if (!bd_status)
1560 break;
1561
1562 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1563 dma_rmb(); /* for reading other rxbd fields */
1564
1565 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1566 &rxbd, &i))
1567 break;
1568
1569 orig_rxbd = rxbd;
1570 orig_cleaned_cnt = cleaned_cnt;
1571 orig_i = i;
1572
1573 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1574 &cleaned_cnt, &xdp_buff);
1575
1576 /* When set, the outer VLAN header is extracted and reported
1577 * in the receive buffer descriptor. So rx_byte_cnt should
1578 * add the length of the extracted VLAN header.
1579 */
1580 if (bd_status & ENETC_RXBD_FLAG_VLAN)
1581 rx_byte_cnt += VLAN_HLEN;
1582 rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1583
1584 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1585
1586 switch (xdp_act) {
1587 default:
1588 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1589 fallthrough;
1590 case XDP_ABORTED:
1591 trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1592 fallthrough;
1593 case XDP_DROP:
1594 enetc_xdp_drop(rx_ring, orig_i, i);
1595 rx_ring->stats.xdp_drops++;
1596 break;
1597 case XDP_PASS:
1598 rxbd = orig_rxbd;
1599 cleaned_cnt = orig_cleaned_cnt;
1600 i = orig_i;
1601
1602 skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1603 &i, &cleaned_cnt,
1604 ENETC_RXB_DMA_SIZE_XDP);
1605 if (unlikely(!skb))
1606 goto out;
1607
1608 napi_gro_receive(napi, skb);
1609 break;
1610 case XDP_TX:
1611 tx_ring = priv->xdp_tx_ring[rx_ring->index];
1612 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) {
1613 enetc_xdp_drop(rx_ring, orig_i, i);
1614 tx_ring->stats.xdp_tx_drops++;
1615 break;
1616 }
1617
1618 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1619 rx_ring,
1620 orig_i, i);
1621
1622 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1623 enetc_xdp_drop(rx_ring, orig_i, i);
1624 tx_ring->stats.xdp_tx_drops++;
1625 } else {
1626 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1627 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1628 xdp_tx_frm_cnt++;
1629 /* The XDP_TX enqueue was successful, so we
1630 * need to scrub the RX software BDs because
1631 * the ownership of the buffers no longer
1632 * belongs to the RX ring, and we must prevent
1633 * enetc_refill_rx_ring() from reusing
1634 * rx_swbd->page.
1635 */
1636 while (orig_i != i) {
1637 rx_ring->rx_swbd[orig_i].page = NULL;
1638 enetc_bdr_idx_inc(rx_ring, &orig_i);
1639 }
1640 }
1641 break;
1642 case XDP_REDIRECT:
1643 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1644 if (unlikely(err)) {
1645 enetc_xdp_drop(rx_ring, orig_i, i);
1646 rx_ring->stats.xdp_redirect_failures++;
1647 } else {
1648 while (orig_i != i) {
1649 enetc_flip_rx_buff(rx_ring,
1650 &rx_ring->rx_swbd[orig_i]);
1651 enetc_bdr_idx_inc(rx_ring, &orig_i);
1652 }
1653 xdp_redirect_frm_cnt++;
1654 rx_ring->stats.xdp_redirect++;
1655 }
1656 }
1657
1658 rx_frm_cnt++;
1659 }
1660
1661 out:
1662 rx_ring->next_to_clean = i;
1663
1664 rx_ring->stats.packets += rx_frm_cnt;
1665 rx_ring->stats.bytes += rx_byte_cnt;
1666
1667 if (xdp_redirect_frm_cnt)
1668 xdp_do_flush_map();
1669
1670 if (xdp_tx_frm_cnt)
1671 enetc_update_tx_ring_tail(tx_ring);
1672
1673 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1674 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1675 rx_ring->xdp.xdp_tx_in_flight);
1676
1677 return rx_frm_cnt;
1678 }
1679
enetc_poll(struct napi_struct * napi,int budget)1680 static int enetc_poll(struct napi_struct *napi, int budget)
1681 {
1682 struct enetc_int_vector
1683 *v = container_of(napi, struct enetc_int_vector, napi);
1684 struct enetc_bdr *rx_ring = &v->rx_ring;
1685 struct bpf_prog *prog;
1686 bool complete = true;
1687 int work_done;
1688 int i;
1689
1690 enetc_lock_mdio();
1691
1692 for (i = 0; i < v->count_tx_rings; i++)
1693 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1694 complete = false;
1695
1696 prog = rx_ring->xdp.prog;
1697 if (prog)
1698 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1699 else
1700 work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1701 if (work_done == budget)
1702 complete = false;
1703 if (work_done)
1704 v->rx_napi_work = true;
1705
1706 if (!complete) {
1707 enetc_unlock_mdio();
1708 return budget;
1709 }
1710
1711 napi_complete_done(napi, work_done);
1712
1713 if (likely(v->rx_dim_en))
1714 enetc_rx_net_dim(v);
1715
1716 v->rx_napi_work = false;
1717
1718 /* enable interrupts */
1719 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1720
1721 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1722 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1723 ENETC_TBIER_TXTIE);
1724
1725 enetc_unlock_mdio();
1726
1727 return work_done;
1728 }
1729
1730 /* Probing and Init */
1731 #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)1732 void enetc_get_si_caps(struct enetc_si *si)
1733 {
1734 struct enetc_hw *hw = &si->hw;
1735 u32 val;
1736
1737 /* find out how many of various resources we have to work with */
1738 val = enetc_rd(hw, ENETC_SICAPR0);
1739 si->num_rx_rings = (val >> 16) & 0xff;
1740 si->num_tx_rings = val & 0xff;
1741
1742 val = enetc_rd(hw, ENETC_SIRFSCAPR);
1743 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1744 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1745
1746 si->num_rss = 0;
1747 val = enetc_rd(hw, ENETC_SIPCAPR0);
1748 if (val & ENETC_SIPCAPR0_RSS) {
1749 u32 rss;
1750
1751 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1752 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1753 }
1754
1755 if (val & ENETC_SIPCAPR0_QBV)
1756 si->hw_features |= ENETC_SI_F_QBV;
1757
1758 if (val & ENETC_SIPCAPR0_QBU)
1759 si->hw_features |= ENETC_SI_F_QBU;
1760
1761 if (val & ENETC_SIPCAPR0_PSFP)
1762 si->hw_features |= ENETC_SI_F_PSFP;
1763 }
1764 EXPORT_SYMBOL_GPL(enetc_get_si_caps);
1765
enetc_dma_alloc_bdr(struct enetc_bdr_resource * res)1766 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
1767 {
1768 size_t bd_base_size = res->bd_count * res->bd_size;
1769
1770 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
1771 &res->bd_dma_base, GFP_KERNEL);
1772 if (!res->bd_base)
1773 return -ENOMEM;
1774
1775 /* h/w requires 128B alignment */
1776 if (!IS_ALIGNED(res->bd_dma_base, 128)) {
1777 dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1778 res->bd_dma_base);
1779 return -EINVAL;
1780 }
1781
1782 return 0;
1783 }
1784
enetc_dma_free_bdr(const struct enetc_bdr_resource * res)1785 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
1786 {
1787 size_t bd_base_size = res->bd_count * res->bd_size;
1788
1789 dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1790 res->bd_dma_base);
1791 }
1792
enetc_alloc_tx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count)1793 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
1794 struct device *dev, size_t bd_count)
1795 {
1796 int err;
1797
1798 res->dev = dev;
1799 res->bd_count = bd_count;
1800 res->bd_size = sizeof(union enetc_tx_bd);
1801
1802 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
1803 if (!res->tx_swbd)
1804 return -ENOMEM;
1805
1806 err = enetc_dma_alloc_bdr(res);
1807 if (err)
1808 goto err_alloc_bdr;
1809
1810 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
1811 &res->tso_headers_dma,
1812 GFP_KERNEL);
1813 if (!res->tso_headers) {
1814 err = -ENOMEM;
1815 goto err_alloc_tso;
1816 }
1817
1818 return 0;
1819
1820 err_alloc_tso:
1821 enetc_dma_free_bdr(res);
1822 err_alloc_bdr:
1823 vfree(res->tx_swbd);
1824 res->tx_swbd = NULL;
1825
1826 return err;
1827 }
1828
enetc_free_tx_resource(const struct enetc_bdr_resource * res)1829 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
1830 {
1831 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
1832 res->tso_headers, res->tso_headers_dma);
1833 enetc_dma_free_bdr(res);
1834 vfree(res->tx_swbd);
1835 }
1836
1837 static struct enetc_bdr_resource *
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)1838 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1839 {
1840 struct enetc_bdr_resource *tx_res;
1841 int i, err;
1842
1843 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
1844 if (!tx_res)
1845 return ERR_PTR(-ENOMEM);
1846
1847 for (i = 0; i < priv->num_tx_rings; i++) {
1848 struct enetc_bdr *tx_ring = priv->tx_ring[i];
1849
1850 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
1851 tx_ring->bd_count);
1852 if (err)
1853 goto fail;
1854 }
1855
1856 return tx_res;
1857
1858 fail:
1859 while (i-- > 0)
1860 enetc_free_tx_resource(&tx_res[i]);
1861
1862 kfree(tx_res);
1863
1864 return ERR_PTR(err);
1865 }
1866
enetc_free_tx_resources(const struct enetc_bdr_resource * tx_res,size_t num_resources)1867 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
1868 size_t num_resources)
1869 {
1870 size_t i;
1871
1872 for (i = 0; i < num_resources; i++)
1873 enetc_free_tx_resource(&tx_res[i]);
1874
1875 kfree(tx_res);
1876 }
1877
enetc_alloc_rx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count,bool extended)1878 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
1879 struct device *dev, size_t bd_count,
1880 bool extended)
1881 {
1882 int err;
1883
1884 res->dev = dev;
1885 res->bd_count = bd_count;
1886 res->bd_size = sizeof(union enetc_rx_bd);
1887 if (extended)
1888 res->bd_size *= 2;
1889
1890 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
1891 if (!res->rx_swbd)
1892 return -ENOMEM;
1893
1894 err = enetc_dma_alloc_bdr(res);
1895 if (err) {
1896 vfree(res->rx_swbd);
1897 return err;
1898 }
1899
1900 return 0;
1901 }
1902
enetc_free_rx_resource(const struct enetc_bdr_resource * res)1903 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
1904 {
1905 enetc_dma_free_bdr(res);
1906 vfree(res->rx_swbd);
1907 }
1908
1909 static struct enetc_bdr_resource *
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv,bool extended)1910 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
1911 {
1912 struct enetc_bdr_resource *rx_res;
1913 int i, err;
1914
1915 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
1916 if (!rx_res)
1917 return ERR_PTR(-ENOMEM);
1918
1919 for (i = 0; i < priv->num_rx_rings; i++) {
1920 struct enetc_bdr *rx_ring = priv->rx_ring[i];
1921
1922 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
1923 rx_ring->bd_count, extended);
1924 if (err)
1925 goto fail;
1926 }
1927
1928 return rx_res;
1929
1930 fail:
1931 while (i-- > 0)
1932 enetc_free_rx_resource(&rx_res[i]);
1933
1934 kfree(rx_res);
1935
1936 return ERR_PTR(err);
1937 }
1938
enetc_free_rx_resources(const struct enetc_bdr_resource * rx_res,size_t num_resources)1939 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
1940 size_t num_resources)
1941 {
1942 size_t i;
1943
1944 for (i = 0; i < num_resources; i++)
1945 enetc_free_rx_resource(&rx_res[i]);
1946
1947 kfree(rx_res);
1948 }
1949
enetc_assign_tx_resource(struct enetc_bdr * tx_ring,const struct enetc_bdr_resource * res)1950 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
1951 const struct enetc_bdr_resource *res)
1952 {
1953 tx_ring->bd_base = res ? res->bd_base : NULL;
1954 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1955 tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
1956 tx_ring->tso_headers = res ? res->tso_headers : NULL;
1957 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
1958 }
1959
enetc_assign_rx_resource(struct enetc_bdr * rx_ring,const struct enetc_bdr_resource * res)1960 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
1961 const struct enetc_bdr_resource *res)
1962 {
1963 rx_ring->bd_base = res ? res->bd_base : NULL;
1964 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1965 rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
1966 }
1967
enetc_assign_tx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)1968 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
1969 const struct enetc_bdr_resource *res)
1970 {
1971 int i;
1972
1973 if (priv->tx_res)
1974 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
1975
1976 for (i = 0; i < priv->num_tx_rings; i++) {
1977 enetc_assign_tx_resource(priv->tx_ring[i],
1978 res ? &res[i] : NULL);
1979 }
1980
1981 priv->tx_res = res;
1982 }
1983
enetc_assign_rx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)1984 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
1985 const struct enetc_bdr_resource *res)
1986 {
1987 int i;
1988
1989 if (priv->rx_res)
1990 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
1991
1992 for (i = 0; i < priv->num_rx_rings; i++) {
1993 enetc_assign_rx_resource(priv->rx_ring[i],
1994 res ? &res[i] : NULL);
1995 }
1996
1997 priv->rx_res = res;
1998 }
1999
enetc_free_tx_ring(struct enetc_bdr * tx_ring)2000 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
2001 {
2002 int i;
2003
2004 for (i = 0; i < tx_ring->bd_count; i++) {
2005 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
2006
2007 enetc_free_tx_frame(tx_ring, tx_swbd);
2008 }
2009 }
2010
enetc_free_rx_ring(struct enetc_bdr * rx_ring)2011 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2012 {
2013 int i;
2014
2015 for (i = 0; i < rx_ring->bd_count; i++) {
2016 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2017
2018 if (!rx_swbd->page)
2019 continue;
2020
2021 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
2022 rx_swbd->dir);
2023 __free_page(rx_swbd->page);
2024 rx_swbd->page = NULL;
2025 }
2026 }
2027
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)2028 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2029 {
2030 int i;
2031
2032 for (i = 0; i < priv->num_rx_rings; i++)
2033 enetc_free_rx_ring(priv->rx_ring[i]);
2034
2035 for (i = 0; i < priv->num_tx_rings; i++)
2036 enetc_free_tx_ring(priv->tx_ring[i]);
2037 }
2038
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)2039 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2040 {
2041 int *rss_table;
2042 int i;
2043
2044 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2045 if (!rss_table)
2046 return -ENOMEM;
2047
2048 /* Set up RSS table defaults */
2049 for (i = 0; i < si->num_rss; i++)
2050 rss_table[i] = i % num_groups;
2051
2052 enetc_set_rss_table(si, rss_table, si->num_rss);
2053
2054 kfree(rss_table);
2055
2056 return 0;
2057 }
2058
enetc_configure_si(struct enetc_ndev_priv * priv)2059 int enetc_configure_si(struct enetc_ndev_priv *priv)
2060 {
2061 struct enetc_si *si = priv->si;
2062 struct enetc_hw *hw = &si->hw;
2063 int err;
2064
2065 /* set SI cache attributes */
2066 enetc_wr(hw, ENETC_SICAR0,
2067 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2068 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2069 /* enable SI */
2070 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2071
2072 if (si->num_rss) {
2073 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2074 if (err)
2075 return err;
2076 }
2077
2078 return 0;
2079 }
2080 EXPORT_SYMBOL_GPL(enetc_configure_si);
2081
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)2082 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2083 {
2084 struct enetc_si *si = priv->si;
2085 int cpus = num_online_cpus();
2086
2087 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2088 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2089
2090 /* Enable all available TX rings in order to configure as many
2091 * priorities as possible, when needed.
2092 * TODO: Make # of TX rings run-time configurable
2093 */
2094 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2095 priv->num_tx_rings = si->num_tx_rings;
2096 priv->bdr_int_num = cpus;
2097 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2098 priv->tx_ictt = ENETC_TXIC_TIMETHR;
2099 }
2100 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2101
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)2102 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2103 {
2104 struct enetc_si *si = priv->si;
2105
2106 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2107 GFP_KERNEL);
2108 if (!priv->cls_rules)
2109 return -ENOMEM;
2110
2111 return 0;
2112 }
2113 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2114
enetc_free_si_resources(struct enetc_ndev_priv * priv)2115 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2116 {
2117 kfree(priv->cls_rules);
2118 }
2119 EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2120
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2121 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2122 {
2123 int idx = tx_ring->index;
2124 u32 tbmr;
2125
2126 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2127 lower_32_bits(tx_ring->bd_dma_base));
2128
2129 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2130 upper_32_bits(tx_ring->bd_dma_base));
2131
2132 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2133 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2134 ENETC_RTBLENR_LEN(tx_ring->bd_count));
2135
2136 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2137 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2138 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2139
2140 /* enable Tx ints by setting pkt thr to 1 */
2141 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2142
2143 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2144 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2145 tbmr |= ENETC_TBMR_VIH;
2146
2147 /* enable ring */
2148 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2149
2150 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2151 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2152 tx_ring->idr = hw->reg + ENETC_SITXIDR;
2153 }
2154
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring,bool extended)2155 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2156 bool extended)
2157 {
2158 int idx = rx_ring->index;
2159 u32 rbmr = 0;
2160
2161 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2162 lower_32_bits(rx_ring->bd_dma_base));
2163
2164 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2165 upper_32_bits(rx_ring->bd_dma_base));
2166
2167 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2168 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2169 ENETC_RTBLENR_LEN(rx_ring->bd_count));
2170
2171 if (rx_ring->xdp.prog)
2172 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2173 else
2174 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2175
2176 /* Also prepare the consumer index in case page allocation never
2177 * succeeds. In that case, hardware will never advance producer index
2178 * to match consumer index, and will drop all frames.
2179 */
2180 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2181 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2182
2183 /* enable Rx ints by setting pkt thr to 1 */
2184 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2185
2186 rx_ring->ext_en = extended;
2187 if (rx_ring->ext_en)
2188 rbmr |= ENETC_RBMR_BDS;
2189
2190 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2191 rbmr |= ENETC_RBMR_VTE;
2192
2193 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2194 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2195
2196 rx_ring->next_to_clean = 0;
2197 rx_ring->next_to_use = 0;
2198 rx_ring->next_to_alloc = 0;
2199
2200 enetc_lock_mdio();
2201 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2202 enetc_unlock_mdio();
2203
2204 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2205 }
2206
enetc_setup_bdrs(struct enetc_ndev_priv * priv,bool extended)2207 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2208 {
2209 struct enetc_hw *hw = &priv->si->hw;
2210 int i;
2211
2212 for (i = 0; i < priv->num_tx_rings; i++)
2213 enetc_setup_txbdr(hw, priv->tx_ring[i]);
2214
2215 for (i = 0; i < priv->num_rx_rings; i++)
2216 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2217 }
2218
enetc_enable_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2219 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2220 {
2221 int idx = tx_ring->index;
2222 u32 tbmr;
2223
2224 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2225 tbmr |= ENETC_TBMR_EN;
2226 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2227 }
2228
enetc_enable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2229 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2230 {
2231 int idx = rx_ring->index;
2232 u32 rbmr;
2233
2234 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2235 rbmr |= ENETC_RBMR_EN;
2236 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2237 }
2238
enetc_enable_rx_bdrs(struct enetc_ndev_priv * priv)2239 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv)
2240 {
2241 struct enetc_hw *hw = &priv->si->hw;
2242 int i;
2243
2244 for (i = 0; i < priv->num_rx_rings; i++)
2245 enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2246 }
2247
enetc_enable_tx_bdrs(struct enetc_ndev_priv * priv)2248 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv)
2249 {
2250 struct enetc_hw *hw = &priv->si->hw;
2251 int i;
2252
2253 for (i = 0; i < priv->num_tx_rings; i++)
2254 enetc_enable_txbdr(hw, priv->tx_ring[i]);
2255 }
2256
enetc_disable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2257 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2258 {
2259 int idx = rx_ring->index;
2260
2261 /* disable EN bit on ring */
2262 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2263 }
2264
enetc_disable_txbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2265 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2266 {
2267 int idx = rx_ring->index;
2268
2269 /* disable EN bit on ring */
2270 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2271 }
2272
enetc_disable_rx_bdrs(struct enetc_ndev_priv * priv)2273 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv)
2274 {
2275 struct enetc_hw *hw = &priv->si->hw;
2276 int i;
2277
2278 for (i = 0; i < priv->num_rx_rings; i++)
2279 enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2280 }
2281
enetc_disable_tx_bdrs(struct enetc_ndev_priv * priv)2282 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv)
2283 {
2284 struct enetc_hw *hw = &priv->si->hw;
2285 int i;
2286
2287 for (i = 0; i < priv->num_tx_rings; i++)
2288 enetc_disable_txbdr(hw, priv->tx_ring[i]);
2289 }
2290
enetc_wait_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2291 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2292 {
2293 int delay = 8, timeout = 100;
2294 int idx = tx_ring->index;
2295
2296 /* wait for busy to clear */
2297 while (delay < timeout &&
2298 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2299 msleep(delay);
2300 delay *= 2;
2301 }
2302
2303 if (delay >= timeout)
2304 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2305 idx);
2306 }
2307
enetc_wait_bdrs(struct enetc_ndev_priv * priv)2308 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2309 {
2310 struct enetc_hw *hw = &priv->si->hw;
2311 int i;
2312
2313 for (i = 0; i < priv->num_tx_rings; i++)
2314 enetc_wait_txbdr(hw, priv->tx_ring[i]);
2315 }
2316
enetc_setup_irqs(struct enetc_ndev_priv * priv)2317 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2318 {
2319 struct pci_dev *pdev = priv->si->pdev;
2320 struct enetc_hw *hw = &priv->si->hw;
2321 int i, j, err;
2322
2323 for (i = 0; i < priv->bdr_int_num; i++) {
2324 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2325 struct enetc_int_vector *v = priv->int_vector[i];
2326 int entry = ENETC_BDR_INT_BASE_IDX + i;
2327
2328 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2329 priv->ndev->name, i);
2330 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);
2331 if (err) {
2332 dev_err(priv->dev, "request_irq() failed!\n");
2333 goto irq_err;
2334 }
2335
2336 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2337 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2338 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2339
2340 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2341
2342 for (j = 0; j < v->count_tx_rings; j++) {
2343 int idx = v->tx_ring[j].index;
2344
2345 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2346 }
2347 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2348 }
2349
2350 return 0;
2351
2352 irq_err:
2353 while (i--) {
2354 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2355
2356 irq_set_affinity_hint(irq, NULL);
2357 free_irq(irq, priv->int_vector[i]);
2358 }
2359
2360 return err;
2361 }
2362
enetc_free_irqs(struct enetc_ndev_priv * priv)2363 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2364 {
2365 struct pci_dev *pdev = priv->si->pdev;
2366 int i;
2367
2368 for (i = 0; i < priv->bdr_int_num; i++) {
2369 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2370
2371 irq_set_affinity_hint(irq, NULL);
2372 free_irq(irq, priv->int_vector[i]);
2373 }
2374 }
2375
enetc_setup_interrupts(struct enetc_ndev_priv * priv)2376 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2377 {
2378 struct enetc_hw *hw = &priv->si->hw;
2379 u32 icpt, ictt;
2380 int i;
2381
2382 /* enable Tx & Rx event indication */
2383 if (priv->ic_mode &
2384 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2385 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2386 /* init to non-0 minimum, will be adjusted later */
2387 ictt = 0x1;
2388 } else {
2389 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2390 ictt = 0;
2391 }
2392
2393 for (i = 0; i < priv->num_rx_rings; i++) {
2394 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2395 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2396 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2397 }
2398
2399 if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2400 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2401 else
2402 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2403
2404 for (i = 0; i < priv->num_tx_rings; i++) {
2405 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2406 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2407 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2408 }
2409 }
2410
enetc_clear_interrupts(struct enetc_ndev_priv * priv)2411 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2412 {
2413 struct enetc_hw *hw = &priv->si->hw;
2414 int i;
2415
2416 for (i = 0; i < priv->num_tx_rings; i++)
2417 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2418
2419 for (i = 0; i < priv->num_rx_rings; i++)
2420 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2421 }
2422
enetc_phylink_connect(struct net_device * ndev)2423 static int enetc_phylink_connect(struct net_device *ndev)
2424 {
2425 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2426 struct ethtool_eee edata;
2427 int err;
2428
2429 if (!priv->phylink) {
2430 /* phy-less mode */
2431 netif_carrier_on(ndev);
2432 return 0;
2433 }
2434
2435 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2436 if (err) {
2437 dev_err(&ndev->dev, "could not attach to PHY\n");
2438 return err;
2439 }
2440
2441 /* disable EEE autoneg, until ENETC driver supports it */
2442 memset(&edata, 0, sizeof(struct ethtool_eee));
2443 phylink_ethtool_set_eee(priv->phylink, &edata);
2444
2445 phylink_start(priv->phylink);
2446
2447 return 0;
2448 }
2449
enetc_tx_onestep_tstamp(struct work_struct * work)2450 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2451 {
2452 struct enetc_ndev_priv *priv;
2453 struct sk_buff *skb;
2454
2455 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2456
2457 netif_tx_lock_bh(priv->ndev);
2458
2459 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2460 skb = skb_dequeue(&priv->tx_skbs);
2461 if (skb)
2462 enetc_start_xmit(skb, priv->ndev);
2463
2464 netif_tx_unlock_bh(priv->ndev);
2465 }
2466
enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv * priv)2467 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2468 {
2469 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2470 skb_queue_head_init(&priv->tx_skbs);
2471 }
2472
enetc_start(struct net_device * ndev)2473 void enetc_start(struct net_device *ndev)
2474 {
2475 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2476 int i;
2477
2478 enetc_setup_interrupts(priv);
2479
2480 for (i = 0; i < priv->bdr_int_num; i++) {
2481 int irq = pci_irq_vector(priv->si->pdev,
2482 ENETC_BDR_INT_BASE_IDX + i);
2483
2484 napi_enable(&priv->int_vector[i]->napi);
2485 enable_irq(irq);
2486 }
2487
2488 enetc_enable_tx_bdrs(priv);
2489
2490 enetc_enable_rx_bdrs(priv);
2491
2492 netif_tx_start_all_queues(ndev);
2493
2494 clear_bit(ENETC_TX_DOWN, &priv->flags);
2495 }
2496 EXPORT_SYMBOL_GPL(enetc_start);
2497
enetc_open(struct net_device * ndev)2498 int enetc_open(struct net_device *ndev)
2499 {
2500 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2501 struct enetc_bdr_resource *tx_res, *rx_res;
2502 bool extended;
2503 int err;
2504
2505 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2506
2507 err = enetc_setup_irqs(priv);
2508 if (err)
2509 return err;
2510
2511 err = enetc_phylink_connect(ndev);
2512 if (err)
2513 goto err_phy_connect;
2514
2515 tx_res = enetc_alloc_tx_resources(priv);
2516 if (IS_ERR(tx_res)) {
2517 err = PTR_ERR(tx_res);
2518 goto err_alloc_tx;
2519 }
2520
2521 rx_res = enetc_alloc_rx_resources(priv, extended);
2522 if (IS_ERR(rx_res)) {
2523 err = PTR_ERR(rx_res);
2524 goto err_alloc_rx;
2525 }
2526
2527 enetc_tx_onestep_tstamp_init(priv);
2528 enetc_assign_tx_resources(priv, tx_res);
2529 enetc_assign_rx_resources(priv, rx_res);
2530 enetc_setup_bdrs(priv, extended);
2531 enetc_start(ndev);
2532
2533 return 0;
2534
2535 err_alloc_rx:
2536 enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2537 err_alloc_tx:
2538 if (priv->phylink)
2539 phylink_disconnect_phy(priv->phylink);
2540 err_phy_connect:
2541 enetc_free_irqs(priv);
2542
2543 return err;
2544 }
2545 EXPORT_SYMBOL_GPL(enetc_open);
2546
enetc_stop(struct net_device * ndev)2547 void enetc_stop(struct net_device *ndev)
2548 {
2549 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2550 int i;
2551
2552 set_bit(ENETC_TX_DOWN, &priv->flags);
2553
2554 netif_tx_stop_all_queues(ndev);
2555
2556 enetc_disable_rx_bdrs(priv);
2557
2558 enetc_wait_bdrs(priv);
2559
2560 enetc_disable_tx_bdrs(priv);
2561
2562 for (i = 0; i < priv->bdr_int_num; i++) {
2563 int irq = pci_irq_vector(priv->si->pdev,
2564 ENETC_BDR_INT_BASE_IDX + i);
2565
2566 disable_irq(irq);
2567 napi_synchronize(&priv->int_vector[i]->napi);
2568 napi_disable(&priv->int_vector[i]->napi);
2569 }
2570
2571 enetc_clear_interrupts(priv);
2572 }
2573 EXPORT_SYMBOL_GPL(enetc_stop);
2574
enetc_close(struct net_device * ndev)2575 int enetc_close(struct net_device *ndev)
2576 {
2577 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2578
2579 enetc_stop(ndev);
2580
2581 if (priv->phylink) {
2582 phylink_stop(priv->phylink);
2583 phylink_disconnect_phy(priv->phylink);
2584 } else {
2585 netif_carrier_off(ndev);
2586 }
2587
2588 enetc_free_rxtx_rings(priv);
2589
2590 /* Avoids dangling pointers and also frees old resources */
2591 enetc_assign_rx_resources(priv, NULL);
2592 enetc_assign_tx_resources(priv, NULL);
2593
2594 enetc_free_irqs(priv);
2595
2596 return 0;
2597 }
2598 EXPORT_SYMBOL_GPL(enetc_close);
2599
enetc_reconfigure(struct enetc_ndev_priv * priv,bool extended,int (* cb)(struct enetc_ndev_priv * priv,void * ctx),void * ctx)2600 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
2601 int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
2602 void *ctx)
2603 {
2604 struct enetc_bdr_resource *tx_res, *rx_res;
2605 int err;
2606
2607 ASSERT_RTNL();
2608
2609 /* If the interface is down, run the callback right away,
2610 * without reconfiguration.
2611 */
2612 if (!netif_running(priv->ndev)) {
2613 if (cb) {
2614 err = cb(priv, ctx);
2615 if (err)
2616 return err;
2617 }
2618
2619 return 0;
2620 }
2621
2622 tx_res = enetc_alloc_tx_resources(priv);
2623 if (IS_ERR(tx_res)) {
2624 err = PTR_ERR(tx_res);
2625 goto out;
2626 }
2627
2628 rx_res = enetc_alloc_rx_resources(priv, extended);
2629 if (IS_ERR(rx_res)) {
2630 err = PTR_ERR(rx_res);
2631 goto out_free_tx_res;
2632 }
2633
2634 enetc_stop(priv->ndev);
2635 enetc_free_rxtx_rings(priv);
2636
2637 /* Interface is down, run optional callback now */
2638 if (cb) {
2639 err = cb(priv, ctx);
2640 if (err)
2641 goto out_restart;
2642 }
2643
2644 enetc_assign_tx_resources(priv, tx_res);
2645 enetc_assign_rx_resources(priv, rx_res);
2646 enetc_setup_bdrs(priv, extended);
2647 enetc_start(priv->ndev);
2648
2649 return 0;
2650
2651 out_restart:
2652 enetc_setup_bdrs(priv, extended);
2653 enetc_start(priv->ndev);
2654 enetc_free_rx_resources(rx_res, priv->num_rx_rings);
2655 out_free_tx_res:
2656 enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2657 out:
2658 return err;
2659 }
2660
enetc_debug_tx_ring_prios(struct enetc_ndev_priv * priv)2661 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
2662 {
2663 int i;
2664
2665 for (i = 0; i < priv->num_tx_rings; i++)
2666 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
2667 priv->tx_ring[i]->prio);
2668 }
2669
enetc_reset_tc_mqprio(struct net_device * ndev)2670 void enetc_reset_tc_mqprio(struct net_device *ndev)
2671 {
2672 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2673 struct enetc_hw *hw = &priv->si->hw;
2674 struct enetc_bdr *tx_ring;
2675 int num_stack_tx_queues;
2676 int i;
2677
2678 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2679
2680 netdev_reset_tc(ndev);
2681 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2682 priv->min_num_stack_tx_queues = num_possible_cpus();
2683
2684 /* Reset all ring priorities to 0 */
2685 for (i = 0; i < priv->num_tx_rings; i++) {
2686 tx_ring = priv->tx_ring[i];
2687 tx_ring->prio = 0;
2688 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2689 }
2690
2691 enetc_debug_tx_ring_prios(priv);
2692
2693 enetc_change_preemptible_tcs(priv, 0);
2694 }
2695 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
2696
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)2697 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2698 {
2699 struct tc_mqprio_qopt_offload *mqprio = type_data;
2700 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2701 struct tc_mqprio_qopt *qopt = &mqprio->qopt;
2702 struct enetc_hw *hw = &priv->si->hw;
2703 int num_stack_tx_queues = 0;
2704 struct enetc_bdr *tx_ring;
2705 u8 num_tc = qopt->num_tc;
2706 int offset, count;
2707 int err, tc, q;
2708
2709 if (!num_tc) {
2710 enetc_reset_tc_mqprio(ndev);
2711 return 0;
2712 }
2713
2714 err = netdev_set_num_tc(ndev, num_tc);
2715 if (err)
2716 return err;
2717
2718 for (tc = 0; tc < num_tc; tc++) {
2719 offset = qopt->offset[tc];
2720 count = qopt->count[tc];
2721 num_stack_tx_queues += count;
2722
2723 err = netdev_set_tc_queue(ndev, tc, count, offset);
2724 if (err)
2725 goto err_reset_tc;
2726
2727 for (q = offset; q < offset + count; q++) {
2728 tx_ring = priv->tx_ring[q];
2729 /* The prio_tc_map is skb_tx_hash()'s way of selecting
2730 * between TX queues based on skb->priority. As such,
2731 * there's nothing to offload based on it.
2732 * Make the mqprio "traffic class" be the priority of
2733 * this ring group, and leave the Tx IPV to traffic
2734 * class mapping as its default mapping value of 1:1.
2735 */
2736 tx_ring->prio = tc;
2737 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2738 }
2739 }
2740
2741 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2742 if (err)
2743 goto err_reset_tc;
2744
2745 priv->min_num_stack_tx_queues = num_stack_tx_queues;
2746
2747 enetc_debug_tx_ring_prios(priv);
2748
2749 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
2750
2751 return 0;
2752
2753 err_reset_tc:
2754 enetc_reset_tc_mqprio(ndev);
2755 return err;
2756 }
2757 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
2758
enetc_reconfigure_xdp_cb(struct enetc_ndev_priv * priv,void * ctx)2759 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
2760 {
2761 struct bpf_prog *old_prog, *prog = ctx;
2762 int num_stack_tx_queues;
2763 int err, i;
2764
2765 old_prog = xchg(&priv->xdp_prog, prog);
2766
2767 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2768 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
2769 if (err) {
2770 xchg(&priv->xdp_prog, old_prog);
2771 return err;
2772 }
2773
2774 if (old_prog)
2775 bpf_prog_put(old_prog);
2776
2777 for (i = 0; i < priv->num_rx_rings; i++) {
2778 struct enetc_bdr *rx_ring = priv->rx_ring[i];
2779
2780 rx_ring->xdp.prog = prog;
2781
2782 if (prog)
2783 rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2784 else
2785 rx_ring->buffer_offset = ENETC_RXB_PAD;
2786 }
2787
2788 return 0;
2789 }
2790
enetc_setup_xdp_prog(struct net_device * ndev,struct bpf_prog * prog,struct netlink_ext_ack * extack)2791 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
2792 struct netlink_ext_ack *extack)
2793 {
2794 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
2795 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2796 bool extended;
2797
2798 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
2799 priv->num_tx_rings) {
2800 NL_SET_ERR_MSG_FMT_MOD(extack,
2801 "Reserving %d XDP TXQs leaves under %d for stack (total %d)",
2802 num_xdp_tx_queues,
2803 priv->min_num_stack_tx_queues,
2804 priv->num_tx_rings);
2805 return -EBUSY;
2806 }
2807
2808 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2809
2810 /* The buffer layout is changing, so we need to drain the old
2811 * RX buffers and seed new ones.
2812 */
2813 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
2814 }
2815
enetc_setup_bpf(struct net_device * ndev,struct netdev_bpf * bpf)2816 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2817 {
2818 switch (bpf->command) {
2819 case XDP_SETUP_PROG:
2820 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
2821 default:
2822 return -EINVAL;
2823 }
2824
2825 return 0;
2826 }
2827 EXPORT_SYMBOL_GPL(enetc_setup_bpf);
2828
enetc_get_stats(struct net_device * ndev)2829 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2830 {
2831 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2832 struct net_device_stats *stats = &ndev->stats;
2833 unsigned long packets = 0, bytes = 0;
2834 unsigned long tx_dropped = 0;
2835 int i;
2836
2837 for (i = 0; i < priv->num_rx_rings; i++) {
2838 packets += priv->rx_ring[i]->stats.packets;
2839 bytes += priv->rx_ring[i]->stats.bytes;
2840 }
2841
2842 stats->rx_packets = packets;
2843 stats->rx_bytes = bytes;
2844 bytes = 0;
2845 packets = 0;
2846
2847 for (i = 0; i < priv->num_tx_rings; i++) {
2848 packets += priv->tx_ring[i]->stats.packets;
2849 bytes += priv->tx_ring[i]->stats.bytes;
2850 tx_dropped += priv->tx_ring[i]->stats.win_drop;
2851 }
2852
2853 stats->tx_packets = packets;
2854 stats->tx_bytes = bytes;
2855 stats->tx_dropped = tx_dropped;
2856
2857 return stats;
2858 }
2859 EXPORT_SYMBOL_GPL(enetc_get_stats);
2860
enetc_set_rss(struct net_device * ndev,int en)2861 static int enetc_set_rss(struct net_device *ndev, int en)
2862 {
2863 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2864 struct enetc_hw *hw = &priv->si->hw;
2865 u32 reg;
2866
2867 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2868
2869 reg = enetc_rd(hw, ENETC_SIMR);
2870 reg &= ~ENETC_SIMR_RSSE;
2871 reg |= (en) ? ENETC_SIMR_RSSE : 0;
2872 enetc_wr(hw, ENETC_SIMR, reg);
2873
2874 return 0;
2875 }
2876
enetc_enable_rxvlan(struct net_device * ndev,bool en)2877 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2878 {
2879 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2880 struct enetc_hw *hw = &priv->si->hw;
2881 int i;
2882
2883 for (i = 0; i < priv->num_rx_rings; i++)
2884 enetc_bdr_enable_rxvlan(hw, i, en);
2885 }
2886
enetc_enable_txvlan(struct net_device * ndev,bool en)2887 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2888 {
2889 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2890 struct enetc_hw *hw = &priv->si->hw;
2891 int i;
2892
2893 for (i = 0; i < priv->num_tx_rings; i++)
2894 enetc_bdr_enable_txvlan(hw, i, en);
2895 }
2896
enetc_set_features(struct net_device * ndev,netdev_features_t features)2897 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2898 {
2899 netdev_features_t changed = ndev->features ^ features;
2900
2901 if (changed & NETIF_F_RXHASH)
2902 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2903
2904 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2905 enetc_enable_rxvlan(ndev,
2906 !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2907
2908 if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2909 enetc_enable_txvlan(ndev,
2910 !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2911 }
2912 EXPORT_SYMBOL_GPL(enetc_set_features);
2913
2914 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)2915 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2916 {
2917 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2918 int err, new_offloads = priv->active_offloads;
2919 struct hwtstamp_config config;
2920
2921 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2922 return -EFAULT;
2923
2924 switch (config.tx_type) {
2925 case HWTSTAMP_TX_OFF:
2926 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2927 break;
2928 case HWTSTAMP_TX_ON:
2929 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2930 new_offloads |= ENETC_F_TX_TSTAMP;
2931 break;
2932 case HWTSTAMP_TX_ONESTEP_SYNC:
2933 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2934 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2935 break;
2936 default:
2937 return -ERANGE;
2938 }
2939
2940 switch (config.rx_filter) {
2941 case HWTSTAMP_FILTER_NONE:
2942 new_offloads &= ~ENETC_F_RX_TSTAMP;
2943 break;
2944 default:
2945 new_offloads |= ENETC_F_RX_TSTAMP;
2946 config.rx_filter = HWTSTAMP_FILTER_ALL;
2947 }
2948
2949 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
2950 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
2951
2952 err = enetc_reconfigure(priv, extended, NULL, NULL);
2953 if (err)
2954 return err;
2955 }
2956
2957 priv->active_offloads = new_offloads;
2958
2959 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2960 -EFAULT : 0;
2961 }
2962
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)2963 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2964 {
2965 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2966 struct hwtstamp_config config;
2967
2968 config.flags = 0;
2969
2970 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2971 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2972 else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2973 config.tx_type = HWTSTAMP_TX_ON;
2974 else
2975 config.tx_type = HWTSTAMP_TX_OFF;
2976
2977 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2978 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2979
2980 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2981 -EFAULT : 0;
2982 }
2983 #endif
2984
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)2985 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2986 {
2987 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2988 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2989 if (cmd == SIOCSHWTSTAMP)
2990 return enetc_hwtstamp_set(ndev, rq);
2991 if (cmd == SIOCGHWTSTAMP)
2992 return enetc_hwtstamp_get(ndev, rq);
2993 #endif
2994
2995 if (!priv->phylink)
2996 return -EOPNOTSUPP;
2997
2998 return phylink_mii_ioctl(priv->phylink, rq, cmd);
2999 }
3000 EXPORT_SYMBOL_GPL(enetc_ioctl);
3001
enetc_alloc_msix(struct enetc_ndev_priv * priv)3002 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
3003 {
3004 struct pci_dev *pdev = priv->si->pdev;
3005 int num_stack_tx_queues;
3006 int first_xdp_tx_ring;
3007 int i, n, err, nvec;
3008 int v_tx_rings;
3009
3010 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
3011 /* allocate MSIX for both messaging and Rx/Tx interrupts */
3012 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
3013
3014 if (n < 0)
3015 return n;
3016
3017 if (n != nvec)
3018 return -EPERM;
3019
3020 /* # of tx rings per int vector */
3021 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
3022
3023 for (i = 0; i < priv->bdr_int_num; i++) {
3024 struct enetc_int_vector *v;
3025 struct enetc_bdr *bdr;
3026 int j;
3027
3028 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
3029 if (!v) {
3030 err = -ENOMEM;
3031 goto fail;
3032 }
3033
3034 priv->int_vector[i] = v;
3035
3036 bdr = &v->rx_ring;
3037 bdr->index = i;
3038 bdr->ndev = priv->ndev;
3039 bdr->dev = priv->dev;
3040 bdr->bd_count = priv->rx_bd_count;
3041 bdr->buffer_offset = ENETC_RXB_PAD;
3042 priv->rx_ring[i] = bdr;
3043
3044 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
3045 if (err) {
3046 kfree(v);
3047 goto fail;
3048 }
3049
3050 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
3051 MEM_TYPE_PAGE_SHARED, NULL);
3052 if (err) {
3053 xdp_rxq_info_unreg(&bdr->xdp.rxq);
3054 kfree(v);
3055 goto fail;
3056 }
3057
3058 /* init defaults for adaptive IC */
3059 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3060 v->rx_ictt = 0x1;
3061 v->rx_dim_en = true;
3062 }
3063 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3064 netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3065 v->count_tx_rings = v_tx_rings;
3066
3067 for (j = 0; j < v_tx_rings; j++) {
3068 int idx;
3069
3070 /* default tx ring mapping policy */
3071 idx = priv->bdr_int_num * j + i;
3072 __set_bit(idx, &v->tx_rings_map);
3073 bdr = &v->tx_ring[j];
3074 bdr->index = idx;
3075 bdr->ndev = priv->ndev;
3076 bdr->dev = priv->dev;
3077 bdr->bd_count = priv->tx_bd_count;
3078 priv->tx_ring[idx] = bdr;
3079 }
3080 }
3081
3082 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3083
3084 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3085 if (err)
3086 goto fail;
3087
3088 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
3089 if (err)
3090 goto fail;
3091
3092 priv->min_num_stack_tx_queues = num_possible_cpus();
3093 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
3094 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
3095
3096 return 0;
3097
3098 fail:
3099 while (i--) {
3100 struct enetc_int_vector *v = priv->int_vector[i];
3101 struct enetc_bdr *rx_ring = &v->rx_ring;
3102
3103 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3104 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3105 netif_napi_del(&v->napi);
3106 cancel_work_sync(&v->rx_dim.work);
3107 kfree(v);
3108 }
3109
3110 pci_free_irq_vectors(pdev);
3111
3112 return err;
3113 }
3114 EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3115
enetc_free_msix(struct enetc_ndev_priv * priv)3116 void enetc_free_msix(struct enetc_ndev_priv *priv)
3117 {
3118 int i;
3119
3120 for (i = 0; i < priv->bdr_int_num; i++) {
3121 struct enetc_int_vector *v = priv->int_vector[i];
3122 struct enetc_bdr *rx_ring = &v->rx_ring;
3123
3124 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3125 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3126 netif_napi_del(&v->napi);
3127 cancel_work_sync(&v->rx_dim.work);
3128 }
3129
3130 for (i = 0; i < priv->num_rx_rings; i++)
3131 priv->rx_ring[i] = NULL;
3132
3133 for (i = 0; i < priv->num_tx_rings; i++)
3134 priv->tx_ring[i] = NULL;
3135
3136 for (i = 0; i < priv->bdr_int_num; i++) {
3137 kfree(priv->int_vector[i]);
3138 priv->int_vector[i] = NULL;
3139 }
3140
3141 /* disable all MSIX for this device */
3142 pci_free_irq_vectors(priv->si->pdev);
3143 }
3144 EXPORT_SYMBOL_GPL(enetc_free_msix);
3145
enetc_kfree_si(struct enetc_si * si)3146 static void enetc_kfree_si(struct enetc_si *si)
3147 {
3148 char *p = (char *)si - si->pad;
3149
3150 kfree(p);
3151 }
3152
enetc_detect_errata(struct enetc_si * si)3153 static void enetc_detect_errata(struct enetc_si *si)
3154 {
3155 if (si->pdev->revision == ENETC_REV1)
3156 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3157 }
3158
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)3159 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3160 {
3161 struct enetc_si *si, *p;
3162 struct enetc_hw *hw;
3163 size_t alloc_size;
3164 int err, len;
3165
3166 pcie_flr(pdev);
3167 err = pci_enable_device_mem(pdev);
3168 if (err)
3169 return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3170
3171 /* set up for high or low dma */
3172 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3173 if (err) {
3174 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3175 goto err_dma;
3176 }
3177
3178 err = pci_request_mem_regions(pdev, name);
3179 if (err) {
3180 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3181 goto err_pci_mem_reg;
3182 }
3183
3184 pci_set_master(pdev);
3185
3186 alloc_size = sizeof(struct enetc_si);
3187 if (sizeof_priv) {
3188 /* align priv to 32B */
3189 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3190 alloc_size += sizeof_priv;
3191 }
3192 /* force 32B alignment for enetc_si */
3193 alloc_size += ENETC_SI_ALIGN - 1;
3194
3195 p = kzalloc(alloc_size, GFP_KERNEL);
3196 if (!p) {
3197 err = -ENOMEM;
3198 goto err_alloc_si;
3199 }
3200
3201 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3202 si->pad = (char *)si - (char *)p;
3203
3204 pci_set_drvdata(pdev, si);
3205 si->pdev = pdev;
3206 hw = &si->hw;
3207
3208 len = pci_resource_len(pdev, ENETC_BAR_REGS);
3209 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3210 if (!hw->reg) {
3211 err = -ENXIO;
3212 dev_err(&pdev->dev, "ioremap() failed\n");
3213 goto err_ioremap;
3214 }
3215 if (len > ENETC_PORT_BASE)
3216 hw->port = hw->reg + ENETC_PORT_BASE;
3217 if (len > ENETC_GLOBAL_BASE)
3218 hw->global = hw->reg + ENETC_GLOBAL_BASE;
3219
3220 enetc_detect_errata(si);
3221
3222 return 0;
3223
3224 err_ioremap:
3225 enetc_kfree_si(si);
3226 err_alloc_si:
3227 pci_release_mem_regions(pdev);
3228 err_pci_mem_reg:
3229 err_dma:
3230 pci_disable_device(pdev);
3231
3232 return err;
3233 }
3234 EXPORT_SYMBOL_GPL(enetc_pci_probe);
3235
enetc_pci_remove(struct pci_dev * pdev)3236 void enetc_pci_remove(struct pci_dev *pdev)
3237 {
3238 struct enetc_si *si = pci_get_drvdata(pdev);
3239 struct enetc_hw *hw = &si->hw;
3240
3241 iounmap(hw->reg);
3242 enetc_kfree_si(si);
3243 pci_release_mem_regions(pdev);
3244 pci_disable_device(pdev);
3245 }
3246 EXPORT_SYMBOL_GPL(enetc_pci_remove);
3247
3248 MODULE_LICENSE("Dual BSD/GPL");
3249