1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3 
4 #include "enetc.h"
5 #include <linux/tcp.h>
6 #include <linux/udp.h>
7 #include <linux/of_mdio.h>
8 #include <linux/vmalloc.h>
9 
10 /* ENETC overhead: optional extension BD + 1 BD gap */
11 #define ENETC_TXBDS_NEEDED(val)	((val) + 2)
12 /* max # of chained Tx BDs is 15, including head and extension BD */
13 #define ENETC_MAX_SKB_FRAGS	13
14 #define ENETC_TXBDS_MAX_NEEDED	ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
15 
16 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
17 			      int active_offloads);
18 
19 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
20 {
21 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
22 	struct enetc_bdr *tx_ring;
23 	int count;
24 
25 	tx_ring = priv->tx_ring[skb->queue_mapping];
26 
27 	if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
28 		if (unlikely(skb_linearize(skb)))
29 			goto drop_packet_err;
30 
31 	count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
32 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
33 		netif_stop_subqueue(ndev, tx_ring->index);
34 		return NETDEV_TX_BUSY;
35 	}
36 
37 	count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
38 	if (unlikely(!count))
39 		goto drop_packet_err;
40 
41 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
42 		netif_stop_subqueue(ndev, tx_ring->index);
43 
44 	return NETDEV_TX_OK;
45 
46 drop_packet_err:
47 	dev_kfree_skb_any(skb);
48 	return NETDEV_TX_OK;
49 }
50 
51 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
52 {
53 	int l3_start, l3_hsize;
54 	u16 l3_flags, l4_flags;
55 
56 	if (skb->ip_summed != CHECKSUM_PARTIAL)
57 		return false;
58 
59 	switch (skb->csum_offset) {
60 	case offsetof(struct tcphdr, check):
61 		l4_flags = ENETC_TXBD_L4_TCP;
62 		break;
63 	case offsetof(struct udphdr, check):
64 		l4_flags = ENETC_TXBD_L4_UDP;
65 		break;
66 	default:
67 		skb_checksum_help(skb);
68 		return false;
69 	}
70 
71 	l3_start = skb_network_offset(skb);
72 	l3_hsize = skb_network_header_len(skb);
73 
74 	l3_flags = 0;
75 	if (skb->protocol == htons(ETH_P_IPV6))
76 		l3_flags = ENETC_TXBD_L3_IPV6;
77 
78 	/* write BD fields */
79 	txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
80 	txbd->l4_csoff = l4_flags;
81 
82 	return true;
83 }
84 
85 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
86 				struct enetc_tx_swbd *tx_swbd)
87 {
88 	if (tx_swbd->is_dma_page)
89 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
90 			       tx_swbd->len, DMA_TO_DEVICE);
91 	else
92 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
93 				 tx_swbd->len, DMA_TO_DEVICE);
94 	tx_swbd->dma = 0;
95 }
96 
97 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
98 			      struct enetc_tx_swbd *tx_swbd)
99 {
100 	if (tx_swbd->dma)
101 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
102 
103 	if (tx_swbd->skb) {
104 		dev_kfree_skb_any(tx_swbd->skb);
105 		tx_swbd->skb = NULL;
106 	}
107 }
108 
109 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
110 			      int active_offloads)
111 {
112 	struct enetc_tx_swbd *tx_swbd;
113 	struct skb_frag_struct *frag;
114 	int len = skb_headlen(skb);
115 	union enetc_tx_bd temp_bd;
116 	union enetc_tx_bd *txbd;
117 	bool do_vlan, do_tstamp;
118 	int i, count = 0;
119 	unsigned int f;
120 	dma_addr_t dma;
121 	u8 flags = 0;
122 
123 	i = tx_ring->next_to_use;
124 	txbd = ENETC_TXBD(*tx_ring, i);
125 	prefetchw(txbd);
126 
127 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
128 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
129 		goto dma_err;
130 
131 	temp_bd.addr = cpu_to_le64(dma);
132 	temp_bd.buf_len = cpu_to_le16(len);
133 	temp_bd.lstatus = 0;
134 
135 	tx_swbd = &tx_ring->tx_swbd[i];
136 	tx_swbd->dma = dma;
137 	tx_swbd->len = len;
138 	tx_swbd->is_dma_page = 0;
139 	count++;
140 
141 	do_vlan = skb_vlan_tag_present(skb);
142 	do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
143 		    (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
144 	tx_swbd->do_tstamp = do_tstamp;
145 	tx_swbd->check_wb = tx_swbd->do_tstamp;
146 
147 	if (do_vlan || do_tstamp)
148 		flags |= ENETC_TXBD_FLAGS_EX;
149 
150 	if (enetc_tx_csum(skb, &temp_bd))
151 		flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
152 
153 	/* first BD needs frm_len and offload flags set */
154 	temp_bd.frm_len = cpu_to_le16(skb->len);
155 	temp_bd.flags = flags;
156 
157 	if (flags & ENETC_TXBD_FLAGS_EX) {
158 		u8 e_flags = 0;
159 		*txbd = temp_bd;
160 		enetc_clear_tx_bd(&temp_bd);
161 
162 		/* add extension BD for VLAN and/or timestamping */
163 		flags = 0;
164 		tx_swbd++;
165 		txbd++;
166 		i++;
167 		if (unlikely(i == tx_ring->bd_count)) {
168 			i = 0;
169 			tx_swbd = tx_ring->tx_swbd;
170 			txbd = ENETC_TXBD(*tx_ring, 0);
171 		}
172 		prefetchw(txbd);
173 
174 		if (do_vlan) {
175 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
176 			temp_bd.ext.tpid = 0; /* < C-TAG */
177 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
178 		}
179 
180 		if (do_tstamp) {
181 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
182 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
183 		}
184 
185 		temp_bd.ext.e_flags = e_flags;
186 		count++;
187 	}
188 
189 	frag = &skb_shinfo(skb)->frags[0];
190 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
191 		len = skb_frag_size(frag);
192 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
193 				       DMA_TO_DEVICE);
194 		if (dma_mapping_error(tx_ring->dev, dma))
195 			goto dma_err;
196 
197 		*txbd = temp_bd;
198 		enetc_clear_tx_bd(&temp_bd);
199 
200 		flags = 0;
201 		tx_swbd++;
202 		txbd++;
203 		i++;
204 		if (unlikely(i == tx_ring->bd_count)) {
205 			i = 0;
206 			tx_swbd = tx_ring->tx_swbd;
207 			txbd = ENETC_TXBD(*tx_ring, 0);
208 		}
209 		prefetchw(txbd);
210 
211 		temp_bd.addr = cpu_to_le64(dma);
212 		temp_bd.buf_len = cpu_to_le16(len);
213 
214 		tx_swbd->dma = dma;
215 		tx_swbd->len = len;
216 		tx_swbd->is_dma_page = 1;
217 		count++;
218 	}
219 
220 	/* last BD needs 'F' bit set */
221 	flags |= ENETC_TXBD_FLAGS_F;
222 	temp_bd.flags = flags;
223 	*txbd = temp_bd;
224 
225 	tx_ring->tx_swbd[i].skb = skb;
226 
227 	enetc_bdr_idx_inc(tx_ring, &i);
228 	tx_ring->next_to_use = i;
229 
230 	/* let H/W know BD ring has been updated */
231 	enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */
232 
233 	return count;
234 
235 dma_err:
236 	dev_err(tx_ring->dev, "DMA map error");
237 
238 	do {
239 		tx_swbd = &tx_ring->tx_swbd[i];
240 		enetc_free_tx_skb(tx_ring, tx_swbd);
241 		if (i == 0)
242 			i = tx_ring->bd_count;
243 		i--;
244 	} while (count--);
245 
246 	return 0;
247 }
248 
249 static irqreturn_t enetc_msix(int irq, void *data)
250 {
251 	struct enetc_int_vector	*v = data;
252 	int i;
253 
254 	/* disable interrupts */
255 	enetc_wr_reg(v->rbier, 0);
256 
257 	for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings)
258 		enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
259 
260 	napi_schedule_irqoff(&v->napi);
261 
262 	return IRQ_HANDLED;
263 }
264 
265 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
266 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
267 			       struct napi_struct *napi, int work_limit);
268 
269 static int enetc_poll(struct napi_struct *napi, int budget)
270 {
271 	struct enetc_int_vector
272 		*v = container_of(napi, struct enetc_int_vector, napi);
273 	bool complete = true;
274 	int work_done;
275 	int i;
276 
277 	for (i = 0; i < v->count_tx_rings; i++)
278 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
279 			complete = false;
280 
281 	work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
282 	if (work_done == budget)
283 		complete = false;
284 
285 	if (!complete)
286 		return budget;
287 
288 	napi_complete_done(napi, work_done);
289 
290 	/* enable interrupts */
291 	enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
292 
293 	for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings)
294 		enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i),
295 			     ENETC_TBIER_TXTIE);
296 
297 	return work_done;
298 }
299 
300 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
301 {
302 	int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
303 
304 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
305 }
306 
307 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
308 				u64 *tstamp)
309 {
310 	u32 lo, hi, tstamp_lo;
311 
312 	lo = enetc_rd(hw, ENETC_SICTR0);
313 	hi = enetc_rd(hw, ENETC_SICTR1);
314 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
315 	if (lo <= tstamp_lo)
316 		hi -= 1;
317 	*tstamp = (u64)hi << 32 | tstamp_lo;
318 }
319 
320 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
321 {
322 	struct skb_shared_hwtstamps shhwtstamps;
323 
324 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
325 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
326 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
327 		skb_tstamp_tx(skb, &shhwtstamps);
328 	}
329 }
330 
331 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
332 {
333 	struct net_device *ndev = tx_ring->ndev;
334 	int tx_frm_cnt = 0, tx_byte_cnt = 0;
335 	struct enetc_tx_swbd *tx_swbd;
336 	int i, bds_to_clean;
337 	bool do_tstamp;
338 	u64 tstamp = 0;
339 
340 	i = tx_ring->next_to_clean;
341 	tx_swbd = &tx_ring->tx_swbd[i];
342 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
343 
344 	do_tstamp = false;
345 
346 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
347 		bool is_eof = !!tx_swbd->skb;
348 
349 		if (unlikely(tx_swbd->check_wb)) {
350 			struct enetc_ndev_priv *priv = netdev_priv(ndev);
351 			union enetc_tx_bd *txbd;
352 
353 			txbd = ENETC_TXBD(*tx_ring, i);
354 
355 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
356 			    tx_swbd->do_tstamp) {
357 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
358 						    &tstamp);
359 				do_tstamp = true;
360 			}
361 		}
362 
363 		if (likely(tx_swbd->dma))
364 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
365 
366 		if (is_eof) {
367 			if (unlikely(do_tstamp)) {
368 				enetc_tstamp_tx(tx_swbd->skb, tstamp);
369 				do_tstamp = false;
370 			}
371 			napi_consume_skb(tx_swbd->skb, napi_budget);
372 			tx_swbd->skb = NULL;
373 		}
374 
375 		tx_byte_cnt += tx_swbd->len;
376 
377 		bds_to_clean--;
378 		tx_swbd++;
379 		i++;
380 		if (unlikely(i == tx_ring->bd_count)) {
381 			i = 0;
382 			tx_swbd = tx_ring->tx_swbd;
383 		}
384 
385 		/* BD iteration loop end */
386 		if (is_eof) {
387 			tx_frm_cnt++;
388 			/* re-arm interrupt source */
389 			enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) |
390 				     BIT(16 + tx_ring->index));
391 		}
392 
393 		if (unlikely(!bds_to_clean))
394 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
395 	}
396 
397 	tx_ring->next_to_clean = i;
398 	tx_ring->stats.packets += tx_frm_cnt;
399 	tx_ring->stats.bytes += tx_byte_cnt;
400 
401 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
402 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
403 		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
404 		netif_wake_subqueue(ndev, tx_ring->index);
405 	}
406 
407 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
408 }
409 
410 static bool enetc_new_page(struct enetc_bdr *rx_ring,
411 			   struct enetc_rx_swbd *rx_swbd)
412 {
413 	struct page *page;
414 	dma_addr_t addr;
415 
416 	page = dev_alloc_page();
417 	if (unlikely(!page))
418 		return false;
419 
420 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
421 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
422 		__free_page(page);
423 
424 		return false;
425 	}
426 
427 	rx_swbd->dma = addr;
428 	rx_swbd->page = page;
429 	rx_swbd->page_offset = ENETC_RXB_PAD;
430 
431 	return true;
432 }
433 
434 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
435 {
436 	struct enetc_rx_swbd *rx_swbd;
437 	union enetc_rx_bd *rxbd;
438 	int i, j;
439 
440 	i = rx_ring->next_to_use;
441 	rx_swbd = &rx_ring->rx_swbd[i];
442 	rxbd = ENETC_RXBD(*rx_ring, i);
443 
444 	for (j = 0; j < buff_cnt; j++) {
445 		/* try reuse page */
446 		if (unlikely(!rx_swbd->page)) {
447 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
448 				rx_ring->stats.rx_alloc_errs++;
449 				break;
450 			}
451 		}
452 
453 		/* update RxBD */
454 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
455 					   rx_swbd->page_offset);
456 		/* clear 'R" as well */
457 		rxbd->r.lstatus = 0;
458 
459 		rx_swbd++;
460 		rxbd++;
461 		i++;
462 		if (unlikely(i == rx_ring->bd_count)) {
463 			i = 0;
464 			rx_swbd = rx_ring->rx_swbd;
465 			rxbd = ENETC_RXBD(*rx_ring, 0);
466 		}
467 	}
468 
469 	if (likely(j)) {
470 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
471 		rx_ring->next_to_use = i;
472 		/* update ENETC's consumer index */
473 		enetc_wr_reg(rx_ring->rcir, i);
474 	}
475 
476 	return j;
477 }
478 
479 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
480 static void enetc_get_rx_tstamp(struct net_device *ndev,
481 				union enetc_rx_bd *rxbd,
482 				struct sk_buff *skb)
483 {
484 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
485 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
486 	struct enetc_hw *hw = &priv->si->hw;
487 	u32 lo, hi, tstamp_lo;
488 	u64 tstamp;
489 
490 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
491 		lo = enetc_rd(hw, ENETC_SICTR0);
492 		hi = enetc_rd(hw, ENETC_SICTR1);
493 		tstamp_lo = le32_to_cpu(rxbd->r.tstamp);
494 		if (lo <= tstamp_lo)
495 			hi -= 1;
496 
497 		tstamp = (u64)hi << 32 | tstamp_lo;
498 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
499 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
500 	}
501 }
502 #endif
503 
504 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
505 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
506 {
507 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
508 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
509 #endif
510 	/* TODO: hashing */
511 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
512 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
513 
514 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
515 		skb->ip_summed = CHECKSUM_COMPLETE;
516 	}
517 
518 	/* copy VLAN to skb, if one is extracted, for now we assume it's a
519 	 * standard TPID, but HW also supports custom values
520 	 */
521 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN)
522 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
523 				       le16_to_cpu(rxbd->r.vlan_opt));
524 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
525 	if (priv->active_offloads & ENETC_F_RX_TSTAMP)
526 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
527 #endif
528 }
529 
530 static void enetc_process_skb(struct enetc_bdr *rx_ring,
531 			      struct sk_buff *skb)
532 {
533 	skb_record_rx_queue(skb, rx_ring->index);
534 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
535 }
536 
537 static bool enetc_page_reusable(struct page *page)
538 {
539 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
540 }
541 
542 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
543 			     struct enetc_rx_swbd *old)
544 {
545 	struct enetc_rx_swbd *new;
546 
547 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
548 
549 	/* next buf that may reuse a page */
550 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
551 
552 	/* copy page reference */
553 	*new = *old;
554 }
555 
556 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
557 					       int i, u16 size)
558 {
559 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
560 
561 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
562 				      rx_swbd->page_offset,
563 				      size, DMA_FROM_DEVICE);
564 	return rx_swbd;
565 }
566 
567 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
568 			      struct enetc_rx_swbd *rx_swbd)
569 {
570 	if (likely(enetc_page_reusable(rx_swbd->page))) {
571 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
572 		page_ref_inc(rx_swbd->page);
573 
574 		enetc_reuse_page(rx_ring, rx_swbd);
575 
576 		/* sync for use by the device */
577 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
578 						 rx_swbd->page_offset,
579 						 ENETC_RXB_DMA_SIZE,
580 						 DMA_FROM_DEVICE);
581 	} else {
582 		dma_unmap_page(rx_ring->dev, rx_swbd->dma,
583 			       PAGE_SIZE, DMA_FROM_DEVICE);
584 	}
585 
586 	rx_swbd->page = NULL;
587 }
588 
589 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
590 						int i, u16 size)
591 {
592 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
593 	struct sk_buff *skb;
594 	void *ba;
595 
596 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
597 	skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
598 	if (unlikely(!skb)) {
599 		rx_ring->stats.rx_alloc_errs++;
600 		return NULL;
601 	}
602 
603 	skb_reserve(skb, ENETC_RXB_PAD);
604 	__skb_put(skb, size);
605 
606 	enetc_put_rx_buff(rx_ring, rx_swbd);
607 
608 	return skb;
609 }
610 
611 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
612 				     u16 size, struct sk_buff *skb)
613 {
614 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
615 
616 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
617 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
618 
619 	enetc_put_rx_buff(rx_ring, rx_swbd);
620 }
621 
622 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
623 
624 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
625 			       struct napi_struct *napi, int work_limit)
626 {
627 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
628 	int cleaned_cnt, i;
629 
630 	cleaned_cnt = enetc_bd_unused(rx_ring);
631 	/* next descriptor to process */
632 	i = rx_ring->next_to_clean;
633 
634 	while (likely(rx_frm_cnt < work_limit)) {
635 		union enetc_rx_bd *rxbd;
636 		struct sk_buff *skb;
637 		u32 bd_status;
638 		u16 size;
639 
640 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
641 			int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
642 
643 			cleaned_cnt -= count;
644 		}
645 
646 		rxbd = ENETC_RXBD(*rx_ring, i);
647 		bd_status = le32_to_cpu(rxbd->r.lstatus);
648 		if (!bd_status)
649 			break;
650 
651 		enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index));
652 		dma_rmb(); /* for reading other rxbd fields */
653 		size = le16_to_cpu(rxbd->r.buf_len);
654 		skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
655 		if (!skb)
656 			break;
657 
658 		enetc_get_offloads(rx_ring, rxbd, skb);
659 
660 		cleaned_cnt++;
661 		rxbd++;
662 		i++;
663 		if (unlikely(i == rx_ring->bd_count)) {
664 			i = 0;
665 			rxbd = ENETC_RXBD(*rx_ring, 0);
666 		}
667 
668 		if (unlikely(bd_status &
669 			     ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
670 			dev_kfree_skb(skb);
671 			while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
672 				dma_rmb();
673 				bd_status = le32_to_cpu(rxbd->r.lstatus);
674 				rxbd++;
675 				i++;
676 				if (unlikely(i == rx_ring->bd_count)) {
677 					i = 0;
678 					rxbd = ENETC_RXBD(*rx_ring, 0);
679 				}
680 			}
681 
682 			rx_ring->ndev->stats.rx_dropped++;
683 			rx_ring->ndev->stats.rx_errors++;
684 
685 			break;
686 		}
687 
688 		/* not last BD in frame? */
689 		while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
690 			bd_status = le32_to_cpu(rxbd->r.lstatus);
691 			size = ENETC_RXB_DMA_SIZE;
692 
693 			if (bd_status & ENETC_RXBD_LSTATUS_F) {
694 				dma_rmb();
695 				size = le16_to_cpu(rxbd->r.buf_len);
696 			}
697 
698 			enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
699 
700 			cleaned_cnt++;
701 			rxbd++;
702 			i++;
703 			if (unlikely(i == rx_ring->bd_count)) {
704 				i = 0;
705 				rxbd = ENETC_RXBD(*rx_ring, 0);
706 			}
707 		}
708 
709 		rx_byte_cnt += skb->len;
710 
711 		enetc_process_skb(rx_ring, skb);
712 
713 		napi_gro_receive(napi, skb);
714 
715 		rx_frm_cnt++;
716 	}
717 
718 	rx_ring->next_to_clean = i;
719 
720 	rx_ring->stats.packets += rx_frm_cnt;
721 	rx_ring->stats.bytes += rx_byte_cnt;
722 
723 	return rx_frm_cnt;
724 }
725 
726 /* Probing and Init */
727 #define ENETC_MAX_RFS_SIZE 64
728 void enetc_get_si_caps(struct enetc_si *si)
729 {
730 	struct enetc_hw *hw = &si->hw;
731 	u32 val;
732 
733 	/* find out how many of various resources we have to work with */
734 	val = enetc_rd(hw, ENETC_SICAPR0);
735 	si->num_rx_rings = (val >> 16) & 0xff;
736 	si->num_tx_rings = val & 0xff;
737 
738 	val = enetc_rd(hw, ENETC_SIRFSCAPR);
739 	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
740 	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
741 
742 	si->num_rss = 0;
743 	val = enetc_rd(hw, ENETC_SIPCAPR0);
744 	if (val & ENETC_SIPCAPR0_RSS) {
745 		val = enetc_rd(hw, ENETC_SIRSSCAPR);
746 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(val);
747 	}
748 }
749 
750 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
751 {
752 	r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
753 					&r->bd_dma_base, GFP_KERNEL);
754 	if (!r->bd_base)
755 		return -ENOMEM;
756 
757 	/* h/w requires 128B alignment */
758 	if (!IS_ALIGNED(r->bd_dma_base, 128)) {
759 		dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
760 				  r->bd_dma_base);
761 		return -EINVAL;
762 	}
763 
764 	return 0;
765 }
766 
767 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
768 {
769 	int err;
770 
771 	txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
772 	if (!txr->tx_swbd)
773 		return -ENOMEM;
774 
775 	err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
776 	if (err) {
777 		vfree(txr->tx_swbd);
778 		return err;
779 	}
780 
781 	txr->next_to_clean = 0;
782 	txr->next_to_use = 0;
783 
784 	return 0;
785 }
786 
787 static void enetc_free_txbdr(struct enetc_bdr *txr)
788 {
789 	int size, i;
790 
791 	for (i = 0; i < txr->bd_count; i++)
792 		enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
793 
794 	size = txr->bd_count * sizeof(union enetc_tx_bd);
795 
796 	dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
797 	txr->bd_base = NULL;
798 
799 	vfree(txr->tx_swbd);
800 	txr->tx_swbd = NULL;
801 }
802 
803 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
804 {
805 	int i, err;
806 
807 	for (i = 0; i < priv->num_tx_rings; i++) {
808 		err = enetc_alloc_txbdr(priv->tx_ring[i]);
809 
810 		if (err)
811 			goto fail;
812 	}
813 
814 	return 0;
815 
816 fail:
817 	while (i-- > 0)
818 		enetc_free_txbdr(priv->tx_ring[i]);
819 
820 	return err;
821 }
822 
823 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
824 {
825 	int i;
826 
827 	for (i = 0; i < priv->num_tx_rings; i++)
828 		enetc_free_txbdr(priv->tx_ring[i]);
829 }
830 
831 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr)
832 {
833 	int err;
834 
835 	rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
836 	if (!rxr->rx_swbd)
837 		return -ENOMEM;
838 
839 	err = enetc_dma_alloc_bdr(rxr, sizeof(union enetc_rx_bd));
840 	if (err) {
841 		vfree(rxr->rx_swbd);
842 		return err;
843 	}
844 
845 	rxr->next_to_clean = 0;
846 	rxr->next_to_use = 0;
847 	rxr->next_to_alloc = 0;
848 
849 	return 0;
850 }
851 
852 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
853 {
854 	int size;
855 
856 	size = rxr->bd_count * sizeof(union enetc_rx_bd);
857 
858 	dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
859 	rxr->bd_base = NULL;
860 
861 	vfree(rxr->rx_swbd);
862 	rxr->rx_swbd = NULL;
863 }
864 
865 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
866 {
867 	int i, err;
868 
869 	for (i = 0; i < priv->num_rx_rings; i++) {
870 		err = enetc_alloc_rxbdr(priv->rx_ring[i]);
871 
872 		if (err)
873 			goto fail;
874 	}
875 
876 	return 0;
877 
878 fail:
879 	while (i-- > 0)
880 		enetc_free_rxbdr(priv->rx_ring[i]);
881 
882 	return err;
883 }
884 
885 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
886 {
887 	int i;
888 
889 	for (i = 0; i < priv->num_rx_rings; i++)
890 		enetc_free_rxbdr(priv->rx_ring[i]);
891 }
892 
893 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
894 {
895 	int i;
896 
897 	if (!tx_ring->tx_swbd)
898 		return;
899 
900 	for (i = 0; i < tx_ring->bd_count; i++) {
901 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
902 
903 		enetc_free_tx_skb(tx_ring, tx_swbd);
904 	}
905 
906 	tx_ring->next_to_clean = 0;
907 	tx_ring->next_to_use = 0;
908 }
909 
910 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
911 {
912 	int i;
913 
914 	if (!rx_ring->rx_swbd)
915 		return;
916 
917 	for (i = 0; i < rx_ring->bd_count; i++) {
918 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
919 
920 		if (!rx_swbd->page)
921 			continue;
922 
923 		dma_unmap_page(rx_ring->dev, rx_swbd->dma,
924 			       PAGE_SIZE, DMA_FROM_DEVICE);
925 		__free_page(rx_swbd->page);
926 		rx_swbd->page = NULL;
927 	}
928 
929 	rx_ring->next_to_clean = 0;
930 	rx_ring->next_to_use = 0;
931 	rx_ring->next_to_alloc = 0;
932 }
933 
934 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
935 {
936 	int i;
937 
938 	for (i = 0; i < priv->num_rx_rings; i++)
939 		enetc_free_rx_ring(priv->rx_ring[i]);
940 
941 	for (i = 0; i < priv->num_tx_rings; i++)
942 		enetc_free_tx_ring(priv->tx_ring[i]);
943 }
944 
945 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
946 {
947 	int size = cbdr->bd_count * sizeof(struct enetc_cbd);
948 
949 	cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
950 					   GFP_KERNEL);
951 	if (!cbdr->bd_base)
952 		return -ENOMEM;
953 
954 	/* h/w requires 128B alignment */
955 	if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
956 		dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
957 		return -EINVAL;
958 	}
959 
960 	cbdr->next_to_clean = 0;
961 	cbdr->next_to_use = 0;
962 
963 	return 0;
964 }
965 
966 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
967 {
968 	int size = cbdr->bd_count * sizeof(struct enetc_cbd);
969 
970 	dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
971 	cbdr->bd_base = NULL;
972 }
973 
974 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
975 {
976 	/* set CBDR cache attributes */
977 	enetc_wr(hw, ENETC_SICAR2,
978 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
979 
980 	enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
981 	enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
982 	enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
983 
984 	enetc_wr(hw, ENETC_SICBDRPIR, 0);
985 	enetc_wr(hw, ENETC_SICBDRCIR, 0);
986 
987 	/* enable ring */
988 	enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
989 
990 	cbdr->pir = hw->reg + ENETC_SICBDRPIR;
991 	cbdr->cir = hw->reg + ENETC_SICBDRCIR;
992 }
993 
994 static void enetc_clear_cbdr(struct enetc_hw *hw)
995 {
996 	enetc_wr(hw, ENETC_SICBDRMR, 0);
997 }
998 
999 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1000 {
1001 	int *rss_table;
1002 	int i;
1003 
1004 	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1005 	if (!rss_table)
1006 		return -ENOMEM;
1007 
1008 	/* Set up RSS table defaults */
1009 	for (i = 0; i < si->num_rss; i++)
1010 		rss_table[i] = i % num_groups;
1011 
1012 	enetc_set_rss_table(si, rss_table, si->num_rss);
1013 
1014 	kfree(rss_table);
1015 
1016 	return 0;
1017 }
1018 
1019 static int enetc_configure_si(struct enetc_ndev_priv *priv)
1020 {
1021 	struct enetc_si *si = priv->si;
1022 	struct enetc_hw *hw = &si->hw;
1023 	int err;
1024 
1025 	enetc_setup_cbdr(hw, &si->cbd_ring);
1026 	/* set SI cache attributes */
1027 	enetc_wr(hw, ENETC_SICAR0,
1028 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1029 	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1030 	/* enable SI */
1031 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1032 
1033 	if (si->num_rss) {
1034 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1035 		if (err)
1036 			return err;
1037 	}
1038 
1039 	return 0;
1040 }
1041 
1042 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1043 {
1044 	struct enetc_si *si = priv->si;
1045 	int cpus = num_online_cpus();
1046 
1047 	priv->tx_bd_count = ENETC_BDR_DEFAULT_SIZE;
1048 	priv->rx_bd_count = ENETC_BDR_DEFAULT_SIZE;
1049 
1050 	/* Enable all available TX rings in order to configure as many
1051 	 * priorities as possible, when needed.
1052 	 * TODO: Make # of TX rings run-time configurable
1053 	 */
1054 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1055 	priv->num_tx_rings = si->num_tx_rings;
1056 	priv->bdr_int_num = cpus;
1057 
1058 	/* SI specific */
1059 	si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1060 }
1061 
1062 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1063 {
1064 	struct enetc_si *si = priv->si;
1065 	int err;
1066 
1067 	err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1068 	if (err)
1069 		return err;
1070 
1071 	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1072 				  GFP_KERNEL);
1073 	if (!priv->cls_rules) {
1074 		err = -ENOMEM;
1075 		goto err_alloc_cls;
1076 	}
1077 
1078 	err = enetc_configure_si(priv);
1079 	if (err)
1080 		goto err_config_si;
1081 
1082 	return 0;
1083 
1084 err_config_si:
1085 	kfree(priv->cls_rules);
1086 err_alloc_cls:
1087 	enetc_clear_cbdr(&si->hw);
1088 	enetc_free_cbdr(priv->dev, &si->cbd_ring);
1089 
1090 	return err;
1091 }
1092 
1093 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1094 {
1095 	struct enetc_si *si = priv->si;
1096 
1097 	enetc_clear_cbdr(&si->hw);
1098 	enetc_free_cbdr(priv->dev, &si->cbd_ring);
1099 
1100 	kfree(priv->cls_rules);
1101 }
1102 
1103 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1104 {
1105 	int idx = tx_ring->index;
1106 	u32 tbmr;
1107 
1108 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1109 		       lower_32_bits(tx_ring->bd_dma_base));
1110 
1111 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1112 		       upper_32_bits(tx_ring->bd_dma_base));
1113 
1114 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1115 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1116 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
1117 
1118 	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1119 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1120 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1121 
1122 	/* enable Tx ints by setting pkt thr to 1 */
1123 	enetc_txbdr_wr(hw, idx, ENETC_TBICIR0, ENETC_TBICIR0_ICEN | 0x1);
1124 
1125 	tbmr = ENETC_TBMR_EN;
1126 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1127 		tbmr |= ENETC_TBMR_VIH;
1128 
1129 	/* enable ring */
1130 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1131 
1132 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1133 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1134 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
1135 }
1136 
1137 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1138 {
1139 	int idx = rx_ring->index;
1140 	u32 rbmr;
1141 
1142 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1143 		       lower_32_bits(rx_ring->bd_dma_base));
1144 
1145 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1146 		       upper_32_bits(rx_ring->bd_dma_base));
1147 
1148 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1149 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1150 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
1151 
1152 	enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1153 
1154 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1155 
1156 	/* enable Rx ints by setting pkt thr to 1 */
1157 	enetc_rxbdr_wr(hw, idx, ENETC_RBICIR0, ENETC_RBICIR0_ICEN | 0x1);
1158 
1159 	rbmr = ENETC_RBMR_EN;
1160 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1161 	rbmr |= ENETC_RBMR_BDS;
1162 #endif
1163 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1164 		rbmr |= ENETC_RBMR_VTE;
1165 
1166 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1167 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1168 
1169 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1170 
1171 	/* enable ring */
1172 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1173 }
1174 
1175 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1176 {
1177 	int i;
1178 
1179 	for (i = 0; i < priv->num_tx_rings; i++)
1180 		enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
1181 
1182 	for (i = 0; i < priv->num_rx_rings; i++)
1183 		enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1184 }
1185 
1186 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1187 {
1188 	int idx = rx_ring->index;
1189 
1190 	/* disable EN bit on ring */
1191 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1192 }
1193 
1194 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1195 {
1196 	int delay = 8, timeout = 100;
1197 	int idx = tx_ring->index;
1198 
1199 	/* disable EN bit on ring */
1200 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1201 
1202 	/* wait for busy to clear */
1203 	while (delay < timeout &&
1204 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1205 		msleep(delay);
1206 		delay *= 2;
1207 	}
1208 
1209 	if (delay >= timeout)
1210 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1211 			    idx);
1212 }
1213 
1214 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1215 {
1216 	int i;
1217 
1218 	for (i = 0; i < priv->num_tx_rings; i++)
1219 		enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
1220 
1221 	for (i = 0; i < priv->num_rx_rings; i++)
1222 		enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1223 
1224 	udelay(1);
1225 }
1226 
1227 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1228 {
1229 	struct pci_dev *pdev = priv->si->pdev;
1230 	cpumask_t cpu_mask;
1231 	int i, j, err;
1232 
1233 	for (i = 0; i < priv->bdr_int_num; i++) {
1234 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1235 		struct enetc_int_vector *v = priv->int_vector[i];
1236 		int entry = ENETC_BDR_INT_BASE_IDX + i;
1237 		struct enetc_hw *hw = &priv->si->hw;
1238 
1239 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1240 			 priv->ndev->name, i);
1241 		err = request_irq(irq, enetc_msix, 0, v->name, v);
1242 		if (err) {
1243 			dev_err(priv->dev, "request_irq() failed!\n");
1244 			goto irq_err;
1245 		}
1246 
1247 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1248 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1249 
1250 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1251 
1252 		for (j = 0; j < v->count_tx_rings; j++) {
1253 			int idx = v->tx_ring[j].index;
1254 
1255 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1256 		}
1257 		cpumask_clear(&cpu_mask);
1258 		cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
1259 		irq_set_affinity_hint(irq, &cpu_mask);
1260 	}
1261 
1262 	return 0;
1263 
1264 irq_err:
1265 	while (i--) {
1266 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1267 
1268 		irq_set_affinity_hint(irq, NULL);
1269 		free_irq(irq, priv->int_vector[i]);
1270 	}
1271 
1272 	return err;
1273 }
1274 
1275 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1276 {
1277 	struct pci_dev *pdev = priv->si->pdev;
1278 	int i;
1279 
1280 	for (i = 0; i < priv->bdr_int_num; i++) {
1281 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1282 
1283 		irq_set_affinity_hint(irq, NULL);
1284 		free_irq(irq, priv->int_vector[i]);
1285 	}
1286 }
1287 
1288 static void enetc_enable_interrupts(struct enetc_ndev_priv *priv)
1289 {
1290 	int i;
1291 
1292 	/* enable Tx & Rx event indication */
1293 	for (i = 0; i < priv->num_rx_rings; i++) {
1294 		enetc_rxbdr_wr(&priv->si->hw, i,
1295 			       ENETC_RBIER, ENETC_RBIER_RXTIE);
1296 	}
1297 
1298 	for (i = 0; i < priv->num_tx_rings; i++) {
1299 		enetc_txbdr_wr(&priv->si->hw, i,
1300 			       ENETC_TBIER, ENETC_TBIER_TXTIE);
1301 	}
1302 }
1303 
1304 static void enetc_disable_interrupts(struct enetc_ndev_priv *priv)
1305 {
1306 	int i;
1307 
1308 	for (i = 0; i < priv->num_tx_rings; i++)
1309 		enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
1310 
1311 	for (i = 0; i < priv->num_rx_rings; i++)
1312 		enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
1313 }
1314 
1315 static void adjust_link(struct net_device *ndev)
1316 {
1317 	struct phy_device *phydev = ndev->phydev;
1318 
1319 	phy_print_status(phydev);
1320 }
1321 
1322 static int enetc_phy_connect(struct net_device *ndev)
1323 {
1324 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1325 	struct phy_device *phydev;
1326 
1327 	if (!priv->phy_node)
1328 		return 0; /* phy-less mode */
1329 
1330 	phydev = of_phy_connect(ndev, priv->phy_node, &adjust_link,
1331 				0, priv->if_mode);
1332 	if (!phydev) {
1333 		dev_err(&ndev->dev, "could not attach to PHY\n");
1334 		return -ENODEV;
1335 	}
1336 
1337 	phy_attached_info(phydev);
1338 
1339 	return 0;
1340 }
1341 
1342 int enetc_open(struct net_device *ndev)
1343 {
1344 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1345 	int i, err;
1346 
1347 	err = enetc_setup_irqs(priv);
1348 	if (err)
1349 		return err;
1350 
1351 	err = enetc_phy_connect(ndev);
1352 	if (err)
1353 		goto err_phy_connect;
1354 
1355 	err = enetc_alloc_tx_resources(priv);
1356 	if (err)
1357 		goto err_alloc_tx;
1358 
1359 	err = enetc_alloc_rx_resources(priv);
1360 	if (err)
1361 		goto err_alloc_rx;
1362 
1363 	enetc_setup_bdrs(priv);
1364 
1365 	err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1366 	if (err)
1367 		goto err_set_queues;
1368 
1369 	err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1370 	if (err)
1371 		goto err_set_queues;
1372 
1373 	for (i = 0; i < priv->bdr_int_num; i++)
1374 		napi_enable(&priv->int_vector[i]->napi);
1375 
1376 	enetc_enable_interrupts(priv);
1377 
1378 	if (ndev->phydev)
1379 		phy_start(ndev->phydev);
1380 	else
1381 		netif_carrier_on(ndev);
1382 
1383 	netif_tx_start_all_queues(ndev);
1384 
1385 	return 0;
1386 
1387 err_set_queues:
1388 	enetc_free_rx_resources(priv);
1389 err_alloc_rx:
1390 	enetc_free_tx_resources(priv);
1391 err_alloc_tx:
1392 	if (ndev->phydev)
1393 		phy_disconnect(ndev->phydev);
1394 err_phy_connect:
1395 	enetc_free_irqs(priv);
1396 
1397 	return err;
1398 }
1399 
1400 int enetc_close(struct net_device *ndev)
1401 {
1402 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1403 	int i;
1404 
1405 	netif_tx_stop_all_queues(ndev);
1406 
1407 	if (ndev->phydev) {
1408 		phy_stop(ndev->phydev);
1409 		phy_disconnect(ndev->phydev);
1410 	} else {
1411 		netif_carrier_off(ndev);
1412 	}
1413 
1414 	for (i = 0; i < priv->bdr_int_num; i++) {
1415 		napi_synchronize(&priv->int_vector[i]->napi);
1416 		napi_disable(&priv->int_vector[i]->napi);
1417 	}
1418 
1419 	enetc_disable_interrupts(priv);
1420 	enetc_clear_bdrs(priv);
1421 
1422 	enetc_free_rxtx_rings(priv);
1423 	enetc_free_rx_resources(priv);
1424 	enetc_free_tx_resources(priv);
1425 	enetc_free_irqs(priv);
1426 
1427 	return 0;
1428 }
1429 
1430 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1431 		   void *type_data)
1432 {
1433 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1434 	struct tc_mqprio_qopt *mqprio = type_data;
1435 	struct enetc_bdr *tx_ring;
1436 	u8 num_tc;
1437 	int i;
1438 
1439 	if (type != TC_SETUP_QDISC_MQPRIO)
1440 		return -EOPNOTSUPP;
1441 
1442 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1443 	num_tc = mqprio->num_tc;
1444 
1445 	if (!num_tc) {
1446 		netdev_reset_tc(ndev);
1447 		netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1448 
1449 		/* Reset all ring priorities to 0 */
1450 		for (i = 0; i < priv->num_tx_rings; i++) {
1451 			tx_ring = priv->tx_ring[i];
1452 			enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
1453 		}
1454 
1455 		return 0;
1456 	}
1457 
1458 	/* Check if we have enough BD rings available to accommodate all TCs */
1459 	if (num_tc > priv->num_tx_rings) {
1460 		netdev_err(ndev, "Max %d traffic classes supported\n",
1461 			   priv->num_tx_rings);
1462 		return -EINVAL;
1463 	}
1464 
1465 	/* For the moment, we use only one BD ring per TC.
1466 	 *
1467 	 * Configure num_tc BD rings with increasing priorities.
1468 	 */
1469 	for (i = 0; i < num_tc; i++) {
1470 		tx_ring = priv->tx_ring[i];
1471 		enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
1472 	}
1473 
1474 	/* Reset the number of netdev queues based on the TC count */
1475 	netif_set_real_num_tx_queues(ndev, num_tc);
1476 
1477 	netdev_set_num_tc(ndev, num_tc);
1478 
1479 	/* Each TC is associated with one netdev queue */
1480 	for (i = 0; i < num_tc; i++)
1481 		netdev_set_tc_queue(ndev, i, 1, i);
1482 
1483 	return 0;
1484 }
1485 
1486 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1487 {
1488 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1489 	struct net_device_stats *stats = &ndev->stats;
1490 	unsigned long packets = 0, bytes = 0;
1491 	int i;
1492 
1493 	for (i = 0; i < priv->num_rx_rings; i++) {
1494 		packets += priv->rx_ring[i]->stats.packets;
1495 		bytes	+= priv->rx_ring[i]->stats.bytes;
1496 	}
1497 
1498 	stats->rx_packets = packets;
1499 	stats->rx_bytes = bytes;
1500 	bytes = 0;
1501 	packets = 0;
1502 
1503 	for (i = 0; i < priv->num_tx_rings; i++) {
1504 		packets += priv->tx_ring[i]->stats.packets;
1505 		bytes	+= priv->tx_ring[i]->stats.bytes;
1506 	}
1507 
1508 	stats->tx_packets = packets;
1509 	stats->tx_bytes = bytes;
1510 
1511 	return stats;
1512 }
1513 
1514 static int enetc_set_rss(struct net_device *ndev, int en)
1515 {
1516 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1517 	struct enetc_hw *hw = &priv->si->hw;
1518 	u32 reg;
1519 
1520 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1521 
1522 	reg = enetc_rd(hw, ENETC_SIMR);
1523 	reg &= ~ENETC_SIMR_RSSE;
1524 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
1525 	enetc_wr(hw, ENETC_SIMR, reg);
1526 
1527 	return 0;
1528 }
1529 
1530 int enetc_set_features(struct net_device *ndev,
1531 		       netdev_features_t features)
1532 {
1533 	netdev_features_t changed = ndev->features ^ features;
1534 
1535 	if (changed & NETIF_F_RXHASH)
1536 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1537 
1538 	return 0;
1539 }
1540 
1541 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1542 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1543 {
1544 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1545 	struct hwtstamp_config config;
1546 
1547 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1548 		return -EFAULT;
1549 
1550 	switch (config.tx_type) {
1551 	case HWTSTAMP_TX_OFF:
1552 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1553 		break;
1554 	case HWTSTAMP_TX_ON:
1555 		priv->active_offloads |= ENETC_F_TX_TSTAMP;
1556 		break;
1557 	default:
1558 		return -ERANGE;
1559 	}
1560 
1561 	switch (config.rx_filter) {
1562 	case HWTSTAMP_FILTER_NONE:
1563 		priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1564 		break;
1565 	default:
1566 		priv->active_offloads |= ENETC_F_RX_TSTAMP;
1567 		config.rx_filter = HWTSTAMP_FILTER_ALL;
1568 	}
1569 
1570 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1571 	       -EFAULT : 0;
1572 }
1573 
1574 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1575 {
1576 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1577 	struct hwtstamp_config config;
1578 
1579 	config.flags = 0;
1580 
1581 	if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1582 		config.tx_type = HWTSTAMP_TX_ON;
1583 	else
1584 		config.tx_type = HWTSTAMP_TX_OFF;
1585 
1586 	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1587 			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1588 
1589 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1590 	       -EFAULT : 0;
1591 }
1592 #endif
1593 
1594 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1595 {
1596 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1597 	if (cmd == SIOCSHWTSTAMP)
1598 		return enetc_hwtstamp_set(ndev, rq);
1599 	if (cmd == SIOCGHWTSTAMP)
1600 		return enetc_hwtstamp_get(ndev, rq);
1601 #endif
1602 	return -EINVAL;
1603 }
1604 
1605 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1606 {
1607 	struct pci_dev *pdev = priv->si->pdev;
1608 	int size, v_tx_rings;
1609 	int i, n, err, nvec;
1610 
1611 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1612 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
1613 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1614 
1615 	if (n < 0)
1616 		return n;
1617 
1618 	if (n != nvec)
1619 		return -EPERM;
1620 
1621 	/* # of tx rings per int vector */
1622 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1623 	size = sizeof(struct enetc_int_vector) +
1624 	       sizeof(struct enetc_bdr) * v_tx_rings;
1625 
1626 	for (i = 0; i < priv->bdr_int_num; i++) {
1627 		struct enetc_int_vector *v;
1628 		struct enetc_bdr *bdr;
1629 		int j;
1630 
1631 		v = kzalloc(size, GFP_KERNEL);
1632 		if (!v) {
1633 			err = -ENOMEM;
1634 			goto fail;
1635 		}
1636 
1637 		priv->int_vector[i] = v;
1638 
1639 		netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1640 			       NAPI_POLL_WEIGHT);
1641 		v->count_tx_rings = v_tx_rings;
1642 
1643 		for (j = 0; j < v_tx_rings; j++) {
1644 			int idx;
1645 
1646 			/* default tx ring mapping policy */
1647 			if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1648 				idx = 2 * j + i; /* 2 CPUs */
1649 			else
1650 				idx = j + i * v_tx_rings; /* default */
1651 
1652 			__set_bit(idx, &v->tx_rings_map);
1653 			bdr = &v->tx_ring[j];
1654 			bdr->index = idx;
1655 			bdr->ndev = priv->ndev;
1656 			bdr->dev = priv->dev;
1657 			bdr->bd_count = priv->tx_bd_count;
1658 			priv->tx_ring[idx] = bdr;
1659 		}
1660 
1661 		bdr = &v->rx_ring;
1662 		bdr->index = i;
1663 		bdr->ndev = priv->ndev;
1664 		bdr->dev = priv->dev;
1665 		bdr->bd_count = priv->rx_bd_count;
1666 		priv->rx_ring[i] = bdr;
1667 	}
1668 
1669 	return 0;
1670 
1671 fail:
1672 	while (i--) {
1673 		netif_napi_del(&priv->int_vector[i]->napi);
1674 		kfree(priv->int_vector[i]);
1675 	}
1676 
1677 	pci_free_irq_vectors(pdev);
1678 
1679 	return err;
1680 }
1681 
1682 void enetc_free_msix(struct enetc_ndev_priv *priv)
1683 {
1684 	int i;
1685 
1686 	for (i = 0; i < priv->bdr_int_num; i++) {
1687 		struct enetc_int_vector *v = priv->int_vector[i];
1688 
1689 		netif_napi_del(&v->napi);
1690 	}
1691 
1692 	for (i = 0; i < priv->num_rx_rings; i++)
1693 		priv->rx_ring[i] = NULL;
1694 
1695 	for (i = 0; i < priv->num_tx_rings; i++)
1696 		priv->tx_ring[i] = NULL;
1697 
1698 	for (i = 0; i < priv->bdr_int_num; i++) {
1699 		kfree(priv->int_vector[i]);
1700 		priv->int_vector[i] = NULL;
1701 	}
1702 
1703 	/* disable all MSIX for this device */
1704 	pci_free_irq_vectors(priv->si->pdev);
1705 }
1706 
1707 static void enetc_kfree_si(struct enetc_si *si)
1708 {
1709 	char *p = (char *)si - si->pad;
1710 
1711 	kfree(p);
1712 }
1713 
1714 static void enetc_detect_errata(struct enetc_si *si)
1715 {
1716 	if (si->pdev->revision == ENETC_REV1)
1717 		si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1718 			     ENETC_ERR_UCMCSWP;
1719 }
1720 
1721 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1722 {
1723 	struct enetc_si *si, *p;
1724 	struct enetc_hw *hw;
1725 	size_t alloc_size;
1726 	int err, len;
1727 
1728 	pcie_flr(pdev);
1729 	err = pci_enable_device_mem(pdev);
1730 	if (err) {
1731 		dev_err(&pdev->dev, "device enable failed\n");
1732 		return err;
1733 	}
1734 
1735 	/* set up for high or low dma */
1736 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1737 	if (err) {
1738 		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1739 		if (err) {
1740 			dev_err(&pdev->dev,
1741 				"DMA configuration failed: 0x%x\n", err);
1742 			goto err_dma;
1743 		}
1744 	}
1745 
1746 	err = pci_request_mem_regions(pdev, name);
1747 	if (err) {
1748 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1749 		goto err_pci_mem_reg;
1750 	}
1751 
1752 	pci_set_master(pdev);
1753 
1754 	alloc_size = sizeof(struct enetc_si);
1755 	if (sizeof_priv) {
1756 		/* align priv to 32B */
1757 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1758 		alloc_size += sizeof_priv;
1759 	}
1760 	/* force 32B alignment for enetc_si */
1761 	alloc_size += ENETC_SI_ALIGN - 1;
1762 
1763 	p = kzalloc(alloc_size, GFP_KERNEL);
1764 	if (!p) {
1765 		err = -ENOMEM;
1766 		goto err_alloc_si;
1767 	}
1768 
1769 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1770 	si->pad = (char *)si - (char *)p;
1771 
1772 	pci_set_drvdata(pdev, si);
1773 	si->pdev = pdev;
1774 	hw = &si->hw;
1775 
1776 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
1777 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1778 	if (!hw->reg) {
1779 		err = -ENXIO;
1780 		dev_err(&pdev->dev, "ioremap() failed\n");
1781 		goto err_ioremap;
1782 	}
1783 	if (len > ENETC_PORT_BASE)
1784 		hw->port = hw->reg + ENETC_PORT_BASE;
1785 	if (len > ENETC_GLOBAL_BASE)
1786 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
1787 
1788 	enetc_detect_errata(si);
1789 
1790 	return 0;
1791 
1792 err_ioremap:
1793 	enetc_kfree_si(si);
1794 err_alloc_si:
1795 	pci_release_mem_regions(pdev);
1796 err_pci_mem_reg:
1797 err_dma:
1798 	pci_disable_device(pdev);
1799 
1800 	return err;
1801 }
1802 
1803 void enetc_pci_remove(struct pci_dev *pdev)
1804 {
1805 	struct enetc_si *si = pci_get_drvdata(pdev);
1806 	struct enetc_hw *hw = &si->hw;
1807 
1808 	iounmap(hw->reg);
1809 	enetc_kfree_si(si);
1810 	pci_release_mem_regions(pdev);
1811 	pci_disable_device(pdev);
1812 }
1813