1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3 
4 #include "enetc.h"
5 #include <linux/tcp.h>
6 #include <linux/udp.h>
7 #include <linux/vmalloc.h>
8 
9 /* ENETC overhead: optional extension BD + 1 BD gap */
10 #define ENETC_TXBDS_NEEDED(val)	((val) + 2)
11 /* max # of chained Tx BDs is 15, including head and extension BD */
12 #define ENETC_MAX_SKB_FRAGS	13
13 #define ENETC_TXBDS_MAX_NEEDED	ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
14 
15 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
16 			      int active_offloads);
17 
18 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
19 {
20 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
21 	struct enetc_bdr *tx_ring;
22 	int count;
23 
24 	tx_ring = priv->tx_ring[skb->queue_mapping];
25 
26 	if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
27 		if (unlikely(skb_linearize(skb)))
28 			goto drop_packet_err;
29 
30 	count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
31 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
32 		netif_stop_subqueue(ndev, tx_ring->index);
33 		return NETDEV_TX_BUSY;
34 	}
35 
36 	count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
37 	if (unlikely(!count))
38 		goto drop_packet_err;
39 
40 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
41 		netif_stop_subqueue(ndev, tx_ring->index);
42 
43 	return NETDEV_TX_OK;
44 
45 drop_packet_err:
46 	dev_kfree_skb_any(skb);
47 	return NETDEV_TX_OK;
48 }
49 
50 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
51 {
52 	int l3_start, l3_hsize;
53 	u16 l3_flags, l4_flags;
54 
55 	if (skb->ip_summed != CHECKSUM_PARTIAL)
56 		return false;
57 
58 	switch (skb->csum_offset) {
59 	case offsetof(struct tcphdr, check):
60 		l4_flags = ENETC_TXBD_L4_TCP;
61 		break;
62 	case offsetof(struct udphdr, check):
63 		l4_flags = ENETC_TXBD_L4_UDP;
64 		break;
65 	default:
66 		skb_checksum_help(skb);
67 		return false;
68 	}
69 
70 	l3_start = skb_network_offset(skb);
71 	l3_hsize = skb_network_header_len(skb);
72 
73 	l3_flags = 0;
74 	if (skb->protocol == htons(ETH_P_IPV6))
75 		l3_flags = ENETC_TXBD_L3_IPV6;
76 
77 	/* write BD fields */
78 	txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
79 	txbd->l4_csoff = l4_flags;
80 
81 	return true;
82 }
83 
84 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
85 				struct enetc_tx_swbd *tx_swbd)
86 {
87 	if (tx_swbd->is_dma_page)
88 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
89 			       tx_swbd->len, DMA_TO_DEVICE);
90 	else
91 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
92 				 tx_swbd->len, DMA_TO_DEVICE);
93 	tx_swbd->dma = 0;
94 }
95 
96 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
97 			      struct enetc_tx_swbd *tx_swbd)
98 {
99 	if (tx_swbd->dma)
100 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
101 
102 	if (tx_swbd->skb) {
103 		dev_kfree_skb_any(tx_swbd->skb);
104 		tx_swbd->skb = NULL;
105 	}
106 }
107 
108 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
109 			      int active_offloads)
110 {
111 	struct enetc_tx_swbd *tx_swbd;
112 	skb_frag_t *frag;
113 	int len = skb_headlen(skb);
114 	union enetc_tx_bd temp_bd;
115 	union enetc_tx_bd *txbd;
116 	bool do_vlan, do_tstamp;
117 	int i, count = 0;
118 	unsigned int f;
119 	dma_addr_t dma;
120 	u8 flags = 0;
121 
122 	i = tx_ring->next_to_use;
123 	txbd = ENETC_TXBD(*tx_ring, i);
124 	prefetchw(txbd);
125 
126 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
127 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
128 		goto dma_err;
129 
130 	temp_bd.addr = cpu_to_le64(dma);
131 	temp_bd.buf_len = cpu_to_le16(len);
132 	temp_bd.lstatus = 0;
133 
134 	tx_swbd = &tx_ring->tx_swbd[i];
135 	tx_swbd->dma = dma;
136 	tx_swbd->len = len;
137 	tx_swbd->is_dma_page = 0;
138 	count++;
139 
140 	do_vlan = skb_vlan_tag_present(skb);
141 	do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
142 		    (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
143 	tx_swbd->do_tstamp = do_tstamp;
144 	tx_swbd->check_wb = tx_swbd->do_tstamp;
145 
146 	if (do_vlan || do_tstamp)
147 		flags |= ENETC_TXBD_FLAGS_EX;
148 
149 	if (enetc_tx_csum(skb, &temp_bd))
150 		flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
151 	else if (tx_ring->tsd_enable)
152 		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
153 
154 	/* first BD needs frm_len and offload flags set */
155 	temp_bd.frm_len = cpu_to_le16(skb->len);
156 	temp_bd.flags = flags;
157 
158 	if (flags & ENETC_TXBD_FLAGS_TSE) {
159 		u32 temp;
160 
161 		temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK)
162 			| (flags << ENETC_TXBD_FLAGS_OFFSET);
163 		temp_bd.txstart = cpu_to_le32(temp);
164 	}
165 
166 	if (flags & ENETC_TXBD_FLAGS_EX) {
167 		u8 e_flags = 0;
168 		*txbd = temp_bd;
169 		enetc_clear_tx_bd(&temp_bd);
170 
171 		/* add extension BD for VLAN and/or timestamping */
172 		flags = 0;
173 		tx_swbd++;
174 		txbd++;
175 		i++;
176 		if (unlikely(i == tx_ring->bd_count)) {
177 			i = 0;
178 			tx_swbd = tx_ring->tx_swbd;
179 			txbd = ENETC_TXBD(*tx_ring, 0);
180 		}
181 		prefetchw(txbd);
182 
183 		if (do_vlan) {
184 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
185 			temp_bd.ext.tpid = 0; /* < C-TAG */
186 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
187 		}
188 
189 		if (do_tstamp) {
190 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
191 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
192 		}
193 
194 		temp_bd.ext.e_flags = e_flags;
195 		count++;
196 	}
197 
198 	frag = &skb_shinfo(skb)->frags[0];
199 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
200 		len = skb_frag_size(frag);
201 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
202 				       DMA_TO_DEVICE);
203 		if (dma_mapping_error(tx_ring->dev, dma))
204 			goto dma_err;
205 
206 		*txbd = temp_bd;
207 		enetc_clear_tx_bd(&temp_bd);
208 
209 		flags = 0;
210 		tx_swbd++;
211 		txbd++;
212 		i++;
213 		if (unlikely(i == tx_ring->bd_count)) {
214 			i = 0;
215 			tx_swbd = tx_ring->tx_swbd;
216 			txbd = ENETC_TXBD(*tx_ring, 0);
217 		}
218 		prefetchw(txbd);
219 
220 		temp_bd.addr = cpu_to_le64(dma);
221 		temp_bd.buf_len = cpu_to_le16(len);
222 
223 		tx_swbd->dma = dma;
224 		tx_swbd->len = len;
225 		tx_swbd->is_dma_page = 1;
226 		count++;
227 	}
228 
229 	/* last BD needs 'F' bit set */
230 	flags |= ENETC_TXBD_FLAGS_F;
231 	temp_bd.flags = flags;
232 	*txbd = temp_bd;
233 
234 	tx_ring->tx_swbd[i].skb = skb;
235 
236 	enetc_bdr_idx_inc(tx_ring, &i);
237 	tx_ring->next_to_use = i;
238 
239 	skb_tx_timestamp(skb);
240 
241 	/* let H/W know BD ring has been updated */
242 	enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */
243 
244 	return count;
245 
246 dma_err:
247 	dev_err(tx_ring->dev, "DMA map error");
248 
249 	do {
250 		tx_swbd = &tx_ring->tx_swbd[i];
251 		enetc_free_tx_skb(tx_ring, tx_swbd);
252 		if (i == 0)
253 			i = tx_ring->bd_count;
254 		i--;
255 	} while (count--);
256 
257 	return 0;
258 }
259 
260 static irqreturn_t enetc_msix(int irq, void *data)
261 {
262 	struct enetc_int_vector	*v = data;
263 	int i;
264 
265 	/* disable interrupts */
266 	enetc_wr_reg(v->rbier, 0);
267 	enetc_wr_reg(v->ricr1, v->rx_ictt);
268 
269 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
270 		enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
271 
272 	napi_schedule(&v->napi);
273 
274 	return IRQ_HANDLED;
275 }
276 
277 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
278 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
279 			       struct napi_struct *napi, int work_limit);
280 
281 static void enetc_rx_dim_work(struct work_struct *w)
282 {
283 	struct dim *dim = container_of(w, struct dim, work);
284 	struct dim_cq_moder moder =
285 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
286 	struct enetc_int_vector	*v =
287 		container_of(dim, struct enetc_int_vector, rx_dim);
288 
289 	v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
290 	dim->state = DIM_START_MEASURE;
291 }
292 
293 static void enetc_rx_net_dim(struct enetc_int_vector *v)
294 {
295 	struct dim_sample dim_sample;
296 
297 	v->comp_cnt++;
298 
299 	if (!v->rx_napi_work)
300 		return;
301 
302 	dim_update_sample(v->comp_cnt,
303 			  v->rx_ring.stats.packets,
304 			  v->rx_ring.stats.bytes,
305 			  &dim_sample);
306 	net_dim(&v->rx_dim, dim_sample);
307 }
308 
309 static int enetc_poll(struct napi_struct *napi, int budget)
310 {
311 	struct enetc_int_vector
312 		*v = container_of(napi, struct enetc_int_vector, napi);
313 	bool complete = true;
314 	int work_done;
315 	int i;
316 
317 	for (i = 0; i < v->count_tx_rings; i++)
318 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
319 			complete = false;
320 
321 	work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
322 	if (work_done == budget)
323 		complete = false;
324 	if (work_done)
325 		v->rx_napi_work = true;
326 
327 	if (!complete)
328 		return budget;
329 
330 	napi_complete_done(napi, work_done);
331 
332 	if (likely(v->rx_dim_en))
333 		enetc_rx_net_dim(v);
334 
335 	v->rx_napi_work = false;
336 
337 	/* enable interrupts */
338 	enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
339 
340 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
341 		enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i),
342 			     ENETC_TBIER_TXTIE);
343 
344 	return work_done;
345 }
346 
347 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
348 {
349 	int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
350 
351 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
352 }
353 
354 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
355 				u64 *tstamp)
356 {
357 	u32 lo, hi, tstamp_lo;
358 
359 	lo = enetc_rd(hw, ENETC_SICTR0);
360 	hi = enetc_rd(hw, ENETC_SICTR1);
361 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
362 	if (lo <= tstamp_lo)
363 		hi -= 1;
364 	*tstamp = (u64)hi << 32 | tstamp_lo;
365 }
366 
367 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
368 {
369 	struct skb_shared_hwtstamps shhwtstamps;
370 
371 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
372 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
373 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
374 		skb_tstamp_tx(skb, &shhwtstamps);
375 	}
376 }
377 
378 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
379 {
380 	struct net_device *ndev = tx_ring->ndev;
381 	int tx_frm_cnt = 0, tx_byte_cnt = 0;
382 	struct enetc_tx_swbd *tx_swbd;
383 	int i, bds_to_clean;
384 	bool do_tstamp;
385 	u64 tstamp = 0;
386 
387 	i = tx_ring->next_to_clean;
388 	tx_swbd = &tx_ring->tx_swbd[i];
389 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
390 
391 	do_tstamp = false;
392 
393 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
394 		bool is_eof = !!tx_swbd->skb;
395 
396 		if (unlikely(tx_swbd->check_wb)) {
397 			struct enetc_ndev_priv *priv = netdev_priv(ndev);
398 			union enetc_tx_bd *txbd;
399 
400 			txbd = ENETC_TXBD(*tx_ring, i);
401 
402 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
403 			    tx_swbd->do_tstamp) {
404 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
405 						    &tstamp);
406 				do_tstamp = true;
407 			}
408 		}
409 
410 		if (likely(tx_swbd->dma))
411 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
412 
413 		if (is_eof) {
414 			if (unlikely(do_tstamp)) {
415 				enetc_tstamp_tx(tx_swbd->skb, tstamp);
416 				do_tstamp = false;
417 			}
418 			napi_consume_skb(tx_swbd->skb, napi_budget);
419 			tx_swbd->skb = NULL;
420 		}
421 
422 		tx_byte_cnt += tx_swbd->len;
423 
424 		bds_to_clean--;
425 		tx_swbd++;
426 		i++;
427 		if (unlikely(i == tx_ring->bd_count)) {
428 			i = 0;
429 			tx_swbd = tx_ring->tx_swbd;
430 		}
431 
432 		/* BD iteration loop end */
433 		if (is_eof) {
434 			tx_frm_cnt++;
435 			/* re-arm interrupt source */
436 			enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) |
437 				     BIT(16 + tx_ring->index));
438 		}
439 
440 		if (unlikely(!bds_to_clean))
441 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
442 	}
443 
444 	tx_ring->next_to_clean = i;
445 	tx_ring->stats.packets += tx_frm_cnt;
446 	tx_ring->stats.bytes += tx_byte_cnt;
447 
448 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
449 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
450 		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
451 		netif_wake_subqueue(ndev, tx_ring->index);
452 	}
453 
454 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
455 }
456 
457 static bool enetc_new_page(struct enetc_bdr *rx_ring,
458 			   struct enetc_rx_swbd *rx_swbd)
459 {
460 	struct page *page;
461 	dma_addr_t addr;
462 
463 	page = dev_alloc_page();
464 	if (unlikely(!page))
465 		return false;
466 
467 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
468 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
469 		__free_page(page);
470 
471 		return false;
472 	}
473 
474 	rx_swbd->dma = addr;
475 	rx_swbd->page = page;
476 	rx_swbd->page_offset = ENETC_RXB_PAD;
477 
478 	return true;
479 }
480 
481 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
482 {
483 	struct enetc_rx_swbd *rx_swbd;
484 	union enetc_rx_bd *rxbd;
485 	int i, j;
486 
487 	i = rx_ring->next_to_use;
488 	rx_swbd = &rx_ring->rx_swbd[i];
489 	rxbd = enetc_rxbd(rx_ring, i);
490 
491 	for (j = 0; j < buff_cnt; j++) {
492 		/* try reuse page */
493 		if (unlikely(!rx_swbd->page)) {
494 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
495 				rx_ring->stats.rx_alloc_errs++;
496 				break;
497 			}
498 		}
499 
500 		/* update RxBD */
501 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
502 					   rx_swbd->page_offset);
503 		/* clear 'R" as well */
504 		rxbd->r.lstatus = 0;
505 
506 		rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
507 		rx_swbd++;
508 		i++;
509 		if (unlikely(i == rx_ring->bd_count)) {
510 			i = 0;
511 			rx_swbd = rx_ring->rx_swbd;
512 		}
513 	}
514 
515 	if (likely(j)) {
516 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
517 		rx_ring->next_to_use = i;
518 		/* update ENETC's consumer index */
519 		enetc_wr_reg(rx_ring->rcir, i);
520 	}
521 
522 	return j;
523 }
524 
525 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
526 static void enetc_get_rx_tstamp(struct net_device *ndev,
527 				union enetc_rx_bd *rxbd,
528 				struct sk_buff *skb)
529 {
530 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
531 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
532 	struct enetc_hw *hw = &priv->si->hw;
533 	u32 lo, hi, tstamp_lo;
534 	u64 tstamp;
535 
536 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
537 		lo = enetc_rd(hw, ENETC_SICTR0);
538 		hi = enetc_rd(hw, ENETC_SICTR1);
539 		rxbd = enetc_rxbd_ext(rxbd);
540 		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
541 		if (lo <= tstamp_lo)
542 			hi -= 1;
543 
544 		tstamp = (u64)hi << 32 | tstamp_lo;
545 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
546 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
547 	}
548 }
549 #endif
550 
551 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
552 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
553 {
554 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
555 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
556 #endif
557 	/* TODO: hashing */
558 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
559 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
560 
561 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
562 		skb->ip_summed = CHECKSUM_COMPLETE;
563 	}
564 
565 	/* copy VLAN to skb, if one is extracted, for now we assume it's a
566 	 * standard TPID, but HW also supports custom values
567 	 */
568 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN)
569 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
570 				       le16_to_cpu(rxbd->r.vlan_opt));
571 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
572 	if (priv->active_offloads & ENETC_F_RX_TSTAMP)
573 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
574 #endif
575 }
576 
577 static void enetc_process_skb(struct enetc_bdr *rx_ring,
578 			      struct sk_buff *skb)
579 {
580 	skb_record_rx_queue(skb, rx_ring->index);
581 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
582 }
583 
584 static bool enetc_page_reusable(struct page *page)
585 {
586 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
587 }
588 
589 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
590 			     struct enetc_rx_swbd *old)
591 {
592 	struct enetc_rx_swbd *new;
593 
594 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
595 
596 	/* next buf that may reuse a page */
597 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
598 
599 	/* copy page reference */
600 	*new = *old;
601 }
602 
603 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
604 					       int i, u16 size)
605 {
606 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
607 
608 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
609 				      rx_swbd->page_offset,
610 				      size, DMA_FROM_DEVICE);
611 	return rx_swbd;
612 }
613 
614 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
615 			      struct enetc_rx_swbd *rx_swbd)
616 {
617 	if (likely(enetc_page_reusable(rx_swbd->page))) {
618 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
619 		page_ref_inc(rx_swbd->page);
620 
621 		enetc_reuse_page(rx_ring, rx_swbd);
622 
623 		/* sync for use by the device */
624 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
625 						 rx_swbd->page_offset,
626 						 ENETC_RXB_DMA_SIZE,
627 						 DMA_FROM_DEVICE);
628 	} else {
629 		dma_unmap_page(rx_ring->dev, rx_swbd->dma,
630 			       PAGE_SIZE, DMA_FROM_DEVICE);
631 	}
632 
633 	rx_swbd->page = NULL;
634 }
635 
636 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
637 						int i, u16 size)
638 {
639 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
640 	struct sk_buff *skb;
641 	void *ba;
642 
643 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
644 	skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
645 	if (unlikely(!skb)) {
646 		rx_ring->stats.rx_alloc_errs++;
647 		return NULL;
648 	}
649 
650 	skb_reserve(skb, ENETC_RXB_PAD);
651 	__skb_put(skb, size);
652 
653 	enetc_put_rx_buff(rx_ring, rx_swbd);
654 
655 	return skb;
656 }
657 
658 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
659 				     u16 size, struct sk_buff *skb)
660 {
661 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
662 
663 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
664 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
665 
666 	enetc_put_rx_buff(rx_ring, rx_swbd);
667 }
668 
669 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
670 
671 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
672 			       struct napi_struct *napi, int work_limit)
673 {
674 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
675 	int cleaned_cnt, i;
676 
677 	cleaned_cnt = enetc_bd_unused(rx_ring);
678 	/* next descriptor to process */
679 	i = rx_ring->next_to_clean;
680 
681 	while (likely(rx_frm_cnt < work_limit)) {
682 		union enetc_rx_bd *rxbd;
683 		struct sk_buff *skb;
684 		u32 bd_status;
685 		u16 size;
686 
687 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
688 			int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
689 
690 			cleaned_cnt -= count;
691 		}
692 
693 		rxbd = enetc_rxbd(rx_ring, i);
694 		bd_status = le32_to_cpu(rxbd->r.lstatus);
695 		if (!bd_status)
696 			break;
697 
698 		enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index));
699 		dma_rmb(); /* for reading other rxbd fields */
700 		size = le16_to_cpu(rxbd->r.buf_len);
701 		skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
702 		if (!skb)
703 			break;
704 
705 		enetc_get_offloads(rx_ring, rxbd, skb);
706 
707 		cleaned_cnt++;
708 
709 		rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
710 		if (unlikely(++i == rx_ring->bd_count))
711 			i = 0;
712 
713 		if (unlikely(bd_status &
714 			     ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
715 			dev_kfree_skb(skb);
716 			while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
717 				dma_rmb();
718 				bd_status = le32_to_cpu(rxbd->r.lstatus);
719 
720 				rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
721 				if (unlikely(++i == rx_ring->bd_count))
722 					i = 0;
723 			}
724 
725 			rx_ring->ndev->stats.rx_dropped++;
726 			rx_ring->ndev->stats.rx_errors++;
727 
728 			break;
729 		}
730 
731 		/* not last BD in frame? */
732 		while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
733 			bd_status = le32_to_cpu(rxbd->r.lstatus);
734 			size = ENETC_RXB_DMA_SIZE;
735 
736 			if (bd_status & ENETC_RXBD_LSTATUS_F) {
737 				dma_rmb();
738 				size = le16_to_cpu(rxbd->r.buf_len);
739 			}
740 
741 			enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
742 
743 			cleaned_cnt++;
744 
745 			rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
746 			if (unlikely(++i == rx_ring->bd_count))
747 				i = 0;
748 		}
749 
750 		rx_byte_cnt += skb->len;
751 
752 		enetc_process_skb(rx_ring, skb);
753 
754 		napi_gro_receive(napi, skb);
755 
756 		rx_frm_cnt++;
757 	}
758 
759 	rx_ring->next_to_clean = i;
760 
761 	rx_ring->stats.packets += rx_frm_cnt;
762 	rx_ring->stats.bytes += rx_byte_cnt;
763 
764 	return rx_frm_cnt;
765 }
766 
767 /* Probing and Init */
768 #define ENETC_MAX_RFS_SIZE 64
769 void enetc_get_si_caps(struct enetc_si *si)
770 {
771 	struct enetc_hw *hw = &si->hw;
772 	u32 val;
773 
774 	/* find out how many of various resources we have to work with */
775 	val = enetc_rd(hw, ENETC_SICAPR0);
776 	si->num_rx_rings = (val >> 16) & 0xff;
777 	si->num_tx_rings = val & 0xff;
778 
779 	val = enetc_rd(hw, ENETC_SIRFSCAPR);
780 	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
781 	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
782 
783 	si->num_rss = 0;
784 	val = enetc_rd(hw, ENETC_SIPCAPR0);
785 	if (val & ENETC_SIPCAPR0_RSS) {
786 		u32 rss;
787 
788 		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
789 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
790 	}
791 
792 	if (val & ENETC_SIPCAPR0_QBV)
793 		si->hw_features |= ENETC_SI_F_QBV;
794 
795 	if (val & ENETC_SIPCAPR0_PSFP)
796 		si->hw_features |= ENETC_SI_F_PSFP;
797 }
798 
799 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
800 {
801 	r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
802 					&r->bd_dma_base, GFP_KERNEL);
803 	if (!r->bd_base)
804 		return -ENOMEM;
805 
806 	/* h/w requires 128B alignment */
807 	if (!IS_ALIGNED(r->bd_dma_base, 128)) {
808 		dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
809 				  r->bd_dma_base);
810 		return -EINVAL;
811 	}
812 
813 	return 0;
814 }
815 
816 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
817 {
818 	int err;
819 
820 	txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
821 	if (!txr->tx_swbd)
822 		return -ENOMEM;
823 
824 	err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
825 	if (err) {
826 		vfree(txr->tx_swbd);
827 		return err;
828 	}
829 
830 	txr->next_to_clean = 0;
831 	txr->next_to_use = 0;
832 
833 	return 0;
834 }
835 
836 static void enetc_free_txbdr(struct enetc_bdr *txr)
837 {
838 	int size, i;
839 
840 	for (i = 0; i < txr->bd_count; i++)
841 		enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
842 
843 	size = txr->bd_count * sizeof(union enetc_tx_bd);
844 
845 	dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
846 	txr->bd_base = NULL;
847 
848 	vfree(txr->tx_swbd);
849 	txr->tx_swbd = NULL;
850 }
851 
852 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
853 {
854 	int i, err;
855 
856 	for (i = 0; i < priv->num_tx_rings; i++) {
857 		err = enetc_alloc_txbdr(priv->tx_ring[i]);
858 
859 		if (err)
860 			goto fail;
861 	}
862 
863 	return 0;
864 
865 fail:
866 	while (i-- > 0)
867 		enetc_free_txbdr(priv->tx_ring[i]);
868 
869 	return err;
870 }
871 
872 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
873 {
874 	int i;
875 
876 	for (i = 0; i < priv->num_tx_rings; i++)
877 		enetc_free_txbdr(priv->tx_ring[i]);
878 }
879 
880 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
881 {
882 	size_t size = sizeof(union enetc_rx_bd);
883 	int err;
884 
885 	rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
886 	if (!rxr->rx_swbd)
887 		return -ENOMEM;
888 
889 	if (extended)
890 		size *= 2;
891 
892 	err = enetc_dma_alloc_bdr(rxr, size);
893 	if (err) {
894 		vfree(rxr->rx_swbd);
895 		return err;
896 	}
897 
898 	rxr->next_to_clean = 0;
899 	rxr->next_to_use = 0;
900 	rxr->next_to_alloc = 0;
901 	rxr->ext_en = extended;
902 
903 	return 0;
904 }
905 
906 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
907 {
908 	int size;
909 
910 	size = rxr->bd_count * sizeof(union enetc_rx_bd);
911 
912 	dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
913 	rxr->bd_base = NULL;
914 
915 	vfree(rxr->rx_swbd);
916 	rxr->rx_swbd = NULL;
917 }
918 
919 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
920 {
921 	bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
922 	int i, err;
923 
924 	for (i = 0; i < priv->num_rx_rings; i++) {
925 		err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
926 
927 		if (err)
928 			goto fail;
929 	}
930 
931 	return 0;
932 
933 fail:
934 	while (i-- > 0)
935 		enetc_free_rxbdr(priv->rx_ring[i]);
936 
937 	return err;
938 }
939 
940 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
941 {
942 	int i;
943 
944 	for (i = 0; i < priv->num_rx_rings; i++)
945 		enetc_free_rxbdr(priv->rx_ring[i]);
946 }
947 
948 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
949 {
950 	int i;
951 
952 	if (!tx_ring->tx_swbd)
953 		return;
954 
955 	for (i = 0; i < tx_ring->bd_count; i++) {
956 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
957 
958 		enetc_free_tx_skb(tx_ring, tx_swbd);
959 	}
960 
961 	tx_ring->next_to_clean = 0;
962 	tx_ring->next_to_use = 0;
963 }
964 
965 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
966 {
967 	int i;
968 
969 	if (!rx_ring->rx_swbd)
970 		return;
971 
972 	for (i = 0; i < rx_ring->bd_count; i++) {
973 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
974 
975 		if (!rx_swbd->page)
976 			continue;
977 
978 		dma_unmap_page(rx_ring->dev, rx_swbd->dma,
979 			       PAGE_SIZE, DMA_FROM_DEVICE);
980 		__free_page(rx_swbd->page);
981 		rx_swbd->page = NULL;
982 	}
983 
984 	rx_ring->next_to_clean = 0;
985 	rx_ring->next_to_use = 0;
986 	rx_ring->next_to_alloc = 0;
987 }
988 
989 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
990 {
991 	int i;
992 
993 	for (i = 0; i < priv->num_rx_rings; i++)
994 		enetc_free_rx_ring(priv->rx_ring[i]);
995 
996 	for (i = 0; i < priv->num_tx_rings; i++)
997 		enetc_free_tx_ring(priv->tx_ring[i]);
998 }
999 
1000 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1001 {
1002 	int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1003 
1004 	cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
1005 					   GFP_KERNEL);
1006 	if (!cbdr->bd_base)
1007 		return -ENOMEM;
1008 
1009 	/* h/w requires 128B alignment */
1010 	if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
1011 		dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1012 		return -EINVAL;
1013 	}
1014 
1015 	cbdr->next_to_clean = 0;
1016 	cbdr->next_to_use = 0;
1017 
1018 	return 0;
1019 }
1020 
1021 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1022 {
1023 	int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1024 
1025 	dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1026 	cbdr->bd_base = NULL;
1027 }
1028 
1029 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
1030 {
1031 	/* set CBDR cache attributes */
1032 	enetc_wr(hw, ENETC_SICAR2,
1033 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1034 
1035 	enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
1036 	enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
1037 	enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
1038 
1039 	enetc_wr(hw, ENETC_SICBDRPIR, 0);
1040 	enetc_wr(hw, ENETC_SICBDRCIR, 0);
1041 
1042 	/* enable ring */
1043 	enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
1044 
1045 	cbdr->pir = hw->reg + ENETC_SICBDRPIR;
1046 	cbdr->cir = hw->reg + ENETC_SICBDRCIR;
1047 }
1048 
1049 static void enetc_clear_cbdr(struct enetc_hw *hw)
1050 {
1051 	enetc_wr(hw, ENETC_SICBDRMR, 0);
1052 }
1053 
1054 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1055 {
1056 	int *rss_table;
1057 	int i;
1058 
1059 	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1060 	if (!rss_table)
1061 		return -ENOMEM;
1062 
1063 	/* Set up RSS table defaults */
1064 	for (i = 0; i < si->num_rss; i++)
1065 		rss_table[i] = i % num_groups;
1066 
1067 	enetc_set_rss_table(si, rss_table, si->num_rss);
1068 
1069 	kfree(rss_table);
1070 
1071 	return 0;
1072 }
1073 
1074 static int enetc_configure_si(struct enetc_ndev_priv *priv)
1075 {
1076 	struct enetc_si *si = priv->si;
1077 	struct enetc_hw *hw = &si->hw;
1078 	int err;
1079 
1080 	enetc_setup_cbdr(hw, &si->cbd_ring);
1081 	/* set SI cache attributes */
1082 	enetc_wr(hw, ENETC_SICAR0,
1083 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1084 	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1085 	/* enable SI */
1086 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1087 
1088 	if (si->num_rss) {
1089 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1090 		if (err)
1091 			return err;
1092 	}
1093 
1094 	return 0;
1095 }
1096 
1097 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1098 {
1099 	struct enetc_si *si = priv->si;
1100 	int cpus = num_online_cpus();
1101 
1102 	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1103 	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1104 
1105 	/* Enable all available TX rings in order to configure as many
1106 	 * priorities as possible, when needed.
1107 	 * TODO: Make # of TX rings run-time configurable
1108 	 */
1109 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1110 	priv->num_tx_rings = si->num_tx_rings;
1111 	priv->bdr_int_num = cpus;
1112 	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
1113 	priv->tx_ictt = ENETC_TXIC_TIMETHR;
1114 
1115 	/* SI specific */
1116 	si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1117 }
1118 
1119 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1120 {
1121 	struct enetc_si *si = priv->si;
1122 	int err;
1123 
1124 	err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1125 	if (err)
1126 		return err;
1127 
1128 	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1129 				  GFP_KERNEL);
1130 	if (!priv->cls_rules) {
1131 		err = -ENOMEM;
1132 		goto err_alloc_cls;
1133 	}
1134 
1135 	err = enetc_configure_si(priv);
1136 	if (err)
1137 		goto err_config_si;
1138 
1139 	return 0;
1140 
1141 err_config_si:
1142 	kfree(priv->cls_rules);
1143 err_alloc_cls:
1144 	enetc_clear_cbdr(&si->hw);
1145 	enetc_free_cbdr(priv->dev, &si->cbd_ring);
1146 
1147 	return err;
1148 }
1149 
1150 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1151 {
1152 	struct enetc_si *si = priv->si;
1153 
1154 	enetc_clear_cbdr(&si->hw);
1155 	enetc_free_cbdr(priv->dev, &si->cbd_ring);
1156 
1157 	kfree(priv->cls_rules);
1158 }
1159 
1160 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1161 {
1162 	int idx = tx_ring->index;
1163 	u32 tbmr;
1164 
1165 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1166 		       lower_32_bits(tx_ring->bd_dma_base));
1167 
1168 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1169 		       upper_32_bits(tx_ring->bd_dma_base));
1170 
1171 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1172 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1173 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
1174 
1175 	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1176 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1177 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1178 
1179 	/* enable Tx ints by setting pkt thr to 1 */
1180 	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
1181 
1182 	tbmr = ENETC_TBMR_EN;
1183 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1184 		tbmr |= ENETC_TBMR_VIH;
1185 
1186 	/* enable ring */
1187 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1188 
1189 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1190 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1191 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
1192 }
1193 
1194 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1195 {
1196 	int idx = rx_ring->index;
1197 	u32 rbmr;
1198 
1199 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1200 		       lower_32_bits(rx_ring->bd_dma_base));
1201 
1202 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1203 		       upper_32_bits(rx_ring->bd_dma_base));
1204 
1205 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1206 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1207 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
1208 
1209 	enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1210 
1211 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1212 
1213 	/* enable Rx ints by setting pkt thr to 1 */
1214 	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
1215 
1216 	rbmr = ENETC_RBMR_EN;
1217 
1218 	if (rx_ring->ext_en)
1219 		rbmr |= ENETC_RBMR_BDS;
1220 
1221 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1222 		rbmr |= ENETC_RBMR_VTE;
1223 
1224 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1225 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1226 
1227 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1228 
1229 	/* enable ring */
1230 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1231 }
1232 
1233 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1234 {
1235 	int i;
1236 
1237 	for (i = 0; i < priv->num_tx_rings; i++)
1238 		enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
1239 
1240 	for (i = 0; i < priv->num_rx_rings; i++)
1241 		enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1242 }
1243 
1244 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1245 {
1246 	int idx = rx_ring->index;
1247 
1248 	/* disable EN bit on ring */
1249 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1250 }
1251 
1252 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1253 {
1254 	int delay = 8, timeout = 100;
1255 	int idx = tx_ring->index;
1256 
1257 	/* disable EN bit on ring */
1258 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1259 
1260 	/* wait for busy to clear */
1261 	while (delay < timeout &&
1262 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1263 		msleep(delay);
1264 		delay *= 2;
1265 	}
1266 
1267 	if (delay >= timeout)
1268 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1269 			    idx);
1270 }
1271 
1272 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1273 {
1274 	int i;
1275 
1276 	for (i = 0; i < priv->num_tx_rings; i++)
1277 		enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
1278 
1279 	for (i = 0; i < priv->num_rx_rings; i++)
1280 		enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1281 
1282 	udelay(1);
1283 }
1284 
1285 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1286 {
1287 	struct pci_dev *pdev = priv->si->pdev;
1288 	cpumask_t cpu_mask;
1289 	int i, j, err;
1290 
1291 	for (i = 0; i < priv->bdr_int_num; i++) {
1292 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1293 		struct enetc_int_vector *v = priv->int_vector[i];
1294 		int entry = ENETC_BDR_INT_BASE_IDX + i;
1295 		struct enetc_hw *hw = &priv->si->hw;
1296 
1297 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1298 			 priv->ndev->name, i);
1299 		err = request_irq(irq, enetc_msix, 0, v->name, v);
1300 		if (err) {
1301 			dev_err(priv->dev, "request_irq() failed!\n");
1302 			goto irq_err;
1303 		}
1304 		disable_irq(irq);
1305 
1306 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1307 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1308 		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
1309 
1310 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1311 
1312 		for (j = 0; j < v->count_tx_rings; j++) {
1313 			int idx = v->tx_ring[j].index;
1314 
1315 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1316 		}
1317 		cpumask_clear(&cpu_mask);
1318 		cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
1319 		irq_set_affinity_hint(irq, &cpu_mask);
1320 	}
1321 
1322 	return 0;
1323 
1324 irq_err:
1325 	while (i--) {
1326 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1327 
1328 		irq_set_affinity_hint(irq, NULL);
1329 		free_irq(irq, priv->int_vector[i]);
1330 	}
1331 
1332 	return err;
1333 }
1334 
1335 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1336 {
1337 	struct pci_dev *pdev = priv->si->pdev;
1338 	int i;
1339 
1340 	for (i = 0; i < priv->bdr_int_num; i++) {
1341 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1342 
1343 		irq_set_affinity_hint(irq, NULL);
1344 		free_irq(irq, priv->int_vector[i]);
1345 	}
1346 }
1347 
1348 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
1349 {
1350 	struct enetc_hw *hw = &priv->si->hw;
1351 	u32 icpt, ictt;
1352 	int i;
1353 
1354 	/* enable Tx & Rx event indication */
1355 	if (priv->ic_mode &
1356 	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
1357 		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
1358 		/* init to non-0 minimum, will be adjusted later */
1359 		ictt = 0x1;
1360 	} else {
1361 		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
1362 		ictt = 0;
1363 	}
1364 
1365 	for (i = 0; i < priv->num_rx_rings; i++) {
1366 		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
1367 		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
1368 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
1369 	}
1370 
1371 	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
1372 		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
1373 	else
1374 		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
1375 
1376 	for (i = 0; i < priv->num_tx_rings; i++) {
1377 		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
1378 		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
1379 		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
1380 	}
1381 }
1382 
1383 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
1384 {
1385 	int i;
1386 
1387 	for (i = 0; i < priv->num_tx_rings; i++)
1388 		enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
1389 
1390 	for (i = 0; i < priv->num_rx_rings; i++)
1391 		enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
1392 }
1393 
1394 static int enetc_phylink_connect(struct net_device *ndev)
1395 {
1396 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1397 	struct ethtool_eee edata;
1398 	int err;
1399 
1400 	if (!priv->phylink)
1401 		return 0; /* phy-less mode */
1402 
1403 	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
1404 	if (err) {
1405 		dev_err(&ndev->dev, "could not attach to PHY\n");
1406 		return err;
1407 	}
1408 
1409 	/* disable EEE autoneg, until ENETC driver supports it */
1410 	memset(&edata, 0, sizeof(struct ethtool_eee));
1411 	phylink_ethtool_set_eee(priv->phylink, &edata);
1412 
1413 	return 0;
1414 }
1415 
1416 void enetc_start(struct net_device *ndev)
1417 {
1418 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1419 	int i;
1420 
1421 	enetc_setup_interrupts(priv);
1422 
1423 	for (i = 0; i < priv->bdr_int_num; i++) {
1424 		int irq = pci_irq_vector(priv->si->pdev,
1425 					 ENETC_BDR_INT_BASE_IDX + i);
1426 
1427 		napi_enable(&priv->int_vector[i]->napi);
1428 		enable_irq(irq);
1429 	}
1430 
1431 	if (priv->phylink)
1432 		phylink_start(priv->phylink);
1433 	else
1434 		netif_carrier_on(ndev);
1435 
1436 	netif_tx_start_all_queues(ndev);
1437 }
1438 
1439 int enetc_open(struct net_device *ndev)
1440 {
1441 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1442 	int err;
1443 
1444 	err = enetc_setup_irqs(priv);
1445 	if (err)
1446 		return err;
1447 
1448 	err = enetc_phylink_connect(ndev);
1449 	if (err)
1450 		goto err_phy_connect;
1451 
1452 	err = enetc_alloc_tx_resources(priv);
1453 	if (err)
1454 		goto err_alloc_tx;
1455 
1456 	err = enetc_alloc_rx_resources(priv);
1457 	if (err)
1458 		goto err_alloc_rx;
1459 
1460 	err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1461 	if (err)
1462 		goto err_set_queues;
1463 
1464 	err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1465 	if (err)
1466 		goto err_set_queues;
1467 
1468 	enetc_setup_bdrs(priv);
1469 	enetc_start(ndev);
1470 
1471 	return 0;
1472 
1473 err_set_queues:
1474 	enetc_free_rx_resources(priv);
1475 err_alloc_rx:
1476 	enetc_free_tx_resources(priv);
1477 err_alloc_tx:
1478 	if (priv->phylink)
1479 		phylink_disconnect_phy(priv->phylink);
1480 err_phy_connect:
1481 	enetc_free_irqs(priv);
1482 
1483 	return err;
1484 }
1485 
1486 void enetc_stop(struct net_device *ndev)
1487 {
1488 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1489 	int i;
1490 
1491 	netif_tx_stop_all_queues(ndev);
1492 
1493 	for (i = 0; i < priv->bdr_int_num; i++) {
1494 		int irq = pci_irq_vector(priv->si->pdev,
1495 					 ENETC_BDR_INT_BASE_IDX + i);
1496 
1497 		disable_irq(irq);
1498 		napi_synchronize(&priv->int_vector[i]->napi);
1499 		napi_disable(&priv->int_vector[i]->napi);
1500 	}
1501 
1502 	if (priv->phylink)
1503 		phylink_stop(priv->phylink);
1504 	else
1505 		netif_carrier_off(ndev);
1506 
1507 	enetc_clear_interrupts(priv);
1508 }
1509 
1510 int enetc_close(struct net_device *ndev)
1511 {
1512 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1513 
1514 	enetc_stop(ndev);
1515 	enetc_clear_bdrs(priv);
1516 
1517 	if (priv->phylink)
1518 		phylink_disconnect_phy(priv->phylink);
1519 	enetc_free_rxtx_rings(priv);
1520 	enetc_free_rx_resources(priv);
1521 	enetc_free_tx_resources(priv);
1522 	enetc_free_irqs(priv);
1523 
1524 	return 0;
1525 }
1526 
1527 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
1528 {
1529 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1530 	struct tc_mqprio_qopt *mqprio = type_data;
1531 	struct enetc_bdr *tx_ring;
1532 	u8 num_tc;
1533 	int i;
1534 
1535 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1536 	num_tc = mqprio->num_tc;
1537 
1538 	if (!num_tc) {
1539 		netdev_reset_tc(ndev);
1540 		netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1541 
1542 		/* Reset all ring priorities to 0 */
1543 		for (i = 0; i < priv->num_tx_rings; i++) {
1544 			tx_ring = priv->tx_ring[i];
1545 			enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
1546 		}
1547 
1548 		return 0;
1549 	}
1550 
1551 	/* Check if we have enough BD rings available to accommodate all TCs */
1552 	if (num_tc > priv->num_tx_rings) {
1553 		netdev_err(ndev, "Max %d traffic classes supported\n",
1554 			   priv->num_tx_rings);
1555 		return -EINVAL;
1556 	}
1557 
1558 	/* For the moment, we use only one BD ring per TC.
1559 	 *
1560 	 * Configure num_tc BD rings with increasing priorities.
1561 	 */
1562 	for (i = 0; i < num_tc; i++) {
1563 		tx_ring = priv->tx_ring[i];
1564 		enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
1565 	}
1566 
1567 	/* Reset the number of netdev queues based on the TC count */
1568 	netif_set_real_num_tx_queues(ndev, num_tc);
1569 
1570 	netdev_set_num_tc(ndev, num_tc);
1571 
1572 	/* Each TC is associated with one netdev queue */
1573 	for (i = 0; i < num_tc; i++)
1574 		netdev_set_tc_queue(ndev, i, 1, i);
1575 
1576 	return 0;
1577 }
1578 
1579 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1580 		   void *type_data)
1581 {
1582 	switch (type) {
1583 	case TC_SETUP_QDISC_MQPRIO:
1584 		return enetc_setup_tc_mqprio(ndev, type_data);
1585 	case TC_SETUP_QDISC_TAPRIO:
1586 		return enetc_setup_tc_taprio(ndev, type_data);
1587 	case TC_SETUP_QDISC_CBS:
1588 		return enetc_setup_tc_cbs(ndev, type_data);
1589 	case TC_SETUP_QDISC_ETF:
1590 		return enetc_setup_tc_txtime(ndev, type_data);
1591 	case TC_SETUP_BLOCK:
1592 		return enetc_setup_tc_psfp(ndev, type_data);
1593 	default:
1594 		return -EOPNOTSUPP;
1595 	}
1596 }
1597 
1598 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1599 {
1600 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1601 	struct net_device_stats *stats = &ndev->stats;
1602 	unsigned long packets = 0, bytes = 0;
1603 	int i;
1604 
1605 	for (i = 0; i < priv->num_rx_rings; i++) {
1606 		packets += priv->rx_ring[i]->stats.packets;
1607 		bytes	+= priv->rx_ring[i]->stats.bytes;
1608 	}
1609 
1610 	stats->rx_packets = packets;
1611 	stats->rx_bytes = bytes;
1612 	bytes = 0;
1613 	packets = 0;
1614 
1615 	for (i = 0; i < priv->num_tx_rings; i++) {
1616 		packets += priv->tx_ring[i]->stats.packets;
1617 		bytes	+= priv->tx_ring[i]->stats.bytes;
1618 	}
1619 
1620 	stats->tx_packets = packets;
1621 	stats->tx_bytes = bytes;
1622 
1623 	return stats;
1624 }
1625 
1626 static int enetc_set_rss(struct net_device *ndev, int en)
1627 {
1628 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1629 	struct enetc_hw *hw = &priv->si->hw;
1630 	u32 reg;
1631 
1632 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1633 
1634 	reg = enetc_rd(hw, ENETC_SIMR);
1635 	reg &= ~ENETC_SIMR_RSSE;
1636 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
1637 	enetc_wr(hw, ENETC_SIMR, reg);
1638 
1639 	return 0;
1640 }
1641 
1642 static int enetc_set_psfp(struct net_device *ndev, int en)
1643 {
1644 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1645 	int err;
1646 
1647 	if (en) {
1648 		err = enetc_psfp_enable(priv);
1649 		if (err)
1650 			return err;
1651 
1652 		priv->active_offloads |= ENETC_F_QCI;
1653 		return 0;
1654 	}
1655 
1656 	err = enetc_psfp_disable(priv);
1657 	if (err)
1658 		return err;
1659 
1660 	priv->active_offloads &= ~ENETC_F_QCI;
1661 
1662 	return 0;
1663 }
1664 
1665 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
1666 {
1667 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1668 	int i;
1669 
1670 	for (i = 0; i < priv->num_rx_rings; i++)
1671 		enetc_bdr_enable_rxvlan(&priv->si->hw, i, en);
1672 }
1673 
1674 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
1675 {
1676 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1677 	int i;
1678 
1679 	for (i = 0; i < priv->num_tx_rings; i++)
1680 		enetc_bdr_enable_txvlan(&priv->si->hw, i, en);
1681 }
1682 
1683 int enetc_set_features(struct net_device *ndev,
1684 		       netdev_features_t features)
1685 {
1686 	netdev_features_t changed = ndev->features ^ features;
1687 	int err = 0;
1688 
1689 	if (changed & NETIF_F_RXHASH)
1690 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1691 
1692 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1693 		enetc_enable_rxvlan(ndev,
1694 				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
1695 
1696 	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
1697 		enetc_enable_txvlan(ndev,
1698 				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
1699 
1700 	if (changed & NETIF_F_HW_TC)
1701 		err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC));
1702 
1703 	return err;
1704 }
1705 
1706 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1707 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1708 {
1709 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1710 	struct hwtstamp_config config;
1711 	int ao;
1712 
1713 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1714 		return -EFAULT;
1715 
1716 	switch (config.tx_type) {
1717 	case HWTSTAMP_TX_OFF:
1718 		priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1719 		break;
1720 	case HWTSTAMP_TX_ON:
1721 		priv->active_offloads |= ENETC_F_TX_TSTAMP;
1722 		break;
1723 	default:
1724 		return -ERANGE;
1725 	}
1726 
1727 	ao = priv->active_offloads;
1728 	switch (config.rx_filter) {
1729 	case HWTSTAMP_FILTER_NONE:
1730 		priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1731 		break;
1732 	default:
1733 		priv->active_offloads |= ENETC_F_RX_TSTAMP;
1734 		config.rx_filter = HWTSTAMP_FILTER_ALL;
1735 	}
1736 
1737 	if (netif_running(ndev) && ao != priv->active_offloads) {
1738 		enetc_close(ndev);
1739 		enetc_open(ndev);
1740 	}
1741 
1742 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1743 	       -EFAULT : 0;
1744 }
1745 
1746 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1747 {
1748 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1749 	struct hwtstamp_config config;
1750 
1751 	config.flags = 0;
1752 
1753 	if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1754 		config.tx_type = HWTSTAMP_TX_ON;
1755 	else
1756 		config.tx_type = HWTSTAMP_TX_OFF;
1757 
1758 	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1759 			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1760 
1761 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1762 	       -EFAULT : 0;
1763 }
1764 #endif
1765 
1766 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1767 {
1768 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1769 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1770 	if (cmd == SIOCSHWTSTAMP)
1771 		return enetc_hwtstamp_set(ndev, rq);
1772 	if (cmd == SIOCGHWTSTAMP)
1773 		return enetc_hwtstamp_get(ndev, rq);
1774 #endif
1775 
1776 	if (!priv->phylink)
1777 		return -EOPNOTSUPP;
1778 
1779 	return phylink_mii_ioctl(priv->phylink, rq, cmd);
1780 }
1781 
1782 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1783 {
1784 	struct pci_dev *pdev = priv->si->pdev;
1785 	int v_tx_rings;
1786 	int i, n, err, nvec;
1787 
1788 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1789 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
1790 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1791 
1792 	if (n < 0)
1793 		return n;
1794 
1795 	if (n != nvec)
1796 		return -EPERM;
1797 
1798 	/* # of tx rings per int vector */
1799 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1800 
1801 	for (i = 0; i < priv->bdr_int_num; i++) {
1802 		struct enetc_int_vector *v;
1803 		struct enetc_bdr *bdr;
1804 		int j;
1805 
1806 		v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
1807 		if (!v) {
1808 			err = -ENOMEM;
1809 			goto fail;
1810 		}
1811 
1812 		priv->int_vector[i] = v;
1813 
1814 		/* init defaults for adaptive IC */
1815 		if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
1816 			v->rx_ictt = 0x1;
1817 			v->rx_dim_en = true;
1818 		}
1819 		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
1820 		netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1821 			       NAPI_POLL_WEIGHT);
1822 		v->count_tx_rings = v_tx_rings;
1823 
1824 		for (j = 0; j < v_tx_rings; j++) {
1825 			int idx;
1826 
1827 			/* default tx ring mapping policy */
1828 			if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1829 				idx = 2 * j + i; /* 2 CPUs */
1830 			else
1831 				idx = j + i * v_tx_rings; /* default */
1832 
1833 			__set_bit(idx, &v->tx_rings_map);
1834 			bdr = &v->tx_ring[j];
1835 			bdr->index = idx;
1836 			bdr->ndev = priv->ndev;
1837 			bdr->dev = priv->dev;
1838 			bdr->bd_count = priv->tx_bd_count;
1839 			priv->tx_ring[idx] = bdr;
1840 		}
1841 
1842 		bdr = &v->rx_ring;
1843 		bdr->index = i;
1844 		bdr->ndev = priv->ndev;
1845 		bdr->dev = priv->dev;
1846 		bdr->bd_count = priv->rx_bd_count;
1847 		priv->rx_ring[i] = bdr;
1848 	}
1849 
1850 	return 0;
1851 
1852 fail:
1853 	while (i--) {
1854 		netif_napi_del(&priv->int_vector[i]->napi);
1855 		cancel_work_sync(&priv->int_vector[i]->rx_dim.work);
1856 		kfree(priv->int_vector[i]);
1857 	}
1858 
1859 	pci_free_irq_vectors(pdev);
1860 
1861 	return err;
1862 }
1863 
1864 void enetc_free_msix(struct enetc_ndev_priv *priv)
1865 {
1866 	int i;
1867 
1868 	for (i = 0; i < priv->bdr_int_num; i++) {
1869 		struct enetc_int_vector *v = priv->int_vector[i];
1870 
1871 		netif_napi_del(&v->napi);
1872 		cancel_work_sync(&v->rx_dim.work);
1873 	}
1874 
1875 	for (i = 0; i < priv->num_rx_rings; i++)
1876 		priv->rx_ring[i] = NULL;
1877 
1878 	for (i = 0; i < priv->num_tx_rings; i++)
1879 		priv->tx_ring[i] = NULL;
1880 
1881 	for (i = 0; i < priv->bdr_int_num; i++) {
1882 		kfree(priv->int_vector[i]);
1883 		priv->int_vector[i] = NULL;
1884 	}
1885 
1886 	/* disable all MSIX for this device */
1887 	pci_free_irq_vectors(priv->si->pdev);
1888 }
1889 
1890 static void enetc_kfree_si(struct enetc_si *si)
1891 {
1892 	char *p = (char *)si - si->pad;
1893 
1894 	kfree(p);
1895 }
1896 
1897 static void enetc_detect_errata(struct enetc_si *si)
1898 {
1899 	if (si->pdev->revision == ENETC_REV1)
1900 		si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1901 			     ENETC_ERR_UCMCSWP;
1902 }
1903 
1904 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1905 {
1906 	struct enetc_si *si, *p;
1907 	struct enetc_hw *hw;
1908 	size_t alloc_size;
1909 	int err, len;
1910 
1911 	pcie_flr(pdev);
1912 	err = pci_enable_device_mem(pdev);
1913 	if (err) {
1914 		dev_err(&pdev->dev, "device enable failed\n");
1915 		return err;
1916 	}
1917 
1918 	/* set up for high or low dma */
1919 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1920 	if (err) {
1921 		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1922 		if (err) {
1923 			dev_err(&pdev->dev,
1924 				"DMA configuration failed: 0x%x\n", err);
1925 			goto err_dma;
1926 		}
1927 	}
1928 
1929 	err = pci_request_mem_regions(pdev, name);
1930 	if (err) {
1931 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1932 		goto err_pci_mem_reg;
1933 	}
1934 
1935 	pci_set_master(pdev);
1936 
1937 	alloc_size = sizeof(struct enetc_si);
1938 	if (sizeof_priv) {
1939 		/* align priv to 32B */
1940 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1941 		alloc_size += sizeof_priv;
1942 	}
1943 	/* force 32B alignment for enetc_si */
1944 	alloc_size += ENETC_SI_ALIGN - 1;
1945 
1946 	p = kzalloc(alloc_size, GFP_KERNEL);
1947 	if (!p) {
1948 		err = -ENOMEM;
1949 		goto err_alloc_si;
1950 	}
1951 
1952 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1953 	si->pad = (char *)si - (char *)p;
1954 
1955 	pci_set_drvdata(pdev, si);
1956 	si->pdev = pdev;
1957 	hw = &si->hw;
1958 
1959 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
1960 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1961 	if (!hw->reg) {
1962 		err = -ENXIO;
1963 		dev_err(&pdev->dev, "ioremap() failed\n");
1964 		goto err_ioremap;
1965 	}
1966 	if (len > ENETC_PORT_BASE)
1967 		hw->port = hw->reg + ENETC_PORT_BASE;
1968 	if (len > ENETC_GLOBAL_BASE)
1969 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
1970 
1971 	enetc_detect_errata(si);
1972 
1973 	return 0;
1974 
1975 err_ioremap:
1976 	enetc_kfree_si(si);
1977 err_alloc_si:
1978 	pci_release_mem_regions(pdev);
1979 err_pci_mem_reg:
1980 err_dma:
1981 	pci_disable_device(pdev);
1982 
1983 	return err;
1984 }
1985 
1986 void enetc_pci_remove(struct pci_dev *pdev)
1987 {
1988 	struct enetc_si *si = pci_get_drvdata(pdev);
1989 	struct enetc_hw *hw = &si->hw;
1990 
1991 	iounmap(hw->reg);
1992 	enetc_kfree_si(si);
1993 	pci_release_mem_regions(pdev);
1994 	pci_disable_device(pdev);
1995 }
1996