1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
2 /*
3  * Copyright 2008 - 2016 Freescale Semiconductor Inc.
4  * Copyright 2020 NXP
5  */
6 
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 
9 #include <linux/init.h>
10 #include <linux/mod_devicetable.h>
11 #include <linux/module.h>
12 #include <linux/of_mdio.h>
13 #include <linux/of_net.h>
14 #include <linux/io.h>
15 #include <linux/if_arp.h>
16 #include <linux/if_vlan.h>
17 #include <linux/icmp.h>
18 #include <linux/ip.h>
19 #include <linux/ipv6.h>
20 #include <linux/platform_device.h>
21 #include <linux/udp.h>
22 #include <linux/tcp.h>
23 #include <linux/net.h>
24 #include <linux/skbuff.h>
25 #include <linux/etherdevice.h>
26 #include <linux/if_ether.h>
27 #include <linux/highmem.h>
28 #include <linux/percpu.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/sort.h>
31 #include <linux/phy_fixed.h>
32 #include <linux/bpf.h>
33 #include <linux/bpf_trace.h>
34 #include <soc/fsl/bman.h>
35 #include <soc/fsl/qman.h>
36 #include "fman.h"
37 #include "fman_port.h"
38 #include "mac.h"
39 #include "dpaa_eth.h"
40 
41 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpaa files
42  * using trace events only need to #include <trace/events/sched.h>
43  */
44 #define CREATE_TRACE_POINTS
45 #include "dpaa_eth_trace.h"
46 
47 static int debug = -1;
48 module_param(debug, int, 0444);
49 MODULE_PARM_DESC(debug, "Module/Driver verbosity level (0=none,...,16=all)");
50 
51 static u16 tx_timeout = 1000;
52 module_param(tx_timeout, ushort, 0444);
53 MODULE_PARM_DESC(tx_timeout, "The Tx timeout in ms");
54 
55 #define FM_FD_STAT_RX_ERRORS						\
56 	(FM_FD_ERR_DMA | FM_FD_ERR_PHYSICAL	| \
57 	 FM_FD_ERR_SIZE | FM_FD_ERR_CLS_DISCARD | \
58 	 FM_FD_ERR_EXTRACTION | FM_FD_ERR_NO_SCHEME	| \
59 	 FM_FD_ERR_PRS_TIMEOUT | FM_FD_ERR_PRS_ILL_INSTRUCT | \
60 	 FM_FD_ERR_PRS_HDR_ERR)
61 
62 #define FM_FD_STAT_TX_ERRORS \
63 	(FM_FD_ERR_UNSUPPORTED_FORMAT | \
64 	 FM_FD_ERR_LENGTH | FM_FD_ERR_DMA)
65 
66 #define DPAA_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
67 			  NETIF_MSG_LINK | NETIF_MSG_IFUP | \
68 			  NETIF_MSG_IFDOWN | NETIF_MSG_HW)
69 
70 #define DPAA_INGRESS_CS_THRESHOLD 0x10000000
71 /* Ingress congestion threshold on FMan ports
72  * The size in bytes of the ingress tail-drop threshold on FMan ports.
73  * Traffic piling up above this value will be rejected by QMan and discarded
74  * by FMan.
75  */
76 
77 /* Size in bytes of the FQ taildrop threshold */
78 #define DPAA_FQ_TD 0x200000
79 
80 #define DPAA_CS_THRESHOLD_1G 0x06000000
81 /* Egress congestion threshold on 1G ports, range 0x1000 .. 0x10000000
82  * The size in bytes of the egress Congestion State notification threshold on
83  * 1G ports. The 1G dTSECs can quite easily be flooded by cores doing Tx in a
84  * tight loop (e.g. by sending UDP datagrams at "while(1) speed"),
85  * and the larger the frame size, the more acute the problem.
86  * So we have to find a balance between these factors:
87  * - avoiding the device staying congested for a prolonged time (risking
88  *   the netdev watchdog to fire - see also the tx_timeout module param);
89  * - affecting performance of protocols such as TCP, which otherwise
90  *   behave well under the congestion notification mechanism;
91  * - preventing the Tx cores from tightly-looping (as if the congestion
92  *   threshold was too low to be effective);
93  * - running out of memory if the CS threshold is set too high.
94  */
95 
96 #define DPAA_CS_THRESHOLD_10G 0x10000000
97 /* The size in bytes of the egress Congestion State notification threshold on
98  * 10G ports, range 0x1000 .. 0x10000000
99  */
100 
101 /* Largest value that the FQD's OAL field can hold */
102 #define FSL_QMAN_MAX_OAL	127
103 
104 /* Default alignment for start of data in an Rx FD */
105 #ifdef CONFIG_DPAA_ERRATUM_A050385
106 /* aligning data start to 64 avoids DMA transaction splits, unless the buffer
107  * is crossing a 4k page boundary
108  */
109 #define DPAA_FD_DATA_ALIGNMENT  (fman_has_errata_a050385() ? 64 : 16)
110 /* aligning to 256 avoids DMA transaction splits caused by 4k page boundary
111  * crossings; also, all SG fragments except the last must have a size multiple
112  * of 256 to avoid DMA transaction splits
113  */
114 #define DPAA_A050385_ALIGN 256
115 #define DPAA_FD_RX_DATA_ALIGNMENT (fman_has_errata_a050385() ? \
116 				   DPAA_A050385_ALIGN : 16)
117 #else
118 #define DPAA_FD_DATA_ALIGNMENT  16
119 #define DPAA_FD_RX_DATA_ALIGNMENT DPAA_FD_DATA_ALIGNMENT
120 #endif
121 
122 /* The DPAA requires 256 bytes reserved and mapped for the SGT */
123 #define DPAA_SGT_SIZE 256
124 
125 /* Values for the L3R field of the FM Parse Results
126  */
127 /* L3 Type field: First IP Present IPv4 */
128 #define FM_L3_PARSE_RESULT_IPV4	0x8000
129 /* L3 Type field: First IP Present IPv6 */
130 #define FM_L3_PARSE_RESULT_IPV6	0x4000
131 /* Values for the L4R field of the FM Parse Results */
132 /* L4 Type field: UDP */
133 #define FM_L4_PARSE_RESULT_UDP	0x40
134 /* L4 Type field: TCP */
135 #define FM_L4_PARSE_RESULT_TCP	0x20
136 
137 /* FD status field indicating whether the FM Parser has attempted to validate
138  * the L4 csum of the frame.
139  * Note that having this bit set doesn't necessarily imply that the checksum
140  * is valid. One would have to check the parse results to find that out.
141  */
142 #define FM_FD_STAT_L4CV         0x00000004
143 
144 #define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
145 #define DPAA_BUFF_RELEASE_MAX 8 /* maximum number of buffers released at once */
146 
147 #define FSL_DPAA_BPID_INV		0xff
148 #define FSL_DPAA_ETH_MAX_BUF_COUNT	128
149 #define FSL_DPAA_ETH_REFILL_THRESHOLD	80
150 
151 #define DPAA_TX_PRIV_DATA_SIZE	16
152 #define DPAA_PARSE_RESULTS_SIZE sizeof(struct fman_prs_result)
153 #define DPAA_TIME_STAMP_SIZE 8
154 #define DPAA_HASH_RESULTS_SIZE 8
155 #define DPAA_HWA_SIZE (DPAA_PARSE_RESULTS_SIZE + DPAA_TIME_STAMP_SIZE \
156 		       + DPAA_HASH_RESULTS_SIZE)
157 #define DPAA_RX_PRIV_DATA_DEFAULT_SIZE (DPAA_TX_PRIV_DATA_SIZE + \
158 					XDP_PACKET_HEADROOM - DPAA_HWA_SIZE)
159 #ifdef CONFIG_DPAA_ERRATUM_A050385
160 #define DPAA_RX_PRIV_DATA_A050385_SIZE (DPAA_A050385_ALIGN - DPAA_HWA_SIZE)
161 #define DPAA_RX_PRIV_DATA_SIZE (fman_has_errata_a050385() ? \
162 				DPAA_RX_PRIV_DATA_A050385_SIZE : \
163 				DPAA_RX_PRIV_DATA_DEFAULT_SIZE)
164 #else
165 #define DPAA_RX_PRIV_DATA_SIZE DPAA_RX_PRIV_DATA_DEFAULT_SIZE
166 #endif
167 
168 #define DPAA_ETH_PCD_RXQ_NUM	128
169 
170 #define DPAA_ENQUEUE_RETRIES	100000
171 
172 enum port_type {RX, TX};
173 
174 struct fm_port_fqs {
175 	struct dpaa_fq *tx_defq;
176 	struct dpaa_fq *tx_errq;
177 	struct dpaa_fq *rx_defq;
178 	struct dpaa_fq *rx_errq;
179 	struct dpaa_fq *rx_pcdq;
180 };
181 
182 /* All the dpa bps in use at any moment */
183 static struct dpaa_bp *dpaa_bp_array[BM_MAX_NUM_OF_POOLS];
184 
185 #define DPAA_BP_RAW_SIZE 4096
186 
187 #ifdef CONFIG_DPAA_ERRATUM_A050385
188 #define dpaa_bp_size(raw_size) (SKB_WITH_OVERHEAD(raw_size) & \
189 				~(DPAA_A050385_ALIGN - 1))
190 #else
191 #define dpaa_bp_size(raw_size) SKB_WITH_OVERHEAD(raw_size)
192 #endif
193 
194 static int dpaa_max_frm;
195 
196 static int dpaa_rx_extra_headroom;
197 
198 #define dpaa_get_max_mtu()	\
199 	(dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN))
200 
201 static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed);
202 
dpaa_netdev_init(struct net_device * net_dev,const struct net_device_ops * dpaa_ops,u16 tx_timeout)203 static int dpaa_netdev_init(struct net_device *net_dev,
204 			    const struct net_device_ops *dpaa_ops,
205 			    u16 tx_timeout)
206 {
207 	struct dpaa_priv *priv = netdev_priv(net_dev);
208 	struct device *dev = net_dev->dev.parent;
209 	struct mac_device *mac_dev = priv->mac_dev;
210 	struct dpaa_percpu_priv *percpu_priv;
211 	const u8 *mac_addr;
212 	int i, err;
213 
214 	/* Although we access another CPU's private data here
215 	 * we do it at initialization so it is safe
216 	 */
217 	for_each_possible_cpu(i) {
218 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
219 		percpu_priv->net_dev = net_dev;
220 	}
221 
222 	net_dev->netdev_ops = dpaa_ops;
223 	mac_addr = mac_dev->addr;
224 
225 	net_dev->mem_start = (unsigned long)priv->mac_dev->res->start;
226 	net_dev->mem_end = (unsigned long)priv->mac_dev->res->end;
227 
228 	net_dev->min_mtu = ETH_MIN_MTU;
229 	net_dev->max_mtu = dpaa_get_max_mtu();
230 
231 	net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
232 				 NETIF_F_LLTX | NETIF_F_RXHASH);
233 
234 	net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA;
235 	/* The kernels enables GSO automatically, if we declare NETIF_F_SG.
236 	 * For conformity, we'll still declare GSO explicitly.
237 	 */
238 	net_dev->features |= NETIF_F_GSO;
239 	net_dev->features |= NETIF_F_RXCSUM;
240 
241 	net_dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
242 	/* we do not want shared skbs on TX */
243 	net_dev->priv_flags &= ~IFF_TX_SKB_SHARING;
244 
245 	net_dev->features |= net_dev->hw_features;
246 	net_dev->vlan_features = net_dev->features;
247 
248 	net_dev->xdp_features = NETDEV_XDP_ACT_BASIC |
249 				NETDEV_XDP_ACT_REDIRECT |
250 				NETDEV_XDP_ACT_NDO_XMIT;
251 
252 	if (is_valid_ether_addr(mac_addr)) {
253 		memcpy(net_dev->perm_addr, mac_addr, net_dev->addr_len);
254 		eth_hw_addr_set(net_dev, mac_addr);
255 	} else {
256 		eth_hw_addr_random(net_dev);
257 		err = mac_dev->change_addr(mac_dev->fman_mac,
258 			(const enet_addr_t *)net_dev->dev_addr);
259 		if (err) {
260 			dev_err(dev, "Failed to set random MAC address\n");
261 			return -EINVAL;
262 		}
263 		dev_info(dev, "Using random MAC address: %pM\n",
264 			 net_dev->dev_addr);
265 	}
266 
267 	net_dev->ethtool_ops = &dpaa_ethtool_ops;
268 
269 	net_dev->needed_headroom = priv->tx_headroom;
270 	net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout);
271 
272 	/* The rest of the config is filled in by the mac device already */
273 	mac_dev->phylink_config.dev = &net_dev->dev;
274 	mac_dev->phylink_config.type = PHYLINK_NETDEV;
275 	mac_dev->update_speed = dpaa_eth_cgr_set_speed;
276 	mac_dev->phylink = phylink_create(&mac_dev->phylink_config,
277 					  dev_fwnode(mac_dev->dev),
278 					  mac_dev->phy_if,
279 					  mac_dev->phylink_ops);
280 	if (IS_ERR(mac_dev->phylink)) {
281 		err = PTR_ERR(mac_dev->phylink);
282 		dev_err_probe(dev, err, "Could not create phylink\n");
283 		return err;
284 	}
285 
286 	/* start without the RUNNING flag, phylib controls it later */
287 	netif_carrier_off(net_dev);
288 
289 	err = register_netdev(net_dev);
290 	if (err < 0) {
291 		dev_err(dev, "register_netdev() = %d\n", err);
292 		phylink_destroy(mac_dev->phylink);
293 		return err;
294 	}
295 
296 	return 0;
297 }
298 
dpaa_stop(struct net_device * net_dev)299 static int dpaa_stop(struct net_device *net_dev)
300 {
301 	struct mac_device *mac_dev;
302 	struct dpaa_priv *priv;
303 	int i, error;
304 	int err = 0;
305 
306 	priv = netdev_priv(net_dev);
307 	mac_dev = priv->mac_dev;
308 
309 	netif_tx_stop_all_queues(net_dev);
310 	/* Allow the Fman (Tx) port to process in-flight frames before we
311 	 * try switching it off.
312 	 */
313 	msleep(200);
314 
315 	phylink_stop(mac_dev->phylink);
316 	mac_dev->disable(mac_dev->fman_mac);
317 
318 	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
319 		error = fman_port_disable(mac_dev->port[i]);
320 		if (error)
321 			err = error;
322 	}
323 
324 	phylink_disconnect_phy(mac_dev->phylink);
325 	net_dev->phydev = NULL;
326 
327 	msleep(200);
328 
329 	return err;
330 }
331 
dpaa_tx_timeout(struct net_device * net_dev,unsigned int txqueue)332 static void dpaa_tx_timeout(struct net_device *net_dev, unsigned int txqueue)
333 {
334 	struct dpaa_percpu_priv *percpu_priv;
335 	const struct dpaa_priv	*priv;
336 
337 	priv = netdev_priv(net_dev);
338 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
339 
340 	netif_crit(priv, timer, net_dev, "Transmit timeout latency: %u ms\n",
341 		   jiffies_to_msecs(jiffies - dev_trans_start(net_dev)));
342 
343 	percpu_priv->stats.tx_errors++;
344 }
345 
346 /* Calculates the statistics for the given device by adding the statistics
347  * collected by each CPU.
348  */
dpaa_get_stats64(struct net_device * net_dev,struct rtnl_link_stats64 * s)349 static void dpaa_get_stats64(struct net_device *net_dev,
350 			     struct rtnl_link_stats64 *s)
351 {
352 	int numstats = sizeof(struct rtnl_link_stats64) / sizeof(u64);
353 	struct dpaa_priv *priv = netdev_priv(net_dev);
354 	struct dpaa_percpu_priv *percpu_priv;
355 	u64 *netstats = (u64 *)s;
356 	u64 *cpustats;
357 	int i, j;
358 
359 	for_each_possible_cpu(i) {
360 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
361 
362 		cpustats = (u64 *)&percpu_priv->stats;
363 
364 		/* add stats from all CPUs */
365 		for (j = 0; j < numstats; j++)
366 			netstats[j] += cpustats[j];
367 	}
368 }
369 
dpaa_setup_tc(struct net_device * net_dev,enum tc_setup_type type,void * type_data)370 static int dpaa_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
371 			 void *type_data)
372 {
373 	struct dpaa_priv *priv = netdev_priv(net_dev);
374 	struct tc_mqprio_qopt *mqprio = type_data;
375 	u8 num_tc;
376 	int i;
377 
378 	if (type != TC_SETUP_QDISC_MQPRIO)
379 		return -EOPNOTSUPP;
380 
381 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
382 	num_tc = mqprio->num_tc;
383 
384 	if (num_tc == priv->num_tc)
385 		return 0;
386 
387 	if (!num_tc) {
388 		netdev_reset_tc(net_dev);
389 		goto out;
390 	}
391 
392 	if (num_tc > DPAA_TC_NUM) {
393 		netdev_err(net_dev, "Too many traffic classes: max %d supported.\n",
394 			   DPAA_TC_NUM);
395 		return -EINVAL;
396 	}
397 
398 	netdev_set_num_tc(net_dev, num_tc);
399 
400 	for (i = 0; i < num_tc; i++)
401 		netdev_set_tc_queue(net_dev, i, DPAA_TC_TXQ_NUM,
402 				    i * DPAA_TC_TXQ_NUM);
403 
404 out:
405 	priv->num_tc = num_tc ? : 1;
406 	netif_set_real_num_tx_queues(net_dev, priv->num_tc * DPAA_TC_TXQ_NUM);
407 	return 0;
408 }
409 
dpaa_mac_dev_get(struct platform_device * pdev)410 static struct mac_device *dpaa_mac_dev_get(struct platform_device *pdev)
411 {
412 	struct dpaa_eth_data *eth_data;
413 	struct device *dpaa_dev;
414 	struct mac_device *mac_dev;
415 
416 	dpaa_dev = &pdev->dev;
417 	eth_data = dpaa_dev->platform_data;
418 	if (!eth_data) {
419 		dev_err(dpaa_dev, "eth_data missing\n");
420 		return ERR_PTR(-ENODEV);
421 	}
422 	mac_dev = eth_data->mac_dev;
423 	if (!mac_dev) {
424 		dev_err(dpaa_dev, "mac_dev missing\n");
425 		return ERR_PTR(-EINVAL);
426 	}
427 
428 	return mac_dev;
429 }
430 
dpaa_set_mac_address(struct net_device * net_dev,void * addr)431 static int dpaa_set_mac_address(struct net_device *net_dev, void *addr)
432 {
433 	const struct dpaa_priv *priv;
434 	struct mac_device *mac_dev;
435 	struct sockaddr old_addr;
436 	int err;
437 
438 	priv = netdev_priv(net_dev);
439 
440 	memcpy(old_addr.sa_data, net_dev->dev_addr,  ETH_ALEN);
441 
442 	err = eth_mac_addr(net_dev, addr);
443 	if (err < 0) {
444 		netif_err(priv, drv, net_dev, "eth_mac_addr() = %d\n", err);
445 		return err;
446 	}
447 
448 	mac_dev = priv->mac_dev;
449 
450 	err = mac_dev->change_addr(mac_dev->fman_mac,
451 				   (const enet_addr_t *)net_dev->dev_addr);
452 	if (err < 0) {
453 		netif_err(priv, drv, net_dev, "mac_dev->change_addr() = %d\n",
454 			  err);
455 		/* reverting to previous address */
456 		eth_mac_addr(net_dev, &old_addr);
457 
458 		return err;
459 	}
460 
461 	return 0;
462 }
463 
dpaa_set_rx_mode(struct net_device * net_dev)464 static void dpaa_set_rx_mode(struct net_device *net_dev)
465 {
466 	const struct dpaa_priv	*priv;
467 	int err;
468 
469 	priv = netdev_priv(net_dev);
470 
471 	if (!!(net_dev->flags & IFF_PROMISC) != priv->mac_dev->promisc) {
472 		priv->mac_dev->promisc = !priv->mac_dev->promisc;
473 		err = priv->mac_dev->set_promisc(priv->mac_dev->fman_mac,
474 						 priv->mac_dev->promisc);
475 		if (err < 0)
476 			netif_err(priv, drv, net_dev,
477 				  "mac_dev->set_promisc() = %d\n",
478 				  err);
479 	}
480 
481 	if (!!(net_dev->flags & IFF_ALLMULTI) != priv->mac_dev->allmulti) {
482 		priv->mac_dev->allmulti = !priv->mac_dev->allmulti;
483 		err = priv->mac_dev->set_allmulti(priv->mac_dev->fman_mac,
484 						  priv->mac_dev->allmulti);
485 		if (err < 0)
486 			netif_err(priv, drv, net_dev,
487 				  "mac_dev->set_allmulti() = %d\n",
488 				  err);
489 	}
490 
491 	err = priv->mac_dev->set_multi(net_dev, priv->mac_dev);
492 	if (err < 0)
493 		netif_err(priv, drv, net_dev, "mac_dev->set_multi() = %d\n",
494 			  err);
495 }
496 
dpaa_bpid2pool(int bpid)497 static struct dpaa_bp *dpaa_bpid2pool(int bpid)
498 {
499 	if (WARN_ON(bpid < 0 || bpid >= BM_MAX_NUM_OF_POOLS))
500 		return NULL;
501 
502 	return dpaa_bp_array[bpid];
503 }
504 
505 /* checks if this bpool is already allocated */
dpaa_bpid2pool_use(int bpid)506 static bool dpaa_bpid2pool_use(int bpid)
507 {
508 	if (dpaa_bpid2pool(bpid)) {
509 		refcount_inc(&dpaa_bp_array[bpid]->refs);
510 		return true;
511 	}
512 
513 	return false;
514 }
515 
516 /* called only once per bpid by dpaa_bp_alloc_pool() */
dpaa_bpid2pool_map(int bpid,struct dpaa_bp * dpaa_bp)517 static void dpaa_bpid2pool_map(int bpid, struct dpaa_bp *dpaa_bp)
518 {
519 	dpaa_bp_array[bpid] = dpaa_bp;
520 	refcount_set(&dpaa_bp->refs, 1);
521 }
522 
dpaa_bp_alloc_pool(struct dpaa_bp * dpaa_bp)523 static int dpaa_bp_alloc_pool(struct dpaa_bp *dpaa_bp)
524 {
525 	int err;
526 
527 	if (dpaa_bp->size == 0 || dpaa_bp->config_count == 0) {
528 		pr_err("%s: Buffer pool is not properly initialized! Missing size or initial number of buffers\n",
529 		       __func__);
530 		return -EINVAL;
531 	}
532 
533 	/* If the pool is already specified, we only create one per bpid */
534 	if (dpaa_bp->bpid != FSL_DPAA_BPID_INV &&
535 	    dpaa_bpid2pool_use(dpaa_bp->bpid))
536 		return 0;
537 
538 	if (dpaa_bp->bpid == FSL_DPAA_BPID_INV) {
539 		dpaa_bp->pool = bman_new_pool();
540 		if (!dpaa_bp->pool) {
541 			pr_err("%s: bman_new_pool() failed\n",
542 			       __func__);
543 			return -ENODEV;
544 		}
545 
546 		dpaa_bp->bpid = (u8)bman_get_bpid(dpaa_bp->pool);
547 	}
548 
549 	if (dpaa_bp->seed_cb) {
550 		err = dpaa_bp->seed_cb(dpaa_bp);
551 		if (err)
552 			goto pool_seed_failed;
553 	}
554 
555 	dpaa_bpid2pool_map(dpaa_bp->bpid, dpaa_bp);
556 
557 	return 0;
558 
559 pool_seed_failed:
560 	pr_err("%s: pool seeding failed\n", __func__);
561 	bman_free_pool(dpaa_bp->pool);
562 
563 	return err;
564 }
565 
566 /* remove and free all the buffers from the given buffer pool */
dpaa_bp_drain(struct dpaa_bp * bp)567 static void dpaa_bp_drain(struct dpaa_bp *bp)
568 {
569 	u8 num = 8;
570 	int ret;
571 
572 	do {
573 		struct bm_buffer bmb[8];
574 		int i;
575 
576 		ret = bman_acquire(bp->pool, bmb, num);
577 		if (ret < 0) {
578 			if (num == 8) {
579 				/* we have less than 8 buffers left;
580 				 * drain them one by one
581 				 */
582 				num = 1;
583 				ret = 1;
584 				continue;
585 			} else {
586 				/* Pool is fully drained */
587 				break;
588 			}
589 		}
590 
591 		if (bp->free_buf_cb)
592 			for (i = 0; i < num; i++)
593 				bp->free_buf_cb(bp, &bmb[i]);
594 	} while (ret > 0);
595 }
596 
dpaa_bp_free(struct dpaa_bp * dpaa_bp)597 static void dpaa_bp_free(struct dpaa_bp *dpaa_bp)
598 {
599 	struct dpaa_bp *bp = dpaa_bpid2pool(dpaa_bp->bpid);
600 
601 	/* the mapping between bpid and dpaa_bp is done very late in the
602 	 * allocation procedure; if something failed before the mapping, the bp
603 	 * was not configured, therefore we don't need the below instructions
604 	 */
605 	if (!bp)
606 		return;
607 
608 	if (!refcount_dec_and_test(&bp->refs))
609 		return;
610 
611 	if (bp->free_buf_cb)
612 		dpaa_bp_drain(bp);
613 
614 	dpaa_bp_array[bp->bpid] = NULL;
615 	bman_free_pool(bp->pool);
616 }
617 
dpaa_bps_free(struct dpaa_priv * priv)618 static void dpaa_bps_free(struct dpaa_priv *priv)
619 {
620 	dpaa_bp_free(priv->dpaa_bp);
621 }
622 
623 /* Use multiple WQs for FQ assignment:
624  *	- Tx Confirmation queues go to WQ1.
625  *	- Rx Error and Tx Error queues go to WQ5 (giving them a better chance
626  *	  to be scheduled, in case there are many more FQs in WQ6).
627  *	- Rx Default goes to WQ6.
628  *	- Tx queues go to different WQs depending on their priority. Equal
629  *	  chunks of NR_CPUS queues go to WQ6 (lowest priority), WQ2, WQ1 and
630  *	  WQ0 (highest priority).
631  * This ensures that Tx-confirmed buffers are timely released. In particular,
632  * it avoids congestion on the Tx Confirm FQs, which can pile up PFDRs if they
633  * are greatly outnumbered by other FQs in the system, while
634  * dequeue scheduling is round-robin.
635  */
dpaa_assign_wq(struct dpaa_fq * fq,int idx)636 static inline void dpaa_assign_wq(struct dpaa_fq *fq, int idx)
637 {
638 	switch (fq->fq_type) {
639 	case FQ_TYPE_TX_CONFIRM:
640 	case FQ_TYPE_TX_CONF_MQ:
641 		fq->wq = 1;
642 		break;
643 	case FQ_TYPE_RX_ERROR:
644 	case FQ_TYPE_TX_ERROR:
645 		fq->wq = 5;
646 		break;
647 	case FQ_TYPE_RX_DEFAULT:
648 	case FQ_TYPE_RX_PCD:
649 		fq->wq = 6;
650 		break;
651 	case FQ_TYPE_TX:
652 		switch (idx / DPAA_TC_TXQ_NUM) {
653 		case 0:
654 			/* Low priority (best effort) */
655 			fq->wq = 6;
656 			break;
657 		case 1:
658 			/* Medium priority */
659 			fq->wq = 2;
660 			break;
661 		case 2:
662 			/* High priority */
663 			fq->wq = 1;
664 			break;
665 		case 3:
666 			/* Very high priority */
667 			fq->wq = 0;
668 			break;
669 		default:
670 			WARN(1, "Too many TX FQs: more than %d!\n",
671 			     DPAA_ETH_TXQ_NUM);
672 		}
673 		break;
674 	default:
675 		WARN(1, "Invalid FQ type %d for FQID %d!\n",
676 		     fq->fq_type, fq->fqid);
677 	}
678 }
679 
dpaa_fq_alloc(struct device * dev,u32 start,u32 count,struct list_head * list,enum dpaa_fq_type fq_type)680 static struct dpaa_fq *dpaa_fq_alloc(struct device *dev,
681 				     u32 start, u32 count,
682 				     struct list_head *list,
683 				     enum dpaa_fq_type fq_type)
684 {
685 	struct dpaa_fq *dpaa_fq;
686 	int i;
687 
688 	dpaa_fq = devm_kcalloc(dev, count, sizeof(*dpaa_fq),
689 			       GFP_KERNEL);
690 	if (!dpaa_fq)
691 		return NULL;
692 
693 	for (i = 0; i < count; i++) {
694 		dpaa_fq[i].fq_type = fq_type;
695 		dpaa_fq[i].fqid = start ? start + i : 0;
696 		list_add_tail(&dpaa_fq[i].list, list);
697 	}
698 
699 	for (i = 0; i < count; i++)
700 		dpaa_assign_wq(dpaa_fq + i, i);
701 
702 	return dpaa_fq;
703 }
704 
dpaa_alloc_all_fqs(struct device * dev,struct list_head * list,struct fm_port_fqs * port_fqs)705 static int dpaa_alloc_all_fqs(struct device *dev, struct list_head *list,
706 			      struct fm_port_fqs *port_fqs)
707 {
708 	struct dpaa_fq *dpaa_fq;
709 	u32 fq_base, fq_base_aligned, i;
710 
711 	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_ERROR);
712 	if (!dpaa_fq)
713 		goto fq_alloc_failed;
714 
715 	port_fqs->rx_errq = &dpaa_fq[0];
716 
717 	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_DEFAULT);
718 	if (!dpaa_fq)
719 		goto fq_alloc_failed;
720 
721 	port_fqs->rx_defq = &dpaa_fq[0];
722 
723 	/* the PCD FQIDs range needs to be aligned for correct operation */
724 	if (qman_alloc_fqid_range(&fq_base, 2 * DPAA_ETH_PCD_RXQ_NUM))
725 		goto fq_alloc_failed;
726 
727 	fq_base_aligned = ALIGN(fq_base, DPAA_ETH_PCD_RXQ_NUM);
728 
729 	for (i = fq_base; i < fq_base_aligned; i++)
730 		qman_release_fqid(i);
731 
732 	for (i = fq_base_aligned + DPAA_ETH_PCD_RXQ_NUM;
733 	     i < (fq_base + 2 * DPAA_ETH_PCD_RXQ_NUM); i++)
734 		qman_release_fqid(i);
735 
736 	dpaa_fq = dpaa_fq_alloc(dev, fq_base_aligned, DPAA_ETH_PCD_RXQ_NUM,
737 				list, FQ_TYPE_RX_PCD);
738 	if (!dpaa_fq)
739 		goto fq_alloc_failed;
740 
741 	port_fqs->rx_pcdq = &dpaa_fq[0];
742 
743 	if (!dpaa_fq_alloc(dev, 0, DPAA_ETH_TXQ_NUM, list, FQ_TYPE_TX_CONF_MQ))
744 		goto fq_alloc_failed;
745 
746 	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_ERROR);
747 	if (!dpaa_fq)
748 		goto fq_alloc_failed;
749 
750 	port_fqs->tx_errq = &dpaa_fq[0];
751 
752 	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_CONFIRM);
753 	if (!dpaa_fq)
754 		goto fq_alloc_failed;
755 
756 	port_fqs->tx_defq = &dpaa_fq[0];
757 
758 	if (!dpaa_fq_alloc(dev, 0, DPAA_ETH_TXQ_NUM, list, FQ_TYPE_TX))
759 		goto fq_alloc_failed;
760 
761 	return 0;
762 
763 fq_alloc_failed:
764 	dev_err(dev, "dpaa_fq_alloc() failed\n");
765 	return -ENOMEM;
766 }
767 
768 static u32 rx_pool_channel;
769 static DEFINE_SPINLOCK(rx_pool_channel_init);
770 
dpaa_get_channel(void)771 static int dpaa_get_channel(void)
772 {
773 	spin_lock(&rx_pool_channel_init);
774 	if (!rx_pool_channel) {
775 		u32 pool;
776 		int ret;
777 
778 		ret = qman_alloc_pool(&pool);
779 
780 		if (!ret)
781 			rx_pool_channel = pool;
782 	}
783 	spin_unlock(&rx_pool_channel_init);
784 	if (!rx_pool_channel)
785 		return -ENOMEM;
786 	return rx_pool_channel;
787 }
788 
dpaa_release_channel(void)789 static void dpaa_release_channel(void)
790 {
791 	qman_release_pool(rx_pool_channel);
792 }
793 
dpaa_eth_add_channel(u16 channel,struct device * dev)794 static void dpaa_eth_add_channel(u16 channel, struct device *dev)
795 {
796 	u32 pool = QM_SDQCR_CHANNELS_POOL_CONV(channel);
797 	const cpumask_t *cpus = qman_affine_cpus();
798 	struct qman_portal *portal;
799 	int cpu;
800 
801 	for_each_cpu_and(cpu, cpus, cpu_online_mask) {
802 		portal = qman_get_affine_portal(cpu);
803 		qman_p_static_dequeue_add(portal, pool);
804 		qman_start_using_portal(portal, dev);
805 	}
806 }
807 
808 /* Congestion group state change notification callback.
809  * Stops the device's egress queues while they are congested and
810  * wakes them upon exiting congested state.
811  * Also updates some CGR-related stats.
812  */
dpaa_eth_cgscn(struct qman_portal * qm,struct qman_cgr * cgr,int congested)813 static void dpaa_eth_cgscn(struct qman_portal *qm, struct qman_cgr *cgr,
814 			   int congested)
815 {
816 	struct dpaa_priv *priv = (struct dpaa_priv *)container_of(cgr,
817 		struct dpaa_priv, cgr_data.cgr);
818 
819 	if (congested) {
820 		priv->cgr_data.congestion_start_jiffies = jiffies;
821 		netif_tx_stop_all_queues(priv->net_dev);
822 		priv->cgr_data.cgr_congested_count++;
823 	} else {
824 		priv->cgr_data.congested_jiffies +=
825 			(jiffies - priv->cgr_data.congestion_start_jiffies);
826 		netif_tx_wake_all_queues(priv->net_dev);
827 	}
828 }
829 
dpaa_eth_cgr_init(struct dpaa_priv * priv)830 static int dpaa_eth_cgr_init(struct dpaa_priv *priv)
831 {
832 	struct qm_mcc_initcgr initcgr;
833 	u32 cs_th;
834 	int err;
835 
836 	err = qman_alloc_cgrid(&priv->cgr_data.cgr.cgrid);
837 	if (err < 0) {
838 		if (netif_msg_drv(priv))
839 			pr_err("%s: Error %d allocating CGR ID\n",
840 			       __func__, err);
841 		goto out_error;
842 	}
843 	priv->cgr_data.cgr.cb = dpaa_eth_cgscn;
844 
845 	/* Enable Congestion State Change Notifications and CS taildrop */
846 	memset(&initcgr, 0, sizeof(initcgr));
847 	initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES);
848 	initcgr.cgr.cscn_en = QM_CGR_EN;
849 
850 	/* Set different thresholds based on the configured MAC speed.
851 	 * This may turn suboptimal if the MAC is reconfigured at another
852 	 * speed, so MACs must call dpaa_eth_cgr_set_speed in their link_up
853 	 * callback.
854 	 */
855 	if (priv->mac_dev->phylink_config.mac_capabilities & MAC_10000FD)
856 		cs_th = DPAA_CS_THRESHOLD_10G;
857 	else
858 		cs_th = DPAA_CS_THRESHOLD_1G;
859 	qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
860 
861 	initcgr.we_mask |= cpu_to_be16(QM_CGR_WE_CSTD_EN);
862 	initcgr.cgr.cstd_en = QM_CGR_EN;
863 
864 	err = qman_create_cgr(&priv->cgr_data.cgr, QMAN_CGR_FLAG_USE_INIT,
865 			      &initcgr);
866 	if (err < 0) {
867 		if (netif_msg_drv(priv))
868 			pr_err("%s: Error %d creating CGR with ID %d\n",
869 			       __func__, err, priv->cgr_data.cgr.cgrid);
870 		qman_release_cgrid(priv->cgr_data.cgr.cgrid);
871 		goto out_error;
872 	}
873 	if (netif_msg_drv(priv))
874 		pr_debug("Created CGR %d for netdev with hwaddr %pM on QMan channel %d\n",
875 			 priv->cgr_data.cgr.cgrid, priv->mac_dev->addr,
876 			 priv->cgr_data.cgr.chan);
877 
878 out_error:
879 	return err;
880 }
881 
dpaa_eth_cgr_set_speed(struct mac_device * mac_dev,int speed)882 static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed)
883 {
884 	struct net_device *net_dev = to_net_dev(mac_dev->phylink_config.dev);
885 	struct dpaa_priv *priv = netdev_priv(net_dev);
886 	struct qm_mcc_initcgr opts = { };
887 	u32 cs_th;
888 	int err;
889 
890 	opts.we_mask = cpu_to_be16(QM_CGR_WE_CS_THRES);
891 	switch (speed) {
892 	case SPEED_10000:
893 		cs_th = DPAA_CS_THRESHOLD_10G;
894 		break;
895 	case SPEED_1000:
896 	default:
897 		cs_th = DPAA_CS_THRESHOLD_1G;
898 		break;
899 	}
900 	qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, cs_th, 1);
901 
902 	err = qman_update_cgr_safe(&priv->cgr_data.cgr, &opts);
903 	if (err)
904 		netdev_err(net_dev, "could not update speed: %d\n", err);
905 }
906 
dpaa_setup_ingress(const struct dpaa_priv * priv,struct dpaa_fq * fq,const struct qman_fq * template)907 static inline void dpaa_setup_ingress(const struct dpaa_priv *priv,
908 				      struct dpaa_fq *fq,
909 				      const struct qman_fq *template)
910 {
911 	fq->fq_base = *template;
912 	fq->net_dev = priv->net_dev;
913 
914 	fq->flags = QMAN_FQ_FLAG_NO_ENQUEUE;
915 	fq->channel = priv->channel;
916 }
917 
dpaa_setup_egress(const struct dpaa_priv * priv,struct dpaa_fq * fq,struct fman_port * port,const struct qman_fq * template)918 static inline void dpaa_setup_egress(const struct dpaa_priv *priv,
919 				     struct dpaa_fq *fq,
920 				     struct fman_port *port,
921 				     const struct qman_fq *template)
922 {
923 	fq->fq_base = *template;
924 	fq->net_dev = priv->net_dev;
925 
926 	if (port) {
927 		fq->flags = QMAN_FQ_FLAG_TO_DCPORTAL;
928 		fq->channel = (u16)fman_port_get_qman_channel_id(port);
929 	} else {
930 		fq->flags = QMAN_FQ_FLAG_NO_MODIFY;
931 	}
932 }
933 
dpaa_fq_setup(struct dpaa_priv * priv,const struct dpaa_fq_cbs * fq_cbs,struct fman_port * tx_port)934 static int dpaa_fq_setup(struct dpaa_priv *priv,
935 			 const struct dpaa_fq_cbs *fq_cbs,
936 			 struct fman_port *tx_port)
937 {
938 	int egress_cnt = 0, conf_cnt = 0, num_portals = 0, portal_cnt = 0, cpu;
939 	const cpumask_t *affine_cpus = qman_affine_cpus();
940 	struct dpaa_fq *fq;
941 	u16 *channels;
942 
943 	channels = kcalloc(num_possible_cpus(), sizeof(u16), GFP_KERNEL);
944 	if (!channels)
945 		return -ENOMEM;
946 
947 	for_each_cpu_and(cpu, affine_cpus, cpu_online_mask)
948 		channels[num_portals++] = qman_affine_channel(cpu);
949 
950 	if (num_portals == 0)
951 		dev_err(priv->net_dev->dev.parent,
952 			"No Qman software (affine) channels found\n");
953 
954 	/* Initialize each FQ in the list */
955 	list_for_each_entry(fq, &priv->dpaa_fq_list, list) {
956 		switch (fq->fq_type) {
957 		case FQ_TYPE_RX_DEFAULT:
958 			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
959 			break;
960 		case FQ_TYPE_RX_ERROR:
961 			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_errq);
962 			break;
963 		case FQ_TYPE_RX_PCD:
964 			if (!num_portals)
965 				continue;
966 			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
967 			fq->channel = channels[portal_cnt++ % num_portals];
968 			break;
969 		case FQ_TYPE_TX:
970 			dpaa_setup_egress(priv, fq, tx_port,
971 					  &fq_cbs->egress_ern);
972 			/* If we have more Tx queues than the number of cores,
973 			 * just ignore the extra ones.
974 			 */
975 			if (egress_cnt < DPAA_ETH_TXQ_NUM)
976 				priv->egress_fqs[egress_cnt++] = &fq->fq_base;
977 			break;
978 		case FQ_TYPE_TX_CONF_MQ:
979 			priv->conf_fqs[conf_cnt++] = &fq->fq_base;
980 			fallthrough;
981 		case FQ_TYPE_TX_CONFIRM:
982 			dpaa_setup_ingress(priv, fq, &fq_cbs->tx_defq);
983 			break;
984 		case FQ_TYPE_TX_ERROR:
985 			dpaa_setup_ingress(priv, fq, &fq_cbs->tx_errq);
986 			break;
987 		default:
988 			dev_warn(priv->net_dev->dev.parent,
989 				 "Unknown FQ type detected!\n");
990 			break;
991 		}
992 	}
993 
994 	 /* Make sure all CPUs receive a corresponding Tx queue. */
995 	while (egress_cnt < DPAA_ETH_TXQ_NUM) {
996 		list_for_each_entry(fq, &priv->dpaa_fq_list, list) {
997 			if (fq->fq_type != FQ_TYPE_TX)
998 				continue;
999 			priv->egress_fqs[egress_cnt++] = &fq->fq_base;
1000 			if (egress_cnt == DPAA_ETH_TXQ_NUM)
1001 				break;
1002 		}
1003 	}
1004 
1005 	kfree(channels);
1006 
1007 	return 0;
1008 }
1009 
dpaa_tx_fq_to_id(const struct dpaa_priv * priv,struct qman_fq * tx_fq)1010 static inline int dpaa_tx_fq_to_id(const struct dpaa_priv *priv,
1011 				   struct qman_fq *tx_fq)
1012 {
1013 	int i;
1014 
1015 	for (i = 0; i < DPAA_ETH_TXQ_NUM; i++)
1016 		if (priv->egress_fqs[i] == tx_fq)
1017 			return i;
1018 
1019 	return -EINVAL;
1020 }
1021 
dpaa_fq_init(struct dpaa_fq * dpaa_fq,bool td_enable)1022 static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable)
1023 {
1024 	const struct dpaa_priv	*priv;
1025 	struct qman_fq *confq = NULL;
1026 	struct qm_mcc_initfq initfq;
1027 	struct device *dev;
1028 	struct qman_fq *fq;
1029 	int queue_id;
1030 	int err;
1031 
1032 	priv = netdev_priv(dpaa_fq->net_dev);
1033 	dev = dpaa_fq->net_dev->dev.parent;
1034 
1035 	if (dpaa_fq->fqid == 0)
1036 		dpaa_fq->flags |= QMAN_FQ_FLAG_DYNAMIC_FQID;
1037 
1038 	dpaa_fq->init = !(dpaa_fq->flags & QMAN_FQ_FLAG_NO_MODIFY);
1039 
1040 	err = qman_create_fq(dpaa_fq->fqid, dpaa_fq->flags, &dpaa_fq->fq_base);
1041 	if (err) {
1042 		dev_err(dev, "qman_create_fq() failed\n");
1043 		return err;
1044 	}
1045 	fq = &dpaa_fq->fq_base;
1046 
1047 	if (dpaa_fq->init) {
1048 		memset(&initfq, 0, sizeof(initfq));
1049 
1050 		initfq.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL);
1051 		/* Note: we may get to keep an empty FQ in cache */
1052 		initfq.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_PREFERINCACHE);
1053 
1054 		/* Try to reduce the number of portal interrupts for
1055 		 * Tx Confirmation FQs.
1056 		 */
1057 		if (dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM)
1058 			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_AVOIDBLOCK);
1059 
1060 		/* FQ placement */
1061 		initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_DESTWQ);
1062 
1063 		qm_fqd_set_destwq(&initfq.fqd, dpaa_fq->channel, dpaa_fq->wq);
1064 
1065 		/* Put all egress queues in a congestion group of their own.
1066 		 * Sensu stricto, the Tx confirmation queues are Rx FQs,
1067 		 * rather than Tx - but they nonetheless account for the
1068 		 * memory footprint on behalf of egress traffic. We therefore
1069 		 * place them in the netdev's CGR, along with the Tx FQs.
1070 		 */
1071 		if (dpaa_fq->fq_type == FQ_TYPE_TX ||
1072 		    dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM ||
1073 		    dpaa_fq->fq_type == FQ_TYPE_TX_CONF_MQ) {
1074 			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CGID);
1075 			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_CGE);
1076 			initfq.fqd.cgid = (u8)priv->cgr_data.cgr.cgrid;
1077 			/* Set a fixed overhead accounting, in an attempt to
1078 			 * reduce the impact of fixed-size skb shells and the
1079 			 * driver's needed headroom on system memory. This is
1080 			 * especially the case when the egress traffic is
1081 			 * composed of small datagrams.
1082 			 * Unfortunately, QMan's OAL value is capped to an
1083 			 * insufficient value, but even that is better than
1084 			 * no overhead accounting at all.
1085 			 */
1086 			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_OAC);
1087 			qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
1088 			qm_fqd_set_oal(&initfq.fqd,
1089 				       min(sizeof(struct sk_buff) +
1090 				       priv->tx_headroom,
1091 				       (size_t)FSL_QMAN_MAX_OAL));
1092 		}
1093 
1094 		if (td_enable) {
1095 			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_TDTHRESH);
1096 			qm_fqd_set_taildrop(&initfq.fqd, DPAA_FQ_TD, 1);
1097 			initfq.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_TDE);
1098 		}
1099 
1100 		if (dpaa_fq->fq_type == FQ_TYPE_TX) {
1101 			queue_id = dpaa_tx_fq_to_id(priv, &dpaa_fq->fq_base);
1102 			if (queue_id >= 0)
1103 				confq = priv->conf_fqs[queue_id];
1104 			if (confq) {
1105 				initfq.we_mask |=
1106 					cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
1107 			/* ContextA: OVOM=1(use contextA2 bits instead of ICAD)
1108 			 *	     A2V=1 (contextA A2 field is valid)
1109 			 *	     A0V=1 (contextA A0 field is valid)
1110 			 *	     B0V=1 (contextB field is valid)
1111 			 * ContextA A2: EBD=1 (deallocate buffers inside FMan)
1112 			 * ContextB B0(ASPID): 0 (absolute Virtual Storage ID)
1113 			 */
1114 				qm_fqd_context_a_set64(&initfq.fqd,
1115 						       0x1e00000080000000ULL);
1116 			}
1117 		}
1118 
1119 		/* Put all the ingress queues in our "ingress CGR". */
1120 		if (priv->use_ingress_cgr &&
1121 		    (dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
1122 		     dpaa_fq->fq_type == FQ_TYPE_RX_ERROR ||
1123 		     dpaa_fq->fq_type == FQ_TYPE_RX_PCD)) {
1124 			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CGID);
1125 			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_CGE);
1126 			initfq.fqd.cgid = (u8)priv->ingress_cgr.cgrid;
1127 			/* Set a fixed overhead accounting, just like for the
1128 			 * egress CGR.
1129 			 */
1130 			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_OAC);
1131 			qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
1132 			qm_fqd_set_oal(&initfq.fqd,
1133 				       min(sizeof(struct sk_buff) +
1134 				       priv->tx_headroom,
1135 				       (size_t)FSL_QMAN_MAX_OAL));
1136 		}
1137 
1138 		/* Initialization common to all ingress queues */
1139 		if (dpaa_fq->flags & QMAN_FQ_FLAG_NO_ENQUEUE) {
1140 			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
1141 			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_HOLDACTIVE |
1142 						QM_FQCTRL_CTXASTASHING);
1143 			initfq.fqd.context_a.stashing.exclusive =
1144 				QM_STASHING_EXCL_DATA | QM_STASHING_EXCL_CTX |
1145 				QM_STASHING_EXCL_ANNOTATION;
1146 			qm_fqd_set_stashing(&initfq.fqd, 1, 2,
1147 					    DIV_ROUND_UP(sizeof(struct qman_fq),
1148 							 64));
1149 		}
1150 
1151 		err = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &initfq);
1152 		if (err < 0) {
1153 			dev_err(dev, "qman_init_fq(%u) = %d\n",
1154 				qman_fq_fqid(fq), err);
1155 			qman_destroy_fq(fq);
1156 			return err;
1157 		}
1158 	}
1159 
1160 	dpaa_fq->fqid = qman_fq_fqid(fq);
1161 
1162 	if (dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
1163 	    dpaa_fq->fq_type == FQ_TYPE_RX_PCD) {
1164 		err = xdp_rxq_info_reg(&dpaa_fq->xdp_rxq, dpaa_fq->net_dev,
1165 				       dpaa_fq->fqid, 0);
1166 		if (err) {
1167 			dev_err(dev, "xdp_rxq_info_reg() = %d\n", err);
1168 			return err;
1169 		}
1170 
1171 		err = xdp_rxq_info_reg_mem_model(&dpaa_fq->xdp_rxq,
1172 						 MEM_TYPE_PAGE_ORDER0, NULL);
1173 		if (err) {
1174 			dev_err(dev, "xdp_rxq_info_reg_mem_model() = %d\n",
1175 				err);
1176 			xdp_rxq_info_unreg(&dpaa_fq->xdp_rxq);
1177 			return err;
1178 		}
1179 	}
1180 
1181 	return 0;
1182 }
1183 
dpaa_fq_free_entry(struct device * dev,struct qman_fq * fq)1184 static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq)
1185 {
1186 	const struct dpaa_priv  *priv;
1187 	struct dpaa_fq *dpaa_fq;
1188 	int err, error;
1189 
1190 	err = 0;
1191 
1192 	dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
1193 	priv = netdev_priv(dpaa_fq->net_dev);
1194 
1195 	if (dpaa_fq->init) {
1196 		err = qman_retire_fq(fq, NULL);
1197 		if (err < 0 && netif_msg_drv(priv))
1198 			dev_err(dev, "qman_retire_fq(%u) = %d\n",
1199 				qman_fq_fqid(fq), err);
1200 
1201 		error = qman_oos_fq(fq);
1202 		if (error < 0 && netif_msg_drv(priv)) {
1203 			dev_err(dev, "qman_oos_fq(%u) = %d\n",
1204 				qman_fq_fqid(fq), error);
1205 			if (err >= 0)
1206 				err = error;
1207 		}
1208 	}
1209 
1210 	if ((dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
1211 	     dpaa_fq->fq_type == FQ_TYPE_RX_PCD) &&
1212 	    xdp_rxq_info_is_reg(&dpaa_fq->xdp_rxq))
1213 		xdp_rxq_info_unreg(&dpaa_fq->xdp_rxq);
1214 
1215 	qman_destroy_fq(fq);
1216 	list_del(&dpaa_fq->list);
1217 
1218 	return err;
1219 }
1220 
dpaa_fq_free(struct device * dev,struct list_head * list)1221 static int dpaa_fq_free(struct device *dev, struct list_head *list)
1222 {
1223 	struct dpaa_fq *dpaa_fq, *tmp;
1224 	int err, error;
1225 
1226 	err = 0;
1227 	list_for_each_entry_safe(dpaa_fq, tmp, list, list) {
1228 		error = dpaa_fq_free_entry(dev, (struct qman_fq *)dpaa_fq);
1229 		if (error < 0 && err >= 0)
1230 			err = error;
1231 	}
1232 
1233 	return err;
1234 }
1235 
dpaa_eth_init_tx_port(struct fman_port * port,struct dpaa_fq * errq,struct dpaa_fq * defq,struct dpaa_buffer_layout * buf_layout)1236 static int dpaa_eth_init_tx_port(struct fman_port *port, struct dpaa_fq *errq,
1237 				 struct dpaa_fq *defq,
1238 				 struct dpaa_buffer_layout *buf_layout)
1239 {
1240 	struct fman_buffer_prefix_content buf_prefix_content;
1241 	struct fman_port_params params;
1242 	int err;
1243 
1244 	memset(&params, 0, sizeof(params));
1245 	memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
1246 
1247 	buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
1248 	buf_prefix_content.pass_prs_result = true;
1249 	buf_prefix_content.pass_hash_result = true;
1250 	buf_prefix_content.pass_time_stamp = true;
1251 	buf_prefix_content.data_align = DPAA_FD_DATA_ALIGNMENT;
1252 
1253 	params.specific_params.non_rx_params.err_fqid = errq->fqid;
1254 	params.specific_params.non_rx_params.dflt_fqid = defq->fqid;
1255 
1256 	err = fman_port_config(port, &params);
1257 	if (err) {
1258 		pr_err("%s: fman_port_config failed\n", __func__);
1259 		return err;
1260 	}
1261 
1262 	err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
1263 	if (err) {
1264 		pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1265 		       __func__);
1266 		return err;
1267 	}
1268 
1269 	err = fman_port_init(port);
1270 	if (err)
1271 		pr_err("%s: fm_port_init failed\n", __func__);
1272 
1273 	return err;
1274 }
1275 
dpaa_eth_init_rx_port(struct fman_port * port,struct dpaa_bp * bp,struct dpaa_fq * errq,struct dpaa_fq * defq,struct dpaa_fq * pcdq,struct dpaa_buffer_layout * buf_layout)1276 static int dpaa_eth_init_rx_port(struct fman_port *port, struct dpaa_bp *bp,
1277 				 struct dpaa_fq *errq,
1278 				 struct dpaa_fq *defq, struct dpaa_fq *pcdq,
1279 				 struct dpaa_buffer_layout *buf_layout)
1280 {
1281 	struct fman_buffer_prefix_content buf_prefix_content;
1282 	struct fman_port_rx_params *rx_p;
1283 	struct fman_port_params params;
1284 	int err;
1285 
1286 	memset(&params, 0, sizeof(params));
1287 	memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
1288 
1289 	buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
1290 	buf_prefix_content.pass_prs_result = true;
1291 	buf_prefix_content.pass_hash_result = true;
1292 	buf_prefix_content.pass_time_stamp = true;
1293 	buf_prefix_content.data_align = DPAA_FD_RX_DATA_ALIGNMENT;
1294 
1295 	rx_p = &params.specific_params.rx_params;
1296 	rx_p->err_fqid = errq->fqid;
1297 	rx_p->dflt_fqid = defq->fqid;
1298 	if (pcdq) {
1299 		rx_p->pcd_base_fqid = pcdq->fqid;
1300 		rx_p->pcd_fqs_count = DPAA_ETH_PCD_RXQ_NUM;
1301 	}
1302 
1303 	rx_p->ext_buf_pools.num_of_pools_used = 1;
1304 	rx_p->ext_buf_pools.ext_buf_pool[0].id =  bp->bpid;
1305 	rx_p->ext_buf_pools.ext_buf_pool[0].size = (u16)bp->size;
1306 
1307 	err = fman_port_config(port, &params);
1308 	if (err) {
1309 		pr_err("%s: fman_port_config failed\n", __func__);
1310 		return err;
1311 	}
1312 
1313 	err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
1314 	if (err) {
1315 		pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1316 		       __func__);
1317 		return err;
1318 	}
1319 
1320 	err = fman_port_init(port);
1321 	if (err)
1322 		pr_err("%s: fm_port_init failed\n", __func__);
1323 
1324 	return err;
1325 }
1326 
dpaa_eth_init_ports(struct mac_device * mac_dev,struct dpaa_bp * bp,struct fm_port_fqs * port_fqs,struct dpaa_buffer_layout * buf_layout,struct device * dev)1327 static int dpaa_eth_init_ports(struct mac_device *mac_dev,
1328 			       struct dpaa_bp *bp,
1329 			       struct fm_port_fqs *port_fqs,
1330 			       struct dpaa_buffer_layout *buf_layout,
1331 			       struct device *dev)
1332 {
1333 	struct fman_port *rxport = mac_dev->port[RX];
1334 	struct fman_port *txport = mac_dev->port[TX];
1335 	int err;
1336 
1337 	err = dpaa_eth_init_tx_port(txport, port_fqs->tx_errq,
1338 				    port_fqs->tx_defq, &buf_layout[TX]);
1339 	if (err)
1340 		return err;
1341 
1342 	err = dpaa_eth_init_rx_port(rxport, bp, port_fqs->rx_errq,
1343 				    port_fqs->rx_defq, port_fqs->rx_pcdq,
1344 				    &buf_layout[RX]);
1345 
1346 	return err;
1347 }
1348 
dpaa_bman_release(const struct dpaa_bp * dpaa_bp,struct bm_buffer * bmb,int cnt)1349 static int dpaa_bman_release(const struct dpaa_bp *dpaa_bp,
1350 			     struct bm_buffer *bmb, int cnt)
1351 {
1352 	int err;
1353 
1354 	err = bman_release(dpaa_bp->pool, bmb, cnt);
1355 	/* Should never occur, address anyway to avoid leaking the buffers */
1356 	if (WARN_ON(err) && dpaa_bp->free_buf_cb)
1357 		while (cnt-- > 0)
1358 			dpaa_bp->free_buf_cb(dpaa_bp, &bmb[cnt]);
1359 
1360 	return cnt;
1361 }
1362 
dpaa_release_sgt_members(struct qm_sg_entry * sgt)1363 static void dpaa_release_sgt_members(struct qm_sg_entry *sgt)
1364 {
1365 	struct bm_buffer bmb[DPAA_BUFF_RELEASE_MAX];
1366 	struct dpaa_bp *dpaa_bp;
1367 	int i = 0, j;
1368 
1369 	memset(bmb, 0, sizeof(bmb));
1370 
1371 	do {
1372 		dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1373 		if (!dpaa_bp)
1374 			return;
1375 
1376 		j = 0;
1377 		do {
1378 			WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1379 
1380 			bm_buffer_set64(&bmb[j], qm_sg_entry_get64(&sgt[i]));
1381 
1382 			j++; i++;
1383 		} while (j < ARRAY_SIZE(bmb) &&
1384 				!qm_sg_entry_is_final(&sgt[i - 1]) &&
1385 				sgt[i - 1].bpid == sgt[i].bpid);
1386 
1387 		dpaa_bman_release(dpaa_bp, bmb, j);
1388 	} while (!qm_sg_entry_is_final(&sgt[i - 1]));
1389 }
1390 
dpaa_fd_release(const struct net_device * net_dev,const struct qm_fd * fd)1391 static void dpaa_fd_release(const struct net_device *net_dev,
1392 			    const struct qm_fd *fd)
1393 {
1394 	struct qm_sg_entry *sgt;
1395 	struct dpaa_bp *dpaa_bp;
1396 	struct bm_buffer bmb;
1397 	dma_addr_t addr;
1398 	void *vaddr;
1399 
1400 	bmb.data = 0;
1401 	bm_buffer_set64(&bmb, qm_fd_addr(fd));
1402 
1403 	dpaa_bp = dpaa_bpid2pool(fd->bpid);
1404 	if (!dpaa_bp)
1405 		return;
1406 
1407 	if (qm_fd_get_format(fd) == qm_fd_sg) {
1408 		vaddr = phys_to_virt(qm_fd_addr(fd));
1409 		sgt = vaddr + qm_fd_get_offset(fd);
1410 
1411 		dma_unmap_page(dpaa_bp->priv->rx_dma_dev, qm_fd_addr(fd),
1412 			       DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1413 
1414 		dpaa_release_sgt_members(sgt);
1415 
1416 		addr = dma_map_page(dpaa_bp->priv->rx_dma_dev,
1417 				    virt_to_page(vaddr), 0, DPAA_BP_RAW_SIZE,
1418 				    DMA_FROM_DEVICE);
1419 		if (dma_mapping_error(dpaa_bp->priv->rx_dma_dev, addr)) {
1420 			netdev_err(net_dev, "DMA mapping failed\n");
1421 			return;
1422 		}
1423 		bm_buffer_set64(&bmb, addr);
1424 	}
1425 
1426 	dpaa_bman_release(dpaa_bp, &bmb, 1);
1427 }
1428 
count_ern(struct dpaa_percpu_priv * percpu_priv,const union qm_mr_entry * msg)1429 static void count_ern(struct dpaa_percpu_priv *percpu_priv,
1430 		      const union qm_mr_entry *msg)
1431 {
1432 	switch (msg->ern.rc & QM_MR_RC_MASK) {
1433 	case QM_MR_RC_CGR_TAILDROP:
1434 		percpu_priv->ern_cnt.cg_tdrop++;
1435 		break;
1436 	case QM_MR_RC_WRED:
1437 		percpu_priv->ern_cnt.wred++;
1438 		break;
1439 	case QM_MR_RC_ERROR:
1440 		percpu_priv->ern_cnt.err_cond++;
1441 		break;
1442 	case QM_MR_RC_ORPWINDOW_EARLY:
1443 		percpu_priv->ern_cnt.early_window++;
1444 		break;
1445 	case QM_MR_RC_ORPWINDOW_LATE:
1446 		percpu_priv->ern_cnt.late_window++;
1447 		break;
1448 	case QM_MR_RC_FQ_TAILDROP:
1449 		percpu_priv->ern_cnt.fq_tdrop++;
1450 		break;
1451 	case QM_MR_RC_ORPWINDOW_RETIRED:
1452 		percpu_priv->ern_cnt.fq_retired++;
1453 		break;
1454 	case QM_MR_RC_ORP_ZERO:
1455 		percpu_priv->ern_cnt.orp_zero++;
1456 		break;
1457 	}
1458 }
1459 
1460 /* Turn on HW checksum computation for this outgoing frame.
1461  * If the current protocol is not something we support in this regard
1462  * (or if the stack has already computed the SW checksum), we do nothing.
1463  *
1464  * Returns 0 if all goes well (or HW csum doesn't apply), and a negative value
1465  * otherwise.
1466  *
1467  * Note that this function may modify the fd->cmd field and the skb data buffer
1468  * (the Parse Results area).
1469  */
dpaa_enable_tx_csum(struct dpaa_priv * priv,struct sk_buff * skb,struct qm_fd * fd,void * parse_results)1470 static int dpaa_enable_tx_csum(struct dpaa_priv *priv,
1471 			       struct sk_buff *skb,
1472 			       struct qm_fd *fd,
1473 			       void *parse_results)
1474 {
1475 	struct fman_prs_result *parse_result;
1476 	u16 ethertype = ntohs(skb->protocol);
1477 	struct ipv6hdr *ipv6h = NULL;
1478 	struct iphdr *iph;
1479 	int retval = 0;
1480 	u8 l4_proto;
1481 
1482 	if (skb->ip_summed != CHECKSUM_PARTIAL)
1483 		return 0;
1484 
1485 	/* Note: L3 csum seems to be already computed in sw, but we can't choose
1486 	 * L4 alone from the FM configuration anyway.
1487 	 */
1488 
1489 	/* Fill in some fields of the Parse Results array, so the FMan
1490 	 * can find them as if they came from the FMan Parser.
1491 	 */
1492 	parse_result = (struct fman_prs_result *)parse_results;
1493 
1494 	/* If we're dealing with VLAN, get the real Ethernet type */
1495 	if (ethertype == ETH_P_8021Q)
1496 		ethertype = ntohs(skb_vlan_eth_hdr(skb)->h_vlan_encapsulated_proto);
1497 
1498 	/* Fill in the relevant L3 parse result fields
1499 	 * and read the L4 protocol type
1500 	 */
1501 	switch (ethertype) {
1502 	case ETH_P_IP:
1503 		parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV4);
1504 		iph = ip_hdr(skb);
1505 		WARN_ON(!iph);
1506 		l4_proto = iph->protocol;
1507 		break;
1508 	case ETH_P_IPV6:
1509 		parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV6);
1510 		ipv6h = ipv6_hdr(skb);
1511 		WARN_ON(!ipv6h);
1512 		l4_proto = ipv6h->nexthdr;
1513 		break;
1514 	default:
1515 		/* We shouldn't even be here */
1516 		if (net_ratelimit())
1517 			netif_alert(priv, tx_err, priv->net_dev,
1518 				    "Can't compute HW csum for L3 proto 0x%x\n",
1519 				    ntohs(skb->protocol));
1520 		retval = -EIO;
1521 		goto return_error;
1522 	}
1523 
1524 	/* Fill in the relevant L4 parse result fields */
1525 	switch (l4_proto) {
1526 	case IPPROTO_UDP:
1527 		parse_result->l4r = FM_L4_PARSE_RESULT_UDP;
1528 		break;
1529 	case IPPROTO_TCP:
1530 		parse_result->l4r = FM_L4_PARSE_RESULT_TCP;
1531 		break;
1532 	default:
1533 		if (net_ratelimit())
1534 			netif_alert(priv, tx_err, priv->net_dev,
1535 				    "Can't compute HW csum for L4 proto 0x%x\n",
1536 				    l4_proto);
1537 		retval = -EIO;
1538 		goto return_error;
1539 	}
1540 
1541 	/* At index 0 is IPOffset_1 as defined in the Parse Results */
1542 	parse_result->ip_off[0] = (u8)skb_network_offset(skb);
1543 	parse_result->l4_off = (u8)skb_transport_offset(skb);
1544 
1545 	/* Enable L3 (and L4, if TCP or UDP) HW checksum. */
1546 	fd->cmd |= cpu_to_be32(FM_FD_CMD_RPD | FM_FD_CMD_DTC);
1547 
1548 	/* On P1023 and similar platforms fd->cmd interpretation could
1549 	 * be disabled by setting CONTEXT_A bit ICMD; currently this bit
1550 	 * is not set so we do not need to check; in the future, if/when
1551 	 * using context_a we need to check this bit
1552 	 */
1553 
1554 return_error:
1555 	return retval;
1556 }
1557 
dpaa_bp_add_8_bufs(const struct dpaa_bp * dpaa_bp)1558 static int dpaa_bp_add_8_bufs(const struct dpaa_bp *dpaa_bp)
1559 {
1560 	struct net_device *net_dev = dpaa_bp->priv->net_dev;
1561 	struct bm_buffer bmb[8];
1562 	dma_addr_t addr;
1563 	struct page *p;
1564 	u8 i;
1565 
1566 	for (i = 0; i < 8; i++) {
1567 		p = dev_alloc_pages(0);
1568 		if (unlikely(!p)) {
1569 			netdev_err(net_dev, "dev_alloc_pages() failed\n");
1570 			goto release_previous_buffs;
1571 		}
1572 
1573 		addr = dma_map_page(dpaa_bp->priv->rx_dma_dev, p, 0,
1574 				    DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1575 		if (unlikely(dma_mapping_error(dpaa_bp->priv->rx_dma_dev,
1576 					       addr))) {
1577 			netdev_err(net_dev, "DMA map failed\n");
1578 			goto release_previous_buffs;
1579 		}
1580 
1581 		bmb[i].data = 0;
1582 		bm_buffer_set64(&bmb[i], addr);
1583 	}
1584 
1585 release_bufs:
1586 	return dpaa_bman_release(dpaa_bp, bmb, i);
1587 
1588 release_previous_buffs:
1589 	WARN_ONCE(1, "dpaa_eth: failed to add buffers on Rx\n");
1590 
1591 	bm_buffer_set64(&bmb[i], 0);
1592 	/* Avoid releasing a completely null buffer; bman_release() requires
1593 	 * at least one buffer.
1594 	 */
1595 	if (likely(i))
1596 		goto release_bufs;
1597 
1598 	return 0;
1599 }
1600 
dpaa_bp_seed(struct dpaa_bp * dpaa_bp)1601 static int dpaa_bp_seed(struct dpaa_bp *dpaa_bp)
1602 {
1603 	int i;
1604 
1605 	/* Give each CPU an allotment of "config_count" buffers */
1606 	for_each_possible_cpu(i) {
1607 		int *count_ptr = per_cpu_ptr(dpaa_bp->percpu_count, i);
1608 		int j;
1609 
1610 		/* Although we access another CPU's counters here
1611 		 * we do it at boot time so it is safe
1612 		 */
1613 		for (j = 0; j < dpaa_bp->config_count; j += 8)
1614 			*count_ptr += dpaa_bp_add_8_bufs(dpaa_bp);
1615 	}
1616 	return 0;
1617 }
1618 
1619 /* Add buffers/(pages) for Rx processing whenever bpool count falls below
1620  * REFILL_THRESHOLD.
1621  */
dpaa_eth_refill_bpool(struct dpaa_bp * dpaa_bp,int * countptr)1622 static int dpaa_eth_refill_bpool(struct dpaa_bp *dpaa_bp, int *countptr)
1623 {
1624 	int count = *countptr;
1625 	int new_bufs;
1626 
1627 	if (unlikely(count < FSL_DPAA_ETH_REFILL_THRESHOLD)) {
1628 		do {
1629 			new_bufs = dpaa_bp_add_8_bufs(dpaa_bp);
1630 			if (unlikely(!new_bufs)) {
1631 				/* Avoid looping forever if we've temporarily
1632 				 * run out of memory. We'll try again at the
1633 				 * next NAPI cycle.
1634 				 */
1635 				break;
1636 			}
1637 			count += new_bufs;
1638 		} while (count < FSL_DPAA_ETH_MAX_BUF_COUNT);
1639 
1640 		*countptr = count;
1641 		if (unlikely(count < FSL_DPAA_ETH_MAX_BUF_COUNT))
1642 			return -ENOMEM;
1643 	}
1644 
1645 	return 0;
1646 }
1647 
dpaa_eth_refill_bpools(struct dpaa_priv * priv)1648 static int dpaa_eth_refill_bpools(struct dpaa_priv *priv)
1649 {
1650 	struct dpaa_bp *dpaa_bp;
1651 	int *countptr;
1652 
1653 	dpaa_bp = priv->dpaa_bp;
1654 	if (!dpaa_bp)
1655 		return -EINVAL;
1656 	countptr = this_cpu_ptr(dpaa_bp->percpu_count);
1657 
1658 	return dpaa_eth_refill_bpool(dpaa_bp, countptr);
1659 }
1660 
1661 /* Cleanup function for outgoing frame descriptors that were built on Tx path,
1662  * either contiguous frames or scatter/gather ones.
1663  * Skb freeing is not handled here.
1664  *
1665  * This function may be called on error paths in the Tx function, so guard
1666  * against cases when not all fd relevant fields were filled in. To avoid
1667  * reading the invalid transmission timestamp for the error paths set ts to
1668  * false.
1669  *
1670  * Return the skb backpointer, since for S/G frames the buffer containing it
1671  * gets freed here.
1672  *
1673  * No skb backpointer is set when transmitting XDP frames. Cleanup the buffer
1674  * and return NULL in this case.
1675  */
dpaa_cleanup_tx_fd(const struct dpaa_priv * priv,const struct qm_fd * fd,bool ts)1676 static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,
1677 					  const struct qm_fd *fd, bool ts)
1678 {
1679 	const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
1680 	struct device *dev = priv->net_dev->dev.parent;
1681 	struct skb_shared_hwtstamps shhwtstamps;
1682 	dma_addr_t addr = qm_fd_addr(fd);
1683 	void *vaddr = phys_to_virt(addr);
1684 	const struct qm_sg_entry *sgt;
1685 	struct dpaa_eth_swbp *swbp;
1686 	struct sk_buff *skb;
1687 	u64 ns;
1688 	int i;
1689 
1690 	if (unlikely(qm_fd_get_format(fd) == qm_fd_sg)) {
1691 		dma_unmap_page(priv->tx_dma_dev, addr,
1692 			       qm_fd_get_offset(fd) + DPAA_SGT_SIZE,
1693 			       dma_dir);
1694 
1695 		/* The sgt buffer has been allocated with netdev_alloc_frag(),
1696 		 * it's from lowmem.
1697 		 */
1698 		sgt = vaddr + qm_fd_get_offset(fd);
1699 
1700 		/* sgt[0] is from lowmem, was dma_map_single()-ed */
1701 		dma_unmap_single(priv->tx_dma_dev, qm_sg_addr(&sgt[0]),
1702 				 qm_sg_entry_get_len(&sgt[0]), dma_dir);
1703 
1704 		/* remaining pages were mapped with skb_frag_dma_map() */
1705 		for (i = 1; (i < DPAA_SGT_MAX_ENTRIES) &&
1706 		     !qm_sg_entry_is_final(&sgt[i - 1]); i++) {
1707 			WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1708 
1709 			dma_unmap_page(priv->tx_dma_dev, qm_sg_addr(&sgt[i]),
1710 				       qm_sg_entry_get_len(&sgt[i]), dma_dir);
1711 		}
1712 	} else {
1713 		dma_unmap_single(priv->tx_dma_dev, addr,
1714 				 qm_fd_get_offset(fd) + qm_fd_get_length(fd),
1715 				 dma_dir);
1716 	}
1717 
1718 	swbp = (struct dpaa_eth_swbp *)vaddr;
1719 	skb = swbp->skb;
1720 
1721 	/* No skb backpointer is set when running XDP. An xdp_frame
1722 	 * backpointer is saved instead.
1723 	 */
1724 	if (!skb) {
1725 		xdp_return_frame(swbp->xdpf);
1726 		return NULL;
1727 	}
1728 
1729 	/* DMA unmapping is required before accessing the HW provided info */
1730 	if (ts && priv->tx_tstamp &&
1731 	    skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
1732 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
1733 
1734 		if (!fman_port_get_tstamp(priv->mac_dev->port[TX], vaddr,
1735 					  &ns)) {
1736 			shhwtstamps.hwtstamp = ns_to_ktime(ns);
1737 			skb_tstamp_tx(skb, &shhwtstamps);
1738 		} else {
1739 			dev_warn(dev, "fman_port_get_tstamp failed!\n");
1740 		}
1741 	}
1742 
1743 	if (qm_fd_get_format(fd) == qm_fd_sg)
1744 		/* Free the page that we allocated on Tx for the SGT */
1745 		free_pages((unsigned long)vaddr, 0);
1746 
1747 	return skb;
1748 }
1749 
rx_csum_offload(const struct dpaa_priv * priv,const struct qm_fd * fd)1750 static u8 rx_csum_offload(const struct dpaa_priv *priv, const struct qm_fd *fd)
1751 {
1752 	/* The parser has run and performed L4 checksum validation.
1753 	 * We know there were no parser errors (and implicitly no
1754 	 * L4 csum error), otherwise we wouldn't be here.
1755 	 */
1756 	if ((priv->net_dev->features & NETIF_F_RXCSUM) &&
1757 	    (be32_to_cpu(fd->status) & FM_FD_STAT_L4CV))
1758 		return CHECKSUM_UNNECESSARY;
1759 
1760 	/* We're here because either the parser didn't run or the L4 checksum
1761 	 * was not verified. This may include the case of a UDP frame with
1762 	 * checksum zero or an L4 proto other than TCP/UDP
1763 	 */
1764 	return CHECKSUM_NONE;
1765 }
1766 
1767 #define PTR_IS_ALIGNED(x, a) (IS_ALIGNED((unsigned long)(x), (a)))
1768 
1769 /* Build a linear skb around the received buffer.
1770  * We are guaranteed there is enough room at the end of the data buffer to
1771  * accommodate the shared info area of the skb.
1772  */
contig_fd_to_skb(const struct dpaa_priv * priv,const struct qm_fd * fd)1773 static struct sk_buff *contig_fd_to_skb(const struct dpaa_priv *priv,
1774 					const struct qm_fd *fd)
1775 {
1776 	ssize_t fd_off = qm_fd_get_offset(fd);
1777 	dma_addr_t addr = qm_fd_addr(fd);
1778 	struct dpaa_bp *dpaa_bp;
1779 	struct sk_buff *skb;
1780 	void *vaddr;
1781 
1782 	vaddr = phys_to_virt(addr);
1783 	WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
1784 
1785 	dpaa_bp = dpaa_bpid2pool(fd->bpid);
1786 	if (!dpaa_bp)
1787 		goto free_buffer;
1788 
1789 	skb = build_skb(vaddr, dpaa_bp->size +
1790 			SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
1791 	if (WARN_ONCE(!skb, "Build skb failure on Rx\n"))
1792 		goto free_buffer;
1793 	skb_reserve(skb, fd_off);
1794 	skb_put(skb, qm_fd_get_length(fd));
1795 
1796 	skb->ip_summed = rx_csum_offload(priv, fd);
1797 
1798 	return skb;
1799 
1800 free_buffer:
1801 	free_pages((unsigned long)vaddr, 0);
1802 	return NULL;
1803 }
1804 
1805 /* Build an skb with the data of the first S/G entry in the linear portion and
1806  * the rest of the frame as skb fragments.
1807  *
1808  * The page fragment holding the S/G Table is recycled here.
1809  */
sg_fd_to_skb(const struct dpaa_priv * priv,const struct qm_fd * fd)1810 static struct sk_buff *sg_fd_to_skb(const struct dpaa_priv *priv,
1811 				    const struct qm_fd *fd)
1812 {
1813 	ssize_t fd_off = qm_fd_get_offset(fd);
1814 	dma_addr_t addr = qm_fd_addr(fd);
1815 	const struct qm_sg_entry *sgt;
1816 	struct page *page, *head_page;
1817 	struct dpaa_bp *dpaa_bp;
1818 	void *vaddr, *sg_vaddr;
1819 	int frag_off, frag_len;
1820 	struct sk_buff *skb;
1821 	dma_addr_t sg_addr;
1822 	int page_offset;
1823 	unsigned int sz;
1824 	int *count_ptr;
1825 	int i, j;
1826 
1827 	vaddr = phys_to_virt(addr);
1828 	WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
1829 
1830 	/* Iterate through the SGT entries and add data buffers to the skb */
1831 	sgt = vaddr + fd_off;
1832 	skb = NULL;
1833 	for (i = 0; i < DPAA_SGT_MAX_ENTRIES; i++) {
1834 		/* Extension bit is not supported */
1835 		WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1836 
1837 		sg_addr = qm_sg_addr(&sgt[i]);
1838 		sg_vaddr = phys_to_virt(sg_addr);
1839 		WARN_ON(!PTR_IS_ALIGNED(sg_vaddr, SMP_CACHE_BYTES));
1840 
1841 		dma_unmap_page(priv->rx_dma_dev, sg_addr,
1842 			       DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1843 
1844 		/* We may use multiple Rx pools */
1845 		dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1846 		if (!dpaa_bp)
1847 			goto free_buffers;
1848 
1849 		if (!skb) {
1850 			sz = dpaa_bp->size +
1851 				SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1852 			skb = build_skb(sg_vaddr, sz);
1853 			if (WARN_ON(!skb))
1854 				goto free_buffers;
1855 
1856 			skb->ip_summed = rx_csum_offload(priv, fd);
1857 
1858 			/* Make sure forwarded skbs will have enough space
1859 			 * on Tx, if extra headers are added.
1860 			 */
1861 			WARN_ON(fd_off != priv->rx_headroom);
1862 			skb_reserve(skb, fd_off);
1863 			skb_put(skb, qm_sg_entry_get_len(&sgt[i]));
1864 		} else {
1865 			/* Not the first S/G entry; all data from buffer will
1866 			 * be added in an skb fragment; fragment index is offset
1867 			 * by one since first S/G entry was incorporated in the
1868 			 * linear part of the skb.
1869 			 *
1870 			 * Caution: 'page' may be a tail page.
1871 			 */
1872 			page = virt_to_page(sg_vaddr);
1873 			head_page = virt_to_head_page(sg_vaddr);
1874 
1875 			/* Compute offset in (possibly tail) page */
1876 			page_offset = ((unsigned long)sg_vaddr &
1877 					(PAGE_SIZE - 1)) +
1878 				(page_address(page) - page_address(head_page));
1879 			/* page_offset only refers to the beginning of sgt[i];
1880 			 * but the buffer itself may have an internal offset.
1881 			 */
1882 			frag_off = qm_sg_entry_get_off(&sgt[i]) + page_offset;
1883 			frag_len = qm_sg_entry_get_len(&sgt[i]);
1884 			/* skb_add_rx_frag() does no checking on the page; if
1885 			 * we pass it a tail page, we'll end up with
1886 			 * bad page accounting and eventually with segafults.
1887 			 */
1888 			skb_add_rx_frag(skb, i - 1, head_page, frag_off,
1889 					frag_len, dpaa_bp->size);
1890 		}
1891 
1892 		/* Update the pool count for the current {cpu x bpool} */
1893 		count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
1894 		(*count_ptr)--;
1895 
1896 		if (qm_sg_entry_is_final(&sgt[i]))
1897 			break;
1898 	}
1899 	WARN_ONCE(i == DPAA_SGT_MAX_ENTRIES, "No final bit on SGT\n");
1900 
1901 	/* free the SG table buffer */
1902 	free_pages((unsigned long)vaddr, 0);
1903 
1904 	return skb;
1905 
1906 free_buffers:
1907 	/* free all the SG entries */
1908 	for (j = 0; j < DPAA_SGT_MAX_ENTRIES ; j++) {
1909 		sg_addr = qm_sg_addr(&sgt[j]);
1910 		sg_vaddr = phys_to_virt(sg_addr);
1911 		/* all pages 0..i were unmaped */
1912 		if (j > i)
1913 			dma_unmap_page(priv->rx_dma_dev, qm_sg_addr(&sgt[j]),
1914 				       DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1915 		free_pages((unsigned long)sg_vaddr, 0);
1916 		/* counters 0..i-1 were decremented */
1917 		if (j >= i) {
1918 			dpaa_bp = dpaa_bpid2pool(sgt[j].bpid);
1919 			if (dpaa_bp) {
1920 				count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
1921 				(*count_ptr)--;
1922 			}
1923 		}
1924 
1925 		if (qm_sg_entry_is_final(&sgt[j]))
1926 			break;
1927 	}
1928 	/* free the SGT fragment */
1929 	free_pages((unsigned long)vaddr, 0);
1930 
1931 	return NULL;
1932 }
1933 
skb_to_contig_fd(struct dpaa_priv * priv,struct sk_buff * skb,struct qm_fd * fd,int * offset)1934 static int skb_to_contig_fd(struct dpaa_priv *priv,
1935 			    struct sk_buff *skb, struct qm_fd *fd,
1936 			    int *offset)
1937 {
1938 	struct net_device *net_dev = priv->net_dev;
1939 	enum dma_data_direction dma_dir;
1940 	struct dpaa_eth_swbp *swbp;
1941 	unsigned char *buff_start;
1942 	dma_addr_t addr;
1943 	int err;
1944 
1945 	/* We are guaranteed to have at least tx_headroom bytes
1946 	 * available, so just use that for offset.
1947 	 */
1948 	fd->bpid = FSL_DPAA_BPID_INV;
1949 	buff_start = skb->data - priv->tx_headroom;
1950 	dma_dir = DMA_TO_DEVICE;
1951 
1952 	swbp = (struct dpaa_eth_swbp *)buff_start;
1953 	swbp->skb = skb;
1954 
1955 	/* Enable L3/L4 hardware checksum computation.
1956 	 *
1957 	 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1958 	 * need to write into the skb.
1959 	 */
1960 	err = dpaa_enable_tx_csum(priv, skb, fd,
1961 				  buff_start + DPAA_TX_PRIV_DATA_SIZE);
1962 	if (unlikely(err < 0)) {
1963 		if (net_ratelimit())
1964 			netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
1965 				  err);
1966 		return err;
1967 	}
1968 
1969 	/* Fill in the rest of the FD fields */
1970 	qm_fd_set_contig(fd, priv->tx_headroom, skb->len);
1971 	fd->cmd |= cpu_to_be32(FM_FD_CMD_FCO);
1972 
1973 	/* Map the entire buffer size that may be seen by FMan, but no more */
1974 	addr = dma_map_single(priv->tx_dma_dev, buff_start,
1975 			      priv->tx_headroom + skb->len, dma_dir);
1976 	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
1977 		if (net_ratelimit())
1978 			netif_err(priv, tx_err, net_dev, "dma_map_single() failed\n");
1979 		return -EINVAL;
1980 	}
1981 	qm_fd_addr_set64(fd, addr);
1982 
1983 	return 0;
1984 }
1985 
skb_to_sg_fd(struct dpaa_priv * priv,struct sk_buff * skb,struct qm_fd * fd)1986 static int skb_to_sg_fd(struct dpaa_priv *priv,
1987 			struct sk_buff *skb, struct qm_fd *fd)
1988 {
1989 	const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
1990 	const int nr_frags = skb_shinfo(skb)->nr_frags;
1991 	struct net_device *net_dev = priv->net_dev;
1992 	struct dpaa_eth_swbp *swbp;
1993 	struct qm_sg_entry *sgt;
1994 	void *buff_start;
1995 	skb_frag_t *frag;
1996 	dma_addr_t addr;
1997 	size_t frag_len;
1998 	struct page *p;
1999 	int i, j, err;
2000 
2001 	/* get a page to store the SGTable */
2002 	p = dev_alloc_pages(0);
2003 	if (unlikely(!p)) {
2004 		netdev_err(net_dev, "dev_alloc_pages() failed\n");
2005 		return -ENOMEM;
2006 	}
2007 	buff_start = page_address(p);
2008 
2009 	/* Enable L3/L4 hardware checksum computation.
2010 	 *
2011 	 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
2012 	 * need to write into the skb.
2013 	 */
2014 	err = dpaa_enable_tx_csum(priv, skb, fd,
2015 				  buff_start + DPAA_TX_PRIV_DATA_SIZE);
2016 	if (unlikely(err < 0)) {
2017 		if (net_ratelimit())
2018 			netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
2019 				  err);
2020 		goto csum_failed;
2021 	}
2022 
2023 	/* SGT[0] is used by the linear part */
2024 	sgt = (struct qm_sg_entry *)(buff_start + priv->tx_headroom);
2025 	frag_len = skb_headlen(skb);
2026 	qm_sg_entry_set_len(&sgt[0], frag_len);
2027 	sgt[0].bpid = FSL_DPAA_BPID_INV;
2028 	sgt[0].offset = 0;
2029 	addr = dma_map_single(priv->tx_dma_dev, skb->data,
2030 			      skb_headlen(skb), dma_dir);
2031 	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2032 		netdev_err(priv->net_dev, "DMA mapping failed\n");
2033 		err = -EINVAL;
2034 		goto sg0_map_failed;
2035 	}
2036 	qm_sg_entry_set64(&sgt[0], addr);
2037 
2038 	/* populate the rest of SGT entries */
2039 	for (i = 0; i < nr_frags; i++) {
2040 		frag = &skb_shinfo(skb)->frags[i];
2041 		frag_len = skb_frag_size(frag);
2042 		WARN_ON(!skb_frag_page(frag));
2043 		addr = skb_frag_dma_map(priv->tx_dma_dev, frag, 0,
2044 					frag_len, dma_dir);
2045 		if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2046 			netdev_err(priv->net_dev, "DMA mapping failed\n");
2047 			err = -EINVAL;
2048 			goto sg_map_failed;
2049 		}
2050 
2051 		qm_sg_entry_set_len(&sgt[i + 1], frag_len);
2052 		sgt[i + 1].bpid = FSL_DPAA_BPID_INV;
2053 		sgt[i + 1].offset = 0;
2054 
2055 		/* keep the offset in the address */
2056 		qm_sg_entry_set64(&sgt[i + 1], addr);
2057 	}
2058 
2059 	/* Set the final bit in the last used entry of the SGT */
2060 	qm_sg_entry_set_f(&sgt[nr_frags], frag_len);
2061 
2062 	/* set fd offset to priv->tx_headroom */
2063 	qm_fd_set_sg(fd, priv->tx_headroom, skb->len);
2064 
2065 	/* DMA map the SGT page */
2066 	swbp = (struct dpaa_eth_swbp *)buff_start;
2067 	swbp->skb = skb;
2068 
2069 	addr = dma_map_page(priv->tx_dma_dev, p, 0,
2070 			    priv->tx_headroom + DPAA_SGT_SIZE, dma_dir);
2071 	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2072 		netdev_err(priv->net_dev, "DMA mapping failed\n");
2073 		err = -EINVAL;
2074 		goto sgt_map_failed;
2075 	}
2076 
2077 	fd->bpid = FSL_DPAA_BPID_INV;
2078 	fd->cmd |= cpu_to_be32(FM_FD_CMD_FCO);
2079 	qm_fd_addr_set64(fd, addr);
2080 
2081 	return 0;
2082 
2083 sgt_map_failed:
2084 sg_map_failed:
2085 	for (j = 0; j < i; j++)
2086 		dma_unmap_page(priv->tx_dma_dev, qm_sg_addr(&sgt[j]),
2087 			       qm_sg_entry_get_len(&sgt[j]), dma_dir);
2088 sg0_map_failed:
2089 csum_failed:
2090 	free_pages((unsigned long)buff_start, 0);
2091 
2092 	return err;
2093 }
2094 
dpaa_xmit(struct dpaa_priv * priv,struct rtnl_link_stats64 * percpu_stats,int queue,struct qm_fd * fd)2095 static inline int dpaa_xmit(struct dpaa_priv *priv,
2096 			    struct rtnl_link_stats64 *percpu_stats,
2097 			    int queue,
2098 			    struct qm_fd *fd)
2099 {
2100 	struct qman_fq *egress_fq;
2101 	int err, i;
2102 
2103 	egress_fq = priv->egress_fqs[queue];
2104 	if (fd->bpid == FSL_DPAA_BPID_INV)
2105 		fd->cmd |= cpu_to_be32(qman_fq_fqid(priv->conf_fqs[queue]));
2106 
2107 	/* Trace this Tx fd */
2108 	trace_dpaa_tx_fd(priv->net_dev, egress_fq, fd);
2109 
2110 	for (i = 0; i < DPAA_ENQUEUE_RETRIES; i++) {
2111 		err = qman_enqueue(egress_fq, fd);
2112 		if (err != -EBUSY)
2113 			break;
2114 	}
2115 
2116 	if (unlikely(err < 0)) {
2117 		percpu_stats->tx_fifo_errors++;
2118 		return err;
2119 	}
2120 
2121 	percpu_stats->tx_packets++;
2122 	percpu_stats->tx_bytes += qm_fd_get_length(fd);
2123 
2124 	return 0;
2125 }
2126 
2127 #ifdef CONFIG_DPAA_ERRATUM_A050385
dpaa_a050385_wa_skb(struct net_device * net_dev,struct sk_buff ** s)2128 static int dpaa_a050385_wa_skb(struct net_device *net_dev, struct sk_buff **s)
2129 {
2130 	struct dpaa_priv *priv = netdev_priv(net_dev);
2131 	struct sk_buff *new_skb, *skb = *s;
2132 	unsigned char *start, i;
2133 
2134 	/* check linear buffer alignment */
2135 	if (!PTR_IS_ALIGNED(skb->data, DPAA_A050385_ALIGN))
2136 		goto workaround;
2137 
2138 	/* linear buffers just need to have an aligned start */
2139 	if (!skb_is_nonlinear(skb))
2140 		return 0;
2141 
2142 	/* linear data size for nonlinear skbs needs to be aligned */
2143 	if (!IS_ALIGNED(skb_headlen(skb), DPAA_A050385_ALIGN))
2144 		goto workaround;
2145 
2146 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2147 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2148 
2149 		/* all fragments need to have aligned start addresses */
2150 		if (!IS_ALIGNED(skb_frag_off(frag), DPAA_A050385_ALIGN))
2151 			goto workaround;
2152 
2153 		/* all but last fragment need to have aligned sizes */
2154 		if (!IS_ALIGNED(skb_frag_size(frag), DPAA_A050385_ALIGN) &&
2155 		    (i < skb_shinfo(skb)->nr_frags - 1))
2156 			goto workaround;
2157 	}
2158 
2159 	return 0;
2160 
2161 workaround:
2162 	/* copy all the skb content into a new linear buffer */
2163 	new_skb = netdev_alloc_skb(net_dev, skb->len + DPAA_A050385_ALIGN - 1 +
2164 						priv->tx_headroom);
2165 	if (!new_skb)
2166 		return -ENOMEM;
2167 
2168 	/* NET_SKB_PAD bytes already reserved, adding up to tx_headroom */
2169 	skb_reserve(new_skb, priv->tx_headroom - NET_SKB_PAD);
2170 
2171 	/* Workaround for DPAA_A050385 requires data start to be aligned */
2172 	start = PTR_ALIGN(new_skb->data, DPAA_A050385_ALIGN);
2173 	if (start - new_skb->data)
2174 		skb_reserve(new_skb, start - new_skb->data);
2175 
2176 	skb_put(new_skb, skb->len);
2177 	skb_copy_bits(skb, 0, new_skb->data, skb->len);
2178 	skb_copy_header(new_skb, skb);
2179 	new_skb->dev = skb->dev;
2180 
2181 	/* Copy relevant timestamp info from the old skb to the new */
2182 	if (priv->tx_tstamp) {
2183 		skb_shinfo(new_skb)->tx_flags = skb_shinfo(skb)->tx_flags;
2184 		skb_shinfo(new_skb)->hwtstamps = skb_shinfo(skb)->hwtstamps;
2185 		skb_shinfo(new_skb)->tskey = skb_shinfo(skb)->tskey;
2186 		if (skb->sk)
2187 			skb_set_owner_w(new_skb, skb->sk);
2188 	}
2189 
2190 	/* We move the headroom when we align it so we have to reset the
2191 	 * network and transport header offsets relative to the new data
2192 	 * pointer. The checksum offload relies on these offsets.
2193 	 */
2194 	skb_set_network_header(new_skb, skb_network_offset(skb));
2195 	skb_set_transport_header(new_skb, skb_transport_offset(skb));
2196 
2197 	dev_kfree_skb(skb);
2198 	*s = new_skb;
2199 
2200 	return 0;
2201 }
2202 
dpaa_a050385_wa_xdpf(struct dpaa_priv * priv,struct xdp_frame ** init_xdpf)2203 static int dpaa_a050385_wa_xdpf(struct dpaa_priv *priv,
2204 				struct xdp_frame **init_xdpf)
2205 {
2206 	struct xdp_frame *new_xdpf, *xdpf = *init_xdpf;
2207 	void *new_buff, *aligned_data;
2208 	struct page *p;
2209 	u32 data_shift;
2210 	int headroom;
2211 
2212 	/* Check the data alignment and make sure the headroom is large
2213 	 * enough to store the xdpf backpointer. Use an aligned headroom
2214 	 * value.
2215 	 *
2216 	 * Due to alignment constraints, we give XDP access to the full 256
2217 	 * byte frame headroom. If the XDP program uses all of it, copy the
2218 	 * data to a new buffer and make room for storing the backpointer.
2219 	 */
2220 	if (PTR_IS_ALIGNED(xdpf->data, DPAA_FD_DATA_ALIGNMENT) &&
2221 	    xdpf->headroom >= priv->tx_headroom) {
2222 		xdpf->headroom = priv->tx_headroom;
2223 		return 0;
2224 	}
2225 
2226 	/* Try to move the data inside the buffer just enough to align it and
2227 	 * store the xdpf backpointer. If the available headroom isn't large
2228 	 * enough, resort to allocating a new buffer and copying the data.
2229 	 */
2230 	aligned_data = PTR_ALIGN_DOWN(xdpf->data, DPAA_FD_DATA_ALIGNMENT);
2231 	data_shift = xdpf->data - aligned_data;
2232 
2233 	/* The XDP frame's headroom needs to be large enough to accommodate
2234 	 * shifting the data as well as storing the xdpf backpointer.
2235 	 */
2236 	if (xdpf->headroom  >= data_shift + priv->tx_headroom) {
2237 		memmove(aligned_data, xdpf->data, xdpf->len);
2238 		xdpf->data = aligned_data;
2239 		xdpf->headroom = priv->tx_headroom;
2240 		return 0;
2241 	}
2242 
2243 	/* The new xdp_frame is stored in the new buffer. Reserve enough space
2244 	 * in the headroom for storing it along with the driver's private
2245 	 * info. The headroom needs to be aligned to DPAA_FD_DATA_ALIGNMENT to
2246 	 * guarantee the data's alignment in the buffer.
2247 	 */
2248 	headroom = ALIGN(sizeof(*new_xdpf) + priv->tx_headroom,
2249 			 DPAA_FD_DATA_ALIGNMENT);
2250 
2251 	/* Assure the extended headroom and data don't overflow the buffer,
2252 	 * while maintaining the mandatory tailroom.
2253 	 */
2254 	if (headroom + xdpf->len > DPAA_BP_RAW_SIZE -
2255 			SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
2256 		return -ENOMEM;
2257 
2258 	p = dev_alloc_pages(0);
2259 	if (unlikely(!p))
2260 		return -ENOMEM;
2261 
2262 	/* Copy the data to the new buffer at a properly aligned offset */
2263 	new_buff = page_address(p);
2264 	memcpy(new_buff + headroom, xdpf->data, xdpf->len);
2265 
2266 	/* Create an XDP frame around the new buffer in a similar fashion
2267 	 * to xdp_convert_buff_to_frame.
2268 	 */
2269 	new_xdpf = new_buff;
2270 	new_xdpf->data = new_buff + headroom;
2271 	new_xdpf->len = xdpf->len;
2272 	new_xdpf->headroom = priv->tx_headroom;
2273 	new_xdpf->frame_sz = DPAA_BP_RAW_SIZE;
2274 	new_xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
2275 
2276 	/* Release the initial buffer */
2277 	xdp_return_frame_rx_napi(xdpf);
2278 
2279 	*init_xdpf = new_xdpf;
2280 	return 0;
2281 }
2282 #endif
2283 
2284 static netdev_tx_t
dpaa_start_xmit(struct sk_buff * skb,struct net_device * net_dev)2285 dpaa_start_xmit(struct sk_buff *skb, struct net_device *net_dev)
2286 {
2287 	const int queue_mapping = skb_get_queue_mapping(skb);
2288 	struct rtnl_link_stats64 *percpu_stats;
2289 	struct dpaa_percpu_priv *percpu_priv;
2290 	struct netdev_queue *txq;
2291 	struct dpaa_priv *priv;
2292 	struct qm_fd fd;
2293 	bool nonlinear;
2294 	int offset = 0;
2295 	int err = 0;
2296 
2297 	priv = netdev_priv(net_dev);
2298 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2299 	percpu_stats = &percpu_priv->stats;
2300 
2301 	qm_fd_clear_fd(&fd);
2302 
2303 	/* Packet data is always read as 32-bit words, so zero out any part of
2304 	 * the skb which might be sent if we have to pad the packet
2305 	 */
2306 	if (__skb_put_padto(skb, ETH_ZLEN, false))
2307 		goto enomem;
2308 
2309 	nonlinear = skb_is_nonlinear(skb);
2310 	if (!nonlinear) {
2311 		/* We're going to store the skb backpointer at the beginning
2312 		 * of the data buffer, so we need a privately owned skb
2313 		 *
2314 		 * We've made sure skb is not shared in dev->priv_flags,
2315 		 * we need to verify the skb head is not cloned
2316 		 */
2317 		if (skb_cow_head(skb, priv->tx_headroom))
2318 			goto enomem;
2319 
2320 		WARN_ON(skb_is_nonlinear(skb));
2321 	}
2322 
2323 	/* MAX_SKB_FRAGS is equal or larger than our dpaa_SGT_MAX_ENTRIES;
2324 	 * make sure we don't feed FMan with more fragments than it supports.
2325 	 */
2326 	if (unlikely(nonlinear &&
2327 		     (skb_shinfo(skb)->nr_frags >= DPAA_SGT_MAX_ENTRIES))) {
2328 		/* If the egress skb contains more fragments than we support
2329 		 * we have no choice but to linearize it ourselves.
2330 		 */
2331 		if (__skb_linearize(skb))
2332 			goto enomem;
2333 
2334 		nonlinear = skb_is_nonlinear(skb);
2335 	}
2336 
2337 #ifdef CONFIG_DPAA_ERRATUM_A050385
2338 	if (unlikely(fman_has_errata_a050385())) {
2339 		if (dpaa_a050385_wa_skb(net_dev, &skb))
2340 			goto enomem;
2341 		nonlinear = skb_is_nonlinear(skb);
2342 	}
2343 #endif
2344 
2345 	if (nonlinear) {
2346 		/* Just create a S/G fd based on the skb */
2347 		err = skb_to_sg_fd(priv, skb, &fd);
2348 		percpu_priv->tx_frag_skbuffs++;
2349 	} else {
2350 		/* Create a contig FD from this skb */
2351 		err = skb_to_contig_fd(priv, skb, &fd, &offset);
2352 	}
2353 	if (unlikely(err < 0))
2354 		goto skb_to_fd_failed;
2355 
2356 	txq = netdev_get_tx_queue(net_dev, queue_mapping);
2357 
2358 	/* LLTX requires to do our own update of trans_start */
2359 	txq_trans_cond_update(txq);
2360 
2361 	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
2362 		fd.cmd |= cpu_to_be32(FM_FD_CMD_UPD);
2363 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2364 	}
2365 
2366 	if (likely(dpaa_xmit(priv, percpu_stats, queue_mapping, &fd) == 0))
2367 		return NETDEV_TX_OK;
2368 
2369 	dpaa_cleanup_tx_fd(priv, &fd, false);
2370 skb_to_fd_failed:
2371 enomem:
2372 	percpu_stats->tx_errors++;
2373 	dev_kfree_skb(skb);
2374 	return NETDEV_TX_OK;
2375 }
2376 
dpaa_rx_error(struct net_device * net_dev,const struct dpaa_priv * priv,struct dpaa_percpu_priv * percpu_priv,const struct qm_fd * fd,u32 fqid)2377 static void dpaa_rx_error(struct net_device *net_dev,
2378 			  const struct dpaa_priv *priv,
2379 			  struct dpaa_percpu_priv *percpu_priv,
2380 			  const struct qm_fd *fd,
2381 			  u32 fqid)
2382 {
2383 	if (net_ratelimit())
2384 		netif_err(priv, hw, net_dev, "Err FD status = 0x%08x\n",
2385 			  be32_to_cpu(fd->status) & FM_FD_STAT_RX_ERRORS);
2386 
2387 	percpu_priv->stats.rx_errors++;
2388 
2389 	if (be32_to_cpu(fd->status) & FM_FD_ERR_DMA)
2390 		percpu_priv->rx_errors.dme++;
2391 	if (be32_to_cpu(fd->status) & FM_FD_ERR_PHYSICAL)
2392 		percpu_priv->rx_errors.fpe++;
2393 	if (be32_to_cpu(fd->status) & FM_FD_ERR_SIZE)
2394 		percpu_priv->rx_errors.fse++;
2395 	if (be32_to_cpu(fd->status) & FM_FD_ERR_PRS_HDR_ERR)
2396 		percpu_priv->rx_errors.phe++;
2397 
2398 	dpaa_fd_release(net_dev, fd);
2399 }
2400 
dpaa_tx_error(struct net_device * net_dev,const struct dpaa_priv * priv,struct dpaa_percpu_priv * percpu_priv,const struct qm_fd * fd,u32 fqid)2401 static void dpaa_tx_error(struct net_device *net_dev,
2402 			  const struct dpaa_priv *priv,
2403 			  struct dpaa_percpu_priv *percpu_priv,
2404 			  const struct qm_fd *fd,
2405 			  u32 fqid)
2406 {
2407 	struct sk_buff *skb;
2408 
2409 	if (net_ratelimit())
2410 		netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
2411 			   be32_to_cpu(fd->status) & FM_FD_STAT_TX_ERRORS);
2412 
2413 	percpu_priv->stats.tx_errors++;
2414 
2415 	skb = dpaa_cleanup_tx_fd(priv, fd, false);
2416 	dev_kfree_skb(skb);
2417 }
2418 
dpaa_eth_poll(struct napi_struct * napi,int budget)2419 static int dpaa_eth_poll(struct napi_struct *napi, int budget)
2420 {
2421 	struct dpaa_napi_portal *np =
2422 			container_of(napi, struct dpaa_napi_portal, napi);
2423 	int cleaned;
2424 
2425 	np->xdp_act = 0;
2426 
2427 	cleaned = qman_p_poll_dqrr(np->p, budget);
2428 
2429 	if (np->xdp_act & XDP_REDIRECT)
2430 		xdp_do_flush();
2431 
2432 	if (cleaned < budget) {
2433 		napi_complete_done(napi, cleaned);
2434 		qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
2435 	} else if (np->down) {
2436 		qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
2437 	}
2438 
2439 	return cleaned;
2440 }
2441 
dpaa_tx_conf(struct net_device * net_dev,const struct dpaa_priv * priv,struct dpaa_percpu_priv * percpu_priv,const struct qm_fd * fd,u32 fqid)2442 static void dpaa_tx_conf(struct net_device *net_dev,
2443 			 const struct dpaa_priv *priv,
2444 			 struct dpaa_percpu_priv *percpu_priv,
2445 			 const struct qm_fd *fd,
2446 			 u32 fqid)
2447 {
2448 	struct sk_buff	*skb;
2449 
2450 	if (unlikely(be32_to_cpu(fd->status) & FM_FD_STAT_TX_ERRORS)) {
2451 		if (net_ratelimit())
2452 			netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
2453 				   be32_to_cpu(fd->status) &
2454 				   FM_FD_STAT_TX_ERRORS);
2455 
2456 		percpu_priv->stats.tx_errors++;
2457 	}
2458 
2459 	percpu_priv->tx_confirm++;
2460 
2461 	skb = dpaa_cleanup_tx_fd(priv, fd, true);
2462 
2463 	consume_skb(skb);
2464 }
2465 
dpaa_eth_napi_schedule(struct dpaa_percpu_priv * percpu_priv,struct qman_portal * portal,bool sched_napi)2466 static inline int dpaa_eth_napi_schedule(struct dpaa_percpu_priv *percpu_priv,
2467 					 struct qman_portal *portal, bool sched_napi)
2468 {
2469 	if (sched_napi) {
2470 		/* Disable QMan IRQ and invoke NAPI */
2471 		qman_p_irqsource_remove(portal, QM_PIRQ_DQRI);
2472 
2473 		percpu_priv->np.p = portal;
2474 		napi_schedule(&percpu_priv->np.napi);
2475 		percpu_priv->in_interrupt++;
2476 		return 1;
2477 	}
2478 	return 0;
2479 }
2480 
rx_error_dqrr(struct qman_portal * portal,struct qman_fq * fq,const struct qm_dqrr_entry * dq,bool sched_napi)2481 static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
2482 					      struct qman_fq *fq,
2483 					      const struct qm_dqrr_entry *dq,
2484 					      bool sched_napi)
2485 {
2486 	struct dpaa_fq *dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
2487 	struct dpaa_percpu_priv *percpu_priv;
2488 	struct net_device *net_dev;
2489 	struct dpaa_bp *dpaa_bp;
2490 	struct dpaa_priv *priv;
2491 
2492 	net_dev = dpaa_fq->net_dev;
2493 	priv = netdev_priv(net_dev);
2494 	dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
2495 	if (!dpaa_bp)
2496 		return qman_cb_dqrr_consume;
2497 
2498 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2499 
2500 	if (dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi))
2501 		return qman_cb_dqrr_stop;
2502 
2503 	dpaa_eth_refill_bpools(priv);
2504 	dpaa_rx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2505 
2506 	return qman_cb_dqrr_consume;
2507 }
2508 
dpaa_xdp_xmit_frame(struct net_device * net_dev,struct xdp_frame * xdpf)2509 static int dpaa_xdp_xmit_frame(struct net_device *net_dev,
2510 			       struct xdp_frame *xdpf)
2511 {
2512 	struct dpaa_priv *priv = netdev_priv(net_dev);
2513 	struct rtnl_link_stats64 *percpu_stats;
2514 	struct dpaa_percpu_priv *percpu_priv;
2515 	struct dpaa_eth_swbp *swbp;
2516 	struct netdev_queue *txq;
2517 	void *buff_start;
2518 	struct qm_fd fd;
2519 	dma_addr_t addr;
2520 	int err;
2521 
2522 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2523 	percpu_stats = &percpu_priv->stats;
2524 
2525 #ifdef CONFIG_DPAA_ERRATUM_A050385
2526 	if (unlikely(fman_has_errata_a050385())) {
2527 		if (dpaa_a050385_wa_xdpf(priv, &xdpf)) {
2528 			err = -ENOMEM;
2529 			goto out_error;
2530 		}
2531 	}
2532 #endif
2533 
2534 	if (xdpf->headroom < DPAA_TX_PRIV_DATA_SIZE) {
2535 		err = -EINVAL;
2536 		goto out_error;
2537 	}
2538 
2539 	buff_start = xdpf->data - xdpf->headroom;
2540 
2541 	/* Leave empty the skb backpointer at the start of the buffer.
2542 	 * Save the XDP frame for easy cleanup on confirmation.
2543 	 */
2544 	swbp = (struct dpaa_eth_swbp *)buff_start;
2545 	swbp->skb = NULL;
2546 	swbp->xdpf = xdpf;
2547 
2548 	qm_fd_clear_fd(&fd);
2549 	fd.bpid = FSL_DPAA_BPID_INV;
2550 	fd.cmd |= cpu_to_be32(FM_FD_CMD_FCO);
2551 	qm_fd_set_contig(&fd, xdpf->headroom, xdpf->len);
2552 
2553 	addr = dma_map_single(priv->tx_dma_dev, buff_start,
2554 			      xdpf->headroom + xdpf->len,
2555 			      DMA_TO_DEVICE);
2556 	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2557 		err = -EINVAL;
2558 		goto out_error;
2559 	}
2560 
2561 	qm_fd_addr_set64(&fd, addr);
2562 
2563 	/* Bump the trans_start */
2564 	txq = netdev_get_tx_queue(net_dev, smp_processor_id());
2565 	txq_trans_cond_update(txq);
2566 
2567 	err = dpaa_xmit(priv, percpu_stats, smp_processor_id(), &fd);
2568 	if (err) {
2569 		dma_unmap_single(priv->tx_dma_dev, addr,
2570 				 qm_fd_get_offset(&fd) + qm_fd_get_length(&fd),
2571 				 DMA_TO_DEVICE);
2572 		goto out_error;
2573 	}
2574 
2575 	return 0;
2576 
2577 out_error:
2578 	percpu_stats->tx_errors++;
2579 	return err;
2580 }
2581 
dpaa_run_xdp(struct dpaa_priv * priv,struct qm_fd * fd,void * vaddr,struct dpaa_fq * dpaa_fq,unsigned int * xdp_meta_len)2582 static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr,
2583 			struct dpaa_fq *dpaa_fq, unsigned int *xdp_meta_len)
2584 {
2585 	ssize_t fd_off = qm_fd_get_offset(fd);
2586 	struct bpf_prog *xdp_prog;
2587 	struct xdp_frame *xdpf;
2588 	struct xdp_buff xdp;
2589 	u32 xdp_act;
2590 	int err;
2591 
2592 	xdp_prog = READ_ONCE(priv->xdp_prog);
2593 	if (!xdp_prog)
2594 		return XDP_PASS;
2595 
2596 	xdp_init_buff(&xdp, DPAA_BP_RAW_SIZE - DPAA_TX_PRIV_DATA_SIZE,
2597 		      &dpaa_fq->xdp_rxq);
2598 	xdp_prepare_buff(&xdp, vaddr + fd_off - XDP_PACKET_HEADROOM,
2599 			 XDP_PACKET_HEADROOM, qm_fd_get_length(fd), true);
2600 
2601 	/* We reserve a fixed headroom of 256 bytes under the erratum and we
2602 	 * offer it all to XDP programs to use. If no room is left for the
2603 	 * xdpf backpointer on TX, we will need to copy the data.
2604 	 * Disable metadata support since data realignments might be required
2605 	 * and the information can be lost.
2606 	 */
2607 #ifdef CONFIG_DPAA_ERRATUM_A050385
2608 	if (unlikely(fman_has_errata_a050385())) {
2609 		xdp_set_data_meta_invalid(&xdp);
2610 		xdp.data_hard_start = vaddr;
2611 		xdp.frame_sz = DPAA_BP_RAW_SIZE;
2612 	}
2613 #endif
2614 
2615 	xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp);
2616 
2617 	/* Update the length and the offset of the FD */
2618 	qm_fd_set_contig(fd, xdp.data - vaddr, xdp.data_end - xdp.data);
2619 
2620 	switch (xdp_act) {
2621 	case XDP_PASS:
2622 #ifdef CONFIG_DPAA_ERRATUM_A050385
2623 		*xdp_meta_len = xdp_data_meta_unsupported(&xdp) ? 0 :
2624 				xdp.data - xdp.data_meta;
2625 #else
2626 		*xdp_meta_len = xdp.data - xdp.data_meta;
2627 #endif
2628 		break;
2629 	case XDP_TX:
2630 		/* We can access the full headroom when sending the frame
2631 		 * back out
2632 		 */
2633 		xdp.data_hard_start = vaddr;
2634 		xdp.frame_sz = DPAA_BP_RAW_SIZE;
2635 		xdpf = xdp_convert_buff_to_frame(&xdp);
2636 		if (unlikely(!xdpf)) {
2637 			free_pages((unsigned long)vaddr, 0);
2638 			break;
2639 		}
2640 
2641 		if (dpaa_xdp_xmit_frame(priv->net_dev, xdpf))
2642 			xdp_return_frame_rx_napi(xdpf);
2643 
2644 		break;
2645 	case XDP_REDIRECT:
2646 		/* Allow redirect to use the full headroom */
2647 		xdp.data_hard_start = vaddr;
2648 		xdp.frame_sz = DPAA_BP_RAW_SIZE;
2649 
2650 		err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog);
2651 		if (err) {
2652 			trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
2653 			free_pages((unsigned long)vaddr, 0);
2654 		}
2655 		break;
2656 	default:
2657 		bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);
2658 		fallthrough;
2659 	case XDP_ABORTED:
2660 		trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
2661 		fallthrough;
2662 	case XDP_DROP:
2663 		/* Free the buffer */
2664 		free_pages((unsigned long)vaddr, 0);
2665 		break;
2666 	}
2667 
2668 	return xdp_act;
2669 }
2670 
rx_default_dqrr(struct qman_portal * portal,struct qman_fq * fq,const struct qm_dqrr_entry * dq,bool sched_napi)2671 static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
2672 						struct qman_fq *fq,
2673 						const struct qm_dqrr_entry *dq,
2674 						bool sched_napi)
2675 {
2676 	bool ts_valid = false, hash_valid = false;
2677 	struct skb_shared_hwtstamps *shhwtstamps;
2678 	unsigned int skb_len, xdp_meta_len = 0;
2679 	struct rtnl_link_stats64 *percpu_stats;
2680 	struct dpaa_percpu_priv *percpu_priv;
2681 	const struct qm_fd *fd = &dq->fd;
2682 	dma_addr_t addr = qm_fd_addr(fd);
2683 	struct dpaa_napi_portal *np;
2684 	enum qm_fd_format fd_format;
2685 	struct net_device *net_dev;
2686 	u32 fd_status, hash_offset;
2687 	struct qm_sg_entry *sgt;
2688 	struct dpaa_bp *dpaa_bp;
2689 	struct dpaa_fq *dpaa_fq;
2690 	struct dpaa_priv *priv;
2691 	struct sk_buff *skb;
2692 	int *count_ptr;
2693 	u32 xdp_act;
2694 	void *vaddr;
2695 	u32 hash;
2696 	u64 ns;
2697 
2698 	dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
2699 	fd_status = be32_to_cpu(fd->status);
2700 	fd_format = qm_fd_get_format(fd);
2701 	net_dev = dpaa_fq->net_dev;
2702 	priv = netdev_priv(net_dev);
2703 	dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
2704 	if (!dpaa_bp)
2705 		return qman_cb_dqrr_consume;
2706 
2707 	/* Trace the Rx fd */
2708 	trace_dpaa_rx_fd(net_dev, fq, &dq->fd);
2709 
2710 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2711 	percpu_stats = &percpu_priv->stats;
2712 	np = &percpu_priv->np;
2713 
2714 	if (unlikely(dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi)))
2715 		return qman_cb_dqrr_stop;
2716 
2717 	/* Make sure we didn't run out of buffers */
2718 	if (unlikely(dpaa_eth_refill_bpools(priv))) {
2719 		/* Unable to refill the buffer pool due to insufficient
2720 		 * system memory. Just release the frame back into the pool,
2721 		 * otherwise we'll soon end up with an empty buffer pool.
2722 		 */
2723 		dpaa_fd_release(net_dev, &dq->fd);
2724 		return qman_cb_dqrr_consume;
2725 	}
2726 
2727 	if (unlikely(fd_status & FM_FD_STAT_RX_ERRORS) != 0) {
2728 		if (net_ratelimit())
2729 			netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
2730 				   fd_status & FM_FD_STAT_RX_ERRORS);
2731 
2732 		percpu_stats->rx_errors++;
2733 		dpaa_fd_release(net_dev, fd);
2734 		return qman_cb_dqrr_consume;
2735 	}
2736 
2737 	dma_unmap_page(dpaa_bp->priv->rx_dma_dev, addr, DPAA_BP_RAW_SIZE,
2738 		       DMA_FROM_DEVICE);
2739 
2740 	/* prefetch the first 64 bytes of the frame or the SGT start */
2741 	vaddr = phys_to_virt(addr);
2742 	prefetch(vaddr + qm_fd_get_offset(fd));
2743 
2744 	/* The only FD types that we may receive are contig and S/G */
2745 	WARN_ON((fd_format != qm_fd_contig) && (fd_format != qm_fd_sg));
2746 
2747 	/* Account for either the contig buffer or the SGT buffer (depending on
2748 	 * which case we were in) having been removed from the pool.
2749 	 */
2750 	count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
2751 	(*count_ptr)--;
2752 
2753 	/* Extract the timestamp stored in the headroom before running XDP */
2754 	if (priv->rx_tstamp) {
2755 		if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr, &ns))
2756 			ts_valid = true;
2757 		else
2758 			WARN_ONCE(1, "fman_port_get_tstamp failed!\n");
2759 	}
2760 
2761 	/* Extract the hash stored in the headroom before running XDP */
2762 	if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use &&
2763 	    !fman_port_get_hash_result_offset(priv->mac_dev->port[RX],
2764 					      &hash_offset)) {
2765 		hash = be32_to_cpu(*(u32 *)(vaddr + hash_offset));
2766 		hash_valid = true;
2767 	}
2768 
2769 	if (likely(fd_format == qm_fd_contig)) {
2770 		xdp_act = dpaa_run_xdp(priv, (struct qm_fd *)fd, vaddr,
2771 				       dpaa_fq, &xdp_meta_len);
2772 		np->xdp_act |= xdp_act;
2773 		if (xdp_act != XDP_PASS) {
2774 			percpu_stats->rx_packets++;
2775 			percpu_stats->rx_bytes += qm_fd_get_length(fd);
2776 			return qman_cb_dqrr_consume;
2777 		}
2778 		skb = contig_fd_to_skb(priv, fd);
2779 	} else {
2780 		/* XDP doesn't support S/G frames. Return the fragments to the
2781 		 * buffer pool and release the SGT.
2782 		 */
2783 		if (READ_ONCE(priv->xdp_prog)) {
2784 			WARN_ONCE(1, "S/G frames not supported under XDP\n");
2785 			sgt = vaddr + qm_fd_get_offset(fd);
2786 			dpaa_release_sgt_members(sgt);
2787 			free_pages((unsigned long)vaddr, 0);
2788 			return qman_cb_dqrr_consume;
2789 		}
2790 		skb = sg_fd_to_skb(priv, fd);
2791 	}
2792 	if (!skb)
2793 		return qman_cb_dqrr_consume;
2794 
2795 	if (xdp_meta_len)
2796 		skb_metadata_set(skb, xdp_meta_len);
2797 
2798 	/* Set the previously extracted timestamp */
2799 	if (ts_valid) {
2800 		shhwtstamps = skb_hwtstamps(skb);
2801 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
2802 		shhwtstamps->hwtstamp = ns_to_ktime(ns);
2803 	}
2804 
2805 	skb->protocol = eth_type_trans(skb, net_dev);
2806 
2807 	/* Set the previously extracted hash */
2808 	if (hash_valid) {
2809 		enum pkt_hash_types type;
2810 
2811 		/* if L4 exists, it was used in the hash generation */
2812 		type = be32_to_cpu(fd->status) & FM_FD_STAT_L4CV ?
2813 			PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
2814 		skb_set_hash(skb, hash, type);
2815 	}
2816 
2817 	skb_len = skb->len;
2818 
2819 	if (unlikely(netif_receive_skb(skb) == NET_RX_DROP)) {
2820 		percpu_stats->rx_dropped++;
2821 		return qman_cb_dqrr_consume;
2822 	}
2823 
2824 	percpu_stats->rx_packets++;
2825 	percpu_stats->rx_bytes += skb_len;
2826 
2827 	return qman_cb_dqrr_consume;
2828 }
2829 
conf_error_dqrr(struct qman_portal * portal,struct qman_fq * fq,const struct qm_dqrr_entry * dq,bool sched_napi)2830 static enum qman_cb_dqrr_result conf_error_dqrr(struct qman_portal *portal,
2831 						struct qman_fq *fq,
2832 						const struct qm_dqrr_entry *dq,
2833 						bool sched_napi)
2834 {
2835 	struct dpaa_percpu_priv *percpu_priv;
2836 	struct net_device *net_dev;
2837 	struct dpaa_priv *priv;
2838 
2839 	net_dev = ((struct dpaa_fq *)fq)->net_dev;
2840 	priv = netdev_priv(net_dev);
2841 
2842 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2843 
2844 	if (dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi))
2845 		return qman_cb_dqrr_stop;
2846 
2847 	dpaa_tx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2848 
2849 	return qman_cb_dqrr_consume;
2850 }
2851 
conf_dflt_dqrr(struct qman_portal * portal,struct qman_fq * fq,const struct qm_dqrr_entry * dq,bool sched_napi)2852 static enum qman_cb_dqrr_result conf_dflt_dqrr(struct qman_portal *portal,
2853 					       struct qman_fq *fq,
2854 					       const struct qm_dqrr_entry *dq,
2855 					       bool sched_napi)
2856 {
2857 	struct dpaa_percpu_priv *percpu_priv;
2858 	struct net_device *net_dev;
2859 	struct dpaa_priv *priv;
2860 
2861 	net_dev = ((struct dpaa_fq *)fq)->net_dev;
2862 	priv = netdev_priv(net_dev);
2863 
2864 	/* Trace the fd */
2865 	trace_dpaa_tx_conf_fd(net_dev, fq, &dq->fd);
2866 
2867 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2868 
2869 	if (dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi))
2870 		return qman_cb_dqrr_stop;
2871 
2872 	dpaa_tx_conf(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2873 
2874 	return qman_cb_dqrr_consume;
2875 }
2876 
egress_ern(struct qman_portal * portal,struct qman_fq * fq,const union qm_mr_entry * msg)2877 static void egress_ern(struct qman_portal *portal,
2878 		       struct qman_fq *fq,
2879 		       const union qm_mr_entry *msg)
2880 {
2881 	const struct qm_fd *fd = &msg->ern.fd;
2882 	struct dpaa_percpu_priv *percpu_priv;
2883 	const struct dpaa_priv *priv;
2884 	struct net_device *net_dev;
2885 	struct sk_buff *skb;
2886 
2887 	net_dev = ((struct dpaa_fq *)fq)->net_dev;
2888 	priv = netdev_priv(net_dev);
2889 	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2890 
2891 	percpu_priv->stats.tx_dropped++;
2892 	percpu_priv->stats.tx_fifo_errors++;
2893 	count_ern(percpu_priv, msg);
2894 
2895 	skb = dpaa_cleanup_tx_fd(priv, fd, false);
2896 	dev_kfree_skb_any(skb);
2897 }
2898 
2899 static const struct dpaa_fq_cbs dpaa_fq_cbs = {
2900 	.rx_defq = { .cb = { .dqrr = rx_default_dqrr } },
2901 	.tx_defq = { .cb = { .dqrr = conf_dflt_dqrr } },
2902 	.rx_errq = { .cb = { .dqrr = rx_error_dqrr } },
2903 	.tx_errq = { .cb = { .dqrr = conf_error_dqrr } },
2904 	.egress_ern = { .cb = { .ern = egress_ern } }
2905 };
2906 
dpaa_eth_napi_enable(struct dpaa_priv * priv)2907 static void dpaa_eth_napi_enable(struct dpaa_priv *priv)
2908 {
2909 	struct dpaa_percpu_priv *percpu_priv;
2910 	int i;
2911 
2912 	for_each_online_cpu(i) {
2913 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
2914 
2915 		percpu_priv->np.down = false;
2916 		napi_enable(&percpu_priv->np.napi);
2917 	}
2918 }
2919 
dpaa_eth_napi_disable(struct dpaa_priv * priv)2920 static void dpaa_eth_napi_disable(struct dpaa_priv *priv)
2921 {
2922 	struct dpaa_percpu_priv *percpu_priv;
2923 	int i;
2924 
2925 	for_each_online_cpu(i) {
2926 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
2927 
2928 		percpu_priv->np.down = true;
2929 		napi_disable(&percpu_priv->np.napi);
2930 	}
2931 }
2932 
dpaa_open(struct net_device * net_dev)2933 static int dpaa_open(struct net_device *net_dev)
2934 {
2935 	struct mac_device *mac_dev;
2936 	struct dpaa_priv *priv;
2937 	int err, i;
2938 
2939 	priv = netdev_priv(net_dev);
2940 	mac_dev = priv->mac_dev;
2941 	dpaa_eth_napi_enable(priv);
2942 
2943 	err = phylink_of_phy_connect(mac_dev->phylink,
2944 				     mac_dev->dev->of_node, 0);
2945 	if (err)
2946 		goto phy_init_failed;
2947 
2948 	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
2949 		err = fman_port_enable(mac_dev->port[i]);
2950 		if (err)
2951 			goto mac_start_failed;
2952 	}
2953 
2954 	err = priv->mac_dev->enable(mac_dev->fman_mac);
2955 	if (err < 0) {
2956 		netif_err(priv, ifup, net_dev, "mac_dev->enable() = %d\n", err);
2957 		goto mac_start_failed;
2958 	}
2959 	phylink_start(mac_dev->phylink);
2960 
2961 	netif_tx_start_all_queues(net_dev);
2962 
2963 	return 0;
2964 
2965 mac_start_failed:
2966 	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++)
2967 		fman_port_disable(mac_dev->port[i]);
2968 	phylink_disconnect_phy(mac_dev->phylink);
2969 
2970 phy_init_failed:
2971 	dpaa_eth_napi_disable(priv);
2972 
2973 	return err;
2974 }
2975 
dpaa_eth_stop(struct net_device * net_dev)2976 static int dpaa_eth_stop(struct net_device *net_dev)
2977 {
2978 	struct dpaa_priv *priv;
2979 	int err;
2980 
2981 	err = dpaa_stop(net_dev);
2982 
2983 	priv = netdev_priv(net_dev);
2984 	dpaa_eth_napi_disable(priv);
2985 
2986 	return err;
2987 }
2988 
xdp_validate_mtu(struct dpaa_priv * priv,int mtu)2989 static bool xdp_validate_mtu(struct dpaa_priv *priv, int mtu)
2990 {
2991 	int max_contig_data = priv->dpaa_bp->size - priv->rx_headroom;
2992 
2993 	/* We do not support S/G fragments when XDP is enabled.
2994 	 * Limit the MTU in relation to the buffer size.
2995 	 */
2996 	if (mtu + VLAN_ETH_HLEN + ETH_FCS_LEN > max_contig_data) {
2997 		dev_warn(priv->net_dev->dev.parent,
2998 			 "The maximum MTU for XDP is %d\n",
2999 			 max_contig_data - VLAN_ETH_HLEN - ETH_FCS_LEN);
3000 		return false;
3001 	}
3002 
3003 	return true;
3004 }
3005 
dpaa_change_mtu(struct net_device * net_dev,int new_mtu)3006 static int dpaa_change_mtu(struct net_device *net_dev, int new_mtu)
3007 {
3008 	struct dpaa_priv *priv = netdev_priv(net_dev);
3009 
3010 	if (priv->xdp_prog && !xdp_validate_mtu(priv, new_mtu))
3011 		return -EINVAL;
3012 
3013 	net_dev->mtu = new_mtu;
3014 	return 0;
3015 }
3016 
dpaa_setup_xdp(struct net_device * net_dev,struct netdev_bpf * bpf)3017 static int dpaa_setup_xdp(struct net_device *net_dev, struct netdev_bpf *bpf)
3018 {
3019 	struct dpaa_priv *priv = netdev_priv(net_dev);
3020 	struct bpf_prog *old_prog;
3021 	int err;
3022 	bool up;
3023 
3024 	/* S/G fragments are not supported in XDP-mode */
3025 	if (bpf->prog && !xdp_validate_mtu(priv, net_dev->mtu)) {
3026 		NL_SET_ERR_MSG_MOD(bpf->extack, "MTU too large for XDP");
3027 		return -EINVAL;
3028 	}
3029 
3030 	up = netif_running(net_dev);
3031 
3032 	if (up)
3033 		dpaa_eth_stop(net_dev);
3034 
3035 	old_prog = xchg(&priv->xdp_prog, bpf->prog);
3036 	if (old_prog)
3037 		bpf_prog_put(old_prog);
3038 
3039 	if (up) {
3040 		err = dpaa_open(net_dev);
3041 		if (err) {
3042 			NL_SET_ERR_MSG_MOD(bpf->extack, "dpaa_open() failed");
3043 			return err;
3044 		}
3045 	}
3046 
3047 	return 0;
3048 }
3049 
dpaa_xdp(struct net_device * net_dev,struct netdev_bpf * xdp)3050 static int dpaa_xdp(struct net_device *net_dev, struct netdev_bpf *xdp)
3051 {
3052 	switch (xdp->command) {
3053 	case XDP_SETUP_PROG:
3054 		return dpaa_setup_xdp(net_dev, xdp);
3055 	default:
3056 		return -EINVAL;
3057 	}
3058 }
3059 
dpaa_xdp_xmit(struct net_device * net_dev,int n,struct xdp_frame ** frames,u32 flags)3060 static int dpaa_xdp_xmit(struct net_device *net_dev, int n,
3061 			 struct xdp_frame **frames, u32 flags)
3062 {
3063 	struct xdp_frame *xdpf;
3064 	int i, nxmit = 0;
3065 
3066 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
3067 		return -EINVAL;
3068 
3069 	if (!netif_running(net_dev))
3070 		return -ENETDOWN;
3071 
3072 	for (i = 0; i < n; i++) {
3073 		xdpf = frames[i];
3074 		if (dpaa_xdp_xmit_frame(net_dev, xdpf))
3075 			break;
3076 		nxmit++;
3077 	}
3078 
3079 	return nxmit;
3080 }
3081 
dpaa_ts_ioctl(struct net_device * dev,struct ifreq * rq,int cmd)3082 static int dpaa_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3083 {
3084 	struct dpaa_priv *priv = netdev_priv(dev);
3085 	struct hwtstamp_config config;
3086 
3087 	if (copy_from_user(&config, rq->ifr_data, sizeof(config)))
3088 		return -EFAULT;
3089 
3090 	switch (config.tx_type) {
3091 	case HWTSTAMP_TX_OFF:
3092 		/* Couldn't disable rx/tx timestamping separately.
3093 		 * Do nothing here.
3094 		 */
3095 		priv->tx_tstamp = false;
3096 		break;
3097 	case HWTSTAMP_TX_ON:
3098 		priv->mac_dev->set_tstamp(priv->mac_dev->fman_mac, true);
3099 		priv->tx_tstamp = true;
3100 		break;
3101 	default:
3102 		return -ERANGE;
3103 	}
3104 
3105 	if (config.rx_filter == HWTSTAMP_FILTER_NONE) {
3106 		/* Couldn't disable rx/tx timestamping separately.
3107 		 * Do nothing here.
3108 		 */
3109 		priv->rx_tstamp = false;
3110 	} else {
3111 		priv->mac_dev->set_tstamp(priv->mac_dev->fman_mac, true);
3112 		priv->rx_tstamp = true;
3113 		/* TS is set for all frame types, not only those requested */
3114 		config.rx_filter = HWTSTAMP_FILTER_ALL;
3115 	}
3116 
3117 	return copy_to_user(rq->ifr_data, &config, sizeof(config)) ?
3118 			-EFAULT : 0;
3119 }
3120 
dpaa_ioctl(struct net_device * net_dev,struct ifreq * rq,int cmd)3121 static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd)
3122 {
3123 	int ret = -EINVAL;
3124 	struct dpaa_priv *priv = netdev_priv(net_dev);
3125 
3126 	if (cmd == SIOCGMIIREG) {
3127 		if (net_dev->phydev)
3128 			return phylink_mii_ioctl(priv->mac_dev->phylink, rq,
3129 						 cmd);
3130 	}
3131 
3132 	if (cmd == SIOCSHWTSTAMP)
3133 		return dpaa_ts_ioctl(net_dev, rq, cmd);
3134 
3135 	return ret;
3136 }
3137 
3138 static const struct net_device_ops dpaa_ops = {
3139 	.ndo_open = dpaa_open,
3140 	.ndo_start_xmit = dpaa_start_xmit,
3141 	.ndo_stop = dpaa_eth_stop,
3142 	.ndo_tx_timeout = dpaa_tx_timeout,
3143 	.ndo_get_stats64 = dpaa_get_stats64,
3144 	.ndo_change_carrier = fixed_phy_change_carrier,
3145 	.ndo_set_mac_address = dpaa_set_mac_address,
3146 	.ndo_validate_addr = eth_validate_addr,
3147 	.ndo_set_rx_mode = dpaa_set_rx_mode,
3148 	.ndo_eth_ioctl = dpaa_ioctl,
3149 	.ndo_setup_tc = dpaa_setup_tc,
3150 	.ndo_change_mtu = dpaa_change_mtu,
3151 	.ndo_bpf = dpaa_xdp,
3152 	.ndo_xdp_xmit = dpaa_xdp_xmit,
3153 };
3154 
dpaa_napi_add(struct net_device * net_dev)3155 static int dpaa_napi_add(struct net_device *net_dev)
3156 {
3157 	struct dpaa_priv *priv = netdev_priv(net_dev);
3158 	struct dpaa_percpu_priv *percpu_priv;
3159 	int cpu;
3160 
3161 	for_each_possible_cpu(cpu) {
3162 		percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
3163 
3164 		netif_napi_add(net_dev, &percpu_priv->np.napi, dpaa_eth_poll);
3165 	}
3166 
3167 	return 0;
3168 }
3169 
dpaa_napi_del(struct net_device * net_dev)3170 static void dpaa_napi_del(struct net_device *net_dev)
3171 {
3172 	struct dpaa_priv *priv = netdev_priv(net_dev);
3173 	struct dpaa_percpu_priv *percpu_priv;
3174 	int cpu;
3175 
3176 	for_each_possible_cpu(cpu) {
3177 		percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
3178 
3179 		netif_napi_del(&percpu_priv->np.napi);
3180 	}
3181 }
3182 
dpaa_bp_free_pf(const struct dpaa_bp * bp,struct bm_buffer * bmb)3183 static inline void dpaa_bp_free_pf(const struct dpaa_bp *bp,
3184 				   struct bm_buffer *bmb)
3185 {
3186 	dma_addr_t addr = bm_buf_addr(bmb);
3187 
3188 	dma_unmap_page(bp->priv->rx_dma_dev, addr, DPAA_BP_RAW_SIZE,
3189 		       DMA_FROM_DEVICE);
3190 
3191 	skb_free_frag(phys_to_virt(addr));
3192 }
3193 
3194 /* Alloc the dpaa_bp struct and configure default values */
dpaa_bp_alloc(struct device * dev)3195 static struct dpaa_bp *dpaa_bp_alloc(struct device *dev)
3196 {
3197 	struct dpaa_bp *dpaa_bp;
3198 
3199 	dpaa_bp = devm_kzalloc(dev, sizeof(*dpaa_bp), GFP_KERNEL);
3200 	if (!dpaa_bp)
3201 		return ERR_PTR(-ENOMEM);
3202 
3203 	dpaa_bp->bpid = FSL_DPAA_BPID_INV;
3204 	dpaa_bp->percpu_count = devm_alloc_percpu(dev, *dpaa_bp->percpu_count);
3205 	if (!dpaa_bp->percpu_count)
3206 		return ERR_PTR(-ENOMEM);
3207 
3208 	dpaa_bp->config_count = FSL_DPAA_ETH_MAX_BUF_COUNT;
3209 
3210 	dpaa_bp->seed_cb = dpaa_bp_seed;
3211 	dpaa_bp->free_buf_cb = dpaa_bp_free_pf;
3212 
3213 	return dpaa_bp;
3214 }
3215 
3216 /* Place all ingress FQs (Rx Default, Rx Error) in a dedicated CGR.
3217  * We won't be sending congestion notifications to FMan; for now, we just use
3218  * this CGR to generate enqueue rejections to FMan in order to drop the frames
3219  * before they reach our ingress queues and eat up memory.
3220  */
dpaa_ingress_cgr_init(struct dpaa_priv * priv)3221 static int dpaa_ingress_cgr_init(struct dpaa_priv *priv)
3222 {
3223 	struct qm_mcc_initcgr initcgr;
3224 	u32 cs_th;
3225 	int err;
3226 
3227 	err = qman_alloc_cgrid(&priv->ingress_cgr.cgrid);
3228 	if (err < 0) {
3229 		if (netif_msg_drv(priv))
3230 			pr_err("Error %d allocating CGR ID\n", err);
3231 		goto out_error;
3232 	}
3233 
3234 	/* Enable CS TD, but disable Congestion State Change Notifications. */
3235 	memset(&initcgr, 0, sizeof(initcgr));
3236 	initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CS_THRES);
3237 	initcgr.cgr.cscn_en = QM_CGR_EN;
3238 	cs_th = DPAA_INGRESS_CS_THRESHOLD;
3239 	qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
3240 
3241 	initcgr.we_mask |= cpu_to_be16(QM_CGR_WE_CSTD_EN);
3242 	initcgr.cgr.cstd_en = QM_CGR_EN;
3243 
3244 	/* This CGR will be associated with the SWP affined to the current CPU.
3245 	 * However, we'll place all our ingress FQs in it.
3246 	 */
3247 	err = qman_create_cgr(&priv->ingress_cgr, QMAN_CGR_FLAG_USE_INIT,
3248 			      &initcgr);
3249 	if (err < 0) {
3250 		if (netif_msg_drv(priv))
3251 			pr_err("Error %d creating ingress CGR with ID %d\n",
3252 			       err, priv->ingress_cgr.cgrid);
3253 		qman_release_cgrid(priv->ingress_cgr.cgrid);
3254 		goto out_error;
3255 	}
3256 	if (netif_msg_drv(priv))
3257 		pr_debug("Created ingress CGR %d for netdev with hwaddr %pM\n",
3258 			 priv->ingress_cgr.cgrid, priv->mac_dev->addr);
3259 
3260 	priv->use_ingress_cgr = true;
3261 
3262 out_error:
3263 	return err;
3264 }
3265 
dpaa_get_headroom(struct dpaa_buffer_layout * bl,enum port_type port)3266 static u16 dpaa_get_headroom(struct dpaa_buffer_layout *bl,
3267 			     enum port_type port)
3268 {
3269 	u16 headroom;
3270 
3271 	/* The frame headroom must accommodate:
3272 	 * - the driver private data area
3273 	 * - parse results, hash results, timestamp if selected
3274 	 * If either hash results or time stamp are selected, both will
3275 	 * be copied to/from the frame headroom, as TS is located between PR and
3276 	 * HR in the IC and IC copy size has a granularity of 16bytes
3277 	 * (see description of FMBM_RICP and FMBM_TICP registers in DPAARM)
3278 	 *
3279 	 * Also make sure the headroom is a multiple of data_align bytes
3280 	 */
3281 	headroom = (u16)(bl[port].priv_data_size + DPAA_HWA_SIZE);
3282 
3283 	if (port == RX) {
3284 #ifdef CONFIG_DPAA_ERRATUM_A050385
3285 		if (unlikely(fman_has_errata_a050385()))
3286 			headroom = XDP_PACKET_HEADROOM;
3287 #endif
3288 
3289 		return ALIGN(headroom, DPAA_FD_RX_DATA_ALIGNMENT);
3290 	} else {
3291 		return ALIGN(headroom, DPAA_FD_DATA_ALIGNMENT);
3292 	}
3293 }
3294 
dpaa_eth_probe(struct platform_device * pdev)3295 static int dpaa_eth_probe(struct platform_device *pdev)
3296 {
3297 	struct net_device *net_dev = NULL;
3298 	struct dpaa_bp *dpaa_bp = NULL;
3299 	struct dpaa_fq *dpaa_fq, *tmp;
3300 	struct dpaa_priv *priv = NULL;
3301 	struct fm_port_fqs port_fqs;
3302 	struct mac_device *mac_dev;
3303 	int err = 0, channel;
3304 	struct device *dev;
3305 
3306 	dev = &pdev->dev;
3307 
3308 	err = bman_is_probed();
3309 	if (!err)
3310 		return -EPROBE_DEFER;
3311 	if (err < 0) {
3312 		dev_err(dev, "failing probe due to bman probe error\n");
3313 		return -ENODEV;
3314 	}
3315 	err = qman_is_probed();
3316 	if (!err)
3317 		return -EPROBE_DEFER;
3318 	if (err < 0) {
3319 		dev_err(dev, "failing probe due to qman probe error\n");
3320 		return -ENODEV;
3321 	}
3322 	err = bman_portals_probed();
3323 	if (!err)
3324 		return -EPROBE_DEFER;
3325 	if (err < 0) {
3326 		dev_err(dev,
3327 			"failing probe due to bman portals probe error\n");
3328 		return -ENODEV;
3329 	}
3330 	err = qman_portals_probed();
3331 	if (!err)
3332 		return -EPROBE_DEFER;
3333 	if (err < 0) {
3334 		dev_err(dev,
3335 			"failing probe due to qman portals probe error\n");
3336 		return -ENODEV;
3337 	}
3338 
3339 	/* Allocate this early, so we can store relevant information in
3340 	 * the private area
3341 	 */
3342 	net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA_ETH_TXQ_NUM);
3343 	if (!net_dev) {
3344 		dev_err(dev, "alloc_etherdev_mq() failed\n");
3345 		return -ENOMEM;
3346 	}
3347 
3348 	/* Do this here, so we can be verbose early */
3349 	SET_NETDEV_DEV(net_dev, dev->parent);
3350 	dev_set_drvdata(dev, net_dev);
3351 
3352 	priv = netdev_priv(net_dev);
3353 	priv->net_dev = net_dev;
3354 
3355 	priv->msg_enable = netif_msg_init(debug, DPAA_MSG_DEFAULT);
3356 
3357 	mac_dev = dpaa_mac_dev_get(pdev);
3358 	if (IS_ERR(mac_dev)) {
3359 		netdev_err(net_dev, "dpaa_mac_dev_get() failed\n");
3360 		err = PTR_ERR(mac_dev);
3361 		goto free_netdev;
3362 	}
3363 
3364 	/* Devices used for DMA mapping */
3365 	priv->rx_dma_dev = fman_port_get_device(mac_dev->port[RX]);
3366 	priv->tx_dma_dev = fman_port_get_device(mac_dev->port[TX]);
3367 	err = dma_coerce_mask_and_coherent(priv->rx_dma_dev, DMA_BIT_MASK(40));
3368 	if (!err)
3369 		err = dma_coerce_mask_and_coherent(priv->tx_dma_dev,
3370 						   DMA_BIT_MASK(40));
3371 	if (err) {
3372 		netdev_err(net_dev, "dma_coerce_mask_and_coherent() failed\n");
3373 		goto free_netdev;
3374 	}
3375 
3376 	/* If fsl_fm_max_frm is set to a higher value than the all-common 1500,
3377 	 * we choose conservatively and let the user explicitly set a higher
3378 	 * MTU via ifconfig. Otherwise, the user may end up with different MTUs
3379 	 * in the same LAN.
3380 	 * If on the other hand fsl_fm_max_frm has been chosen below 1500,
3381 	 * start with the maximum allowed.
3382 	 */
3383 	net_dev->mtu = min(dpaa_get_max_mtu(), ETH_DATA_LEN);
3384 
3385 	netdev_dbg(net_dev, "Setting initial MTU on net device: %d\n",
3386 		   net_dev->mtu);
3387 
3388 	priv->buf_layout[RX].priv_data_size = DPAA_RX_PRIV_DATA_SIZE; /* Rx */
3389 	priv->buf_layout[TX].priv_data_size = DPAA_TX_PRIV_DATA_SIZE; /* Tx */
3390 
3391 	/* bp init */
3392 	dpaa_bp = dpaa_bp_alloc(dev);
3393 	if (IS_ERR(dpaa_bp)) {
3394 		err = PTR_ERR(dpaa_bp);
3395 		goto free_dpaa_bps;
3396 	}
3397 	/* the raw size of the buffers used for reception */
3398 	dpaa_bp->raw_size = DPAA_BP_RAW_SIZE;
3399 	/* avoid runtime computations by keeping the usable size here */
3400 	dpaa_bp->size = dpaa_bp_size(dpaa_bp->raw_size);
3401 	dpaa_bp->priv = priv;
3402 
3403 	err = dpaa_bp_alloc_pool(dpaa_bp);
3404 	if (err < 0)
3405 		goto free_dpaa_bps;
3406 	priv->dpaa_bp = dpaa_bp;
3407 
3408 	INIT_LIST_HEAD(&priv->dpaa_fq_list);
3409 
3410 	memset(&port_fqs, 0, sizeof(port_fqs));
3411 
3412 	err = dpaa_alloc_all_fqs(dev, &priv->dpaa_fq_list, &port_fqs);
3413 	if (err < 0) {
3414 		dev_err(dev, "dpaa_alloc_all_fqs() failed\n");
3415 		goto free_dpaa_bps;
3416 	}
3417 
3418 	priv->mac_dev = mac_dev;
3419 
3420 	channel = dpaa_get_channel();
3421 	if (channel < 0) {
3422 		dev_err(dev, "dpaa_get_channel() failed\n");
3423 		err = channel;
3424 		goto free_dpaa_bps;
3425 	}
3426 
3427 	priv->channel = (u16)channel;
3428 
3429 	/* Walk the CPUs with affine portals
3430 	 * and add this pool channel to each's dequeue mask.
3431 	 */
3432 	dpaa_eth_add_channel(priv->channel, &pdev->dev);
3433 
3434 	err = dpaa_fq_setup(priv, &dpaa_fq_cbs, priv->mac_dev->port[TX]);
3435 	if (err)
3436 		goto free_dpaa_bps;
3437 
3438 	/* Create a congestion group for this netdev, with
3439 	 * dynamically-allocated CGR ID.
3440 	 * Must be executed after probing the MAC, but before
3441 	 * assigning the egress FQs to the CGRs.
3442 	 */
3443 	err = dpaa_eth_cgr_init(priv);
3444 	if (err < 0) {
3445 		dev_err(dev, "Error initializing CGR\n");
3446 		goto free_dpaa_bps;
3447 	}
3448 
3449 	err = dpaa_ingress_cgr_init(priv);
3450 	if (err < 0) {
3451 		dev_err(dev, "Error initializing ingress CGR\n");
3452 		goto delete_egress_cgr;
3453 	}
3454 
3455 	/* Add the FQs to the interface, and make them active */
3456 	list_for_each_entry_safe(dpaa_fq, tmp, &priv->dpaa_fq_list, list) {
3457 		err = dpaa_fq_init(dpaa_fq, false);
3458 		if (err < 0)
3459 			goto free_dpaa_fqs;
3460 	}
3461 
3462 	priv->tx_headroom = dpaa_get_headroom(priv->buf_layout, TX);
3463 	priv->rx_headroom = dpaa_get_headroom(priv->buf_layout, RX);
3464 
3465 	/* All real interfaces need their ports initialized */
3466 	err = dpaa_eth_init_ports(mac_dev, dpaa_bp, &port_fqs,
3467 				  &priv->buf_layout[0], dev);
3468 	if (err)
3469 		goto free_dpaa_fqs;
3470 
3471 	/* Rx traffic distribution based on keygen hashing defaults to on */
3472 	priv->keygen_in_use = true;
3473 
3474 	priv->percpu_priv = devm_alloc_percpu(dev, *priv->percpu_priv);
3475 	if (!priv->percpu_priv) {
3476 		dev_err(dev, "devm_alloc_percpu() failed\n");
3477 		err = -ENOMEM;
3478 		goto free_dpaa_fqs;
3479 	}
3480 
3481 	priv->num_tc = 1;
3482 	netif_set_real_num_tx_queues(net_dev, priv->num_tc * DPAA_TC_TXQ_NUM);
3483 
3484 	/* Initialize NAPI */
3485 	err = dpaa_napi_add(net_dev);
3486 	if (err < 0)
3487 		goto delete_dpaa_napi;
3488 
3489 	err = dpaa_netdev_init(net_dev, &dpaa_ops, tx_timeout);
3490 	if (err < 0)
3491 		goto delete_dpaa_napi;
3492 
3493 	dpaa_eth_sysfs_init(&net_dev->dev);
3494 
3495 	netif_info(priv, probe, net_dev, "Probed interface %s\n",
3496 		   net_dev->name);
3497 
3498 	return 0;
3499 
3500 delete_dpaa_napi:
3501 	dpaa_napi_del(net_dev);
3502 free_dpaa_fqs:
3503 	dpaa_fq_free(dev, &priv->dpaa_fq_list);
3504 	qman_delete_cgr_safe(&priv->ingress_cgr);
3505 	qman_release_cgrid(priv->ingress_cgr.cgrid);
3506 delete_egress_cgr:
3507 	qman_delete_cgr_safe(&priv->cgr_data.cgr);
3508 	qman_release_cgrid(priv->cgr_data.cgr.cgrid);
3509 free_dpaa_bps:
3510 	dpaa_bps_free(priv);
3511 free_netdev:
3512 	dev_set_drvdata(dev, NULL);
3513 	free_netdev(net_dev);
3514 
3515 	return err;
3516 }
3517 
dpaa_remove(struct platform_device * pdev)3518 static void dpaa_remove(struct platform_device *pdev)
3519 {
3520 	struct net_device *net_dev;
3521 	struct dpaa_priv *priv;
3522 	struct device *dev;
3523 	int err;
3524 
3525 	dev = &pdev->dev;
3526 	net_dev = dev_get_drvdata(dev);
3527 
3528 	priv = netdev_priv(net_dev);
3529 
3530 	dpaa_eth_sysfs_remove(dev);
3531 
3532 	dev_set_drvdata(dev, NULL);
3533 	unregister_netdev(net_dev);
3534 	phylink_destroy(priv->mac_dev->phylink);
3535 
3536 	err = dpaa_fq_free(dev, &priv->dpaa_fq_list);
3537 	if (err)
3538 		dev_err(dev, "Failed to free FQs on remove (%pE)\n",
3539 			ERR_PTR(err));
3540 
3541 	qman_delete_cgr_safe(&priv->ingress_cgr);
3542 	qman_release_cgrid(priv->ingress_cgr.cgrid);
3543 	qman_delete_cgr_safe(&priv->cgr_data.cgr);
3544 	qman_release_cgrid(priv->cgr_data.cgr.cgrid);
3545 
3546 	dpaa_napi_del(net_dev);
3547 
3548 	dpaa_bps_free(priv);
3549 
3550 	free_netdev(net_dev);
3551 }
3552 
3553 static const struct platform_device_id dpaa_devtype[] = {
3554 	{
3555 		.name = "dpaa-ethernet",
3556 		.driver_data = 0,
3557 	}, {
3558 	}
3559 };
3560 MODULE_DEVICE_TABLE(platform, dpaa_devtype);
3561 
3562 static struct platform_driver dpaa_driver = {
3563 	.driver = {
3564 		.name = KBUILD_MODNAME,
3565 	},
3566 	.id_table = dpaa_devtype,
3567 	.probe = dpaa_eth_probe,
3568 	.remove_new = dpaa_remove
3569 };
3570 
dpaa_load(void)3571 static int __init dpaa_load(void)
3572 {
3573 	int err;
3574 
3575 	pr_debug("FSL DPAA Ethernet driver\n");
3576 
3577 	/* initialize dpaa_eth mirror values */
3578 	dpaa_rx_extra_headroom = fman_get_rx_extra_headroom();
3579 	dpaa_max_frm = fman_get_max_frm();
3580 
3581 	err = platform_driver_register(&dpaa_driver);
3582 	if (err < 0)
3583 		pr_err("Error, platform_driver_register() = %d\n", err);
3584 
3585 	return err;
3586 }
3587 module_init(dpaa_load);
3588 
dpaa_unload(void)3589 static void __exit dpaa_unload(void)
3590 {
3591 	platform_driver_unregister(&dpaa_driver);
3592 
3593 	/* Only one channel is used and needs to be released after all
3594 	 * interfaces are removed
3595 	 */
3596 	dpaa_release_channel();
3597 }
3598 module_exit(dpaa_unload);
3599 
3600 MODULE_LICENSE("Dual BSD/GPL");
3601 MODULE_DESCRIPTION("FSL DPAA Ethernet driver");
3602