1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2018 Netronome Systems, Inc. */
3 
4 #include <linux/bitfield.h>
5 #include <linux/bitmap.h>
6 #include <linux/etherdevice.h>
7 #include <linux/lockdep.h>
8 #include <linux/netdevice.h>
9 #include <linux/rcupdate.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/slab.h>
12 
13 #include "../nfpcore/nfp.h"
14 #include "../nfpcore/nfp_cpp.h"
15 #include "../nfpcore/nfp_nsp.h"
16 #include "../nfp_app.h"
17 #include "../nfp_main.h"
18 #include "../nfp_net.h"
19 #include "../nfp_net_repr.h"
20 #include "../nfp_port.h"
21 #include "main.h"
22 
23 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id)
24 {
25 	return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) |
26 	       FIELD_PREP(NFP_ABM_PORTID_ID, id);
27 }
28 
29 static int
30 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev,
31 		 enum tc_setup_type type, void *type_data)
32 {
33 	struct nfp_repr *repr = netdev_priv(netdev);
34 	struct nfp_port *port;
35 
36 	port = nfp_port_from_netdev(netdev);
37 	if (!port || port->type != NFP_PORT_PF_PORT)
38 		return -EOPNOTSUPP;
39 
40 	switch (type) {
41 	case TC_SETUP_ROOT_QDISC:
42 		return nfp_abm_setup_root(netdev, repr->app_priv, type_data);
43 	case TC_SETUP_QDISC_MQ:
44 		return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data);
45 	case TC_SETUP_QDISC_RED:
46 		return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data);
47 	case TC_SETUP_QDISC_GRED:
48 		return nfp_abm_setup_tc_gred(netdev, repr->app_priv, type_data);
49 	case TC_SETUP_BLOCK:
50 		return nfp_abm_setup_cls_block(netdev, repr, type_data);
51 	default:
52 		return -EOPNOTSUPP;
53 	}
54 }
55 
56 static struct net_device *
57 nfp_abm_repr_get(struct nfp_app *app, u32 port_id, bool *redir_egress)
58 {
59 	enum nfp_repr_type rtype;
60 	struct nfp_reprs *reprs;
61 	u8 port;
62 
63 	rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id);
64 	port = FIELD_GET(NFP_ABM_PORTID_ID, port_id);
65 
66 	reprs = rcu_dereference(app->reprs[rtype]);
67 	if (!reprs)
68 		return NULL;
69 
70 	if (port >= reprs->num_reprs)
71 		return NULL;
72 
73 	return rcu_dereference(reprs->reprs[port]);
74 }
75 
76 static int
77 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink,
78 		   enum nfp_port_type ptype)
79 {
80 	struct net_device *netdev;
81 	enum nfp_repr_type rtype;
82 	struct nfp_reprs *reprs;
83 	struct nfp_repr *repr;
84 	struct nfp_port *port;
85 	unsigned int txqs;
86 	int err;
87 
88 	if (ptype == NFP_PORT_PHYS_PORT) {
89 		rtype = NFP_REPR_TYPE_PHYS_PORT;
90 		txqs = 1;
91 	} else {
92 		rtype = NFP_REPR_TYPE_PF;
93 		txqs = alink->vnic->max_rx_rings;
94 	}
95 
96 	netdev = nfp_repr_alloc_mqs(app, txqs, 1);
97 	if (!netdev)
98 		return -ENOMEM;
99 	repr = netdev_priv(netdev);
100 	repr->app_priv = alink;
101 
102 	port = nfp_port_alloc(app, ptype, netdev);
103 	if (IS_ERR(port)) {
104 		err = PTR_ERR(port);
105 		goto err_free_repr;
106 	}
107 
108 	if (ptype == NFP_PORT_PHYS_PORT) {
109 		port->eth_forced = true;
110 		err = nfp_port_init_phy_port(app->pf, app, port, alink->id);
111 		if (err)
112 			goto err_free_port;
113 	} else {
114 		port->pf_id = alink->abm->pf_id;
115 		port->pf_split = app->pf->max_data_vnics > 1;
116 		port->pf_split_id = alink->id;
117 		port->vnic = alink->vnic->dp.ctrl_bar;
118 	}
119 
120 	SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev);
121 	eth_hw_addr_random(netdev);
122 
123 	err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id),
124 			    port, alink->vnic->dp.netdev);
125 	if (err)
126 		goto err_free_port;
127 
128 	reprs = nfp_reprs_get_locked(app, rtype);
129 	WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr");
130 	rtnl_lock();
131 	rcu_assign_pointer(reprs->reprs[alink->id], netdev);
132 	rtnl_unlock();
133 
134 	nfp_info(app->cpp, "%s Port %d Representor(%s) created\n",
135 		 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys",
136 		 alink->id, netdev->name);
137 
138 	return 0;
139 
140 err_free_port:
141 	nfp_port_free(port);
142 err_free_repr:
143 	nfp_repr_free(netdev);
144 	return err;
145 }
146 
147 static void
148 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink,
149 		  enum nfp_repr_type rtype)
150 {
151 	struct net_device *netdev;
152 	struct nfp_reprs *reprs;
153 
154 	reprs = nfp_reprs_get_locked(app, rtype);
155 	netdev = nfp_repr_get_locked(app, reprs, alink->id);
156 	if (!netdev)
157 		return;
158 	rtnl_lock();
159 	rcu_assign_pointer(reprs->reprs[alink->id], NULL);
160 	rtnl_unlock();
161 	synchronize_rcu();
162 	/* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */
163 	nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev));
164 }
165 
166 static void
167 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink)
168 {
169 	nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF);
170 	nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT);
171 }
172 
173 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm)
174 {
175 	struct nfp_pf *pf = abm->app->pf;
176 	struct nfp_net *nn;
177 
178 	list_for_each_entry(nn, &pf->vnics, vnic_list)
179 		nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv);
180 }
181 
182 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app)
183 {
184 	struct nfp_abm *abm = app->priv;
185 
186 	return abm->eswitch_mode;
187 }
188 
189 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm)
190 {
191 	nfp_abm_kill_reprs_all(abm);
192 	nfp_abm_ctrl_qm_disable(abm);
193 
194 	abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
195 	return 0;
196 }
197 
198 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm)
199 {
200 	if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY)
201 		WARN_ON(nfp_abm_eswitch_set_legacy(abm));
202 }
203 
204 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm)
205 {
206 	struct nfp_app *app = abm->app;
207 	struct nfp_pf *pf = app->pf;
208 	struct nfp_net *nn;
209 	int err;
210 
211 	if (!abm->red_support)
212 		return -EOPNOTSUPP;
213 
214 	err = nfp_abm_ctrl_qm_enable(abm);
215 	if (err)
216 		return err;
217 
218 	list_for_each_entry(nn, &pf->vnics, vnic_list) {
219 		struct nfp_abm_link *alink = nn->app_priv;
220 
221 		err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT);
222 		if (err)
223 			goto err_kill_all_reprs;
224 
225 		err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT);
226 		if (err)
227 			goto err_kill_all_reprs;
228 	}
229 
230 	abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
231 	return 0;
232 
233 err_kill_all_reprs:
234 	nfp_abm_kill_reprs_all(abm);
235 	nfp_abm_ctrl_qm_disable(abm);
236 	return err;
237 }
238 
239 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode)
240 {
241 	struct nfp_abm *abm = app->priv;
242 
243 	if (abm->eswitch_mode == mode)
244 		return 0;
245 
246 	switch (mode) {
247 	case DEVLINK_ESWITCH_MODE_LEGACY:
248 		return nfp_abm_eswitch_set_legacy(abm);
249 	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
250 		return nfp_abm_eswitch_set_switchdev(abm);
251 	default:
252 		return -EINVAL;
253 	}
254 }
255 
256 static void
257 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn,
258 		     unsigned int id)
259 {
260 	struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id];
261 	u8 mac_addr[ETH_ALEN];
262 	struct nfp_nsp *nsp;
263 	char hwinfo[32];
264 	int err;
265 
266 	if (id > pf->eth_tbl->count) {
267 		nfp_warn(pf->cpp, "No entry for persistent MAC address\n");
268 		eth_hw_addr_random(nn->dp.netdev);
269 		return;
270 	}
271 
272 	snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac.pf%u",
273 		 eth_port->eth_index, abm->pf_id);
274 
275 	nsp = nfp_nsp_open(pf->cpp);
276 	if (IS_ERR(nsp)) {
277 		nfp_warn(pf->cpp, "Failed to access the NSP for persistent MAC address: %ld\n",
278 			 PTR_ERR(nsp));
279 		eth_hw_addr_random(nn->dp.netdev);
280 		return;
281 	}
282 
283 	if (!nfp_nsp_has_hwinfo_lookup(nsp)) {
284 		nfp_warn(pf->cpp, "NSP doesn't support PF MAC generation\n");
285 		eth_hw_addr_random(nn->dp.netdev);
286 		nfp_nsp_close(nsp);
287 		return;
288 	}
289 
290 	err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo));
291 	nfp_nsp_close(nsp);
292 	if (err) {
293 		nfp_warn(pf->cpp, "Reading persistent MAC address failed: %d\n",
294 			 err);
295 		eth_hw_addr_random(nn->dp.netdev);
296 		return;
297 	}
298 
299 	if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
300 		   &mac_addr[0], &mac_addr[1], &mac_addr[2],
301 		   &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) {
302 		nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n",
303 			 hwinfo);
304 		eth_hw_addr_random(nn->dp.netdev);
305 		return;
306 	}
307 
308 	eth_hw_addr_set(nn->dp.netdev, mac_addr);
309 	ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr);
310 }
311 
312 static int
313 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
314 {
315 	struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id];
316 	struct nfp_abm *abm = app->priv;
317 	struct nfp_abm_link *alink;
318 	int err;
319 
320 	alink = kzalloc(sizeof(*alink), GFP_KERNEL);
321 	if (!alink)
322 		return -ENOMEM;
323 	nn->app_priv = alink;
324 	alink->abm = abm;
325 	alink->vnic = nn;
326 	alink->id = id;
327 	alink->total_queues = alink->vnic->max_rx_rings;
328 
329 	INIT_LIST_HEAD(&alink->dscp_map);
330 
331 	err = nfp_abm_ctrl_read_params(alink);
332 	if (err)
333 		goto err_free_alink;
334 
335 	alink->prio_map = kzalloc(abm->prio_map_len, GFP_KERNEL);
336 	if (!alink->prio_map) {
337 		err = -ENOMEM;
338 		goto err_free_alink;
339 	}
340 
341 	/* This is a multi-host app, make sure MAC/PHY is up, but don't
342 	 * make the MAC/PHY state follow the state of any of the ports.
343 	 */
344 	err = nfp_eth_set_configured(app->cpp, eth_port->index, true);
345 	if (err < 0)
346 		goto err_free_priomap;
347 
348 	netif_keep_dst(nn->dp.netdev);
349 
350 	nfp_abm_vnic_set_mac(app->pf, abm, nn, id);
351 	INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL);
352 
353 	return 0;
354 
355 err_free_priomap:
356 	kfree(alink->prio_map);
357 err_free_alink:
358 	kfree(alink);
359 	return err;
360 }
361 
362 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn)
363 {
364 	struct nfp_abm_link *alink = nn->app_priv;
365 
366 	nfp_abm_kill_reprs(alink->abm, alink);
367 	WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n");
368 	kfree(alink->prio_map);
369 	kfree(alink);
370 }
371 
372 static int nfp_abm_vnic_init(struct nfp_app *app, struct nfp_net *nn)
373 {
374 	struct nfp_abm_link *alink = nn->app_priv;
375 
376 	if (nfp_abm_has_prio(alink->abm))
377 		return nfp_abm_ctrl_prio_map_update(alink, alink->prio_map);
378 	return 0;
379 }
380 
381 static u64 *
382 nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data)
383 {
384 	struct nfp_repr *repr = netdev_priv(port->netdev);
385 	struct nfp_abm_link *alink;
386 	unsigned int i;
387 
388 	if (port->type != NFP_PORT_PF_PORT)
389 		return data;
390 	alink = repr->app_priv;
391 	for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {
392 		*data++ = nfp_abm_ctrl_stat_non_sto(alink, i);
393 		*data++ = nfp_abm_ctrl_stat_sto(alink, i);
394 	}
395 	return data;
396 }
397 
398 static int
399 nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port)
400 {
401 	struct nfp_repr *repr = netdev_priv(port->netdev);
402 	struct nfp_abm_link *alink;
403 
404 	if (port->type != NFP_PORT_PF_PORT)
405 		return 0;
406 	alink = repr->app_priv;
407 	return alink->vnic->dp.num_r_vecs * 2;
408 }
409 
410 static u8 *
411 nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port,
412 			       u8 *data)
413 {
414 	struct nfp_repr *repr = netdev_priv(port->netdev);
415 	struct nfp_abm_link *alink;
416 	unsigned int i;
417 
418 	if (port->type != NFP_PORT_PF_PORT)
419 		return data;
420 	alink = repr->app_priv;
421 	for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {
422 		ethtool_sprintf(&data, "q%u_no_wait", i);
423 		ethtool_sprintf(&data, "q%u_delayed", i);
424 	}
425 	return data;
426 }
427 
428 static int nfp_abm_fw_init_reset(struct nfp_abm *abm)
429 {
430 	unsigned int i;
431 
432 	if (!abm->red_support)
433 		return 0;
434 
435 	for (i = 0; i < abm->num_bands * NFP_NET_MAX_RX_RINGS; i++) {
436 		__nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY);
437 		__nfp_abm_ctrl_set_q_act(abm, i, NFP_ABM_ACT_DROP);
438 	}
439 
440 	return nfp_abm_ctrl_qm_disable(abm);
441 }
442 
443 static int nfp_abm_init(struct nfp_app *app)
444 {
445 	struct nfp_pf *pf = app->pf;
446 	struct nfp_reprs *reprs;
447 	struct nfp_abm *abm;
448 	int err;
449 
450 	if (!pf->eth_tbl) {
451 		nfp_err(pf->cpp, "ABM NIC requires ETH table\n");
452 		return -EINVAL;
453 	}
454 	if (pf->max_data_vnics != pf->eth_tbl->count) {
455 		nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
456 			pf->max_data_vnics, pf->eth_tbl->count);
457 		return -EINVAL;
458 	}
459 	if (!pf->mac_stats_bar) {
460 		nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n");
461 		return -EINVAL;
462 	}
463 
464 	abm = kzalloc(sizeof(*abm), GFP_KERNEL);
465 	if (!abm)
466 		return -ENOMEM;
467 	app->priv = abm;
468 	abm->app = app;
469 
470 	err = nfp_abm_ctrl_find_addrs(abm);
471 	if (err)
472 		goto err_free_abm;
473 
474 	err = -ENOMEM;
475 	abm->num_thresholds = array_size(abm->num_bands, NFP_NET_MAX_RX_RINGS);
476 	abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL);
477 	if (!abm->threshold_undef)
478 		goto err_free_abm;
479 
480 	abm->thresholds = kvcalloc(abm->num_thresholds,
481 				   sizeof(*abm->thresholds), GFP_KERNEL);
482 	if (!abm->thresholds)
483 		goto err_free_thresh_umap;
484 
485 	abm->actions = kvcalloc(abm->num_thresholds, sizeof(*abm->actions),
486 				GFP_KERNEL);
487 	if (!abm->actions)
488 		goto err_free_thresh;
489 
490 	/* We start in legacy mode, make sure advanced queuing is disabled */
491 	err = nfp_abm_fw_init_reset(abm);
492 	if (err)
493 		goto err_free_act;
494 
495 	err = -ENOMEM;
496 	reprs = nfp_reprs_alloc(pf->max_data_vnics);
497 	if (!reprs)
498 		goto err_free_act;
499 	RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs);
500 
501 	reprs = nfp_reprs_alloc(pf->max_data_vnics);
502 	if (!reprs)
503 		goto err_free_phys;
504 	RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs);
505 
506 	return 0;
507 
508 err_free_phys:
509 	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
510 err_free_act:
511 	kvfree(abm->actions);
512 err_free_thresh:
513 	kvfree(abm->thresholds);
514 err_free_thresh_umap:
515 	bitmap_free(abm->threshold_undef);
516 err_free_abm:
517 	kfree(abm);
518 	app->priv = NULL;
519 	return err;
520 }
521 
522 static void nfp_abm_clean(struct nfp_app *app)
523 {
524 	struct nfp_abm *abm = app->priv;
525 
526 	nfp_abm_eswitch_clean_up(abm);
527 	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
528 	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
529 	bitmap_free(abm->threshold_undef);
530 	kvfree(abm->actions);
531 	kvfree(abm->thresholds);
532 	kfree(abm);
533 	app->priv = NULL;
534 }
535 
536 const struct nfp_app_type app_abm = {
537 	.id		= NFP_APP_ACTIVE_BUFFER_MGMT_NIC,
538 	.name		= "abm",
539 
540 	.init		= nfp_abm_init,
541 	.clean		= nfp_abm_clean,
542 
543 	.vnic_alloc	= nfp_abm_vnic_alloc,
544 	.vnic_free	= nfp_abm_vnic_free,
545 	.vnic_init	= nfp_abm_vnic_init,
546 
547 	.port_get_stats		= nfp_abm_port_get_stats,
548 	.port_get_stats_count	= nfp_abm_port_get_stats_count,
549 	.port_get_stats_strings	= nfp_abm_port_get_stats_strings,
550 
551 	.setup_tc	= nfp_abm_setup_tc,
552 
553 	.eswitch_mode_get	= nfp_abm_eswitch_mode_get,
554 	.eswitch_mode_set	= nfp_abm_eswitch_mode_set,
555 
556 	.dev_get	= nfp_abm_repr_get,
557 };
558