1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/etherdevice.h> 7 #include <linux/lockdep.h> 8 #include <linux/netdevice.h> 9 #include <linux/rcupdate.h> 10 #include <linux/rtnetlink.h> 11 #include <linux/slab.h> 12 13 #include "../nfpcore/nfp.h" 14 #include "../nfpcore/nfp_cpp.h" 15 #include "../nfpcore/nfp_nsp.h" 16 #include "../nfp_app.h" 17 #include "../nfp_main.h" 18 #include "../nfp_net.h" 19 #include "../nfp_net_repr.h" 20 #include "../nfp_port.h" 21 #include "main.h" 22 23 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) 24 { 25 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) | 26 FIELD_PREP(NFP_ABM_PORTID_ID, id); 27 } 28 29 static int 30 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, 31 enum tc_setup_type type, void *type_data) 32 { 33 struct nfp_repr *repr = netdev_priv(netdev); 34 struct nfp_port *port; 35 36 port = nfp_port_from_netdev(netdev); 37 if (!port || port->type != NFP_PORT_PF_PORT) 38 return -EOPNOTSUPP; 39 40 switch (type) { 41 case TC_SETUP_ROOT_QDISC: 42 return nfp_abm_setup_root(netdev, repr->app_priv, type_data); 43 case TC_SETUP_QDISC_MQ: 44 return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data); 45 case TC_SETUP_QDISC_RED: 46 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data); 47 case TC_SETUP_QDISC_GRED: 48 return nfp_abm_setup_tc_gred(netdev, repr->app_priv, type_data); 49 case TC_SETUP_BLOCK: 50 return nfp_abm_setup_cls_block(netdev, repr, type_data); 51 default: 52 return -EOPNOTSUPP; 53 } 54 } 55 56 static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id) 57 { 58 enum nfp_repr_type rtype; 59 struct nfp_reprs *reprs; 60 u8 port; 61 62 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id); 63 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id); 64 65 reprs = rcu_dereference(app->reprs[rtype]); 66 if (!reprs) 67 return NULL; 68 69 if (port >= reprs->num_reprs) 70 return NULL; 71 72 return rcu_dereference(reprs->reprs[port]); 73 } 74 75 static int 76 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink, 77 enum nfp_port_type ptype) 78 { 79 struct net_device *netdev; 80 enum nfp_repr_type rtype; 81 struct nfp_reprs *reprs; 82 struct nfp_repr *repr; 83 struct nfp_port *port; 84 unsigned int txqs; 85 int err; 86 87 if (ptype == NFP_PORT_PHYS_PORT) { 88 rtype = NFP_REPR_TYPE_PHYS_PORT; 89 txqs = 1; 90 } else { 91 rtype = NFP_REPR_TYPE_PF; 92 txqs = alink->vnic->max_rx_rings; 93 } 94 95 netdev = nfp_repr_alloc_mqs(app, txqs, 1); 96 if (!netdev) 97 return -ENOMEM; 98 repr = netdev_priv(netdev); 99 repr->app_priv = alink; 100 101 port = nfp_port_alloc(app, ptype, netdev); 102 if (IS_ERR(port)) { 103 err = PTR_ERR(port); 104 goto err_free_repr; 105 } 106 107 if (ptype == NFP_PORT_PHYS_PORT) { 108 port->eth_forced = true; 109 err = nfp_port_init_phy_port(app->pf, app, port, alink->id); 110 if (err) 111 goto err_free_port; 112 } else { 113 port->pf_id = alink->abm->pf_id; 114 port->pf_split = app->pf->max_data_vnics > 1; 115 port->pf_split_id = alink->id; 116 port->vnic = alink->vnic->dp.ctrl_bar; 117 } 118 119 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev); 120 eth_hw_addr_random(netdev); 121 122 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id), 123 port, alink->vnic->dp.netdev); 124 if (err) 125 goto err_free_port; 126 127 reprs = nfp_reprs_get_locked(app, rtype); 128 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr"); 129 rtnl_lock(); 130 rcu_assign_pointer(reprs->reprs[alink->id], netdev); 131 rtnl_unlock(); 132 133 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n", 134 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys", 135 alink->id, netdev->name); 136 137 return 0; 138 139 err_free_port: 140 nfp_port_free(port); 141 err_free_repr: 142 nfp_repr_free(netdev); 143 return err; 144 } 145 146 static void 147 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink, 148 enum nfp_repr_type rtype) 149 { 150 struct net_device *netdev; 151 struct nfp_reprs *reprs; 152 153 reprs = nfp_reprs_get_locked(app, rtype); 154 netdev = nfp_repr_get_locked(app, reprs, alink->id); 155 if (!netdev) 156 return; 157 rtnl_lock(); 158 rcu_assign_pointer(reprs->reprs[alink->id], NULL); 159 rtnl_unlock(); 160 synchronize_rcu(); 161 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */ 162 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev)); 163 } 164 165 static void 166 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink) 167 { 168 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF); 169 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT); 170 } 171 172 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm) 173 { 174 struct nfp_pf *pf = abm->app->pf; 175 struct nfp_net *nn; 176 177 list_for_each_entry(nn, &pf->vnics, vnic_list) 178 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv); 179 } 180 181 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app) 182 { 183 struct nfp_abm *abm = app->priv; 184 185 return abm->eswitch_mode; 186 } 187 188 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm) 189 { 190 nfp_abm_kill_reprs_all(abm); 191 nfp_abm_ctrl_qm_disable(abm); 192 193 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY; 194 return 0; 195 } 196 197 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm) 198 { 199 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY) 200 WARN_ON(nfp_abm_eswitch_set_legacy(abm)); 201 } 202 203 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm) 204 { 205 struct nfp_app *app = abm->app; 206 struct nfp_pf *pf = app->pf; 207 struct nfp_net *nn; 208 int err; 209 210 if (!abm->red_support) 211 return -EOPNOTSUPP; 212 213 err = nfp_abm_ctrl_qm_enable(abm); 214 if (err) 215 return err; 216 217 list_for_each_entry(nn, &pf->vnics, vnic_list) { 218 struct nfp_abm_link *alink = nn->app_priv; 219 220 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT); 221 if (err) 222 goto err_kill_all_reprs; 223 224 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT); 225 if (err) 226 goto err_kill_all_reprs; 227 } 228 229 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 230 return 0; 231 232 err_kill_all_reprs: 233 nfp_abm_kill_reprs_all(abm); 234 nfp_abm_ctrl_qm_disable(abm); 235 return err; 236 } 237 238 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode) 239 { 240 struct nfp_abm *abm = app->priv; 241 242 if (abm->eswitch_mode == mode) 243 return 0; 244 245 switch (mode) { 246 case DEVLINK_ESWITCH_MODE_LEGACY: 247 return nfp_abm_eswitch_set_legacy(abm); 248 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 249 return nfp_abm_eswitch_set_switchdev(abm); 250 default: 251 return -EINVAL; 252 } 253 } 254 255 static void 256 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn, 257 unsigned int id) 258 { 259 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id]; 260 u8 mac_addr[ETH_ALEN]; 261 struct nfp_nsp *nsp; 262 char hwinfo[32]; 263 int err; 264 265 if (id > pf->eth_tbl->count) { 266 nfp_warn(pf->cpp, "No entry for persistent MAC address\n"); 267 eth_hw_addr_random(nn->dp.netdev); 268 return; 269 } 270 271 snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac.pf%u", 272 eth_port->eth_index, abm->pf_id); 273 274 nsp = nfp_nsp_open(pf->cpp); 275 if (IS_ERR(nsp)) { 276 nfp_warn(pf->cpp, "Failed to access the NSP for persistent MAC address: %ld\n", 277 PTR_ERR(nsp)); 278 eth_hw_addr_random(nn->dp.netdev); 279 return; 280 } 281 282 if (!nfp_nsp_has_hwinfo_lookup(nsp)) { 283 nfp_warn(pf->cpp, "NSP doesn't support PF MAC generation\n"); 284 eth_hw_addr_random(nn->dp.netdev); 285 return; 286 } 287 288 err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo)); 289 nfp_nsp_close(nsp); 290 if (err) { 291 nfp_warn(pf->cpp, "Reading persistent MAC address failed: %d\n", 292 err); 293 eth_hw_addr_random(nn->dp.netdev); 294 return; 295 } 296 297 if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", 298 &mac_addr[0], &mac_addr[1], &mac_addr[2], 299 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { 300 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n", 301 hwinfo); 302 eth_hw_addr_random(nn->dp.netdev); 303 return; 304 } 305 306 ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr); 307 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr); 308 } 309 310 static int 311 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 312 { 313 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id]; 314 struct nfp_abm *abm = app->priv; 315 struct nfp_abm_link *alink; 316 int err; 317 318 alink = kzalloc(sizeof(*alink), GFP_KERNEL); 319 if (!alink) 320 return -ENOMEM; 321 nn->app_priv = alink; 322 alink->abm = abm; 323 alink->vnic = nn; 324 alink->id = id; 325 alink->total_queues = alink->vnic->max_rx_rings; 326 327 INIT_LIST_HEAD(&alink->dscp_map); 328 329 err = nfp_abm_ctrl_read_params(alink); 330 if (err) 331 goto err_free_alink; 332 333 alink->prio_map = kzalloc(abm->prio_map_len, GFP_KERNEL); 334 if (!alink->prio_map) 335 goto err_free_alink; 336 337 /* This is a multi-host app, make sure MAC/PHY is up, but don't 338 * make the MAC/PHY state follow the state of any of the ports. 339 */ 340 err = nfp_eth_set_configured(app->cpp, eth_port->index, true); 341 if (err < 0) 342 goto err_free_priomap; 343 344 netif_keep_dst(nn->dp.netdev); 345 346 nfp_abm_vnic_set_mac(app->pf, abm, nn, id); 347 INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL); 348 349 return 0; 350 351 err_free_priomap: 352 kfree(alink->prio_map); 353 err_free_alink: 354 kfree(alink); 355 return err; 356 } 357 358 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) 359 { 360 struct nfp_abm_link *alink = nn->app_priv; 361 362 nfp_abm_kill_reprs(alink->abm, alink); 363 WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n"); 364 kfree(alink->prio_map); 365 kfree(alink); 366 } 367 368 static int nfp_abm_vnic_init(struct nfp_app *app, struct nfp_net *nn) 369 { 370 struct nfp_abm_link *alink = nn->app_priv; 371 372 if (nfp_abm_has_prio(alink->abm)) 373 return nfp_abm_ctrl_prio_map_update(alink, alink->prio_map); 374 return 0; 375 } 376 377 static u64 * 378 nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data) 379 { 380 struct nfp_repr *repr = netdev_priv(port->netdev); 381 struct nfp_abm_link *alink; 382 unsigned int i; 383 384 if (port->type != NFP_PORT_PF_PORT) 385 return data; 386 alink = repr->app_priv; 387 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 388 *data++ = nfp_abm_ctrl_stat_non_sto(alink, i); 389 *data++ = nfp_abm_ctrl_stat_sto(alink, i); 390 } 391 return data; 392 } 393 394 static int 395 nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port) 396 { 397 struct nfp_repr *repr = netdev_priv(port->netdev); 398 struct nfp_abm_link *alink; 399 400 if (port->type != NFP_PORT_PF_PORT) 401 return 0; 402 alink = repr->app_priv; 403 return alink->vnic->dp.num_r_vecs * 2; 404 } 405 406 static u8 * 407 nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port, 408 u8 *data) 409 { 410 struct nfp_repr *repr = netdev_priv(port->netdev); 411 struct nfp_abm_link *alink; 412 unsigned int i; 413 414 if (port->type != NFP_PORT_PF_PORT) 415 return data; 416 alink = repr->app_priv; 417 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 418 data = nfp_pr_et(data, "q%u_no_wait", i); 419 data = nfp_pr_et(data, "q%u_delayed", i); 420 } 421 return data; 422 } 423 424 static int nfp_abm_fw_init_reset(struct nfp_abm *abm) 425 { 426 unsigned int i; 427 428 if (!abm->red_support) 429 return 0; 430 431 for (i = 0; i < abm->num_bands * NFP_NET_MAX_RX_RINGS; i++) { 432 __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); 433 __nfp_abm_ctrl_set_q_act(abm, i, NFP_ABM_ACT_DROP); 434 } 435 436 return nfp_abm_ctrl_qm_disable(abm); 437 } 438 439 static int nfp_abm_init(struct nfp_app *app) 440 { 441 struct nfp_pf *pf = app->pf; 442 struct nfp_reprs *reprs; 443 struct nfp_abm *abm; 444 int err; 445 446 if (!pf->eth_tbl) { 447 nfp_err(pf->cpp, "ABM NIC requires ETH table\n"); 448 return -EINVAL; 449 } 450 if (pf->max_data_vnics != pf->eth_tbl->count) { 451 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 452 pf->max_data_vnics, pf->eth_tbl->count); 453 return -EINVAL; 454 } 455 if (!pf->mac_stats_bar) { 456 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n"); 457 return -EINVAL; 458 } 459 460 abm = kzalloc(sizeof(*abm), GFP_KERNEL); 461 if (!abm) 462 return -ENOMEM; 463 app->priv = abm; 464 abm->app = app; 465 466 err = nfp_abm_ctrl_find_addrs(abm); 467 if (err) 468 goto err_free_abm; 469 470 err = -ENOMEM; 471 abm->num_thresholds = array_size(abm->num_bands, NFP_NET_MAX_RX_RINGS); 472 abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL); 473 if (!abm->threshold_undef) 474 goto err_free_abm; 475 476 abm->thresholds = kvcalloc(abm->num_thresholds, 477 sizeof(*abm->thresholds), GFP_KERNEL); 478 if (!abm->thresholds) 479 goto err_free_thresh_umap; 480 481 abm->actions = kvcalloc(abm->num_thresholds, sizeof(*abm->actions), 482 GFP_KERNEL); 483 if (!abm->actions) 484 goto err_free_thresh; 485 486 /* We start in legacy mode, make sure advanced queuing is disabled */ 487 err = nfp_abm_fw_init_reset(abm); 488 if (err) 489 goto err_free_act; 490 491 err = -ENOMEM; 492 reprs = nfp_reprs_alloc(pf->max_data_vnics); 493 if (!reprs) 494 goto err_free_act; 495 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); 496 497 reprs = nfp_reprs_alloc(pf->max_data_vnics); 498 if (!reprs) 499 goto err_free_phys; 500 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs); 501 502 return 0; 503 504 err_free_phys: 505 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 506 err_free_act: 507 kvfree(abm->actions); 508 err_free_thresh: 509 kvfree(abm->thresholds); 510 err_free_thresh_umap: 511 bitmap_free(abm->threshold_undef); 512 err_free_abm: 513 kfree(abm); 514 app->priv = NULL; 515 return err; 516 } 517 518 static void nfp_abm_clean(struct nfp_app *app) 519 { 520 struct nfp_abm *abm = app->priv; 521 522 nfp_abm_eswitch_clean_up(abm); 523 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 524 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 525 bitmap_free(abm->threshold_undef); 526 kvfree(abm->actions); 527 kvfree(abm->thresholds); 528 kfree(abm); 529 app->priv = NULL; 530 } 531 532 const struct nfp_app_type app_abm = { 533 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC, 534 .name = "abm", 535 536 .init = nfp_abm_init, 537 .clean = nfp_abm_clean, 538 539 .vnic_alloc = nfp_abm_vnic_alloc, 540 .vnic_free = nfp_abm_vnic_free, 541 .vnic_init = nfp_abm_vnic_init, 542 543 .port_get_stats = nfp_abm_port_get_stats, 544 .port_get_stats_count = nfp_abm_port_get_stats_count, 545 .port_get_stats_strings = nfp_abm_port_get_stats_strings, 546 547 .setup_tc = nfp_abm_setup_tc, 548 549 .eswitch_mode_get = nfp_abm_eswitch_mode_get, 550 .eswitch_mode_set = nfp_abm_eswitch_mode_set, 551 552 .repr_get = nfp_abm_repr_get, 553 }; 554