1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/etherdevice.h> 7 #include <linux/lockdep.h> 8 #include <linux/netdevice.h> 9 #include <linux/rcupdate.h> 10 #include <linux/rtnetlink.h> 11 #include <linux/slab.h> 12 13 #include "../nfpcore/nfp.h" 14 #include "../nfpcore/nfp_cpp.h" 15 #include "../nfpcore/nfp_nsp.h" 16 #include "../nfp_app.h" 17 #include "../nfp_main.h" 18 #include "../nfp_net.h" 19 #include "../nfp_net_repr.h" 20 #include "../nfp_port.h" 21 #include "main.h" 22 23 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) 24 { 25 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) | 26 FIELD_PREP(NFP_ABM_PORTID_ID, id); 27 } 28 29 static int 30 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, 31 enum tc_setup_type type, void *type_data) 32 { 33 struct nfp_repr *repr = netdev_priv(netdev); 34 struct nfp_port *port; 35 36 port = nfp_port_from_netdev(netdev); 37 if (!port || port->type != NFP_PORT_PF_PORT) 38 return -EOPNOTSUPP; 39 40 switch (type) { 41 case TC_SETUP_ROOT_QDISC: 42 return nfp_abm_setup_root(netdev, repr->app_priv, type_data); 43 case TC_SETUP_QDISC_MQ: 44 return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data); 45 case TC_SETUP_QDISC_RED: 46 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data); 47 case TC_SETUP_QDISC_GRED: 48 return nfp_abm_setup_tc_gred(netdev, repr->app_priv, type_data); 49 case TC_SETUP_BLOCK: 50 return nfp_abm_setup_cls_block(netdev, repr, type_data); 51 default: 52 return -EOPNOTSUPP; 53 } 54 } 55 56 static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id) 57 { 58 enum nfp_repr_type rtype; 59 struct nfp_reprs *reprs; 60 u8 port; 61 62 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id); 63 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id); 64 65 reprs = rcu_dereference(app->reprs[rtype]); 66 if (!reprs) 67 return NULL; 68 69 if (port >= reprs->num_reprs) 70 return NULL; 71 72 return rcu_dereference(reprs->reprs[port]); 73 } 74 75 static int 76 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink, 77 enum nfp_port_type ptype) 78 { 79 struct net_device *netdev; 80 enum nfp_repr_type rtype; 81 struct nfp_reprs *reprs; 82 struct nfp_repr *repr; 83 struct nfp_port *port; 84 unsigned int txqs; 85 int err; 86 87 if (ptype == NFP_PORT_PHYS_PORT) { 88 rtype = NFP_REPR_TYPE_PHYS_PORT; 89 txqs = 1; 90 } else { 91 rtype = NFP_REPR_TYPE_PF; 92 txqs = alink->vnic->max_rx_rings; 93 } 94 95 netdev = nfp_repr_alloc_mqs(app, txqs, 1); 96 if (!netdev) 97 return -ENOMEM; 98 repr = netdev_priv(netdev); 99 repr->app_priv = alink; 100 101 port = nfp_port_alloc(app, ptype, netdev); 102 if (IS_ERR(port)) { 103 err = PTR_ERR(port); 104 goto err_free_repr; 105 } 106 107 if (ptype == NFP_PORT_PHYS_PORT) { 108 port->eth_forced = true; 109 err = nfp_port_init_phy_port(app->pf, app, port, alink->id); 110 if (err) 111 goto err_free_port; 112 } else { 113 port->pf_id = alink->abm->pf_id; 114 port->pf_split = app->pf->max_data_vnics > 1; 115 port->pf_split_id = alink->id; 116 port->vnic = alink->vnic->dp.ctrl_bar; 117 } 118 119 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev); 120 eth_hw_addr_random(netdev); 121 122 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id), 123 port, alink->vnic->dp.netdev); 124 if (err) 125 goto err_free_port; 126 127 reprs = nfp_reprs_get_locked(app, rtype); 128 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr"); 129 rcu_assign_pointer(reprs->reprs[alink->id], netdev); 130 131 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n", 132 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys", 133 alink->id, netdev->name); 134 135 return 0; 136 137 err_free_port: 138 nfp_port_free(port); 139 err_free_repr: 140 nfp_repr_free(netdev); 141 return err; 142 } 143 144 static void 145 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink, 146 enum nfp_repr_type rtype) 147 { 148 struct net_device *netdev; 149 struct nfp_reprs *reprs; 150 151 reprs = nfp_reprs_get_locked(app, rtype); 152 netdev = nfp_repr_get_locked(app, reprs, alink->id); 153 if (!netdev) 154 return; 155 rcu_assign_pointer(reprs->reprs[alink->id], NULL); 156 synchronize_rcu(); 157 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */ 158 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev)); 159 } 160 161 static void 162 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink) 163 { 164 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF); 165 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT); 166 } 167 168 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm) 169 { 170 struct nfp_pf *pf = abm->app->pf; 171 struct nfp_net *nn; 172 173 list_for_each_entry(nn, &pf->vnics, vnic_list) 174 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv); 175 } 176 177 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app) 178 { 179 struct nfp_abm *abm = app->priv; 180 181 return abm->eswitch_mode; 182 } 183 184 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm) 185 { 186 nfp_abm_kill_reprs_all(abm); 187 nfp_abm_ctrl_qm_disable(abm); 188 189 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY; 190 return 0; 191 } 192 193 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm) 194 { 195 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY) 196 WARN_ON(nfp_abm_eswitch_set_legacy(abm)); 197 } 198 199 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm) 200 { 201 struct nfp_app *app = abm->app; 202 struct nfp_pf *pf = app->pf; 203 struct nfp_net *nn; 204 int err; 205 206 err = nfp_abm_ctrl_qm_enable(abm); 207 if (err) 208 return err; 209 210 list_for_each_entry(nn, &pf->vnics, vnic_list) { 211 struct nfp_abm_link *alink = nn->app_priv; 212 213 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT); 214 if (err) 215 goto err_kill_all_reprs; 216 217 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT); 218 if (err) 219 goto err_kill_all_reprs; 220 } 221 222 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 223 return 0; 224 225 err_kill_all_reprs: 226 nfp_abm_kill_reprs_all(abm); 227 nfp_abm_ctrl_qm_disable(abm); 228 return err; 229 } 230 231 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode) 232 { 233 struct nfp_abm *abm = app->priv; 234 235 if (abm->eswitch_mode == mode) 236 return 0; 237 238 switch (mode) { 239 case DEVLINK_ESWITCH_MODE_LEGACY: 240 return nfp_abm_eswitch_set_legacy(abm); 241 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 242 return nfp_abm_eswitch_set_switchdev(abm); 243 default: 244 return -EINVAL; 245 } 246 } 247 248 static void 249 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn, 250 unsigned int id) 251 { 252 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id]; 253 u8 mac_addr[ETH_ALEN]; 254 struct nfp_nsp *nsp; 255 char hwinfo[32]; 256 int err; 257 258 if (id > pf->eth_tbl->count) { 259 nfp_warn(pf->cpp, "No entry for persistent MAC address\n"); 260 eth_hw_addr_random(nn->dp.netdev); 261 return; 262 } 263 264 snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac.pf%u", 265 eth_port->eth_index, abm->pf_id); 266 267 nsp = nfp_nsp_open(pf->cpp); 268 if (IS_ERR(nsp)) { 269 nfp_warn(pf->cpp, "Failed to access the NSP for persistent MAC address: %ld\n", 270 PTR_ERR(nsp)); 271 eth_hw_addr_random(nn->dp.netdev); 272 return; 273 } 274 275 if (!nfp_nsp_has_hwinfo_lookup(nsp)) { 276 nfp_warn(pf->cpp, "NSP doesn't support PF MAC generation\n"); 277 eth_hw_addr_random(nn->dp.netdev); 278 return; 279 } 280 281 err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo)); 282 nfp_nsp_close(nsp); 283 if (err) { 284 nfp_warn(pf->cpp, "Reading persistent MAC address failed: %d\n", 285 err); 286 eth_hw_addr_random(nn->dp.netdev); 287 return; 288 } 289 290 if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", 291 &mac_addr[0], &mac_addr[1], &mac_addr[2], 292 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { 293 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n", 294 hwinfo); 295 eth_hw_addr_random(nn->dp.netdev); 296 return; 297 } 298 299 ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr); 300 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr); 301 } 302 303 static int 304 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 305 { 306 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id]; 307 struct nfp_abm *abm = app->priv; 308 struct nfp_abm_link *alink; 309 int err; 310 311 alink = kzalloc(sizeof(*alink), GFP_KERNEL); 312 if (!alink) 313 return -ENOMEM; 314 nn->app_priv = alink; 315 alink->abm = abm; 316 alink->vnic = nn; 317 alink->id = id; 318 alink->total_queues = alink->vnic->max_rx_rings; 319 320 INIT_LIST_HEAD(&alink->dscp_map); 321 322 err = nfp_abm_ctrl_read_params(alink); 323 if (err) 324 goto err_free_alink; 325 326 alink->prio_map = kzalloc(abm->prio_map_len, GFP_KERNEL); 327 if (!alink->prio_map) 328 goto err_free_alink; 329 330 /* This is a multi-host app, make sure MAC/PHY is up, but don't 331 * make the MAC/PHY state follow the state of any of the ports. 332 */ 333 err = nfp_eth_set_configured(app->cpp, eth_port->index, true); 334 if (err < 0) 335 goto err_free_priomap; 336 337 netif_keep_dst(nn->dp.netdev); 338 339 nfp_abm_vnic_set_mac(app->pf, abm, nn, id); 340 INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL); 341 342 return 0; 343 344 err_free_priomap: 345 kfree(alink->prio_map); 346 err_free_alink: 347 kfree(alink); 348 return err; 349 } 350 351 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) 352 { 353 struct nfp_abm_link *alink = nn->app_priv; 354 355 nfp_abm_kill_reprs(alink->abm, alink); 356 WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n"); 357 kfree(alink->prio_map); 358 kfree(alink); 359 } 360 361 static int nfp_abm_vnic_init(struct nfp_app *app, struct nfp_net *nn) 362 { 363 struct nfp_abm_link *alink = nn->app_priv; 364 365 if (nfp_abm_has_prio(alink->abm)) 366 return nfp_abm_ctrl_prio_map_update(alink, alink->prio_map); 367 return 0; 368 } 369 370 static u64 * 371 nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data) 372 { 373 struct nfp_repr *repr = netdev_priv(port->netdev); 374 struct nfp_abm_link *alink; 375 unsigned int i; 376 377 if (port->type != NFP_PORT_PF_PORT) 378 return data; 379 alink = repr->app_priv; 380 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 381 *data++ = nfp_abm_ctrl_stat_non_sto(alink, i); 382 *data++ = nfp_abm_ctrl_stat_sto(alink, i); 383 } 384 return data; 385 } 386 387 static int 388 nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port) 389 { 390 struct nfp_repr *repr = netdev_priv(port->netdev); 391 struct nfp_abm_link *alink; 392 393 if (port->type != NFP_PORT_PF_PORT) 394 return 0; 395 alink = repr->app_priv; 396 return alink->vnic->dp.num_r_vecs * 2; 397 } 398 399 static u8 * 400 nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port, 401 u8 *data) 402 { 403 struct nfp_repr *repr = netdev_priv(port->netdev); 404 struct nfp_abm_link *alink; 405 unsigned int i; 406 407 if (port->type != NFP_PORT_PF_PORT) 408 return data; 409 alink = repr->app_priv; 410 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 411 data = nfp_pr_et(data, "q%u_no_wait", i); 412 data = nfp_pr_et(data, "q%u_delayed", i); 413 } 414 return data; 415 } 416 417 static int nfp_abm_init(struct nfp_app *app) 418 { 419 struct nfp_pf *pf = app->pf; 420 struct nfp_reprs *reprs; 421 struct nfp_abm *abm; 422 unsigned int i; 423 int err; 424 425 if (!pf->eth_tbl) { 426 nfp_err(pf->cpp, "ABM NIC requires ETH table\n"); 427 return -EINVAL; 428 } 429 if (pf->max_data_vnics != pf->eth_tbl->count) { 430 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 431 pf->max_data_vnics, pf->eth_tbl->count); 432 return -EINVAL; 433 } 434 if (!pf->mac_stats_bar) { 435 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n"); 436 return -EINVAL; 437 } 438 439 abm = kzalloc(sizeof(*abm), GFP_KERNEL); 440 if (!abm) 441 return -ENOMEM; 442 app->priv = abm; 443 abm->app = app; 444 445 err = nfp_abm_ctrl_find_addrs(abm); 446 if (err) 447 goto err_free_abm; 448 449 err = -ENOMEM; 450 abm->num_thresholds = array_size(abm->num_bands, NFP_NET_MAX_RX_RINGS); 451 abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL); 452 if (!abm->threshold_undef) 453 goto err_free_abm; 454 455 abm->thresholds = kvcalloc(abm->num_thresholds, 456 sizeof(*abm->thresholds), GFP_KERNEL); 457 if (!abm->thresholds) 458 goto err_free_thresh_umap; 459 for (i = 0; i < abm->num_bands * NFP_NET_MAX_RX_RINGS; i++) 460 __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); 461 462 /* We start in legacy mode, make sure advanced queuing is disabled */ 463 err = nfp_abm_ctrl_qm_disable(abm); 464 if (err) 465 goto err_free_thresh; 466 467 err = -ENOMEM; 468 reprs = nfp_reprs_alloc(pf->max_data_vnics); 469 if (!reprs) 470 goto err_free_thresh; 471 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); 472 473 reprs = nfp_reprs_alloc(pf->max_data_vnics); 474 if (!reprs) 475 goto err_free_phys; 476 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs); 477 478 return 0; 479 480 err_free_phys: 481 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 482 err_free_thresh: 483 kvfree(abm->thresholds); 484 err_free_thresh_umap: 485 bitmap_free(abm->threshold_undef); 486 err_free_abm: 487 kfree(abm); 488 app->priv = NULL; 489 return err; 490 } 491 492 static void nfp_abm_clean(struct nfp_app *app) 493 { 494 struct nfp_abm *abm = app->priv; 495 496 nfp_abm_eswitch_clean_up(abm); 497 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 498 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 499 bitmap_free(abm->threshold_undef); 500 kvfree(abm->thresholds); 501 kfree(abm); 502 app->priv = NULL; 503 } 504 505 const struct nfp_app_type app_abm = { 506 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC, 507 .name = "abm", 508 509 .init = nfp_abm_init, 510 .clean = nfp_abm_clean, 511 512 .vnic_alloc = nfp_abm_vnic_alloc, 513 .vnic_free = nfp_abm_vnic_free, 514 .vnic_init = nfp_abm_vnic_init, 515 516 .port_get_stats = nfp_abm_port_get_stats, 517 .port_get_stats_count = nfp_abm_port_get_stats_count, 518 .port_get_stats_strings = nfp_abm_port_get_stats_strings, 519 520 .setup_tc = nfp_abm_setup_tc, 521 522 .eswitch_mode_get = nfp_abm_eswitch_mode_get, 523 .eswitch_mode_set = nfp_abm_eswitch_mode_set, 524 525 .repr_get = nfp_abm_repr_get, 526 }; 527