1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/etherdevice.h> 7 #include <linux/lockdep.h> 8 #include <linux/netdevice.h> 9 #include <linux/rcupdate.h> 10 #include <linux/rtnetlink.h> 11 #include <linux/slab.h> 12 13 #include "../nfpcore/nfp.h" 14 #include "../nfpcore/nfp_cpp.h" 15 #include "../nfpcore/nfp_nsp.h" 16 #include "../nfp_app.h" 17 #include "../nfp_main.h" 18 #include "../nfp_net.h" 19 #include "../nfp_net_repr.h" 20 #include "../nfp_port.h" 21 #include "main.h" 22 23 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) 24 { 25 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) | 26 FIELD_PREP(NFP_ABM_PORTID_ID, id); 27 } 28 29 static int 30 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, 31 enum tc_setup_type type, void *type_data) 32 { 33 struct nfp_repr *repr = netdev_priv(netdev); 34 struct nfp_port *port; 35 36 port = nfp_port_from_netdev(netdev); 37 if (!port || port->type != NFP_PORT_PF_PORT) 38 return -EOPNOTSUPP; 39 40 switch (type) { 41 case TC_SETUP_ROOT_QDISC: 42 return nfp_abm_setup_root(netdev, repr->app_priv, type_data); 43 case TC_SETUP_QDISC_MQ: 44 return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data); 45 case TC_SETUP_QDISC_RED: 46 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data); 47 case TC_SETUP_QDISC_GRED: 48 return nfp_abm_setup_tc_gred(netdev, repr->app_priv, type_data); 49 default: 50 return -EOPNOTSUPP; 51 } 52 } 53 54 static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id) 55 { 56 enum nfp_repr_type rtype; 57 struct nfp_reprs *reprs; 58 u8 port; 59 60 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id); 61 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id); 62 63 reprs = rcu_dereference(app->reprs[rtype]); 64 if (!reprs) 65 return NULL; 66 67 if (port >= reprs->num_reprs) 68 return NULL; 69 70 return rcu_dereference(reprs->reprs[port]); 71 } 72 73 static int 74 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink, 75 enum nfp_port_type ptype) 76 { 77 struct net_device *netdev; 78 enum nfp_repr_type rtype; 79 struct nfp_reprs *reprs; 80 struct nfp_repr *repr; 81 struct nfp_port *port; 82 unsigned int txqs; 83 int err; 84 85 if (ptype == NFP_PORT_PHYS_PORT) { 86 rtype = NFP_REPR_TYPE_PHYS_PORT; 87 txqs = 1; 88 } else { 89 rtype = NFP_REPR_TYPE_PF; 90 txqs = alink->vnic->max_rx_rings; 91 } 92 93 netdev = nfp_repr_alloc_mqs(app, txqs, 1); 94 if (!netdev) 95 return -ENOMEM; 96 repr = netdev_priv(netdev); 97 repr->app_priv = alink; 98 99 port = nfp_port_alloc(app, ptype, netdev); 100 if (IS_ERR(port)) { 101 err = PTR_ERR(port); 102 goto err_free_repr; 103 } 104 105 if (ptype == NFP_PORT_PHYS_PORT) { 106 port->eth_forced = true; 107 err = nfp_port_init_phy_port(app->pf, app, port, alink->id); 108 if (err) 109 goto err_free_port; 110 } else { 111 port->pf_id = alink->abm->pf_id; 112 port->pf_split = app->pf->max_data_vnics > 1; 113 port->pf_split_id = alink->id; 114 port->vnic = alink->vnic->dp.ctrl_bar; 115 } 116 117 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev); 118 eth_hw_addr_random(netdev); 119 120 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id), 121 port, alink->vnic->dp.netdev); 122 if (err) 123 goto err_free_port; 124 125 reprs = nfp_reprs_get_locked(app, rtype); 126 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr"); 127 rcu_assign_pointer(reprs->reprs[alink->id], netdev); 128 129 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n", 130 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys", 131 alink->id, netdev->name); 132 133 return 0; 134 135 err_free_port: 136 nfp_port_free(port); 137 err_free_repr: 138 nfp_repr_free(netdev); 139 return err; 140 } 141 142 static void 143 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink, 144 enum nfp_repr_type rtype) 145 { 146 struct net_device *netdev; 147 struct nfp_reprs *reprs; 148 149 reprs = nfp_reprs_get_locked(app, rtype); 150 netdev = nfp_repr_get_locked(app, reprs, alink->id); 151 if (!netdev) 152 return; 153 rcu_assign_pointer(reprs->reprs[alink->id], NULL); 154 synchronize_rcu(); 155 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */ 156 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev)); 157 } 158 159 static void 160 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink) 161 { 162 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF); 163 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT); 164 } 165 166 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm) 167 { 168 struct nfp_pf *pf = abm->app->pf; 169 struct nfp_net *nn; 170 171 list_for_each_entry(nn, &pf->vnics, vnic_list) 172 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv); 173 } 174 175 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app) 176 { 177 struct nfp_abm *abm = app->priv; 178 179 return abm->eswitch_mode; 180 } 181 182 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm) 183 { 184 nfp_abm_kill_reprs_all(abm); 185 nfp_abm_ctrl_qm_disable(abm); 186 187 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY; 188 return 0; 189 } 190 191 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm) 192 { 193 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY) 194 WARN_ON(nfp_abm_eswitch_set_legacy(abm)); 195 } 196 197 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm) 198 { 199 struct nfp_app *app = abm->app; 200 struct nfp_pf *pf = app->pf; 201 struct nfp_net *nn; 202 int err; 203 204 err = nfp_abm_ctrl_qm_enable(abm); 205 if (err) 206 return err; 207 208 list_for_each_entry(nn, &pf->vnics, vnic_list) { 209 struct nfp_abm_link *alink = nn->app_priv; 210 211 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT); 212 if (err) 213 goto err_kill_all_reprs; 214 215 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT); 216 if (err) 217 goto err_kill_all_reprs; 218 } 219 220 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 221 return 0; 222 223 err_kill_all_reprs: 224 nfp_abm_kill_reprs_all(abm); 225 nfp_abm_ctrl_qm_disable(abm); 226 return err; 227 } 228 229 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode) 230 { 231 struct nfp_abm *abm = app->priv; 232 233 if (abm->eswitch_mode == mode) 234 return 0; 235 236 switch (mode) { 237 case DEVLINK_ESWITCH_MODE_LEGACY: 238 return nfp_abm_eswitch_set_legacy(abm); 239 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 240 return nfp_abm_eswitch_set_switchdev(abm); 241 default: 242 return -EINVAL; 243 } 244 } 245 246 static void 247 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn, 248 unsigned int id) 249 { 250 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id]; 251 u8 mac_addr[ETH_ALEN]; 252 struct nfp_nsp *nsp; 253 char hwinfo[32]; 254 int err; 255 256 if (id > pf->eth_tbl->count) { 257 nfp_warn(pf->cpp, "No entry for persistent MAC address\n"); 258 eth_hw_addr_random(nn->dp.netdev); 259 return; 260 } 261 262 snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac.pf%u", 263 eth_port->eth_index, abm->pf_id); 264 265 nsp = nfp_nsp_open(pf->cpp); 266 if (IS_ERR(nsp)) { 267 nfp_warn(pf->cpp, "Failed to access the NSP for persistent MAC address: %ld\n", 268 PTR_ERR(nsp)); 269 eth_hw_addr_random(nn->dp.netdev); 270 return; 271 } 272 273 if (!nfp_nsp_has_hwinfo_lookup(nsp)) { 274 nfp_warn(pf->cpp, "NSP doesn't support PF MAC generation\n"); 275 eth_hw_addr_random(nn->dp.netdev); 276 return; 277 } 278 279 err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo)); 280 nfp_nsp_close(nsp); 281 if (err) { 282 nfp_warn(pf->cpp, "Reading persistent MAC address failed: %d\n", 283 err); 284 eth_hw_addr_random(nn->dp.netdev); 285 return; 286 } 287 288 if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", 289 &mac_addr[0], &mac_addr[1], &mac_addr[2], 290 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { 291 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n", 292 hwinfo); 293 eth_hw_addr_random(nn->dp.netdev); 294 return; 295 } 296 297 ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr); 298 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr); 299 } 300 301 static int 302 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 303 { 304 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id]; 305 struct nfp_abm *abm = app->priv; 306 struct nfp_abm_link *alink; 307 int err; 308 309 alink = kzalloc(sizeof(*alink), GFP_KERNEL); 310 if (!alink) 311 return -ENOMEM; 312 nn->app_priv = alink; 313 alink->abm = abm; 314 alink->vnic = nn; 315 alink->id = id; 316 alink->total_queues = alink->vnic->max_rx_rings; 317 318 err = nfp_abm_ctrl_read_params(alink); 319 if (err) 320 goto err_free_alink; 321 322 /* This is a multi-host app, make sure MAC/PHY is up, but don't 323 * make the MAC/PHY state follow the state of any of the ports. 324 */ 325 err = nfp_eth_set_configured(app->cpp, eth_port->index, true); 326 if (err < 0) 327 goto err_free_alink; 328 329 netif_keep_dst(nn->dp.netdev); 330 331 nfp_abm_vnic_set_mac(app->pf, abm, nn, id); 332 INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL); 333 334 return 0; 335 336 err_free_alink: 337 kfree(alink); 338 return err; 339 } 340 341 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) 342 { 343 struct nfp_abm_link *alink = nn->app_priv; 344 345 nfp_abm_kill_reprs(alink->abm, alink); 346 WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n"); 347 kfree(alink); 348 } 349 350 static u64 * 351 nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data) 352 { 353 struct nfp_repr *repr = netdev_priv(port->netdev); 354 struct nfp_abm_link *alink; 355 unsigned int i; 356 357 if (port->type != NFP_PORT_PF_PORT) 358 return data; 359 alink = repr->app_priv; 360 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 361 *data++ = nfp_abm_ctrl_stat_non_sto(alink, i); 362 *data++ = nfp_abm_ctrl_stat_sto(alink, i); 363 } 364 return data; 365 } 366 367 static int 368 nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port) 369 { 370 struct nfp_repr *repr = netdev_priv(port->netdev); 371 struct nfp_abm_link *alink; 372 373 if (port->type != NFP_PORT_PF_PORT) 374 return 0; 375 alink = repr->app_priv; 376 return alink->vnic->dp.num_r_vecs * 2; 377 } 378 379 static u8 * 380 nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port, 381 u8 *data) 382 { 383 struct nfp_repr *repr = netdev_priv(port->netdev); 384 struct nfp_abm_link *alink; 385 unsigned int i; 386 387 if (port->type != NFP_PORT_PF_PORT) 388 return data; 389 alink = repr->app_priv; 390 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 391 data = nfp_pr_et(data, "q%u_no_wait", i); 392 data = nfp_pr_et(data, "q%u_delayed", i); 393 } 394 return data; 395 } 396 397 static int nfp_abm_init(struct nfp_app *app) 398 { 399 struct nfp_pf *pf = app->pf; 400 struct nfp_reprs *reprs; 401 struct nfp_abm *abm; 402 unsigned int i; 403 int err; 404 405 if (!pf->eth_tbl) { 406 nfp_err(pf->cpp, "ABM NIC requires ETH table\n"); 407 return -EINVAL; 408 } 409 if (pf->max_data_vnics != pf->eth_tbl->count) { 410 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 411 pf->max_data_vnics, pf->eth_tbl->count); 412 return -EINVAL; 413 } 414 if (!pf->mac_stats_bar) { 415 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n"); 416 return -EINVAL; 417 } 418 419 abm = kzalloc(sizeof(*abm), GFP_KERNEL); 420 if (!abm) 421 return -ENOMEM; 422 app->priv = abm; 423 abm->app = app; 424 425 err = nfp_abm_ctrl_find_addrs(abm); 426 if (err) 427 goto err_free_abm; 428 429 err = -ENOMEM; 430 abm->num_thresholds = array_size(abm->num_bands, NFP_NET_MAX_RX_RINGS); 431 abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL); 432 if (!abm->threshold_undef) 433 goto err_free_abm; 434 435 abm->thresholds = kvcalloc(abm->num_thresholds, 436 sizeof(*abm->thresholds), GFP_KERNEL); 437 if (!abm->thresholds) 438 goto err_free_thresh_umap; 439 for (i = 0; i < abm->num_bands * NFP_NET_MAX_RX_RINGS; i++) 440 __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); 441 442 /* We start in legacy mode, make sure advanced queuing is disabled */ 443 err = nfp_abm_ctrl_qm_disable(abm); 444 if (err) 445 goto err_free_thresh; 446 447 err = -ENOMEM; 448 reprs = nfp_reprs_alloc(pf->max_data_vnics); 449 if (!reprs) 450 goto err_free_thresh; 451 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); 452 453 reprs = nfp_reprs_alloc(pf->max_data_vnics); 454 if (!reprs) 455 goto err_free_phys; 456 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs); 457 458 return 0; 459 460 err_free_phys: 461 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 462 err_free_thresh: 463 kvfree(abm->thresholds); 464 err_free_thresh_umap: 465 bitmap_free(abm->threshold_undef); 466 err_free_abm: 467 kfree(abm); 468 app->priv = NULL; 469 return err; 470 } 471 472 static void nfp_abm_clean(struct nfp_app *app) 473 { 474 struct nfp_abm *abm = app->priv; 475 476 nfp_abm_eswitch_clean_up(abm); 477 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 478 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 479 bitmap_free(abm->threshold_undef); 480 kvfree(abm->thresholds); 481 kfree(abm); 482 app->priv = NULL; 483 } 484 485 const struct nfp_app_type app_abm = { 486 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC, 487 .name = "abm", 488 489 .init = nfp_abm_init, 490 .clean = nfp_abm_clean, 491 492 .vnic_alloc = nfp_abm_vnic_alloc, 493 .vnic_free = nfp_abm_vnic_free, 494 495 .port_get_stats = nfp_abm_port_get_stats, 496 .port_get_stats_count = nfp_abm_port_get_stats_count, 497 .port_get_stats_strings = nfp_abm_port_get_stats_strings, 498 499 .setup_tc = nfp_abm_setup_tc, 500 501 .eswitch_mode_get = nfp_abm_eswitch_mode_get, 502 .eswitch_mode_set = nfp_abm_eswitch_mode_set, 503 504 .repr_get = nfp_abm_repr_get, 505 }; 506