1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/etherdevice.h> 7 #include <linux/lockdep.h> 8 #include <linux/netdevice.h> 9 #include <linux/rcupdate.h> 10 #include <linux/rtnetlink.h> 11 #include <linux/slab.h> 12 13 #include "../nfpcore/nfp.h" 14 #include "../nfpcore/nfp_cpp.h" 15 #include "../nfpcore/nfp_nsp.h" 16 #include "../nfp_app.h" 17 #include "../nfp_main.h" 18 #include "../nfp_net.h" 19 #include "../nfp_net_repr.h" 20 #include "../nfp_port.h" 21 #include "main.h" 22 23 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) 24 { 25 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) | 26 FIELD_PREP(NFP_ABM_PORTID_ID, id); 27 } 28 29 static int 30 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, 31 enum tc_setup_type type, void *type_data) 32 { 33 struct nfp_repr *repr = netdev_priv(netdev); 34 struct nfp_port *port; 35 36 port = nfp_port_from_netdev(netdev); 37 if (!port || port->type != NFP_PORT_PF_PORT) 38 return -EOPNOTSUPP; 39 40 switch (type) { 41 case TC_SETUP_ROOT_QDISC: 42 return nfp_abm_setup_root(netdev, repr->app_priv, type_data); 43 case TC_SETUP_QDISC_MQ: 44 return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data); 45 case TC_SETUP_QDISC_RED: 46 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data); 47 default: 48 return -EOPNOTSUPP; 49 } 50 } 51 52 static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id) 53 { 54 enum nfp_repr_type rtype; 55 struct nfp_reprs *reprs; 56 u8 port; 57 58 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id); 59 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id); 60 61 reprs = rcu_dereference(app->reprs[rtype]); 62 if (!reprs) 63 return NULL; 64 65 if (port >= reprs->num_reprs) 66 return NULL; 67 68 return rcu_dereference(reprs->reprs[port]); 69 } 70 71 static int 72 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink, 73 enum nfp_port_type ptype) 74 { 75 struct net_device *netdev; 76 enum nfp_repr_type rtype; 77 struct nfp_reprs *reprs; 78 struct nfp_repr *repr; 79 struct nfp_port *port; 80 unsigned int txqs; 81 int err; 82 83 if (ptype == NFP_PORT_PHYS_PORT) { 84 rtype = NFP_REPR_TYPE_PHYS_PORT; 85 txqs = 1; 86 } else { 87 rtype = NFP_REPR_TYPE_PF; 88 txqs = alink->vnic->max_rx_rings; 89 } 90 91 netdev = nfp_repr_alloc_mqs(app, txqs, 1); 92 if (!netdev) 93 return -ENOMEM; 94 repr = netdev_priv(netdev); 95 repr->app_priv = alink; 96 97 port = nfp_port_alloc(app, ptype, netdev); 98 if (IS_ERR(port)) { 99 err = PTR_ERR(port); 100 goto err_free_repr; 101 } 102 103 if (ptype == NFP_PORT_PHYS_PORT) { 104 port->eth_forced = true; 105 err = nfp_port_init_phy_port(app->pf, app, port, alink->id); 106 if (err) 107 goto err_free_port; 108 } else { 109 port->pf_id = alink->abm->pf_id; 110 port->pf_split = app->pf->max_data_vnics > 1; 111 port->pf_split_id = alink->id; 112 port->vnic = alink->vnic->dp.ctrl_bar; 113 } 114 115 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev); 116 eth_hw_addr_random(netdev); 117 118 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id), 119 port, alink->vnic->dp.netdev); 120 if (err) 121 goto err_free_port; 122 123 reprs = nfp_reprs_get_locked(app, rtype); 124 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr"); 125 rcu_assign_pointer(reprs->reprs[alink->id], netdev); 126 127 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n", 128 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys", 129 alink->id, netdev->name); 130 131 return 0; 132 133 err_free_port: 134 nfp_port_free(port); 135 err_free_repr: 136 nfp_repr_free(netdev); 137 return err; 138 } 139 140 static void 141 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink, 142 enum nfp_repr_type rtype) 143 { 144 struct net_device *netdev; 145 struct nfp_reprs *reprs; 146 147 reprs = nfp_reprs_get_locked(app, rtype); 148 netdev = nfp_repr_get_locked(app, reprs, alink->id); 149 if (!netdev) 150 return; 151 rcu_assign_pointer(reprs->reprs[alink->id], NULL); 152 synchronize_rcu(); 153 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */ 154 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev)); 155 } 156 157 static void 158 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink) 159 { 160 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF); 161 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT); 162 } 163 164 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm) 165 { 166 struct nfp_pf *pf = abm->app->pf; 167 struct nfp_net *nn; 168 169 list_for_each_entry(nn, &pf->vnics, vnic_list) 170 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv); 171 } 172 173 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app) 174 { 175 struct nfp_abm *abm = app->priv; 176 177 return abm->eswitch_mode; 178 } 179 180 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm) 181 { 182 nfp_abm_kill_reprs_all(abm); 183 nfp_abm_ctrl_qm_disable(abm); 184 185 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY; 186 return 0; 187 } 188 189 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm) 190 { 191 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY) 192 WARN_ON(nfp_abm_eswitch_set_legacy(abm)); 193 } 194 195 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm) 196 { 197 struct nfp_app *app = abm->app; 198 struct nfp_pf *pf = app->pf; 199 struct nfp_net *nn; 200 int err; 201 202 err = nfp_abm_ctrl_qm_enable(abm); 203 if (err) 204 return err; 205 206 list_for_each_entry(nn, &pf->vnics, vnic_list) { 207 struct nfp_abm_link *alink = nn->app_priv; 208 209 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT); 210 if (err) 211 goto err_kill_all_reprs; 212 213 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT); 214 if (err) 215 goto err_kill_all_reprs; 216 } 217 218 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 219 return 0; 220 221 err_kill_all_reprs: 222 nfp_abm_kill_reprs_all(abm); 223 nfp_abm_ctrl_qm_disable(abm); 224 return err; 225 } 226 227 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode) 228 { 229 struct nfp_abm *abm = app->priv; 230 231 if (abm->eswitch_mode == mode) 232 return 0; 233 234 switch (mode) { 235 case DEVLINK_ESWITCH_MODE_LEGACY: 236 return nfp_abm_eswitch_set_legacy(abm); 237 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 238 return nfp_abm_eswitch_set_switchdev(abm); 239 default: 240 return -EINVAL; 241 } 242 } 243 244 static void 245 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn, 246 unsigned int id) 247 { 248 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id]; 249 u8 mac_addr[ETH_ALEN]; 250 struct nfp_nsp *nsp; 251 char hwinfo[32]; 252 int err; 253 254 if (id > pf->eth_tbl->count) { 255 nfp_warn(pf->cpp, "No entry for persistent MAC address\n"); 256 eth_hw_addr_random(nn->dp.netdev); 257 return; 258 } 259 260 snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac.pf%u", 261 eth_port->eth_index, abm->pf_id); 262 263 nsp = nfp_nsp_open(pf->cpp); 264 if (IS_ERR(nsp)) { 265 nfp_warn(pf->cpp, "Failed to access the NSP for persistent MAC address: %ld\n", 266 PTR_ERR(nsp)); 267 eth_hw_addr_random(nn->dp.netdev); 268 return; 269 } 270 271 if (!nfp_nsp_has_hwinfo_lookup(nsp)) { 272 nfp_warn(pf->cpp, "NSP doesn't support PF MAC generation\n"); 273 eth_hw_addr_random(nn->dp.netdev); 274 return; 275 } 276 277 err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo)); 278 nfp_nsp_close(nsp); 279 if (err) { 280 nfp_warn(pf->cpp, "Reading persistent MAC address failed: %d\n", 281 err); 282 eth_hw_addr_random(nn->dp.netdev); 283 return; 284 } 285 286 if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", 287 &mac_addr[0], &mac_addr[1], &mac_addr[2], 288 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { 289 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n", 290 hwinfo); 291 eth_hw_addr_random(nn->dp.netdev); 292 return; 293 } 294 295 ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr); 296 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr); 297 } 298 299 static int 300 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 301 { 302 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id]; 303 struct nfp_abm *abm = app->priv; 304 struct nfp_abm_link *alink; 305 int err; 306 307 alink = kzalloc(sizeof(*alink), GFP_KERNEL); 308 if (!alink) 309 return -ENOMEM; 310 nn->app_priv = alink; 311 alink->abm = abm; 312 alink->vnic = nn; 313 alink->id = id; 314 alink->total_queues = alink->vnic->max_rx_rings; 315 316 /* This is a multi-host app, make sure MAC/PHY is up, but don't 317 * make the MAC/PHY state follow the state of any of the ports. 318 */ 319 err = nfp_eth_set_configured(app->cpp, eth_port->index, true); 320 if (err < 0) 321 goto err_free_alink; 322 323 netif_keep_dst(nn->dp.netdev); 324 325 nfp_abm_vnic_set_mac(app->pf, abm, nn, id); 326 nfp_abm_ctrl_read_params(alink); 327 INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL); 328 329 return 0; 330 331 err_free_alink: 332 kfree(alink); 333 return err; 334 } 335 336 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) 337 { 338 struct nfp_abm_link *alink = nn->app_priv; 339 340 nfp_abm_kill_reprs(alink->abm, alink); 341 WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n"); 342 kfree(alink); 343 } 344 345 static u64 * 346 nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data) 347 { 348 struct nfp_repr *repr = netdev_priv(port->netdev); 349 struct nfp_abm_link *alink; 350 unsigned int i; 351 352 if (port->type != NFP_PORT_PF_PORT) 353 return data; 354 alink = repr->app_priv; 355 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 356 *data++ = nfp_abm_ctrl_stat_non_sto(alink, i); 357 *data++ = nfp_abm_ctrl_stat_sto(alink, i); 358 } 359 return data; 360 } 361 362 static int 363 nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port) 364 { 365 struct nfp_repr *repr = netdev_priv(port->netdev); 366 struct nfp_abm_link *alink; 367 368 if (port->type != NFP_PORT_PF_PORT) 369 return 0; 370 alink = repr->app_priv; 371 return alink->vnic->dp.num_r_vecs * 2; 372 } 373 374 static u8 * 375 nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port, 376 u8 *data) 377 { 378 struct nfp_repr *repr = netdev_priv(port->netdev); 379 struct nfp_abm_link *alink; 380 unsigned int i; 381 382 if (port->type != NFP_PORT_PF_PORT) 383 return data; 384 alink = repr->app_priv; 385 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 386 data = nfp_pr_et(data, "q%u_no_wait", i); 387 data = nfp_pr_et(data, "q%u_delayed", i); 388 } 389 return data; 390 } 391 392 static int nfp_abm_init(struct nfp_app *app) 393 { 394 struct nfp_pf *pf = app->pf; 395 struct nfp_reprs *reprs; 396 struct nfp_abm *abm; 397 unsigned int i; 398 int err; 399 400 if (!pf->eth_tbl) { 401 nfp_err(pf->cpp, "ABM NIC requires ETH table\n"); 402 return -EINVAL; 403 } 404 if (pf->max_data_vnics != pf->eth_tbl->count) { 405 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 406 pf->max_data_vnics, pf->eth_tbl->count); 407 return -EINVAL; 408 } 409 if (!pf->mac_stats_bar) { 410 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n"); 411 return -EINVAL; 412 } 413 414 abm = kzalloc(sizeof(*abm), GFP_KERNEL); 415 if (!abm) 416 return -ENOMEM; 417 app->priv = abm; 418 abm->app = app; 419 420 err = nfp_abm_ctrl_find_addrs(abm); 421 if (err) 422 goto err_free_abm; 423 424 err = -ENOMEM; 425 abm->num_thresholds = array_size(abm->num_bands, NFP_NET_MAX_RX_RINGS); 426 abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL); 427 if (!abm->threshold_undef) 428 goto err_free_abm; 429 430 abm->thresholds = kvcalloc(abm->num_thresholds, 431 sizeof(*abm->thresholds), GFP_KERNEL); 432 if (!abm->thresholds) 433 goto err_free_thresh_umap; 434 for (i = 0; i < abm->num_bands * NFP_NET_MAX_RX_RINGS; i++) 435 __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); 436 437 /* We start in legacy mode, make sure advanced queuing is disabled */ 438 err = nfp_abm_ctrl_qm_disable(abm); 439 if (err) 440 goto err_free_thresh; 441 442 err = -ENOMEM; 443 reprs = nfp_reprs_alloc(pf->max_data_vnics); 444 if (!reprs) 445 goto err_free_thresh; 446 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); 447 448 reprs = nfp_reprs_alloc(pf->max_data_vnics); 449 if (!reprs) 450 goto err_free_phys; 451 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs); 452 453 return 0; 454 455 err_free_phys: 456 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 457 err_free_thresh: 458 kvfree(abm->thresholds); 459 err_free_thresh_umap: 460 bitmap_free(abm->threshold_undef); 461 err_free_abm: 462 kfree(abm); 463 app->priv = NULL; 464 return err; 465 } 466 467 static void nfp_abm_clean(struct nfp_app *app) 468 { 469 struct nfp_abm *abm = app->priv; 470 471 nfp_abm_eswitch_clean_up(abm); 472 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 473 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 474 bitmap_free(abm->threshold_undef); 475 kvfree(abm->thresholds); 476 kfree(abm); 477 app->priv = NULL; 478 } 479 480 const struct nfp_app_type app_abm = { 481 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC, 482 .name = "abm", 483 484 .init = nfp_abm_init, 485 .clean = nfp_abm_clean, 486 487 .vnic_alloc = nfp_abm_vnic_alloc, 488 .vnic_free = nfp_abm_vnic_free, 489 490 .port_get_stats = nfp_abm_port_get_stats, 491 .port_get_stats_count = nfp_abm_port_get_stats_count, 492 .port_get_stats_strings = nfp_abm_port_get_stats_strings, 493 494 .setup_tc = nfp_abm_setup_tc, 495 496 .eswitch_mode_get = nfp_abm_eswitch_mode_get, 497 .eswitch_mode_set = nfp_abm_eswitch_mode_set, 498 499 .repr_get = nfp_abm_repr_get, 500 }; 501