196de2506SJakub Kicinski // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
296de2506SJakub Kicinski /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
38aa0cb00SJakub Kicinski
4bb45e51cSJakub Kicinski #include <net/pkt_cls.h>
5bb45e51cSJakub Kicinski
68aa0cb00SJakub Kicinski #include "../nfpcore/nfp_cpp.h"
777a844eeSJakub Kicinski #include "../nfpcore/nfp_nffw.h"
8e3ac6c07SJakub Kicinski #include "../nfpcore/nfp_nsp.h"
98aa0cb00SJakub Kicinski #include "../nfp_app.h"
108aa0cb00SJakub Kicinski #include "../nfp_main.h"
118aa0cb00SJakub Kicinski #include "../nfp_net.h"
128aa0cb00SJakub Kicinski #include "../nfp_port.h"
130d49eaf4SJakub Kicinski #include "fw.h"
14bb45e51cSJakub Kicinski #include "main.h"
15bb45e51cSJakub Kicinski
16630a4d38SJakub Kicinski const struct rhashtable_params nfp_bpf_maps_neutral_params = {
17630a4d38SJakub Kicinski .nelem_hint = 4,
18c593642cSPankaj Bharadiya .key_len = sizeof_field(struct bpf_map, id),
19ab01f4acSJakub Kicinski .key_offset = offsetof(struct nfp_bpf_neutral_map, map_id),
20630a4d38SJakub Kicinski .head_offset = offsetof(struct nfp_bpf_neutral_map, l),
21630a4d38SJakub Kicinski .automatic_shrinking = true,
22630a4d38SJakub Kicinski };
23630a4d38SJakub Kicinski
nfp_net_ebpf_capable(struct nfp_net * nn)24bb45e51cSJakub Kicinski static bool nfp_net_ebpf_capable(struct nfp_net *nn)
25bb45e51cSJakub Kicinski {
260f6cf4ddSJakub Kicinski #ifdef __LITTLE_ENDIAN
2728264eb2SJakub Kicinski struct nfp_app_bpf *bpf = nn->app->priv;
2828264eb2SJakub Kicinski
2928264eb2SJakub Kicinski return nn->cap & NFP_NET_CFG_CTRL_BPF &&
3028264eb2SJakub Kicinski bpf->abi_version &&
3128264eb2SJakub Kicinski nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version;
3228264eb2SJakub Kicinski #else
33bb45e51cSJakub Kicinski return false;
3428264eb2SJakub Kicinski #endif
35bb45e51cSJakub Kicinski }
36bb45e51cSJakub Kicinski
37bb45e51cSJakub Kicinski static int
nfp_bpf_xdp_offload(struct nfp_app * app,struct nfp_net * nn,struct bpf_prog * prog,struct netlink_ext_ack * extack)38bb45e51cSJakub Kicinski nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
39acc2abbbSQuentin Monnet struct bpf_prog *prog, struct netlink_ext_ack *extack)
40bb45e51cSJakub Kicinski {
419ce7a956SJakub Kicinski bool running, xdp_running;
42bb45e51cSJakub Kicinski
43bb45e51cSJakub Kicinski if (!nfp_net_ebpf_capable(nn))
44bb45e51cSJakub Kicinski return -EINVAL;
45bb45e51cSJakub Kicinski
469ce7a956SJakub Kicinski running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
475f428401SJakub Kicinski xdp_running = running && nn->xdp_hw.prog;
48bb45e51cSJakub Kicinski
499ce7a956SJakub Kicinski if (!prog && !xdp_running)
509ce7a956SJakub Kicinski return 0;
519ce7a956SJakub Kicinski if (prog && running && !xdp_running)
529ce7a956SJakub Kicinski return -EBUSY;
539ce7a956SJakub Kicinski
545f428401SJakub Kicinski return nfp_net_bpf_offload(nn, prog, running, extack);
55bb45e51cSJakub Kicinski }
56bb45e51cSJakub Kicinski
nfp_bpf_extra_cap(struct nfp_app * app,struct nfp_net * nn)57bb45e51cSJakub Kicinski static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
58bb45e51cSJakub Kicinski {
59bb45e51cSJakub Kicinski return nfp_net_ebpf_capable(nn) ? "BPF" : "";
60bb45e51cSJakub Kicinski }
618aa0cb00SJakub Kicinski
624f83435aSJakub Kicinski static int
nfp_bpf_vnic_alloc(struct nfp_app * app,struct nfp_net * nn,unsigned int id)634f83435aSJakub Kicinski nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
644f83435aSJakub Kicinski {
65e3ac6c07SJakub Kicinski struct nfp_pf *pf = app->pf;
662314fe9eSJakub Kicinski struct nfp_bpf_vnic *bv;
674f83435aSJakub Kicinski int err;
684f83435aSJakub Kicinski
69e3ac6c07SJakub Kicinski if (!pf->eth_tbl) {
70e3ac6c07SJakub Kicinski nfp_err(pf->cpp, "No ETH table\n");
71e3ac6c07SJakub Kicinski return -EINVAL;
72e3ac6c07SJakub Kicinski }
73e3ac6c07SJakub Kicinski if (pf->max_data_vnics != pf->eth_tbl->count) {
74e3ac6c07SJakub Kicinski nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
75e3ac6c07SJakub Kicinski pf->max_data_vnics, pf->eth_tbl->count);
76e3ac6c07SJakub Kicinski return -EINVAL;
77e3ac6c07SJakub Kicinski }
78e3ac6c07SJakub Kicinski
792314fe9eSJakub Kicinski bv = kzalloc(sizeof(*bv), GFP_KERNEL);
802314fe9eSJakub Kicinski if (!bv)
814f83435aSJakub Kicinski return -ENOMEM;
822314fe9eSJakub Kicinski nn->app_priv = bv;
834f83435aSJakub Kicinski
844f83435aSJakub Kicinski err = nfp_app_nic_vnic_alloc(app, nn, id);
854f83435aSJakub Kicinski if (err)
864f83435aSJakub Kicinski goto err_free_priv;
874f83435aSJakub Kicinski
882314fe9eSJakub Kicinski bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
892314fe9eSJakub Kicinski bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
902314fe9eSJakub Kicinski
914f83435aSJakub Kicinski return 0;
924f83435aSJakub Kicinski err_free_priv:
934f83435aSJakub Kicinski kfree(nn->app_priv);
944f83435aSJakub Kicinski return err;
954f83435aSJakub Kicinski }
964f83435aSJakub Kicinski
nfp_bpf_vnic_free(struct nfp_app * app,struct nfp_net * nn)974f83435aSJakub Kicinski static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
984f83435aSJakub Kicinski {
994f83435aSJakub Kicinski struct nfp_bpf_vnic *bv = nn->app_priv;
1004f83435aSJakub Kicinski
1014f83435aSJakub Kicinski WARN_ON(bv->tc_prog);
1024f83435aSJakub Kicinski kfree(bv);
1034f83435aSJakub Kicinski }
1044f83435aSJakub Kicinski
nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)10590d97315SJiri Pirko static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
10690d97315SJiri Pirko void *type_data, void *cb_priv)
107bb45e51cSJakub Kicinski {
108de4784caSJiri Pirko struct tc_cls_bpf_offload *cls_bpf = type_data;
10990d97315SJiri Pirko struct nfp_net *nn = cb_priv;
110d3f89b98SJakub Kicinski struct bpf_prog *oldprog;
111d3f89b98SJakub Kicinski struct nfp_bpf_vnic *bv;
112d3f89b98SJakub Kicinski int err;
113bb45e51cSJakub Kicinski
11452be9a7cSQuentin Monnet if (type != TC_SETUP_CLSBPF) {
11552be9a7cSQuentin Monnet NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
11652be9a7cSQuentin Monnet "only offload of BPF classifiers supported");
11752be9a7cSQuentin Monnet return -EOPNOTSUPP;
11852be9a7cSQuentin Monnet }
1193107fdc8SJakub Kicinski if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
12052be9a7cSQuentin Monnet return -EOPNOTSUPP;
12152be9a7cSQuentin Monnet if (!nfp_net_ebpf_capable(nn)) {
12252be9a7cSQuentin Monnet NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
12352be9a7cSQuentin Monnet "NFP firmware does not support eBPF offload");
12452be9a7cSQuentin Monnet return -EOPNOTSUPP;
12552be9a7cSQuentin Monnet }
12652be9a7cSQuentin Monnet if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
12752be9a7cSQuentin Monnet NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
12852be9a7cSQuentin Monnet "only ETH_P_ALL supported as filter protocol");
12952be9a7cSQuentin Monnet return -EOPNOTSUPP;
13052be9a7cSQuentin Monnet }
131f449657fSJakub Kicinski
132012bb8a8SJakub Kicinski /* Only support TC direct action */
133012bb8a8SJakub Kicinski if (!cls_bpf->exts_integrated ||
134012bb8a8SJakub Kicinski tcf_exts_has_actions(cls_bpf->exts)) {
13552be9a7cSQuentin Monnet NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
13652be9a7cSQuentin Monnet "only direct action with no legacy actions supported");
137012bb8a8SJakub Kicinski return -EOPNOTSUPP;
138012bb8a8SJakub Kicinski }
139012bb8a8SJakub Kicinski
140102740bdSJakub Kicinski if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
14190d97315SJiri Pirko return -EOPNOTSUPP;
142102740bdSJakub Kicinski
143d3f89b98SJakub Kicinski bv = nn->app_priv;
144d3f89b98SJakub Kicinski oldprog = cls_bpf->oldprog;
145d3f89b98SJakub Kicinski
146d3f89b98SJakub Kicinski /* Don't remove if oldprog doesn't match driver's state */
147d3f89b98SJakub Kicinski if (bv->tc_prog != oldprog) {
148d3f89b98SJakub Kicinski oldprog = NULL;
149d3f89b98SJakub Kicinski if (!cls_bpf->prog)
150d3f89b98SJakub Kicinski return 0;
15190d97315SJiri Pirko }
152d3f89b98SJakub Kicinski
15352be9a7cSQuentin Monnet err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
15452be9a7cSQuentin Monnet cls_bpf->common.extack);
155d3f89b98SJakub Kicinski if (err)
156d3f89b98SJakub Kicinski return err;
157d3f89b98SJakub Kicinski
158d3f89b98SJakub Kicinski bv->tc_prog = cls_bpf->prog;
159d692403eSJakub Kicinski nn->port->tc_offload_cnt = !!bv->tc_prog;
160d3f89b98SJakub Kicinski return 0;
16190d97315SJiri Pirko }
16290d97315SJiri Pirko
163955bcb6eSPablo Neira Ayuso static LIST_HEAD(nfp_bpf_block_cb_list);
164955bcb6eSPablo Neira Ayuso
nfp_bpf_setup_tc(struct nfp_app * app,struct net_device * netdev,enum tc_setup_type type,void * type_data)16590d97315SJiri Pirko static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
16690d97315SJiri Pirko enum tc_setup_type type, void *type_data)
16790d97315SJiri Pirko {
1684e95bc26SPablo Neira Ayuso struct nfp_net *nn = netdev_priv(netdev);
1694e95bc26SPablo Neira Ayuso
17090d97315SJiri Pirko switch (type) {
17190d97315SJiri Pirko case TC_SETUP_BLOCK:
172955bcb6eSPablo Neira Ayuso return flow_block_cb_setup_simple(type_data,
173955bcb6eSPablo Neira Ayuso &nfp_bpf_block_cb_list,
1744e95bc26SPablo Neira Ayuso nfp_bpf_setup_tc_block_cb,
1754e95bc26SPablo Neira Ayuso nn, nn, true);
17690d97315SJiri Pirko default:
17790d97315SJiri Pirko return -EOPNOTSUPP;
17890d97315SJiri Pirko }
179bb45e51cSJakub Kicinski }
180bb45e51cSJakub Kicinski
1810d49eaf4SJakub Kicinski static int
nfp_bpf_check_mtu(struct nfp_app * app,struct net_device * netdev,int new_mtu)182167cebefSJohn Hurley nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
183ccbdc596SJakub Kicinski {
184ccbdc596SJakub Kicinski struct nfp_net *nn = netdev_priv(netdev);
185*90a881fcSYu Xiao struct nfp_bpf_vnic *bv;
186*90a881fcSYu Xiao struct bpf_prog *prog;
187ccbdc596SJakub Kicinski
188ccbdc596SJakub Kicinski if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
189ccbdc596SJakub Kicinski return 0;
190ccbdc596SJakub Kicinski
191*90a881fcSYu Xiao if (nn->xdp_hw.prog) {
192*90a881fcSYu Xiao prog = nn->xdp_hw.prog;
193*90a881fcSYu Xiao } else {
194*90a881fcSYu Xiao bv = nn->app_priv;
195*90a881fcSYu Xiao prog = bv->tc_prog;
196*90a881fcSYu Xiao }
197*90a881fcSYu Xiao
198*90a881fcSYu Xiao if (nfp_bpf_offload_check_mtu(nn, prog, new_mtu)) {
199*90a881fcSYu Xiao nn_info(nn, "BPF offload active, potential packet access beyond hardware packet boundary");
200ccbdc596SJakub Kicinski return -EBUSY;
201ccbdc596SJakub Kicinski }
202ccbdc596SJakub Kicinski return 0;
203ccbdc596SJakub Kicinski }
204ccbdc596SJakub Kicinski
205ccbdc596SJakub Kicinski static int
nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)2060d49eaf4SJakub Kicinski nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
2070d49eaf4SJakub Kicinski u32 length)
2080d49eaf4SJakub Kicinski {
2090d49eaf4SJakub Kicinski struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
2100d49eaf4SJakub Kicinski struct nfp_cpp *cpp = bpf->app->pf->cpp;
2110d49eaf4SJakub Kicinski
2120d49eaf4SJakub Kicinski if (length < sizeof(*cap)) {
2130d49eaf4SJakub Kicinski nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
2140d49eaf4SJakub Kicinski return -EINVAL;
2150d49eaf4SJakub Kicinski }
2160d49eaf4SJakub Kicinski
2170d49eaf4SJakub Kicinski bpf->adjust_head.flags = readl(&cap->flags);
2180d49eaf4SJakub Kicinski bpf->adjust_head.off_min = readl(&cap->off_min);
2190d49eaf4SJakub Kicinski bpf->adjust_head.off_max = readl(&cap->off_max);
2208231f844SJakub Kicinski bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
2218231f844SJakub Kicinski bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
2220d49eaf4SJakub Kicinski
2230d49eaf4SJakub Kicinski if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
2240d49eaf4SJakub Kicinski nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
2250d49eaf4SJakub Kicinski return -EINVAL;
2260d49eaf4SJakub Kicinski }
2270d49eaf4SJakub Kicinski if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
2280d49eaf4SJakub Kicinski !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
2290d49eaf4SJakub Kicinski nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
2300d49eaf4SJakub Kicinski memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
2310d49eaf4SJakub Kicinski return 0;
2320d49eaf4SJakub Kicinski }
2330d49eaf4SJakub Kicinski
2340d49eaf4SJakub Kicinski return 0;
2350d49eaf4SJakub Kicinski }
2360d49eaf4SJakub Kicinski
2379d080d5dSJakub Kicinski static int
nfp_bpf_parse_cap_func(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)2389d080d5dSJakub Kicinski nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
2399d080d5dSJakub Kicinski {
2409d080d5dSJakub Kicinski struct nfp_bpf_cap_tlv_func __iomem *cap = value;
2419d080d5dSJakub Kicinski
2429d080d5dSJakub Kicinski if (length < sizeof(*cap)) {
2439d080d5dSJakub Kicinski nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
2449d080d5dSJakub Kicinski return -EINVAL;
2459d080d5dSJakub Kicinski }
2469d080d5dSJakub Kicinski
2479d080d5dSJakub Kicinski switch (readl(&cap->func_id)) {
2489d080d5dSJakub Kicinski case BPF_FUNC_map_lookup_elem:
2499d080d5dSJakub Kicinski bpf->helpers.map_lookup = readl(&cap->func_addr);
2509d080d5dSJakub Kicinski break;
25144d65a47SJakub Kicinski case BPF_FUNC_map_update_elem:
25244d65a47SJakub Kicinski bpf->helpers.map_update = readl(&cap->func_addr);
25344d65a47SJakub Kicinski break;
254bfee64deSJakub Kicinski case BPF_FUNC_map_delete_elem:
255bfee64deSJakub Kicinski bpf->helpers.map_delete = readl(&cap->func_addr);
256bfee64deSJakub Kicinski break;
2579816dd35SJakub Kicinski case BPF_FUNC_perf_event_output:
2589816dd35SJakub Kicinski bpf->helpers.perf_event_output = readl(&cap->func_addr);
2599816dd35SJakub Kicinski break;
2609d080d5dSJakub Kicinski }
2619d080d5dSJakub Kicinski
2629d080d5dSJakub Kicinski return 0;
2639d080d5dSJakub Kicinski }
2649d080d5dSJakub Kicinski
2659d080d5dSJakub Kicinski static int
nfp_bpf_parse_cap_maps(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)2669d080d5dSJakub Kicinski nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
2679d080d5dSJakub Kicinski {
2689d080d5dSJakub Kicinski struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
2699d080d5dSJakub Kicinski
2709d080d5dSJakub Kicinski if (length < sizeof(*cap)) {
2719d080d5dSJakub Kicinski nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
2729d080d5dSJakub Kicinski return -EINVAL;
2739d080d5dSJakub Kicinski }
2749d080d5dSJakub Kicinski
2759d080d5dSJakub Kicinski bpf->maps.types = readl(&cap->types);
2769d080d5dSJakub Kicinski bpf->maps.max_maps = readl(&cap->max_maps);
2779d080d5dSJakub Kicinski bpf->maps.max_elems = readl(&cap->max_elems);
2789d080d5dSJakub Kicinski bpf->maps.max_key_sz = readl(&cap->max_key_sz);
2799d080d5dSJakub Kicinski bpf->maps.max_val_sz = readl(&cap->max_val_sz);
2809d080d5dSJakub Kicinski bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
2819d080d5dSJakub Kicinski
2829d080d5dSJakub Kicinski return 0;
2839d080d5dSJakub Kicinski }
2849d080d5dSJakub Kicinski
285df4a37d8SJakub Kicinski static int
nfp_bpf_parse_cap_random(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)286df4a37d8SJakub Kicinski nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
287df4a37d8SJakub Kicinski u32 length)
288df4a37d8SJakub Kicinski {
289df4a37d8SJakub Kicinski bpf->pseudo_random = true;
290df4a37d8SJakub Kicinski return 0;
291df4a37d8SJakub Kicinski }
292df4a37d8SJakub Kicinski
293d985888fSJakub Kicinski static int
nfp_bpf_parse_cap_qsel(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)294d985888fSJakub Kicinski nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
295d985888fSJakub Kicinski {
296d985888fSJakub Kicinski bpf->queue_select = true;
297d985888fSJakub Kicinski return 0;
298d985888fSJakub Kicinski }
299d985888fSJakub Kicinski
3000c261593SJakub Kicinski static int
nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)3010c261593SJakub Kicinski nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value,
3020c261593SJakub Kicinski u32 length)
3030c261593SJakub Kicinski {
3040c261593SJakub Kicinski bpf->adjust_tail = true;
3050c261593SJakub Kicinski return 0;
3060c261593SJakub Kicinski }
3070c261593SJakub Kicinski
30828264eb2SJakub Kicinski static int
nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)309f24e2909SJakub Kicinski nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf *bpf, void __iomem *value,
310f24e2909SJakub Kicinski u32 length)
311f24e2909SJakub Kicinski {
312f24e2909SJakub Kicinski bpf->cmsg_multi_ent = true;
313f24e2909SJakub Kicinski return 0;
314f24e2909SJakub Kicinski }
315f24e2909SJakub Kicinski
316f24e2909SJakub Kicinski static int
nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf * bpf,void __iomem * value,u32 length)31728264eb2SJakub Kicinski nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value,
31828264eb2SJakub Kicinski u32 length)
31928264eb2SJakub Kicinski {
32028264eb2SJakub Kicinski if (length < 4) {
32128264eb2SJakub Kicinski nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n",
32228264eb2SJakub Kicinski length);
32328264eb2SJakub Kicinski return -EINVAL;
32428264eb2SJakub Kicinski }
32528264eb2SJakub Kicinski
32628264eb2SJakub Kicinski bpf->abi_version = readl(value);
3270c9864c0SJakub Kicinski if (bpf->abi_version < 2 || bpf->abi_version > 3) {
32828264eb2SJakub Kicinski nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n",
32928264eb2SJakub Kicinski bpf->abi_version);
33028264eb2SJakub Kicinski bpf->abi_version = 0;
33128264eb2SJakub Kicinski }
33228264eb2SJakub Kicinski
33328264eb2SJakub Kicinski return 0;
33428264eb2SJakub Kicinski }
33528264eb2SJakub Kicinski
nfp_bpf_parse_capabilities(struct nfp_app * app)33677a844eeSJakub Kicinski static int nfp_bpf_parse_capabilities(struct nfp_app *app)
33777a844eeSJakub Kicinski {
33877a844eeSJakub Kicinski struct nfp_cpp *cpp = app->pf->cpp;
33977a844eeSJakub Kicinski struct nfp_cpp_area *area;
34077a844eeSJakub Kicinski u8 __iomem *mem, *start;
34177a844eeSJakub Kicinski
34277a844eeSJakub Kicinski mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
34377a844eeSJakub Kicinski 8, &area);
34477a844eeSJakub Kicinski if (IS_ERR(mem))
34577a844eeSJakub Kicinski return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
34677a844eeSJakub Kicinski
34777a844eeSJakub Kicinski start = mem;
34826aeb9daSJakub Kicinski while (mem - start + 8 <= nfp_cpp_area_size(area)) {
3490d49eaf4SJakub Kicinski u8 __iomem *value;
35077a844eeSJakub Kicinski u32 type, length;
35177a844eeSJakub Kicinski
35277a844eeSJakub Kicinski type = readl(mem);
35377a844eeSJakub Kicinski length = readl(mem + 4);
3540d49eaf4SJakub Kicinski value = mem + 8;
35577a844eeSJakub Kicinski
35677a844eeSJakub Kicinski mem += 8 + length;
35777a844eeSJakub Kicinski if (mem - start > nfp_cpp_area_size(area))
35877a844eeSJakub Kicinski goto err_release_free;
35977a844eeSJakub Kicinski
36077a844eeSJakub Kicinski switch (type) {
3619d080d5dSJakub Kicinski case NFP_BPF_CAP_TYPE_FUNC:
3629d080d5dSJakub Kicinski if (nfp_bpf_parse_cap_func(app->priv, value, length))
3639d080d5dSJakub Kicinski goto err_release_free;
3649d080d5dSJakub Kicinski break;
3650d49eaf4SJakub Kicinski case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
3660d49eaf4SJakub Kicinski if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
3670d49eaf4SJakub Kicinski length))
3680d49eaf4SJakub Kicinski goto err_release_free;
3690d49eaf4SJakub Kicinski break;
3709d080d5dSJakub Kicinski case NFP_BPF_CAP_TYPE_MAPS:
3719d080d5dSJakub Kicinski if (nfp_bpf_parse_cap_maps(app->priv, value, length))
3729d080d5dSJakub Kicinski goto err_release_free;
3739d080d5dSJakub Kicinski break;
374df4a37d8SJakub Kicinski case NFP_BPF_CAP_TYPE_RANDOM:
375df4a37d8SJakub Kicinski if (nfp_bpf_parse_cap_random(app->priv, value, length))
376df4a37d8SJakub Kicinski goto err_release_free;
377df4a37d8SJakub Kicinski break;
378d985888fSJakub Kicinski case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
379d985888fSJakub Kicinski if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
380d985888fSJakub Kicinski goto err_release_free;
381d985888fSJakub Kicinski break;
3820c261593SJakub Kicinski case NFP_BPF_CAP_TYPE_ADJUST_TAIL:
3830c261593SJakub Kicinski if (nfp_bpf_parse_cap_adjust_tail(app->priv, value,
3840c261593SJakub Kicinski length))
3850c261593SJakub Kicinski goto err_release_free;
3860c261593SJakub Kicinski break;
38728264eb2SJakub Kicinski case NFP_BPF_CAP_TYPE_ABI_VERSION:
38828264eb2SJakub Kicinski if (nfp_bpf_parse_cap_abi_version(app->priv, value,
38928264eb2SJakub Kicinski length))
39028264eb2SJakub Kicinski goto err_release_free;
39128264eb2SJakub Kicinski break;
392f24e2909SJakub Kicinski case NFP_BPF_CAP_TYPE_CMSG_MULTI_ENT:
393f24e2909SJakub Kicinski if (nfp_bpf_parse_cap_cmsg_multi_ent(app->priv, value,
394f24e2909SJakub Kicinski length))
395f24e2909SJakub Kicinski goto err_release_free;
396f24e2909SJakub Kicinski break;
39777a844eeSJakub Kicinski default:
39877a844eeSJakub Kicinski nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
39977a844eeSJakub Kicinski break;
40077a844eeSJakub Kicinski }
40177a844eeSJakub Kicinski }
40277a844eeSJakub Kicinski if (mem - start != nfp_cpp_area_size(area)) {
4030bce7c9aSJakub Kicinski nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
40477a844eeSJakub Kicinski mem - start, nfp_cpp_area_size(area));
40577a844eeSJakub Kicinski goto err_release_free;
40677a844eeSJakub Kicinski }
40777a844eeSJakub Kicinski
40877a844eeSJakub Kicinski nfp_cpp_area_release_free(area);
40977a844eeSJakub Kicinski
41077a844eeSJakub Kicinski return 0;
41177a844eeSJakub Kicinski
41277a844eeSJakub Kicinski err_release_free:
4130bce7c9aSJakub Kicinski nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
41477a844eeSJakub Kicinski nfp_cpp_area_release_free(area);
41577a844eeSJakub Kicinski return -EINVAL;
41677a844eeSJakub Kicinski }
41777a844eeSJakub Kicinski
nfp_bpf_init_capabilities(struct nfp_app_bpf * bpf)41828264eb2SJakub Kicinski static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf)
41928264eb2SJakub Kicinski {
42028264eb2SJakub Kicinski bpf->abi_version = 2; /* Original BPF ABI version */
42128264eb2SJakub Kicinski }
42228264eb2SJakub Kicinski
nfp_bpf_ndo_init(struct nfp_app * app,struct net_device * netdev)4239fd7c555SJakub Kicinski static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
4249fd7c555SJakub Kicinski {
425602144c2SJakub Kicinski struct nfp_app_bpf *bpf = app->priv;
426602144c2SJakub Kicinski
427602144c2SJakub Kicinski return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
4289fd7c555SJakub Kicinski }
4299fd7c555SJakub Kicinski
nfp_bpf_ndo_uninit(struct nfp_app * app,struct net_device * netdev)4309fd7c555SJakub Kicinski static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
4319fd7c555SJakub Kicinski {
432602144c2SJakub Kicinski struct nfp_app_bpf *bpf = app->priv;
433602144c2SJakub Kicinski
434602144c2SJakub Kicinski bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
4359fd7c555SJakub Kicinski }
4369fd7c555SJakub Kicinski
nfp_bpf_start(struct nfp_app * app)437bc2796dbSJakub Kicinski static int nfp_bpf_start(struct nfp_app *app)
438bc2796dbSJakub Kicinski {
439bc2796dbSJakub Kicinski struct nfp_app_bpf *bpf = app->priv;
440bc2796dbSJakub Kicinski
441bc2796dbSJakub Kicinski if (app->ctrl->dp.mtu < nfp_bpf_ctrl_cmsg_min_mtu(bpf)) {
442bc2796dbSJakub Kicinski nfp_err(bpf->app->cpp,
443bc2796dbSJakub Kicinski "ctrl channel MTU below min required %u < %u\n",
444bc2796dbSJakub Kicinski app->ctrl->dp.mtu, nfp_bpf_ctrl_cmsg_min_mtu(bpf));
445bc2796dbSJakub Kicinski return -EINVAL;
446bc2796dbSJakub Kicinski }
447bc2796dbSJakub Kicinski
448f24e2909SJakub Kicinski if (bpf->cmsg_multi_ent)
449f24e2909SJakub Kicinski bpf->cmsg_cache_cnt = nfp_bpf_ctrl_cmsg_cache_cnt(bpf);
450f24e2909SJakub Kicinski else
451f24e2909SJakub Kicinski bpf->cmsg_cache_cnt = 1;
452f24e2909SJakub Kicinski
453bc2796dbSJakub Kicinski return 0;
454bc2796dbSJakub Kicinski }
455bc2796dbSJakub Kicinski
nfp_bpf_init(struct nfp_app * app)45677a844eeSJakub Kicinski static int nfp_bpf_init(struct nfp_app *app)
45777a844eeSJakub Kicinski {
45877a844eeSJakub Kicinski struct nfp_app_bpf *bpf;
45977a844eeSJakub Kicinski int err;
46077a844eeSJakub Kicinski
46177a844eeSJakub Kicinski bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
46277a844eeSJakub Kicinski if (!bpf)
46377a844eeSJakub Kicinski return -ENOMEM;
46477a844eeSJakub Kicinski bpf->app = app;
46577a844eeSJakub Kicinski app->priv = bpf;
46677a844eeSJakub Kicinski
4674da98eeaSJakub Kicinski INIT_LIST_HEAD(&bpf->map_list);
4684da98eeaSJakub Kicinski
469bcf0cafaSJakub Kicinski err = nfp_ccm_init(&bpf->ccm, app);
470bcf0cafaSJakub Kicinski if (err)
471bcf0cafaSJakub Kicinski goto err_free_bpf;
472bcf0cafaSJakub Kicinski
473630a4d38SJakub Kicinski err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
47477a844eeSJakub Kicinski if (err)
475bcf0cafaSJakub Kicinski goto err_clean_ccm;
47677a844eeSJakub Kicinski
47728264eb2SJakub Kicinski nfp_bpf_init_capabilities(bpf);
47828264eb2SJakub Kicinski
479630a4d38SJakub Kicinski err = nfp_bpf_parse_capabilities(app);
480630a4d38SJakub Kicinski if (err)
481630a4d38SJakub Kicinski goto err_free_neutral_maps;
482630a4d38SJakub Kicinski
4830c9864c0SJakub Kicinski if (bpf->abi_version < 3) {
4840c9864c0SJakub Kicinski bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4;
4850c9864c0SJakub Kicinski bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4;
4860c9864c0SJakub Kicinski } else {
4870c9864c0SJakub Kicinski bpf->cmsg_key_sz = bpf->maps.max_key_sz;
4880c9864c0SJakub Kicinski bpf->cmsg_val_sz = bpf->maps.max_val_sz;
4890c9864c0SJakub Kicinski app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf);
4900c9864c0SJakub Kicinski }
4910c9864c0SJakub Kicinski
492dd27c2e3SJakub Kicinski bpf->bpf_dev = bpf_offload_dev_create(&nfp_bpf_dev_ops, bpf);
493602144c2SJakub Kicinski err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
494602144c2SJakub Kicinski if (err)
495602144c2SJakub Kicinski goto err_free_neutral_maps;
496602144c2SJakub Kicinski
49777a844eeSJakub Kicinski return 0;
49877a844eeSJakub Kicinski
499630a4d38SJakub Kicinski err_free_neutral_maps:
500630a4d38SJakub Kicinski rhashtable_destroy(&bpf->maps_neutral);
501bcf0cafaSJakub Kicinski err_clean_ccm:
502bcf0cafaSJakub Kicinski nfp_ccm_clean(&bpf->ccm);
50377a844eeSJakub Kicinski err_free_bpf:
50477a844eeSJakub Kicinski kfree(bpf);
50577a844eeSJakub Kicinski return err;
50677a844eeSJakub Kicinski }
50777a844eeSJakub Kicinski
nfp_bpf_clean(struct nfp_app * app)50877a844eeSJakub Kicinski static void nfp_bpf_clean(struct nfp_app *app)
50977a844eeSJakub Kicinski {
5104da98eeaSJakub Kicinski struct nfp_app_bpf *bpf = app->priv;
5114da98eeaSJakub Kicinski
512602144c2SJakub Kicinski bpf_offload_dev_destroy(bpf->bpf_dev);
513bcf0cafaSJakub Kicinski nfp_ccm_clean(&bpf->ccm);
5144da98eeaSJakub Kicinski WARN_ON(!list_empty(&bpf->map_list));
5151bba4c41SJakub Kicinski WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
516630a4d38SJakub Kicinski rhashtable_free_and_destroy(&bpf->maps_neutral,
517630a4d38SJakub Kicinski nfp_check_rhashtable_empty, NULL);
5184da98eeaSJakub Kicinski kfree(bpf);
51977a844eeSJakub Kicinski }
52077a844eeSJakub Kicinski
5218aa0cb00SJakub Kicinski const struct nfp_app_type app_bpf = {
5228aa0cb00SJakub Kicinski .id = NFP_APP_BPF_NIC,
5232707d6f1SJakub Kicinski .name = "ebpf",
5248aa0cb00SJakub Kicinski
52581bd5dedSJakub Kicinski .ctrl_cap_mask = 0,
52678a0a65fSJakub Kicinski
52777a844eeSJakub Kicinski .init = nfp_bpf_init,
52877a844eeSJakub Kicinski .clean = nfp_bpf_clean,
529bc2796dbSJakub Kicinski .start = nfp_bpf_start,
53077a844eeSJakub Kicinski
531167cebefSJohn Hurley .check_mtu = nfp_bpf_check_mtu,
532ccbdc596SJakub Kicinski
533bb45e51cSJakub Kicinski .extra_cap = nfp_bpf_extra_cap,
534bb45e51cSJakub Kicinski
5359fd7c555SJakub Kicinski .ndo_init = nfp_bpf_ndo_init,
5369fd7c555SJakub Kicinski .ndo_uninit = nfp_bpf_ndo_uninit,
5379fd7c555SJakub Kicinski
5384f83435aSJakub Kicinski .vnic_alloc = nfp_bpf_vnic_alloc,
5394f83435aSJakub Kicinski .vnic_free = nfp_bpf_vnic_free,
540bb45e51cSJakub Kicinski
541d48ae231SJakub Kicinski .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx,
54209587627SJakub Kicinski .ctrl_msg_rx_raw = nfp_bpf_ctrl_msg_rx_raw,
543d48ae231SJakub Kicinski
544bb45e51cSJakub Kicinski .setup_tc = nfp_bpf_setup_tc,
545af93d15aSJakub Kicinski .bpf = nfp_ndo_bpf,
546bb45e51cSJakub Kicinski .xdp_offload = nfp_bpf_xdp_offload,
5478aa0cb00SJakub Kicinski };
548