1b0588390SAlexei Starovoitov // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2b0588390SAlexei Starovoitov /* Copyright (c) 2019 Facebook */
3b0588390SAlexei Starovoitov
429db4beaSAlexei Starovoitov #ifdef __KERNEL__
529db4beaSAlexei Starovoitov #include <linux/bpf.h>
629db4beaSAlexei Starovoitov #include <linux/btf.h>
729db4beaSAlexei Starovoitov #include <linux/string.h>
829db4beaSAlexei Starovoitov #include <linux/bpf_verifier.h>
929db4beaSAlexei Starovoitov #include "relo_core.h"
1029db4beaSAlexei Starovoitov
btf_kind_str(const struct btf_type * t)1129db4beaSAlexei Starovoitov static const char *btf_kind_str(const struct btf_type *t)
1229db4beaSAlexei Starovoitov {
1329db4beaSAlexei Starovoitov return btf_type_str(t);
1429db4beaSAlexei Starovoitov }
1529db4beaSAlexei Starovoitov
is_ldimm64_insn(struct bpf_insn * insn)1629db4beaSAlexei Starovoitov static bool is_ldimm64_insn(struct bpf_insn *insn)
1729db4beaSAlexei Starovoitov {
1829db4beaSAlexei Starovoitov return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
1929db4beaSAlexei Starovoitov }
2029db4beaSAlexei Starovoitov
2129db4beaSAlexei Starovoitov static const struct btf_type *
skip_mods_and_typedefs(const struct btf * btf,u32 id,u32 * res_id)2229db4beaSAlexei Starovoitov skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id)
2329db4beaSAlexei Starovoitov {
2429db4beaSAlexei Starovoitov return btf_type_skip_modifiers(btf, id, res_id);
2529db4beaSAlexei Starovoitov }
2629db4beaSAlexei Starovoitov
btf__name_by_offset(const struct btf * btf,u32 offset)2729db4beaSAlexei Starovoitov static const char *btf__name_by_offset(const struct btf *btf, u32 offset)
2829db4beaSAlexei Starovoitov {
2929db4beaSAlexei Starovoitov return btf_name_by_offset(btf, offset);
3029db4beaSAlexei Starovoitov }
3129db4beaSAlexei Starovoitov
btf__resolve_size(const struct btf * btf,u32 type_id)3229db4beaSAlexei Starovoitov static s64 btf__resolve_size(const struct btf *btf, u32 type_id)
3329db4beaSAlexei Starovoitov {
3429db4beaSAlexei Starovoitov const struct btf_type *t;
3529db4beaSAlexei Starovoitov int size;
3629db4beaSAlexei Starovoitov
3729db4beaSAlexei Starovoitov t = btf_type_by_id(btf, type_id);
3829db4beaSAlexei Starovoitov t = btf_resolve_size(btf, t, &size);
3929db4beaSAlexei Starovoitov if (IS_ERR(t))
4029db4beaSAlexei Starovoitov return PTR_ERR(t);
4129db4beaSAlexei Starovoitov return size;
4229db4beaSAlexei Starovoitov }
4329db4beaSAlexei Starovoitov
4429db4beaSAlexei Starovoitov enum libbpf_print_level {
4529db4beaSAlexei Starovoitov LIBBPF_WARN,
4629db4beaSAlexei Starovoitov LIBBPF_INFO,
4729db4beaSAlexei Starovoitov LIBBPF_DEBUG,
4829db4beaSAlexei Starovoitov };
4929db4beaSAlexei Starovoitov
5029db4beaSAlexei Starovoitov #undef pr_warn
5129db4beaSAlexei Starovoitov #undef pr_info
5229db4beaSAlexei Starovoitov #undef pr_debug
5329db4beaSAlexei Starovoitov #define pr_warn(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__)
5429db4beaSAlexei Starovoitov #define pr_info(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__)
5529db4beaSAlexei Starovoitov #define pr_debug(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__)
5629db4beaSAlexei Starovoitov #define libbpf_print(level, fmt, ...) bpf_log((void *)prog_name, fmt, ##__VA_ARGS__)
5729db4beaSAlexei Starovoitov #else
58b0588390SAlexei Starovoitov #include <stdio.h>
59b0588390SAlexei Starovoitov #include <string.h>
60b0588390SAlexei Starovoitov #include <errno.h>
61b0588390SAlexei Starovoitov #include <ctype.h>
62b0588390SAlexei Starovoitov #include <linux/err.h>
63b0588390SAlexei Starovoitov
64b0588390SAlexei Starovoitov #include "libbpf.h"
65b0588390SAlexei Starovoitov #include "bpf.h"
66b0588390SAlexei Starovoitov #include "btf.h"
67b0588390SAlexei Starovoitov #include "str_error.h"
68b0588390SAlexei Starovoitov #include "libbpf_internal.h"
6929db4beaSAlexei Starovoitov #endif
70b0588390SAlexei Starovoitov
is_flex_arr(const struct btf * btf,const struct bpf_core_accessor * acc,const struct btf_array * arr)71b0588390SAlexei Starovoitov static bool is_flex_arr(const struct btf *btf,
72b0588390SAlexei Starovoitov const struct bpf_core_accessor *acc,
73b0588390SAlexei Starovoitov const struct btf_array *arr)
74b0588390SAlexei Starovoitov {
75b0588390SAlexei Starovoitov const struct btf_type *t;
76b0588390SAlexei Starovoitov
77b0588390SAlexei Starovoitov /* not a flexible array, if not inside a struct or has non-zero size */
78b0588390SAlexei Starovoitov if (!acc->name || arr->nelems > 0)
79b0588390SAlexei Starovoitov return false;
80b0588390SAlexei Starovoitov
81b0588390SAlexei Starovoitov /* has to be the last member of enclosing struct */
8274753e14SAlexei Starovoitov t = btf_type_by_id(btf, acc->type_id);
83b0588390SAlexei Starovoitov return acc->idx == btf_vlen(t) - 1;
84b0588390SAlexei Starovoitov }
85b0588390SAlexei Starovoitov
core_relo_kind_str(enum bpf_core_relo_kind kind)86b0588390SAlexei Starovoitov static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
87b0588390SAlexei Starovoitov {
88b0588390SAlexei Starovoitov switch (kind) {
8946334a0cSAlexei Starovoitov case BPF_CORE_FIELD_BYTE_OFFSET: return "byte_off";
9046334a0cSAlexei Starovoitov case BPF_CORE_FIELD_BYTE_SIZE: return "byte_sz";
9146334a0cSAlexei Starovoitov case BPF_CORE_FIELD_EXISTS: return "field_exists";
9246334a0cSAlexei Starovoitov case BPF_CORE_FIELD_SIGNED: return "signed";
9346334a0cSAlexei Starovoitov case BPF_CORE_FIELD_LSHIFT_U64: return "lshift_u64";
9446334a0cSAlexei Starovoitov case BPF_CORE_FIELD_RSHIFT_U64: return "rshift_u64";
9546334a0cSAlexei Starovoitov case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id";
9646334a0cSAlexei Starovoitov case BPF_CORE_TYPE_ID_TARGET: return "target_type_id";
9746334a0cSAlexei Starovoitov case BPF_CORE_TYPE_EXISTS: return "type_exists";
98ec6209c8SDaniel Müller case BPF_CORE_TYPE_MATCHES: return "type_matches";
9946334a0cSAlexei Starovoitov case BPF_CORE_TYPE_SIZE: return "type_size";
10046334a0cSAlexei Starovoitov case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists";
10146334a0cSAlexei Starovoitov case BPF_CORE_ENUMVAL_VALUE: return "enumval_value";
102b0588390SAlexei Starovoitov default: return "unknown";
103b0588390SAlexei Starovoitov }
104b0588390SAlexei Starovoitov }
105b0588390SAlexei Starovoitov
core_relo_is_field_based(enum bpf_core_relo_kind kind)106b0588390SAlexei Starovoitov static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)
107b0588390SAlexei Starovoitov {
108b0588390SAlexei Starovoitov switch (kind) {
10946334a0cSAlexei Starovoitov case BPF_CORE_FIELD_BYTE_OFFSET:
11046334a0cSAlexei Starovoitov case BPF_CORE_FIELD_BYTE_SIZE:
11146334a0cSAlexei Starovoitov case BPF_CORE_FIELD_EXISTS:
11246334a0cSAlexei Starovoitov case BPF_CORE_FIELD_SIGNED:
11346334a0cSAlexei Starovoitov case BPF_CORE_FIELD_LSHIFT_U64:
11446334a0cSAlexei Starovoitov case BPF_CORE_FIELD_RSHIFT_U64:
115b0588390SAlexei Starovoitov return true;
116b0588390SAlexei Starovoitov default:
117b0588390SAlexei Starovoitov return false;
118b0588390SAlexei Starovoitov }
119b0588390SAlexei Starovoitov }
120b0588390SAlexei Starovoitov
core_relo_is_type_based(enum bpf_core_relo_kind kind)121b0588390SAlexei Starovoitov static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
122b0588390SAlexei Starovoitov {
123b0588390SAlexei Starovoitov switch (kind) {
12446334a0cSAlexei Starovoitov case BPF_CORE_TYPE_ID_LOCAL:
12546334a0cSAlexei Starovoitov case BPF_CORE_TYPE_ID_TARGET:
12646334a0cSAlexei Starovoitov case BPF_CORE_TYPE_EXISTS:
127ec6209c8SDaniel Müller case BPF_CORE_TYPE_MATCHES:
12846334a0cSAlexei Starovoitov case BPF_CORE_TYPE_SIZE:
129b0588390SAlexei Starovoitov return true;
130b0588390SAlexei Starovoitov default:
131b0588390SAlexei Starovoitov return false;
132b0588390SAlexei Starovoitov }
133b0588390SAlexei Starovoitov }
134b0588390SAlexei Starovoitov
core_relo_is_enumval_based(enum bpf_core_relo_kind kind)135b0588390SAlexei Starovoitov static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
136b0588390SAlexei Starovoitov {
137b0588390SAlexei Starovoitov switch (kind) {
13846334a0cSAlexei Starovoitov case BPF_CORE_ENUMVAL_EXISTS:
13946334a0cSAlexei Starovoitov case BPF_CORE_ENUMVAL_VALUE:
140b0588390SAlexei Starovoitov return true;
141b0588390SAlexei Starovoitov default:
142b0588390SAlexei Starovoitov return false;
143b0588390SAlexei Starovoitov }
144b0588390SAlexei Starovoitov }
145b0588390SAlexei Starovoitov
__bpf_core_types_are_compat(const struct btf * local_btf,__u32 local_id,const struct btf * targ_btf,__u32 targ_id,int level)146fd75733dSDaniel Müller int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
147fd75733dSDaniel Müller const struct btf *targ_btf, __u32 targ_id, int level)
148fd75733dSDaniel Müller {
149fd75733dSDaniel Müller const struct btf_type *local_type, *targ_type;
150fd75733dSDaniel Müller int depth = 32; /* max recursion depth */
151fd75733dSDaniel Müller
152fd75733dSDaniel Müller /* caller made sure that names match (ignoring flavor suffix) */
153fd75733dSDaniel Müller local_type = btf_type_by_id(local_btf, local_id);
154fd75733dSDaniel Müller targ_type = btf_type_by_id(targ_btf, targ_id);
155fd75733dSDaniel Müller if (!btf_kind_core_compat(local_type, targ_type))
156fd75733dSDaniel Müller return 0;
157fd75733dSDaniel Müller
158fd75733dSDaniel Müller recur:
159fd75733dSDaniel Müller depth--;
160fd75733dSDaniel Müller if (depth < 0)
161fd75733dSDaniel Müller return -EINVAL;
162fd75733dSDaniel Müller
163fd75733dSDaniel Müller local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
164fd75733dSDaniel Müller targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
165fd75733dSDaniel Müller if (!local_type || !targ_type)
166fd75733dSDaniel Müller return -EINVAL;
167fd75733dSDaniel Müller
168fd75733dSDaniel Müller if (!btf_kind_core_compat(local_type, targ_type))
169fd75733dSDaniel Müller return 0;
170fd75733dSDaniel Müller
171fd75733dSDaniel Müller switch (btf_kind(local_type)) {
172fd75733dSDaniel Müller case BTF_KIND_UNKN:
173fd75733dSDaniel Müller case BTF_KIND_STRUCT:
174fd75733dSDaniel Müller case BTF_KIND_UNION:
175fd75733dSDaniel Müller case BTF_KIND_ENUM:
176fd75733dSDaniel Müller case BTF_KIND_FWD:
177fd75733dSDaniel Müller case BTF_KIND_ENUM64:
178fd75733dSDaniel Müller return 1;
179fd75733dSDaniel Müller case BTF_KIND_INT:
180fd75733dSDaniel Müller /* just reject deprecated bitfield-like integers; all other
181fd75733dSDaniel Müller * integers are by default compatible between each other
182fd75733dSDaniel Müller */
183fd75733dSDaniel Müller return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
184fd75733dSDaniel Müller case BTF_KIND_PTR:
185fd75733dSDaniel Müller local_id = local_type->type;
186fd75733dSDaniel Müller targ_id = targ_type->type;
187fd75733dSDaniel Müller goto recur;
188fd75733dSDaniel Müller case BTF_KIND_ARRAY:
189fd75733dSDaniel Müller local_id = btf_array(local_type)->type;
190fd75733dSDaniel Müller targ_id = btf_array(targ_type)->type;
191fd75733dSDaniel Müller goto recur;
192fd75733dSDaniel Müller case BTF_KIND_FUNC_PROTO: {
193fd75733dSDaniel Müller struct btf_param *local_p = btf_params(local_type);
194fd75733dSDaniel Müller struct btf_param *targ_p = btf_params(targ_type);
195fd75733dSDaniel Müller __u16 local_vlen = btf_vlen(local_type);
196fd75733dSDaniel Müller __u16 targ_vlen = btf_vlen(targ_type);
197fd75733dSDaniel Müller int i, err;
198fd75733dSDaniel Müller
199fd75733dSDaniel Müller if (local_vlen != targ_vlen)
200fd75733dSDaniel Müller return 0;
201fd75733dSDaniel Müller
202fd75733dSDaniel Müller for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
203fd75733dSDaniel Müller if (level <= 0)
204fd75733dSDaniel Müller return -EINVAL;
205fd75733dSDaniel Müller
206fd75733dSDaniel Müller skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
207fd75733dSDaniel Müller skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
208fd75733dSDaniel Müller err = __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id,
209fd75733dSDaniel Müller level - 1);
210fd75733dSDaniel Müller if (err <= 0)
211fd75733dSDaniel Müller return err;
212fd75733dSDaniel Müller }
213fd75733dSDaniel Müller
214fd75733dSDaniel Müller /* tail recurse for return type check */
215fd75733dSDaniel Müller skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
216fd75733dSDaniel Müller skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
217fd75733dSDaniel Müller goto recur;
218fd75733dSDaniel Müller }
219fd75733dSDaniel Müller default:
220fd75733dSDaniel Müller pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
221fd75733dSDaniel Müller btf_kind_str(local_type), local_id, targ_id);
222fd75733dSDaniel Müller return 0;
223fd75733dSDaniel Müller }
224fd75733dSDaniel Müller }
225fd75733dSDaniel Müller
226b0588390SAlexei Starovoitov /*
227b0588390SAlexei Starovoitov * Turn bpf_core_relo into a low- and high-level spec representation,
228b0588390SAlexei Starovoitov * validating correctness along the way, as well as calculating resulting
229b0588390SAlexei Starovoitov * field bit offset, specified by accessor string. Low-level spec captures
230b0588390SAlexei Starovoitov * every single level of nestedness, including traversing anonymous
231b0588390SAlexei Starovoitov * struct/union members. High-level one only captures semantically meaningful
232b0588390SAlexei Starovoitov * "turning points": named fields and array indicies.
233b0588390SAlexei Starovoitov * E.g., for this case:
234b0588390SAlexei Starovoitov *
235b0588390SAlexei Starovoitov * struct sample {
236b0588390SAlexei Starovoitov * int __unimportant;
237b0588390SAlexei Starovoitov * struct {
238b0588390SAlexei Starovoitov * int __1;
239b0588390SAlexei Starovoitov * int __2;
240b0588390SAlexei Starovoitov * int a[7];
241b0588390SAlexei Starovoitov * };
242b0588390SAlexei Starovoitov * };
243b0588390SAlexei Starovoitov *
244b0588390SAlexei Starovoitov * struct sample *s = ...;
245b0588390SAlexei Starovoitov *
246b0588390SAlexei Starovoitov * int x = &s->a[3]; // access string = '0:1:2:3'
247b0588390SAlexei Starovoitov *
248b0588390SAlexei Starovoitov * Low-level spec has 1:1 mapping with each element of access string (it's
249b0588390SAlexei Starovoitov * just a parsed access string representation): [0, 1, 2, 3].
250b0588390SAlexei Starovoitov *
251b0588390SAlexei Starovoitov * High-level spec will capture only 3 points:
2529bbdfad8SDaniel Müller * - initial zero-index access by pointer (&s->... is the same as &s[0]...);
253b0588390SAlexei Starovoitov * - field 'a' access (corresponds to '2' in low-level spec);
254b0588390SAlexei Starovoitov * - array element #3 access (corresponds to '3' in low-level spec).
255b0588390SAlexei Starovoitov *
256ec6209c8SDaniel Müller * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE,
257b0588390SAlexei Starovoitov * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
258b0588390SAlexei Starovoitov * spec and raw_spec are kept empty.
259b0588390SAlexei Starovoitov *
260b0588390SAlexei Starovoitov * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
261b0588390SAlexei Starovoitov * string to specify enumerator's value index that need to be relocated.
262b0588390SAlexei Starovoitov */
bpf_core_parse_spec(const char * prog_name,const struct btf * btf,const struct bpf_core_relo * relo,struct bpf_core_spec * spec)2639fdc4273SAndrii Nakryiko int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
26414032f26SAndrii Nakryiko const struct bpf_core_relo *relo,
265b0588390SAlexei Starovoitov struct bpf_core_spec *spec)
266b0588390SAlexei Starovoitov {
267b0588390SAlexei Starovoitov int access_idx, parsed_len, i;
268b0588390SAlexei Starovoitov struct bpf_core_accessor *acc;
269b0588390SAlexei Starovoitov const struct btf_type *t;
27014032f26SAndrii Nakryiko const char *name, *spec_str;
27123b2a3a8SYonghong Song __u32 id, name_off;
272b0588390SAlexei Starovoitov __s64 sz;
273b0588390SAlexei Starovoitov
27414032f26SAndrii Nakryiko spec_str = btf__name_by_offset(btf, relo->access_str_off);
275b0588390SAlexei Starovoitov if (str_is_empty(spec_str) || *spec_str == ':')
276b0588390SAlexei Starovoitov return -EINVAL;
277b0588390SAlexei Starovoitov
278b0588390SAlexei Starovoitov memset(spec, 0, sizeof(*spec));
279b0588390SAlexei Starovoitov spec->btf = btf;
28014032f26SAndrii Nakryiko spec->root_type_id = relo->type_id;
28114032f26SAndrii Nakryiko spec->relo_kind = relo->kind;
282b0588390SAlexei Starovoitov
283b0588390SAlexei Starovoitov /* type-based relocations don't have a field access string */
28414032f26SAndrii Nakryiko if (core_relo_is_type_based(relo->kind)) {
285b0588390SAlexei Starovoitov if (strcmp(spec_str, "0"))
286b0588390SAlexei Starovoitov return -EINVAL;
287b0588390SAlexei Starovoitov return 0;
288b0588390SAlexei Starovoitov }
289b0588390SAlexei Starovoitov
290b0588390SAlexei Starovoitov /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
291b0588390SAlexei Starovoitov while (*spec_str) {
292b0588390SAlexei Starovoitov if (*spec_str == ':')
293b0588390SAlexei Starovoitov ++spec_str;
294b0588390SAlexei Starovoitov if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
295b0588390SAlexei Starovoitov return -EINVAL;
296b0588390SAlexei Starovoitov if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
297b0588390SAlexei Starovoitov return -E2BIG;
298b0588390SAlexei Starovoitov spec_str += parsed_len;
299b0588390SAlexei Starovoitov spec->raw_spec[spec->raw_len++] = access_idx;
300b0588390SAlexei Starovoitov }
301b0588390SAlexei Starovoitov
302b0588390SAlexei Starovoitov if (spec->raw_len == 0)
303b0588390SAlexei Starovoitov return -EINVAL;
304b0588390SAlexei Starovoitov
30514032f26SAndrii Nakryiko t = skip_mods_and_typedefs(btf, relo->type_id, &id);
306b0588390SAlexei Starovoitov if (!t)
307b0588390SAlexei Starovoitov return -EINVAL;
308b0588390SAlexei Starovoitov
309b0588390SAlexei Starovoitov access_idx = spec->raw_spec[0];
310b0588390SAlexei Starovoitov acc = &spec->spec[0];
311b0588390SAlexei Starovoitov acc->type_id = id;
312b0588390SAlexei Starovoitov acc->idx = access_idx;
313b0588390SAlexei Starovoitov spec->len++;
314b0588390SAlexei Starovoitov
31514032f26SAndrii Nakryiko if (core_relo_is_enumval_based(relo->kind)) {
31623b2a3a8SYonghong Song if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
317b0588390SAlexei Starovoitov return -EINVAL;
318b0588390SAlexei Starovoitov
319b0588390SAlexei Starovoitov /* record enumerator name in a first accessor */
32023b2a3a8SYonghong Song name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off
32123b2a3a8SYonghong Song : btf_enum64(t)[access_idx].name_off;
32223b2a3a8SYonghong Song acc->name = btf__name_by_offset(btf, name_off);
323b0588390SAlexei Starovoitov return 0;
324b0588390SAlexei Starovoitov }
325b0588390SAlexei Starovoitov
32614032f26SAndrii Nakryiko if (!core_relo_is_field_based(relo->kind))
327b0588390SAlexei Starovoitov return -EINVAL;
328b0588390SAlexei Starovoitov
329b0588390SAlexei Starovoitov sz = btf__resolve_size(btf, id);
330b0588390SAlexei Starovoitov if (sz < 0)
331b0588390SAlexei Starovoitov return sz;
332b0588390SAlexei Starovoitov spec->bit_offset = access_idx * sz * 8;
333b0588390SAlexei Starovoitov
334b0588390SAlexei Starovoitov for (i = 1; i < spec->raw_len; i++) {
335b0588390SAlexei Starovoitov t = skip_mods_and_typedefs(btf, id, &id);
336b0588390SAlexei Starovoitov if (!t)
337b0588390SAlexei Starovoitov return -EINVAL;
338b0588390SAlexei Starovoitov
339b0588390SAlexei Starovoitov access_idx = spec->raw_spec[i];
340b0588390SAlexei Starovoitov acc = &spec->spec[spec->len];
341b0588390SAlexei Starovoitov
342b0588390SAlexei Starovoitov if (btf_is_composite(t)) {
343b0588390SAlexei Starovoitov const struct btf_member *m;
344b0588390SAlexei Starovoitov __u32 bit_offset;
345b0588390SAlexei Starovoitov
346b0588390SAlexei Starovoitov if (access_idx >= btf_vlen(t))
347b0588390SAlexei Starovoitov return -EINVAL;
348b0588390SAlexei Starovoitov
349b0588390SAlexei Starovoitov bit_offset = btf_member_bit_offset(t, access_idx);
350b0588390SAlexei Starovoitov spec->bit_offset += bit_offset;
351b0588390SAlexei Starovoitov
352b0588390SAlexei Starovoitov m = btf_members(t) + access_idx;
353b0588390SAlexei Starovoitov if (m->name_off) {
354b0588390SAlexei Starovoitov name = btf__name_by_offset(btf, m->name_off);
355b0588390SAlexei Starovoitov if (str_is_empty(name))
356b0588390SAlexei Starovoitov return -EINVAL;
357b0588390SAlexei Starovoitov
358b0588390SAlexei Starovoitov acc->type_id = id;
359b0588390SAlexei Starovoitov acc->idx = access_idx;
360b0588390SAlexei Starovoitov acc->name = name;
361b0588390SAlexei Starovoitov spec->len++;
362b0588390SAlexei Starovoitov }
363b0588390SAlexei Starovoitov
364b0588390SAlexei Starovoitov id = m->type;
365b0588390SAlexei Starovoitov } else if (btf_is_array(t)) {
366b0588390SAlexei Starovoitov const struct btf_array *a = btf_array(t);
367b0588390SAlexei Starovoitov bool flex;
368b0588390SAlexei Starovoitov
369b0588390SAlexei Starovoitov t = skip_mods_and_typedefs(btf, a->type, &id);
370b0588390SAlexei Starovoitov if (!t)
371b0588390SAlexei Starovoitov return -EINVAL;
372b0588390SAlexei Starovoitov
373b0588390SAlexei Starovoitov flex = is_flex_arr(btf, acc - 1, a);
374b0588390SAlexei Starovoitov if (!flex && access_idx >= a->nelems)
375b0588390SAlexei Starovoitov return -EINVAL;
376b0588390SAlexei Starovoitov
377b0588390SAlexei Starovoitov spec->spec[spec->len].type_id = id;
378b0588390SAlexei Starovoitov spec->spec[spec->len].idx = access_idx;
379b0588390SAlexei Starovoitov spec->len++;
380b0588390SAlexei Starovoitov
381b0588390SAlexei Starovoitov sz = btf__resolve_size(btf, id);
382b0588390SAlexei Starovoitov if (sz < 0)
383b0588390SAlexei Starovoitov return sz;
384b0588390SAlexei Starovoitov spec->bit_offset += access_idx * sz * 8;
385b0588390SAlexei Starovoitov } else {
38629db4beaSAlexei Starovoitov pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
38714032f26SAndrii Nakryiko prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t));
388b0588390SAlexei Starovoitov return -EINVAL;
389b0588390SAlexei Starovoitov }
390b0588390SAlexei Starovoitov }
391b0588390SAlexei Starovoitov
392b0588390SAlexei Starovoitov return 0;
393b0588390SAlexei Starovoitov }
394b0588390SAlexei Starovoitov
395b0588390SAlexei Starovoitov /* Check two types for compatibility for the purpose of field access
396b0588390SAlexei Starovoitov * relocation. const/volatile/restrict and typedefs are skipped to ensure we
397b0588390SAlexei Starovoitov * are relocating semantically compatible entities:
398b0588390SAlexei Starovoitov * - any two STRUCTs/UNIONs are compatible and can be mixed;
399b0588390SAlexei Starovoitov * - any two FWDs are compatible, if their names match (modulo flavor suffix);
400b0588390SAlexei Starovoitov * - any two PTRs are always compatible;
401b0588390SAlexei Starovoitov * - for ENUMs, names should be the same (ignoring flavor suffix) or at
402b0588390SAlexei Starovoitov * least one of enums should be anonymous;
403b0588390SAlexei Starovoitov * - for ENUMs, check sizes, names are ignored;
404b0588390SAlexei Starovoitov * - for INT, size and signedness are ignored;
405b0588390SAlexei Starovoitov * - any two FLOATs are always compatible;
406b0588390SAlexei Starovoitov * - for ARRAY, dimensionality is ignored, element types are checked for
407b0588390SAlexei Starovoitov * compatibility recursively;
408b0588390SAlexei Starovoitov * - everything else shouldn't be ever a target of relocation.
409b0588390SAlexei Starovoitov * These rules are not set in stone and probably will be adjusted as we get
410b0588390SAlexei Starovoitov * more experience with using BPF CO-RE relocations.
411b0588390SAlexei Starovoitov */
bpf_core_fields_are_compat(const struct btf * local_btf,__u32 local_id,const struct btf * targ_btf,__u32 targ_id)412b0588390SAlexei Starovoitov static int bpf_core_fields_are_compat(const struct btf *local_btf,
413b0588390SAlexei Starovoitov __u32 local_id,
414b0588390SAlexei Starovoitov const struct btf *targ_btf,
415b0588390SAlexei Starovoitov __u32 targ_id)
416b0588390SAlexei Starovoitov {
417b0588390SAlexei Starovoitov const struct btf_type *local_type, *targ_type;
418b0588390SAlexei Starovoitov
419b0588390SAlexei Starovoitov recur:
420b0588390SAlexei Starovoitov local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
421b0588390SAlexei Starovoitov targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
422b0588390SAlexei Starovoitov if (!local_type || !targ_type)
423b0588390SAlexei Starovoitov return -EINVAL;
424b0588390SAlexei Starovoitov
425b0588390SAlexei Starovoitov if (btf_is_composite(local_type) && btf_is_composite(targ_type))
426b0588390SAlexei Starovoitov return 1;
42723b2a3a8SYonghong Song if (!btf_kind_core_compat(local_type, targ_type))
428b0588390SAlexei Starovoitov return 0;
429b0588390SAlexei Starovoitov
430b0588390SAlexei Starovoitov switch (btf_kind(local_type)) {
431b0588390SAlexei Starovoitov case BTF_KIND_PTR:
432b0588390SAlexei Starovoitov case BTF_KIND_FLOAT:
433b0588390SAlexei Starovoitov return 1;
434b0588390SAlexei Starovoitov case BTF_KIND_FWD:
43523b2a3a8SYonghong Song case BTF_KIND_ENUM64:
436b0588390SAlexei Starovoitov case BTF_KIND_ENUM: {
437b0588390SAlexei Starovoitov const char *local_name, *targ_name;
438b0588390SAlexei Starovoitov size_t local_len, targ_len;
439b0588390SAlexei Starovoitov
440b0588390SAlexei Starovoitov local_name = btf__name_by_offset(local_btf,
441b0588390SAlexei Starovoitov local_type->name_off);
442b0588390SAlexei Starovoitov targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
443b0588390SAlexei Starovoitov local_len = bpf_core_essential_name_len(local_name);
444b0588390SAlexei Starovoitov targ_len = bpf_core_essential_name_len(targ_name);
445b0588390SAlexei Starovoitov /* one of them is anonymous or both w/ same flavor-less names */
446b0588390SAlexei Starovoitov return local_len == 0 || targ_len == 0 ||
447b0588390SAlexei Starovoitov (local_len == targ_len &&
448b0588390SAlexei Starovoitov strncmp(local_name, targ_name, local_len) == 0);
449b0588390SAlexei Starovoitov }
450b0588390SAlexei Starovoitov case BTF_KIND_INT:
451b0588390SAlexei Starovoitov /* just reject deprecated bitfield-like integers; all other
452b0588390SAlexei Starovoitov * integers are by default compatible between each other
453b0588390SAlexei Starovoitov */
454b0588390SAlexei Starovoitov return btf_int_offset(local_type) == 0 &&
455b0588390SAlexei Starovoitov btf_int_offset(targ_type) == 0;
456b0588390SAlexei Starovoitov case BTF_KIND_ARRAY:
457b0588390SAlexei Starovoitov local_id = btf_array(local_type)->type;
458b0588390SAlexei Starovoitov targ_id = btf_array(targ_type)->type;
459b0588390SAlexei Starovoitov goto recur;
460b0588390SAlexei Starovoitov default:
461b0588390SAlexei Starovoitov return 0;
462b0588390SAlexei Starovoitov }
463b0588390SAlexei Starovoitov }
464b0588390SAlexei Starovoitov
465b0588390SAlexei Starovoitov /*
466b0588390SAlexei Starovoitov * Given single high-level named field accessor in local type, find
467b0588390SAlexei Starovoitov * corresponding high-level accessor for a target type. Along the way,
468b0588390SAlexei Starovoitov * maintain low-level spec for target as well. Also keep updating target
469b0588390SAlexei Starovoitov * bit offset.
470b0588390SAlexei Starovoitov *
471b0588390SAlexei Starovoitov * Searching is performed through recursive exhaustive enumeration of all
472b0588390SAlexei Starovoitov * fields of a struct/union. If there are any anonymous (embedded)
473b0588390SAlexei Starovoitov * structs/unions, they are recursively searched as well. If field with
474b0588390SAlexei Starovoitov * desired name is found, check compatibility between local and target types,
475b0588390SAlexei Starovoitov * before returning result.
476b0588390SAlexei Starovoitov *
477b0588390SAlexei Starovoitov * 1 is returned, if field is found.
478b0588390SAlexei Starovoitov * 0 is returned if no compatible field is found.
479b0588390SAlexei Starovoitov * <0 is returned on error.
480b0588390SAlexei Starovoitov */
bpf_core_match_member(const struct btf * local_btf,const struct bpf_core_accessor * local_acc,const struct btf * targ_btf,__u32 targ_id,struct bpf_core_spec * spec,__u32 * next_targ_id)481b0588390SAlexei Starovoitov static int bpf_core_match_member(const struct btf *local_btf,
482b0588390SAlexei Starovoitov const struct bpf_core_accessor *local_acc,
483b0588390SAlexei Starovoitov const struct btf *targ_btf,
484b0588390SAlexei Starovoitov __u32 targ_id,
485b0588390SAlexei Starovoitov struct bpf_core_spec *spec,
486b0588390SAlexei Starovoitov __u32 *next_targ_id)
487b0588390SAlexei Starovoitov {
488b0588390SAlexei Starovoitov const struct btf_type *local_type, *targ_type;
489b0588390SAlexei Starovoitov const struct btf_member *local_member, *m;
490b0588390SAlexei Starovoitov const char *local_name, *targ_name;
491b0588390SAlexei Starovoitov __u32 local_id;
492b0588390SAlexei Starovoitov int i, n, found;
493b0588390SAlexei Starovoitov
494b0588390SAlexei Starovoitov targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
495b0588390SAlexei Starovoitov if (!targ_type)
496b0588390SAlexei Starovoitov return -EINVAL;
497b0588390SAlexei Starovoitov if (!btf_is_composite(targ_type))
498b0588390SAlexei Starovoitov return 0;
499b0588390SAlexei Starovoitov
500b0588390SAlexei Starovoitov local_id = local_acc->type_id;
50174753e14SAlexei Starovoitov local_type = btf_type_by_id(local_btf, local_id);
502b0588390SAlexei Starovoitov local_member = btf_members(local_type) + local_acc->idx;
503b0588390SAlexei Starovoitov local_name = btf__name_by_offset(local_btf, local_member->name_off);
504b0588390SAlexei Starovoitov
505b0588390SAlexei Starovoitov n = btf_vlen(targ_type);
506b0588390SAlexei Starovoitov m = btf_members(targ_type);
507b0588390SAlexei Starovoitov for (i = 0; i < n; i++, m++) {
508b0588390SAlexei Starovoitov __u32 bit_offset;
509b0588390SAlexei Starovoitov
510b0588390SAlexei Starovoitov bit_offset = btf_member_bit_offset(targ_type, i);
511b0588390SAlexei Starovoitov
512b0588390SAlexei Starovoitov /* too deep struct/union/array nesting */
513b0588390SAlexei Starovoitov if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
514b0588390SAlexei Starovoitov return -E2BIG;
515b0588390SAlexei Starovoitov
516b0588390SAlexei Starovoitov /* speculate this member will be the good one */
517b0588390SAlexei Starovoitov spec->bit_offset += bit_offset;
518b0588390SAlexei Starovoitov spec->raw_spec[spec->raw_len++] = i;
519b0588390SAlexei Starovoitov
520b0588390SAlexei Starovoitov targ_name = btf__name_by_offset(targ_btf, m->name_off);
521b0588390SAlexei Starovoitov if (str_is_empty(targ_name)) {
522b0588390SAlexei Starovoitov /* embedded struct/union, we need to go deeper */
523b0588390SAlexei Starovoitov found = bpf_core_match_member(local_btf, local_acc,
524b0588390SAlexei Starovoitov targ_btf, m->type,
525b0588390SAlexei Starovoitov spec, next_targ_id);
526b0588390SAlexei Starovoitov if (found) /* either found or error */
527b0588390SAlexei Starovoitov return found;
528b0588390SAlexei Starovoitov } else if (strcmp(local_name, targ_name) == 0) {
529b0588390SAlexei Starovoitov /* matching named field */
530b0588390SAlexei Starovoitov struct bpf_core_accessor *targ_acc;
531b0588390SAlexei Starovoitov
532b0588390SAlexei Starovoitov targ_acc = &spec->spec[spec->len++];
533b0588390SAlexei Starovoitov targ_acc->type_id = targ_id;
534b0588390SAlexei Starovoitov targ_acc->idx = i;
535b0588390SAlexei Starovoitov targ_acc->name = targ_name;
536b0588390SAlexei Starovoitov
537b0588390SAlexei Starovoitov *next_targ_id = m->type;
538b0588390SAlexei Starovoitov found = bpf_core_fields_are_compat(local_btf,
539b0588390SAlexei Starovoitov local_member->type,
540b0588390SAlexei Starovoitov targ_btf, m->type);
541b0588390SAlexei Starovoitov if (!found)
542b0588390SAlexei Starovoitov spec->len--; /* pop accessor */
543b0588390SAlexei Starovoitov return found;
544b0588390SAlexei Starovoitov }
545b0588390SAlexei Starovoitov /* member turned out not to be what we looked for */
546b0588390SAlexei Starovoitov spec->bit_offset -= bit_offset;
547b0588390SAlexei Starovoitov spec->raw_len--;
548b0588390SAlexei Starovoitov }
549b0588390SAlexei Starovoitov
550b0588390SAlexei Starovoitov return 0;
551b0588390SAlexei Starovoitov }
552b0588390SAlexei Starovoitov
553b0588390SAlexei Starovoitov /*
554b0588390SAlexei Starovoitov * Try to match local spec to a target type and, if successful, produce full
555b0588390SAlexei Starovoitov * target spec (high-level, low-level + bit offset).
556b0588390SAlexei Starovoitov */
bpf_core_spec_match(struct bpf_core_spec * local_spec,const struct btf * targ_btf,__u32 targ_id,struct bpf_core_spec * targ_spec)557b0588390SAlexei Starovoitov static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
558b0588390SAlexei Starovoitov const struct btf *targ_btf, __u32 targ_id,
559b0588390SAlexei Starovoitov struct bpf_core_spec *targ_spec)
560b0588390SAlexei Starovoitov {
561b0588390SAlexei Starovoitov const struct btf_type *targ_type;
562b0588390SAlexei Starovoitov const struct bpf_core_accessor *local_acc;
563b0588390SAlexei Starovoitov struct bpf_core_accessor *targ_acc;
564b0588390SAlexei Starovoitov int i, sz, matched;
56523b2a3a8SYonghong Song __u32 name_off;
566b0588390SAlexei Starovoitov
567b0588390SAlexei Starovoitov memset(targ_spec, 0, sizeof(*targ_spec));
568b0588390SAlexei Starovoitov targ_spec->btf = targ_btf;
569b0588390SAlexei Starovoitov targ_spec->root_type_id = targ_id;
570b0588390SAlexei Starovoitov targ_spec->relo_kind = local_spec->relo_kind;
571b0588390SAlexei Starovoitov
572b0588390SAlexei Starovoitov if (core_relo_is_type_based(local_spec->relo_kind)) {
573ec6209c8SDaniel Müller if (local_spec->relo_kind == BPF_CORE_TYPE_MATCHES)
574ec6209c8SDaniel Müller return bpf_core_types_match(local_spec->btf,
575ec6209c8SDaniel Müller local_spec->root_type_id,
576ec6209c8SDaniel Müller targ_btf, targ_id);
577ec6209c8SDaniel Müller else
578b0588390SAlexei Starovoitov return bpf_core_types_are_compat(local_spec->btf,
579b0588390SAlexei Starovoitov local_spec->root_type_id,
580b0588390SAlexei Starovoitov targ_btf, targ_id);
581b0588390SAlexei Starovoitov }
582b0588390SAlexei Starovoitov
583b0588390SAlexei Starovoitov local_acc = &local_spec->spec[0];
584b0588390SAlexei Starovoitov targ_acc = &targ_spec->spec[0];
585b0588390SAlexei Starovoitov
586b0588390SAlexei Starovoitov if (core_relo_is_enumval_based(local_spec->relo_kind)) {
587b0588390SAlexei Starovoitov size_t local_essent_len, targ_essent_len;
588b0588390SAlexei Starovoitov const char *targ_name;
589b0588390SAlexei Starovoitov
590b0588390SAlexei Starovoitov /* has to resolve to an enum */
591b0588390SAlexei Starovoitov targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
59223b2a3a8SYonghong Song if (!btf_is_any_enum(targ_type))
593b0588390SAlexei Starovoitov return 0;
594b0588390SAlexei Starovoitov
595b0588390SAlexei Starovoitov local_essent_len = bpf_core_essential_name_len(local_acc->name);
596b0588390SAlexei Starovoitov
59723b2a3a8SYonghong Song for (i = 0; i < btf_vlen(targ_type); i++) {
59823b2a3a8SYonghong Song if (btf_is_enum(targ_type))
59923b2a3a8SYonghong Song name_off = btf_enum(targ_type)[i].name_off;
60023b2a3a8SYonghong Song else
60123b2a3a8SYonghong Song name_off = btf_enum64(targ_type)[i].name_off;
60223b2a3a8SYonghong Song
60323b2a3a8SYonghong Song targ_name = btf__name_by_offset(targ_spec->btf, name_off);
604b0588390SAlexei Starovoitov targ_essent_len = bpf_core_essential_name_len(targ_name);
605b0588390SAlexei Starovoitov if (targ_essent_len != local_essent_len)
606b0588390SAlexei Starovoitov continue;
607b0588390SAlexei Starovoitov if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
608b0588390SAlexei Starovoitov targ_acc->type_id = targ_id;
609b0588390SAlexei Starovoitov targ_acc->idx = i;
610b0588390SAlexei Starovoitov targ_acc->name = targ_name;
611b0588390SAlexei Starovoitov targ_spec->len++;
612b0588390SAlexei Starovoitov targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
613b0588390SAlexei Starovoitov targ_spec->raw_len++;
614b0588390SAlexei Starovoitov return 1;
615b0588390SAlexei Starovoitov }
616b0588390SAlexei Starovoitov }
617b0588390SAlexei Starovoitov return 0;
618b0588390SAlexei Starovoitov }
619b0588390SAlexei Starovoitov
620b0588390SAlexei Starovoitov if (!core_relo_is_field_based(local_spec->relo_kind))
621b0588390SAlexei Starovoitov return -EINVAL;
622b0588390SAlexei Starovoitov
623b0588390SAlexei Starovoitov for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
624b0588390SAlexei Starovoitov targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
625b0588390SAlexei Starovoitov &targ_id);
626b0588390SAlexei Starovoitov if (!targ_type)
627b0588390SAlexei Starovoitov return -EINVAL;
628b0588390SAlexei Starovoitov
629b0588390SAlexei Starovoitov if (local_acc->name) {
630b0588390SAlexei Starovoitov matched = bpf_core_match_member(local_spec->btf,
631b0588390SAlexei Starovoitov local_acc,
632b0588390SAlexei Starovoitov targ_btf, targ_id,
633b0588390SAlexei Starovoitov targ_spec, &targ_id);
634b0588390SAlexei Starovoitov if (matched <= 0)
635b0588390SAlexei Starovoitov return matched;
636b0588390SAlexei Starovoitov } else {
637b0588390SAlexei Starovoitov /* for i=0, targ_id is already treated as array element
638b0588390SAlexei Starovoitov * type (because it's the original struct), for others
639b0588390SAlexei Starovoitov * we should find array element type first
640b0588390SAlexei Starovoitov */
641b0588390SAlexei Starovoitov if (i > 0) {
642b0588390SAlexei Starovoitov const struct btf_array *a;
643b0588390SAlexei Starovoitov bool flex;
644b0588390SAlexei Starovoitov
645b0588390SAlexei Starovoitov if (!btf_is_array(targ_type))
646b0588390SAlexei Starovoitov return 0;
647b0588390SAlexei Starovoitov
648b0588390SAlexei Starovoitov a = btf_array(targ_type);
649b0588390SAlexei Starovoitov flex = is_flex_arr(targ_btf, targ_acc - 1, a);
650b0588390SAlexei Starovoitov if (!flex && local_acc->idx >= a->nelems)
651b0588390SAlexei Starovoitov return 0;
652b0588390SAlexei Starovoitov if (!skip_mods_and_typedefs(targ_btf, a->type,
653b0588390SAlexei Starovoitov &targ_id))
654b0588390SAlexei Starovoitov return -EINVAL;
655b0588390SAlexei Starovoitov }
656b0588390SAlexei Starovoitov
657b0588390SAlexei Starovoitov /* too deep struct/union/array nesting */
658b0588390SAlexei Starovoitov if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
659b0588390SAlexei Starovoitov return -E2BIG;
660b0588390SAlexei Starovoitov
661b0588390SAlexei Starovoitov targ_acc->type_id = targ_id;
662b0588390SAlexei Starovoitov targ_acc->idx = local_acc->idx;
663b0588390SAlexei Starovoitov targ_acc->name = NULL;
664b0588390SAlexei Starovoitov targ_spec->len++;
665b0588390SAlexei Starovoitov targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
666b0588390SAlexei Starovoitov targ_spec->raw_len++;
667b0588390SAlexei Starovoitov
668b0588390SAlexei Starovoitov sz = btf__resolve_size(targ_btf, targ_id);
669b0588390SAlexei Starovoitov if (sz < 0)
670b0588390SAlexei Starovoitov return sz;
671b0588390SAlexei Starovoitov targ_spec->bit_offset += local_acc->idx * sz * 8;
672b0588390SAlexei Starovoitov }
673b0588390SAlexei Starovoitov }
674b0588390SAlexei Starovoitov
675b0588390SAlexei Starovoitov return 1;
676b0588390SAlexei Starovoitov }
677b0588390SAlexei Starovoitov
bpf_core_calc_field_relo(const char * prog_name,const struct bpf_core_relo * relo,const struct bpf_core_spec * spec,__u64 * val,__u32 * field_sz,__u32 * type_id,bool * validate)678b0588390SAlexei Starovoitov static int bpf_core_calc_field_relo(const char *prog_name,
679b0588390SAlexei Starovoitov const struct bpf_core_relo *relo,
680b0588390SAlexei Starovoitov const struct bpf_core_spec *spec,
68177628165SYonghong Song __u64 *val, __u32 *field_sz, __u32 *type_id,
682b0588390SAlexei Starovoitov bool *validate)
683b0588390SAlexei Starovoitov {
684b0588390SAlexei Starovoitov const struct bpf_core_accessor *acc;
685b0588390SAlexei Starovoitov const struct btf_type *t;
686b0588390SAlexei Starovoitov __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
687b0588390SAlexei Starovoitov const struct btf_member *m;
688b0588390SAlexei Starovoitov const struct btf_type *mt;
689b0588390SAlexei Starovoitov bool bitfield;
690b0588390SAlexei Starovoitov __s64 sz;
691b0588390SAlexei Starovoitov
692b0588390SAlexei Starovoitov *field_sz = 0;
693b0588390SAlexei Starovoitov
69446334a0cSAlexei Starovoitov if (relo->kind == BPF_CORE_FIELD_EXISTS) {
695b0588390SAlexei Starovoitov *val = spec ? 1 : 0;
696b0588390SAlexei Starovoitov return 0;
697b0588390SAlexei Starovoitov }
698b0588390SAlexei Starovoitov
699b0588390SAlexei Starovoitov if (!spec)
700b0588390SAlexei Starovoitov return -EUCLEAN; /* request instruction poisoning */
701b0588390SAlexei Starovoitov
702b0588390SAlexei Starovoitov acc = &spec->spec[spec->len - 1];
70374753e14SAlexei Starovoitov t = btf_type_by_id(spec->btf, acc->type_id);
704b0588390SAlexei Starovoitov
705b0588390SAlexei Starovoitov /* a[n] accessor needs special handling */
706b0588390SAlexei Starovoitov if (!acc->name) {
70746334a0cSAlexei Starovoitov if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) {
708b0588390SAlexei Starovoitov *val = spec->bit_offset / 8;
709b0588390SAlexei Starovoitov /* remember field size for load/store mem size */
710b0588390SAlexei Starovoitov sz = btf__resolve_size(spec->btf, acc->type_id);
711b0588390SAlexei Starovoitov if (sz < 0)
712b0588390SAlexei Starovoitov return -EINVAL;
713b0588390SAlexei Starovoitov *field_sz = sz;
714b0588390SAlexei Starovoitov *type_id = acc->type_id;
71546334a0cSAlexei Starovoitov } else if (relo->kind == BPF_CORE_FIELD_BYTE_SIZE) {
716b0588390SAlexei Starovoitov sz = btf__resolve_size(spec->btf, acc->type_id);
717b0588390SAlexei Starovoitov if (sz < 0)
718b0588390SAlexei Starovoitov return -EINVAL;
719b0588390SAlexei Starovoitov *val = sz;
720b0588390SAlexei Starovoitov } else {
721b0588390SAlexei Starovoitov pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
722b0588390SAlexei Starovoitov prog_name, relo->kind, relo->insn_off / 8);
723b0588390SAlexei Starovoitov return -EINVAL;
724b0588390SAlexei Starovoitov }
725b0588390SAlexei Starovoitov if (validate)
726b0588390SAlexei Starovoitov *validate = true;
727b0588390SAlexei Starovoitov return 0;
728b0588390SAlexei Starovoitov }
729b0588390SAlexei Starovoitov
730b0588390SAlexei Starovoitov m = btf_members(t) + acc->idx;
731b0588390SAlexei Starovoitov mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
732b0588390SAlexei Starovoitov bit_off = spec->bit_offset;
733b0588390SAlexei Starovoitov bit_sz = btf_member_bitfield_size(t, acc->idx);
734b0588390SAlexei Starovoitov
735b0588390SAlexei Starovoitov bitfield = bit_sz > 0;
736b0588390SAlexei Starovoitov if (bitfield) {
737b0588390SAlexei Starovoitov byte_sz = mt->size;
738b0588390SAlexei Starovoitov byte_off = bit_off / 8 / byte_sz * byte_sz;
739b0588390SAlexei Starovoitov /* figure out smallest int size necessary for bitfield load */
740b0588390SAlexei Starovoitov while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
741b0588390SAlexei Starovoitov if (byte_sz >= 8) {
742b0588390SAlexei Starovoitov /* bitfield can't be read with 64-bit read */
743b0588390SAlexei Starovoitov pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
744b0588390SAlexei Starovoitov prog_name, relo->kind, relo->insn_off / 8);
745b0588390SAlexei Starovoitov return -E2BIG;
746b0588390SAlexei Starovoitov }
747b0588390SAlexei Starovoitov byte_sz *= 2;
748b0588390SAlexei Starovoitov byte_off = bit_off / 8 / byte_sz * byte_sz;
749b0588390SAlexei Starovoitov }
750b0588390SAlexei Starovoitov } else {
751b0588390SAlexei Starovoitov sz = btf__resolve_size(spec->btf, field_type_id);
752b0588390SAlexei Starovoitov if (sz < 0)
753b0588390SAlexei Starovoitov return -EINVAL;
754b0588390SAlexei Starovoitov byte_sz = sz;
755b0588390SAlexei Starovoitov byte_off = spec->bit_offset / 8;
756b0588390SAlexei Starovoitov bit_sz = byte_sz * 8;
757b0588390SAlexei Starovoitov }
758b0588390SAlexei Starovoitov
759b0588390SAlexei Starovoitov /* for bitfields, all the relocatable aspects are ambiguous and we
760b0588390SAlexei Starovoitov * might disagree with compiler, so turn off validation of expected
761b0588390SAlexei Starovoitov * value, except for signedness
762b0588390SAlexei Starovoitov */
763b0588390SAlexei Starovoitov if (validate)
764b0588390SAlexei Starovoitov *validate = !bitfield;
765b0588390SAlexei Starovoitov
766b0588390SAlexei Starovoitov switch (relo->kind) {
76746334a0cSAlexei Starovoitov case BPF_CORE_FIELD_BYTE_OFFSET:
768b0588390SAlexei Starovoitov *val = byte_off;
769b0588390SAlexei Starovoitov if (!bitfield) {
770b0588390SAlexei Starovoitov *field_sz = byte_sz;
771b0588390SAlexei Starovoitov *type_id = field_type_id;
772b0588390SAlexei Starovoitov }
773b0588390SAlexei Starovoitov break;
77446334a0cSAlexei Starovoitov case BPF_CORE_FIELD_BYTE_SIZE:
775b0588390SAlexei Starovoitov *val = byte_sz;
776b0588390SAlexei Starovoitov break;
77746334a0cSAlexei Starovoitov case BPF_CORE_FIELD_SIGNED:
77823b2a3a8SYonghong Song *val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) ||
779*f3bdb54fSAndrii Nakryiko (btf_is_int(mt) && (btf_int_encoding(mt) & BTF_INT_SIGNED));
780b0588390SAlexei Starovoitov if (validate)
781b0588390SAlexei Starovoitov *validate = true; /* signedness is never ambiguous */
782b0588390SAlexei Starovoitov break;
78346334a0cSAlexei Starovoitov case BPF_CORE_FIELD_LSHIFT_U64:
7843930198dSIlya Leoshkevich #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
785b0588390SAlexei Starovoitov *val = 64 - (bit_off + bit_sz - byte_off * 8);
786b0588390SAlexei Starovoitov #else
787b0588390SAlexei Starovoitov *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
788b0588390SAlexei Starovoitov #endif
789b0588390SAlexei Starovoitov break;
79046334a0cSAlexei Starovoitov case BPF_CORE_FIELD_RSHIFT_U64:
791b0588390SAlexei Starovoitov *val = 64 - bit_sz;
792b0588390SAlexei Starovoitov if (validate)
793b0588390SAlexei Starovoitov *validate = true; /* right shift is never ambiguous */
794b0588390SAlexei Starovoitov break;
79546334a0cSAlexei Starovoitov case BPF_CORE_FIELD_EXISTS:
796b0588390SAlexei Starovoitov default:
797b0588390SAlexei Starovoitov return -EOPNOTSUPP;
798b0588390SAlexei Starovoitov }
799b0588390SAlexei Starovoitov
800b0588390SAlexei Starovoitov return 0;
801b0588390SAlexei Starovoitov }
802b0588390SAlexei Starovoitov
bpf_core_calc_type_relo(const struct bpf_core_relo * relo,const struct bpf_core_spec * spec,__u64 * val,bool * validate)803b0588390SAlexei Starovoitov static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
804b0588390SAlexei Starovoitov const struct bpf_core_spec *spec,
80577628165SYonghong Song __u64 *val, bool *validate)
806b0588390SAlexei Starovoitov {
807b0588390SAlexei Starovoitov __s64 sz;
808b0588390SAlexei Starovoitov
8094b443bc1SAndrii Nakryiko /* by default, always check expected value in bpf_insn */
8104b443bc1SAndrii Nakryiko if (validate)
8114b443bc1SAndrii Nakryiko *validate = true;
8124b443bc1SAndrii Nakryiko
813b0588390SAlexei Starovoitov /* type-based relos return zero when target type is not found */
814b0588390SAlexei Starovoitov if (!spec) {
815b0588390SAlexei Starovoitov *val = 0;
816b0588390SAlexei Starovoitov return 0;
817b0588390SAlexei Starovoitov }
818b0588390SAlexei Starovoitov
819b0588390SAlexei Starovoitov switch (relo->kind) {
82046334a0cSAlexei Starovoitov case BPF_CORE_TYPE_ID_TARGET:
821b0588390SAlexei Starovoitov *val = spec->root_type_id;
8224b443bc1SAndrii Nakryiko /* type ID, embedded in bpf_insn, might change during linking,
8234b443bc1SAndrii Nakryiko * so enforcing it is pointless
8244b443bc1SAndrii Nakryiko */
8254b443bc1SAndrii Nakryiko if (validate)
8264b443bc1SAndrii Nakryiko *validate = false;
827b0588390SAlexei Starovoitov break;
82846334a0cSAlexei Starovoitov case BPF_CORE_TYPE_EXISTS:
829ec6209c8SDaniel Müller case BPF_CORE_TYPE_MATCHES:
830b0588390SAlexei Starovoitov *val = 1;
831b0588390SAlexei Starovoitov break;
83246334a0cSAlexei Starovoitov case BPF_CORE_TYPE_SIZE:
833b0588390SAlexei Starovoitov sz = btf__resolve_size(spec->btf, spec->root_type_id);
834b0588390SAlexei Starovoitov if (sz < 0)
835b0588390SAlexei Starovoitov return -EINVAL;
836b0588390SAlexei Starovoitov *val = sz;
837b0588390SAlexei Starovoitov break;
83846334a0cSAlexei Starovoitov case BPF_CORE_TYPE_ID_LOCAL:
83946334a0cSAlexei Starovoitov /* BPF_CORE_TYPE_ID_LOCAL is handled specially and shouldn't get here */
840b0588390SAlexei Starovoitov default:
841b0588390SAlexei Starovoitov return -EOPNOTSUPP;
842b0588390SAlexei Starovoitov }
843b0588390SAlexei Starovoitov
844b0588390SAlexei Starovoitov return 0;
845b0588390SAlexei Starovoitov }
846b0588390SAlexei Starovoitov
bpf_core_calc_enumval_relo(const struct bpf_core_relo * relo,const struct bpf_core_spec * spec,__u64 * val)847b0588390SAlexei Starovoitov static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
848b0588390SAlexei Starovoitov const struct bpf_core_spec *spec,
84977628165SYonghong Song __u64 *val)
850b0588390SAlexei Starovoitov {
851b0588390SAlexei Starovoitov const struct btf_type *t;
852b0588390SAlexei Starovoitov
853b0588390SAlexei Starovoitov switch (relo->kind) {
85446334a0cSAlexei Starovoitov case BPF_CORE_ENUMVAL_EXISTS:
855b0588390SAlexei Starovoitov *val = spec ? 1 : 0;
856b0588390SAlexei Starovoitov break;
85746334a0cSAlexei Starovoitov case BPF_CORE_ENUMVAL_VALUE:
858b0588390SAlexei Starovoitov if (!spec)
859b0588390SAlexei Starovoitov return -EUCLEAN; /* request instruction poisoning */
86074753e14SAlexei Starovoitov t = btf_type_by_id(spec->btf, spec->spec[0].type_id);
86123b2a3a8SYonghong Song if (btf_is_enum(t))
86223b2a3a8SYonghong Song *val = btf_enum(t)[spec->spec[0].idx].val;
86323b2a3a8SYonghong Song else
86423b2a3a8SYonghong Song *val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx);
865b0588390SAlexei Starovoitov break;
866b0588390SAlexei Starovoitov default:
867b0588390SAlexei Starovoitov return -EOPNOTSUPP;
868b0588390SAlexei Starovoitov }
869b0588390SAlexei Starovoitov
870b0588390SAlexei Starovoitov return 0;
871b0588390SAlexei Starovoitov }
872b0588390SAlexei Starovoitov
873b0588390SAlexei Starovoitov /* Calculate original and target relocation values, given local and target
874b0588390SAlexei Starovoitov * specs and relocation kind. These values are calculated for each candidate.
875b0588390SAlexei Starovoitov * If there are multiple candidates, resulting values should all be consistent
876b0588390SAlexei Starovoitov * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity.
877b0588390SAlexei Starovoitov * If instruction has to be poisoned, *poison will be set to true.
878b0588390SAlexei Starovoitov */
bpf_core_calc_relo(const char * prog_name,const struct bpf_core_relo * relo,int relo_idx,const struct bpf_core_spec * local_spec,const struct bpf_core_spec * targ_spec,struct bpf_core_relo_res * res)879b0588390SAlexei Starovoitov static int bpf_core_calc_relo(const char *prog_name,
880b0588390SAlexei Starovoitov const struct bpf_core_relo *relo,
881b0588390SAlexei Starovoitov int relo_idx,
882b0588390SAlexei Starovoitov const struct bpf_core_spec *local_spec,
883b0588390SAlexei Starovoitov const struct bpf_core_spec *targ_spec,
884b0588390SAlexei Starovoitov struct bpf_core_relo_res *res)
885b0588390SAlexei Starovoitov {
886b0588390SAlexei Starovoitov int err = -EOPNOTSUPP;
887b0588390SAlexei Starovoitov
888b0588390SAlexei Starovoitov res->orig_val = 0;
889b0588390SAlexei Starovoitov res->new_val = 0;
890b0588390SAlexei Starovoitov res->poison = false;
891b0588390SAlexei Starovoitov res->validate = true;
892b0588390SAlexei Starovoitov res->fail_memsz_adjust = false;
893b0588390SAlexei Starovoitov res->orig_sz = res->new_sz = 0;
894b0588390SAlexei Starovoitov res->orig_type_id = res->new_type_id = 0;
895b0588390SAlexei Starovoitov
896b0588390SAlexei Starovoitov if (core_relo_is_field_based(relo->kind)) {
897b0588390SAlexei Starovoitov err = bpf_core_calc_field_relo(prog_name, relo, local_spec,
898b0588390SAlexei Starovoitov &res->orig_val, &res->orig_sz,
899b0588390SAlexei Starovoitov &res->orig_type_id, &res->validate);
900b0588390SAlexei Starovoitov err = err ?: bpf_core_calc_field_relo(prog_name, relo, targ_spec,
901b0588390SAlexei Starovoitov &res->new_val, &res->new_sz,
902b0588390SAlexei Starovoitov &res->new_type_id, NULL);
903b0588390SAlexei Starovoitov if (err)
904b0588390SAlexei Starovoitov goto done;
905b0588390SAlexei Starovoitov /* Validate if it's safe to adjust load/store memory size.
906b0588390SAlexei Starovoitov * Adjustments are performed only if original and new memory
907b0588390SAlexei Starovoitov * sizes differ.
908b0588390SAlexei Starovoitov */
909b0588390SAlexei Starovoitov res->fail_memsz_adjust = false;
910b0588390SAlexei Starovoitov if (res->orig_sz != res->new_sz) {
911b0588390SAlexei Starovoitov const struct btf_type *orig_t, *new_t;
912b0588390SAlexei Starovoitov
91374753e14SAlexei Starovoitov orig_t = btf_type_by_id(local_spec->btf, res->orig_type_id);
91474753e14SAlexei Starovoitov new_t = btf_type_by_id(targ_spec->btf, res->new_type_id);
915b0588390SAlexei Starovoitov
916b0588390SAlexei Starovoitov /* There are two use cases in which it's safe to
917b0588390SAlexei Starovoitov * adjust load/store's mem size:
918b0588390SAlexei Starovoitov * - reading a 32-bit kernel pointer, while on BPF
919b0588390SAlexei Starovoitov * size pointers are always 64-bit; in this case
920b0588390SAlexei Starovoitov * it's safe to "downsize" instruction size due to
921b0588390SAlexei Starovoitov * pointer being treated as unsigned integer with
922b0588390SAlexei Starovoitov * zero-extended upper 32-bits;
923b0588390SAlexei Starovoitov * - reading unsigned integers, again due to
924b0588390SAlexei Starovoitov * zero-extension is preserving the value correctly.
925b0588390SAlexei Starovoitov *
926b0588390SAlexei Starovoitov * In all other cases it's incorrect to attempt to
927b0588390SAlexei Starovoitov * load/store field because read value will be
928b0588390SAlexei Starovoitov * incorrect, so we poison relocated instruction.
929b0588390SAlexei Starovoitov */
930b0588390SAlexei Starovoitov if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
931b0588390SAlexei Starovoitov goto done;
932b0588390SAlexei Starovoitov if (btf_is_int(orig_t) && btf_is_int(new_t) &&
933b0588390SAlexei Starovoitov btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
934b0588390SAlexei Starovoitov btf_int_encoding(new_t) != BTF_INT_SIGNED)
935b0588390SAlexei Starovoitov goto done;
936b0588390SAlexei Starovoitov
937b0588390SAlexei Starovoitov /* mark as invalid mem size adjustment, but this will
938b0588390SAlexei Starovoitov * only be checked for LDX/STX/ST insns
939b0588390SAlexei Starovoitov */
940b0588390SAlexei Starovoitov res->fail_memsz_adjust = true;
941b0588390SAlexei Starovoitov }
942b0588390SAlexei Starovoitov } else if (core_relo_is_type_based(relo->kind)) {
9434b443bc1SAndrii Nakryiko err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate);
9444b443bc1SAndrii Nakryiko err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL);
945b0588390SAlexei Starovoitov } else if (core_relo_is_enumval_based(relo->kind)) {
946b0588390SAlexei Starovoitov err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
947b0588390SAlexei Starovoitov err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
948b0588390SAlexei Starovoitov }
949b0588390SAlexei Starovoitov
950b0588390SAlexei Starovoitov done:
951b0588390SAlexei Starovoitov if (err == -EUCLEAN) {
952b0588390SAlexei Starovoitov /* EUCLEAN is used to signal instruction poisoning request */
953b0588390SAlexei Starovoitov res->poison = true;
954b0588390SAlexei Starovoitov err = 0;
955b0588390SAlexei Starovoitov } else if (err == -EOPNOTSUPP) {
956b0588390SAlexei Starovoitov /* EOPNOTSUPP means unknown/unsupported relocation */
957b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
958b0588390SAlexei Starovoitov prog_name, relo_idx, core_relo_kind_str(relo->kind),
959b0588390SAlexei Starovoitov relo->kind, relo->insn_off / 8);
960b0588390SAlexei Starovoitov }
961b0588390SAlexei Starovoitov
962b0588390SAlexei Starovoitov return err;
963b0588390SAlexei Starovoitov }
964b0588390SAlexei Starovoitov
965b0588390SAlexei Starovoitov /*
966b0588390SAlexei Starovoitov * Turn instruction for which CO_RE relocation failed into invalid one with
967b0588390SAlexei Starovoitov * distinct signature.
968b0588390SAlexei Starovoitov */
bpf_core_poison_insn(const char * prog_name,int relo_idx,int insn_idx,struct bpf_insn * insn)969b0588390SAlexei Starovoitov static void bpf_core_poison_insn(const char *prog_name, int relo_idx,
970b0588390SAlexei Starovoitov int insn_idx, struct bpf_insn *insn)
971b0588390SAlexei Starovoitov {
972b0588390SAlexei Starovoitov pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
973b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx);
974b0588390SAlexei Starovoitov insn->code = BPF_JMP | BPF_CALL;
975b0588390SAlexei Starovoitov insn->dst_reg = 0;
976b0588390SAlexei Starovoitov insn->src_reg = 0;
977b0588390SAlexei Starovoitov insn->off = 0;
978b0588390SAlexei Starovoitov /* if this instruction is reachable (not a dead code),
979b0588390SAlexei Starovoitov * verifier will complain with the following message:
980b0588390SAlexei Starovoitov * invalid func unknown#195896080
981b0588390SAlexei Starovoitov */
982b0588390SAlexei Starovoitov insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
983b0588390SAlexei Starovoitov }
984b0588390SAlexei Starovoitov
insn_bpf_size_to_bytes(struct bpf_insn * insn)985b0588390SAlexei Starovoitov static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
986b0588390SAlexei Starovoitov {
987b0588390SAlexei Starovoitov switch (BPF_SIZE(insn->code)) {
988b0588390SAlexei Starovoitov case BPF_DW: return 8;
989b0588390SAlexei Starovoitov case BPF_W: return 4;
990b0588390SAlexei Starovoitov case BPF_H: return 2;
991b0588390SAlexei Starovoitov case BPF_B: return 1;
992b0588390SAlexei Starovoitov default: return -1;
993b0588390SAlexei Starovoitov }
994b0588390SAlexei Starovoitov }
995b0588390SAlexei Starovoitov
insn_bytes_to_bpf_size(__u32 sz)996b0588390SAlexei Starovoitov static int insn_bytes_to_bpf_size(__u32 sz)
997b0588390SAlexei Starovoitov {
998b0588390SAlexei Starovoitov switch (sz) {
999b0588390SAlexei Starovoitov case 8: return BPF_DW;
1000b0588390SAlexei Starovoitov case 4: return BPF_W;
1001b0588390SAlexei Starovoitov case 2: return BPF_H;
1002b0588390SAlexei Starovoitov case 1: return BPF_B;
1003b0588390SAlexei Starovoitov default: return -1;
1004b0588390SAlexei Starovoitov }
1005b0588390SAlexei Starovoitov }
1006b0588390SAlexei Starovoitov
1007b0588390SAlexei Starovoitov /*
1008b0588390SAlexei Starovoitov * Patch relocatable BPF instruction.
1009b0588390SAlexei Starovoitov *
1010b0588390SAlexei Starovoitov * Patched value is determined by relocation kind and target specification.
1011b0588390SAlexei Starovoitov * For existence relocations target spec will be NULL if field/type is not found.
1012b0588390SAlexei Starovoitov * Expected insn->imm value is determined using relocation kind and local
1013b0588390SAlexei Starovoitov * spec, and is checked before patching instruction. If actual insn->imm value
1014b0588390SAlexei Starovoitov * is wrong, bail out with error.
1015b0588390SAlexei Starovoitov *
1016b0588390SAlexei Starovoitov * Currently supported classes of BPF instruction are:
1017b0588390SAlexei Starovoitov * 1. rX = <imm> (assignment with immediate operand);
1018b0588390SAlexei Starovoitov * 2. rX += <imm> (arithmetic operations with immediate operand);
1019b0588390SAlexei Starovoitov * 3. rX = <imm64> (load with 64-bit immediate value);
1020b0588390SAlexei Starovoitov * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64};
1021b0588390SAlexei Starovoitov * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
1022b0588390SAlexei Starovoitov * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
1023b0588390SAlexei Starovoitov */
bpf_core_patch_insn(const char * prog_name,struct bpf_insn * insn,int insn_idx,const struct bpf_core_relo * relo,int relo_idx,const struct bpf_core_relo_res * res)1024adb8fa19SMauricio Vásquez int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
1025b0588390SAlexei Starovoitov int insn_idx, const struct bpf_core_relo *relo,
1026b0588390SAlexei Starovoitov int relo_idx, const struct bpf_core_relo_res *res)
1027b0588390SAlexei Starovoitov {
102877628165SYonghong Song __u64 orig_val, new_val;
1029b0588390SAlexei Starovoitov __u8 class;
1030b0588390SAlexei Starovoitov
1031b0588390SAlexei Starovoitov class = BPF_CLASS(insn->code);
1032b0588390SAlexei Starovoitov
1033b0588390SAlexei Starovoitov if (res->poison) {
1034b0588390SAlexei Starovoitov poison:
1035b0588390SAlexei Starovoitov /* poison second part of ldimm64 to avoid confusing error from
1036b0588390SAlexei Starovoitov * verifier about "unknown opcode 00"
1037b0588390SAlexei Starovoitov */
1038b0588390SAlexei Starovoitov if (is_ldimm64_insn(insn))
1039b0588390SAlexei Starovoitov bpf_core_poison_insn(prog_name, relo_idx, insn_idx + 1, insn + 1);
1040b0588390SAlexei Starovoitov bpf_core_poison_insn(prog_name, relo_idx, insn_idx, insn);
1041b0588390SAlexei Starovoitov return 0;
1042b0588390SAlexei Starovoitov }
1043b0588390SAlexei Starovoitov
1044b0588390SAlexei Starovoitov orig_val = res->orig_val;
1045b0588390SAlexei Starovoitov new_val = res->new_val;
1046b0588390SAlexei Starovoitov
1047b0588390SAlexei Starovoitov switch (class) {
1048b0588390SAlexei Starovoitov case BPF_ALU:
1049b0588390SAlexei Starovoitov case BPF_ALU64:
1050b0588390SAlexei Starovoitov if (BPF_SRC(insn->code) != BPF_K)
1051b0588390SAlexei Starovoitov return -EINVAL;
1052b0588390SAlexei Starovoitov if (res->validate && insn->imm != orig_val) {
105377628165SYonghong Song pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n",
1054b0588390SAlexei Starovoitov prog_name, relo_idx,
105577628165SYonghong Song insn_idx, insn->imm, (unsigned long long)orig_val,
105677628165SYonghong Song (unsigned long long)new_val);
1057b0588390SAlexei Starovoitov return -EINVAL;
1058b0588390SAlexei Starovoitov }
1059b0588390SAlexei Starovoitov orig_val = insn->imm;
1060b0588390SAlexei Starovoitov insn->imm = new_val;
106177628165SYonghong Song pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n",
1062b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx,
106377628165SYonghong Song (unsigned long long)orig_val, (unsigned long long)new_val);
1064b0588390SAlexei Starovoitov break;
1065b0588390SAlexei Starovoitov case BPF_LDX:
1066b0588390SAlexei Starovoitov case BPF_ST:
1067b0588390SAlexei Starovoitov case BPF_STX:
1068b0588390SAlexei Starovoitov if (res->validate && insn->off != orig_val) {
106977628165SYonghong Song pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n",
107077628165SYonghong Song prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val,
107177628165SYonghong Song (unsigned long long)new_val);
1072b0588390SAlexei Starovoitov return -EINVAL;
1073b0588390SAlexei Starovoitov }
1074b0588390SAlexei Starovoitov if (new_val > SHRT_MAX) {
107577628165SYonghong Song pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n",
107677628165SYonghong Song prog_name, relo_idx, insn_idx, (unsigned long long)new_val);
1077b0588390SAlexei Starovoitov return -ERANGE;
1078b0588390SAlexei Starovoitov }
1079b0588390SAlexei Starovoitov if (res->fail_memsz_adjust) {
1080b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
1081b0588390SAlexei Starovoitov "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
1082b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx);
1083b0588390SAlexei Starovoitov goto poison;
1084b0588390SAlexei Starovoitov }
1085b0588390SAlexei Starovoitov
1086b0588390SAlexei Starovoitov orig_val = insn->off;
1087b0588390SAlexei Starovoitov insn->off = new_val;
108877628165SYonghong Song pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n",
108977628165SYonghong Song prog_name, relo_idx, insn_idx, (unsigned long long)orig_val,
109077628165SYonghong Song (unsigned long long)new_val);
1091b0588390SAlexei Starovoitov
1092b0588390SAlexei Starovoitov if (res->new_sz != res->orig_sz) {
1093b0588390SAlexei Starovoitov int insn_bytes_sz, insn_bpf_sz;
1094b0588390SAlexei Starovoitov
1095b0588390SAlexei Starovoitov insn_bytes_sz = insn_bpf_size_to_bytes(insn);
1096b0588390SAlexei Starovoitov if (insn_bytes_sz != res->orig_sz) {
1097b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
1098b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
1099b0588390SAlexei Starovoitov return -EINVAL;
1100b0588390SAlexei Starovoitov }
1101b0588390SAlexei Starovoitov
1102b0588390SAlexei Starovoitov insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
1103b0588390SAlexei Starovoitov if (insn_bpf_sz < 0) {
1104b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
1105b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx, res->new_sz);
1106b0588390SAlexei Starovoitov return -EINVAL;
1107b0588390SAlexei Starovoitov }
1108b0588390SAlexei Starovoitov
1109b0588390SAlexei Starovoitov insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);
1110b0588390SAlexei Starovoitov pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
1111b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
1112b0588390SAlexei Starovoitov }
1113b0588390SAlexei Starovoitov break;
1114b0588390SAlexei Starovoitov case BPF_LD: {
1115b0588390SAlexei Starovoitov __u64 imm;
1116b0588390SAlexei Starovoitov
1117b0588390SAlexei Starovoitov if (!is_ldimm64_insn(insn) ||
1118b0588390SAlexei Starovoitov insn[0].src_reg != 0 || insn[0].off != 0 ||
1119b0588390SAlexei Starovoitov insn[1].code != 0 || insn[1].dst_reg != 0 ||
1120b0588390SAlexei Starovoitov insn[1].src_reg != 0 || insn[1].off != 0) {
1121b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
1122b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx);
1123b0588390SAlexei Starovoitov return -EINVAL;
1124b0588390SAlexei Starovoitov }
1125b0588390SAlexei Starovoitov
1126b58b2b3aSYonghong Song imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32);
1127b0588390SAlexei Starovoitov if (res->validate && imm != orig_val) {
112877628165SYonghong Song pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n",
1129b0588390SAlexei Starovoitov prog_name, relo_idx,
1130b0588390SAlexei Starovoitov insn_idx, (unsigned long long)imm,
113177628165SYonghong Song (unsigned long long)orig_val, (unsigned long long)new_val);
1132b0588390SAlexei Starovoitov return -EINVAL;
1133b0588390SAlexei Starovoitov }
1134b0588390SAlexei Starovoitov
1135b0588390SAlexei Starovoitov insn[0].imm = new_val;
113677628165SYonghong Song insn[1].imm = new_val >> 32;
113777628165SYonghong Song pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n",
1138b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx,
113977628165SYonghong Song (unsigned long long)imm, (unsigned long long)new_val);
1140b0588390SAlexei Starovoitov break;
1141b0588390SAlexei Starovoitov }
1142b0588390SAlexei Starovoitov default:
1143b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
1144b0588390SAlexei Starovoitov prog_name, relo_idx, insn_idx, insn->code,
1145b0588390SAlexei Starovoitov insn->src_reg, insn->dst_reg, insn->off, insn->imm);
1146b0588390SAlexei Starovoitov return -EINVAL;
1147b0588390SAlexei Starovoitov }
1148b0588390SAlexei Starovoitov
1149b0588390SAlexei Starovoitov return 0;
1150b0588390SAlexei Starovoitov }
1151b0588390SAlexei Starovoitov
1152b0588390SAlexei Starovoitov /* Output spec definition in the format:
1153b0588390SAlexei Starovoitov * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
1154b0588390SAlexei Starovoitov * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
1155b0588390SAlexei Starovoitov */
bpf_core_format_spec(char * buf,size_t buf_sz,const struct bpf_core_spec * spec)11569fdc4273SAndrii Nakryiko int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec)
1157b0588390SAlexei Starovoitov {
1158b0588390SAlexei Starovoitov const struct btf_type *t;
1159b0588390SAlexei Starovoitov const char *s;
1160b0588390SAlexei Starovoitov __u32 type_id;
1161b58af63aSAndrii Nakryiko int i, len = 0;
1162b58af63aSAndrii Nakryiko
1163b58af63aSAndrii Nakryiko #define append_buf(fmt, args...) \
1164b58af63aSAndrii Nakryiko ({ \
1165b58af63aSAndrii Nakryiko int r; \
1166b58af63aSAndrii Nakryiko r = snprintf(buf, buf_sz, fmt, ##args); \
1167b58af63aSAndrii Nakryiko len += r; \
1168b58af63aSAndrii Nakryiko if (r >= buf_sz) \
1169b58af63aSAndrii Nakryiko r = buf_sz; \
1170b58af63aSAndrii Nakryiko buf += r; \
1171b58af63aSAndrii Nakryiko buf_sz -= r; \
1172b58af63aSAndrii Nakryiko })
1173b0588390SAlexei Starovoitov
1174b0588390SAlexei Starovoitov type_id = spec->root_type_id;
117574753e14SAlexei Starovoitov t = btf_type_by_id(spec->btf, type_id);
1176b0588390SAlexei Starovoitov s = btf__name_by_offset(spec->btf, t->name_off);
1177b0588390SAlexei Starovoitov
1178b58af63aSAndrii Nakryiko append_buf("<%s> [%u] %s %s",
1179b58af63aSAndrii Nakryiko core_relo_kind_str(spec->relo_kind),
1180b58af63aSAndrii Nakryiko type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
1181b0588390SAlexei Starovoitov
1182b0588390SAlexei Starovoitov if (core_relo_is_type_based(spec->relo_kind))
1183b58af63aSAndrii Nakryiko return len;
1184b0588390SAlexei Starovoitov
1185b0588390SAlexei Starovoitov if (core_relo_is_enumval_based(spec->relo_kind)) {
1186b0588390SAlexei Starovoitov t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
118723b2a3a8SYonghong Song if (btf_is_enum(t)) {
118823b2a3a8SYonghong Song const struct btf_enum *e;
118923b2a3a8SYonghong Song const char *fmt_str;
119023b2a3a8SYonghong Song
1191b0588390SAlexei Starovoitov e = btf_enum(t) + spec->raw_spec[0];
1192b0588390SAlexei Starovoitov s = btf__name_by_offset(spec->btf, e->name_off);
119323b2a3a8SYonghong Song fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u";
119423b2a3a8SYonghong Song append_buf(fmt_str, s, e->val);
119523b2a3a8SYonghong Song } else {
119623b2a3a8SYonghong Song const struct btf_enum64 *e;
119723b2a3a8SYonghong Song const char *fmt_str;
1198b0588390SAlexei Starovoitov
119923b2a3a8SYonghong Song e = btf_enum64(t) + spec->raw_spec[0];
120023b2a3a8SYonghong Song s = btf__name_by_offset(spec->btf, e->name_off);
120123b2a3a8SYonghong Song fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu";
120223b2a3a8SYonghong Song append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e));
120323b2a3a8SYonghong Song }
1204b58af63aSAndrii Nakryiko return len;
1205b0588390SAlexei Starovoitov }
1206b0588390SAlexei Starovoitov
1207b0588390SAlexei Starovoitov if (core_relo_is_field_based(spec->relo_kind)) {
1208b0588390SAlexei Starovoitov for (i = 0; i < spec->len; i++) {
1209b0588390SAlexei Starovoitov if (spec->spec[i].name)
1210b58af63aSAndrii Nakryiko append_buf(".%s", spec->spec[i].name);
1211b0588390SAlexei Starovoitov else if (i > 0 || spec->spec[i].idx > 0)
1212b58af63aSAndrii Nakryiko append_buf("[%u]", spec->spec[i].idx);
1213b0588390SAlexei Starovoitov }
1214b0588390SAlexei Starovoitov
1215b58af63aSAndrii Nakryiko append_buf(" (");
1216b0588390SAlexei Starovoitov for (i = 0; i < spec->raw_len; i++)
1217b58af63aSAndrii Nakryiko append_buf("%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
1218b0588390SAlexei Starovoitov
1219b0588390SAlexei Starovoitov if (spec->bit_offset % 8)
1220b58af63aSAndrii Nakryiko append_buf(" @ offset %u.%u)", spec->bit_offset / 8, spec->bit_offset % 8);
1221b0588390SAlexei Starovoitov else
1222b58af63aSAndrii Nakryiko append_buf(" @ offset %u)", spec->bit_offset / 8);
1223b58af63aSAndrii Nakryiko return len;
1224b0588390SAlexei Starovoitov }
1225b58af63aSAndrii Nakryiko
1226b58af63aSAndrii Nakryiko return len;
1227b58af63aSAndrii Nakryiko #undef append_buf
1228b0588390SAlexei Starovoitov }
1229b0588390SAlexei Starovoitov
1230b0588390SAlexei Starovoitov /*
1231adb8fa19SMauricio Vásquez * Calculate CO-RE relocation target result.
1232b0588390SAlexei Starovoitov *
1233b0588390SAlexei Starovoitov * The outline and important points of the algorithm:
1234b0588390SAlexei Starovoitov * 1. For given local type, find corresponding candidate target types.
1235b0588390SAlexei Starovoitov * Candidate type is a type with the same "essential" name, ignoring
1236b0588390SAlexei Starovoitov * everything after last triple underscore (___). E.g., `sample`,
1237b0588390SAlexei Starovoitov * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
1238b0588390SAlexei Starovoitov * for each other. Names with triple underscore are referred to as
1239b0588390SAlexei Starovoitov * "flavors" and are useful, among other things, to allow to
1240b0588390SAlexei Starovoitov * specify/support incompatible variations of the same kernel struct, which
1241b0588390SAlexei Starovoitov * might differ between different kernel versions and/or build
1242b0588390SAlexei Starovoitov * configurations.
1243b0588390SAlexei Starovoitov *
1244b0588390SAlexei Starovoitov * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
1245b0588390SAlexei Starovoitov * converter, when deduplicated BTF of a kernel still contains more than
1246b0588390SAlexei Starovoitov * one different types with the same name. In that case, ___2, ___3, etc
1247b0588390SAlexei Starovoitov * are appended starting from second name conflict. But start flavors are
1248b0588390SAlexei Starovoitov * also useful to be defined "locally", in BPF program, to extract same
1249b0588390SAlexei Starovoitov * data from incompatible changes between different kernel
1250b0588390SAlexei Starovoitov * versions/configurations. For instance, to handle field renames between
1251b0588390SAlexei Starovoitov * kernel versions, one can use two flavors of the struct name with the
1252b0588390SAlexei Starovoitov * same common name and use conditional relocations to extract that field,
1253b0588390SAlexei Starovoitov * depending on target kernel version.
1254b0588390SAlexei Starovoitov * 2. For each candidate type, try to match local specification to this
1255b0588390SAlexei Starovoitov * candidate target type. Matching involves finding corresponding
1256b0588390SAlexei Starovoitov * high-level spec accessors, meaning that all named fields should match,
1257b0588390SAlexei Starovoitov * as well as all array accesses should be within the actual bounds. Also,
1258b0588390SAlexei Starovoitov * types should be compatible (see bpf_core_fields_are_compat for details).
1259b0588390SAlexei Starovoitov * 3. It is supported and expected that there might be multiple flavors
1260b0588390SAlexei Starovoitov * matching the spec. As long as all the specs resolve to the same set of
1261b0588390SAlexei Starovoitov * offsets across all candidates, there is no error. If there is any
12629bbdfad8SDaniel Müller * ambiguity, CO-RE relocation will fail. This is necessary to accommodate
12639bbdfad8SDaniel Müller * imperfection of BTF deduplication, which can cause slight duplication of
1264b0588390SAlexei Starovoitov * the same BTF type, if some directly or indirectly referenced (by
1265b0588390SAlexei Starovoitov * pointer) type gets resolved to different actual types in different
12669bbdfad8SDaniel Müller * object files. If such a situation occurs, deduplicated BTF will end up
1267b0588390SAlexei Starovoitov * with two (or more) structurally identical types, which differ only in
1268b0588390SAlexei Starovoitov * types they refer to through pointer. This should be OK in most cases and
1269b0588390SAlexei Starovoitov * is not an error.
1270b0588390SAlexei Starovoitov * 4. Candidate types search is performed by linearly scanning through all
1271b0588390SAlexei Starovoitov * types in target BTF. It is anticipated that this is overall more
1272b0588390SAlexei Starovoitov * efficient memory-wise and not significantly worse (if not better)
1273b0588390SAlexei Starovoitov * CPU-wise compared to prebuilding a map from all local type names to
1274b0588390SAlexei Starovoitov * a list of candidate type names. It's also sped up by caching resolved
1275b0588390SAlexei Starovoitov * list of matching candidates per each local "root" type ID, that has at
1276b0588390SAlexei Starovoitov * least one bpf_core_relo associated with it. This list is shared
1277b0588390SAlexei Starovoitov * between multiple relocations for the same type ID and is updated as some
1278b0588390SAlexei Starovoitov * of the candidates are pruned due to structural incompatibility.
1279b0588390SAlexei Starovoitov */
bpf_core_calc_relo_insn(const char * prog_name,const struct bpf_core_relo * relo,int relo_idx,const struct btf * local_btf,struct bpf_core_cand_list * cands,struct bpf_core_spec * specs_scratch,struct bpf_core_relo_res * targ_res)1280adb8fa19SMauricio Vásquez int bpf_core_calc_relo_insn(const char *prog_name,
1281b0588390SAlexei Starovoitov const struct bpf_core_relo *relo,
1282b0588390SAlexei Starovoitov int relo_idx,
1283b0588390SAlexei Starovoitov const struct btf *local_btf,
128478c1f8d0SAlexei Starovoitov struct bpf_core_cand_list *cands,
1285adb8fa19SMauricio Vásquez struct bpf_core_spec *specs_scratch,
1286adb8fa19SMauricio Vásquez struct bpf_core_relo_res *targ_res)
1287b0588390SAlexei Starovoitov {
128878c1f8d0SAlexei Starovoitov struct bpf_core_spec *local_spec = &specs_scratch[0];
128978c1f8d0SAlexei Starovoitov struct bpf_core_spec *cand_spec = &specs_scratch[1];
129078c1f8d0SAlexei Starovoitov struct bpf_core_spec *targ_spec = &specs_scratch[2];
1291adb8fa19SMauricio Vásquez struct bpf_core_relo_res cand_res;
1292b0588390SAlexei Starovoitov const struct btf_type *local_type;
1293b0588390SAlexei Starovoitov const char *local_name;
1294b0588390SAlexei Starovoitov __u32 local_id;
1295b58af63aSAndrii Nakryiko char spec_buf[256];
1296b0588390SAlexei Starovoitov int i, j, err;
1297b0588390SAlexei Starovoitov
1298b0588390SAlexei Starovoitov local_id = relo->type_id;
129974753e14SAlexei Starovoitov local_type = btf_type_by_id(local_btf, local_id);
1300b0588390SAlexei Starovoitov local_name = btf__name_by_offset(local_btf, local_type->name_off);
1301b0588390SAlexei Starovoitov if (!local_name)
1302b0588390SAlexei Starovoitov return -EINVAL;
1303b0588390SAlexei Starovoitov
130414032f26SAndrii Nakryiko err = bpf_core_parse_spec(prog_name, local_btf, relo, local_spec);
1305b0588390SAlexei Starovoitov if (err) {
130614032f26SAndrii Nakryiko const char *spec_str;
130714032f26SAndrii Nakryiko
130814032f26SAndrii Nakryiko spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
1309b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
1310b0588390SAlexei Starovoitov prog_name, relo_idx, local_id, btf_kind_str(local_type),
1311b0588390SAlexei Starovoitov str_is_empty(local_name) ? "<anon>" : local_name,
131214032f26SAndrii Nakryiko spec_str ?: "<?>", err);
1313b0588390SAlexei Starovoitov return -EINVAL;
1314b0588390SAlexei Starovoitov }
1315b0588390SAlexei Starovoitov
1316b58af63aSAndrii Nakryiko bpf_core_format_spec(spec_buf, sizeof(spec_buf), local_spec);
1317b58af63aSAndrii Nakryiko pr_debug("prog '%s': relo #%d: %s\n", prog_name, relo_idx, spec_buf);
1318b0588390SAlexei Starovoitov
1319b0588390SAlexei Starovoitov /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
132046334a0cSAlexei Starovoitov if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {
13214b443bc1SAndrii Nakryiko /* bpf_insn's imm value could get out of sync during linking */
1322adb8fa19SMauricio Vásquez memset(targ_res, 0, sizeof(*targ_res));
1323adb8fa19SMauricio Vásquez targ_res->validate = false;
1324adb8fa19SMauricio Vásquez targ_res->poison = false;
1325adb8fa19SMauricio Vásquez targ_res->orig_val = local_spec->root_type_id;
1326adb8fa19SMauricio Vásquez targ_res->new_val = local_spec->root_type_id;
1327adb8fa19SMauricio Vásquez return 0;
1328b0588390SAlexei Starovoitov }
1329b0588390SAlexei Starovoitov
1330b0588390SAlexei Starovoitov /* libbpf doesn't support candidate search for anonymous types */
1331afe98d46SAndrii Nakryiko if (str_is_empty(local_name)) {
1332b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
1333b0588390SAlexei Starovoitov prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
1334b0588390SAlexei Starovoitov return -EOPNOTSUPP;
1335b0588390SAlexei Starovoitov }
1336b0588390SAlexei Starovoitov
1337b0588390SAlexei Starovoitov for (i = 0, j = 0; i < cands->len; i++) {
133878c1f8d0SAlexei Starovoitov err = bpf_core_spec_match(local_spec, cands->cands[i].btf,
133978c1f8d0SAlexei Starovoitov cands->cands[i].id, cand_spec);
1340b0588390SAlexei Starovoitov if (err < 0) {
1341b58af63aSAndrii Nakryiko bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
1342b58af63aSAndrii Nakryiko pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n ",
1343b58af63aSAndrii Nakryiko prog_name, relo_idx, i, spec_buf, err);
1344b0588390SAlexei Starovoitov return err;
1345b0588390SAlexei Starovoitov }
1346b0588390SAlexei Starovoitov
1347b58af63aSAndrii Nakryiko bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
1348b58af63aSAndrii Nakryiko pr_debug("prog '%s': relo #%d: %s candidate #%d %s\n", prog_name,
1349b58af63aSAndrii Nakryiko relo_idx, err == 0 ? "non-matching" : "matching", i, spec_buf);
1350b0588390SAlexei Starovoitov
1351b0588390SAlexei Starovoitov if (err == 0)
1352b0588390SAlexei Starovoitov continue;
1353b0588390SAlexei Starovoitov
135478c1f8d0SAlexei Starovoitov err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res);
1355b0588390SAlexei Starovoitov if (err)
1356b0588390SAlexei Starovoitov return err;
1357b0588390SAlexei Starovoitov
1358b0588390SAlexei Starovoitov if (j == 0) {
1359adb8fa19SMauricio Vásquez *targ_res = cand_res;
136078c1f8d0SAlexei Starovoitov *targ_spec = *cand_spec;
136178c1f8d0SAlexei Starovoitov } else if (cand_spec->bit_offset != targ_spec->bit_offset) {
1362b0588390SAlexei Starovoitov /* if there are many field relo candidates, they
1363b0588390SAlexei Starovoitov * should all resolve to the same bit offset
1364b0588390SAlexei Starovoitov */
1365b0588390SAlexei Starovoitov pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
136678c1f8d0SAlexei Starovoitov prog_name, relo_idx, cand_spec->bit_offset,
136778c1f8d0SAlexei Starovoitov targ_spec->bit_offset);
1368b0588390SAlexei Starovoitov return -EINVAL;
1369adb8fa19SMauricio Vásquez } else if (cand_res.poison != targ_res->poison ||
1370adb8fa19SMauricio Vásquez cand_res.new_val != targ_res->new_val) {
1371b0588390SAlexei Starovoitov /* all candidates should result in the same relocation
1372b0588390SAlexei Starovoitov * decision and value, otherwise it's dangerous to
1373b0588390SAlexei Starovoitov * proceed due to ambiguity
1374b0588390SAlexei Starovoitov */
137577628165SYonghong Song pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n",
1376b0588390SAlexei Starovoitov prog_name, relo_idx,
137777628165SYonghong Song cand_res.poison ? "failure" : "success",
137877628165SYonghong Song (unsigned long long)cand_res.new_val,
137977628165SYonghong Song targ_res->poison ? "failure" : "success",
138077628165SYonghong Song (unsigned long long)targ_res->new_val);
1381b0588390SAlexei Starovoitov return -EINVAL;
1382b0588390SAlexei Starovoitov }
1383b0588390SAlexei Starovoitov
1384b0588390SAlexei Starovoitov cands->cands[j++] = cands->cands[i];
1385b0588390SAlexei Starovoitov }
1386b0588390SAlexei Starovoitov
1387b0588390SAlexei Starovoitov /*
138846334a0cSAlexei Starovoitov * For BPF_CORE_FIELD_EXISTS relo or when used BPF program has field
1389b0588390SAlexei Starovoitov * existence checks or kernel version/config checks, it's expected
1390b0588390SAlexei Starovoitov * that we might not find any candidates. In this case, if field
1391b0588390SAlexei Starovoitov * wasn't found in any candidate, the list of candidates shouldn't
1392b0588390SAlexei Starovoitov * change at all, we'll just handle relocating appropriately,
1393b0588390SAlexei Starovoitov * depending on relo's kind.
1394b0588390SAlexei Starovoitov */
1395b0588390SAlexei Starovoitov if (j > 0)
1396b0588390SAlexei Starovoitov cands->len = j;
1397b0588390SAlexei Starovoitov
1398b0588390SAlexei Starovoitov /*
1399b0588390SAlexei Starovoitov * If no candidates were found, it might be both a programmer error,
1400b0588390SAlexei Starovoitov * as well as expected case, depending whether instruction w/
1401b0588390SAlexei Starovoitov * relocation is guarded in some way that makes it unreachable (dead
1402b0588390SAlexei Starovoitov * code) if relocation can't be resolved. This is handled in
1403b0588390SAlexei Starovoitov * bpf_core_patch_insn() uniformly by replacing that instruction with
1404b0588390SAlexei Starovoitov * BPF helper call insn (using invalid helper ID). If that instruction
1405b0588390SAlexei Starovoitov * is indeed unreachable, then it will be ignored and eliminated by
1406b0588390SAlexei Starovoitov * verifier. If it was an error, then verifier will complain and point
1407b0588390SAlexei Starovoitov * to a specific instruction number in its log.
1408b0588390SAlexei Starovoitov */
1409b0588390SAlexei Starovoitov if (j == 0) {
1410b0588390SAlexei Starovoitov pr_debug("prog '%s': relo #%d: no matching targets found\n",
1411b0588390SAlexei Starovoitov prog_name, relo_idx);
1412b0588390SAlexei Starovoitov
1413b0588390SAlexei Starovoitov /* calculate single target relo result explicitly */
1414adb8fa19SMauricio Vásquez err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, targ_res);
1415b0588390SAlexei Starovoitov if (err)
1416b0588390SAlexei Starovoitov return err;
1417b0588390SAlexei Starovoitov }
1418b0588390SAlexei Starovoitov
1419b0588390SAlexei Starovoitov return 0;
1420b0588390SAlexei Starovoitov }
1421ec6209c8SDaniel Müller
bpf_core_names_match(const struct btf * local_btf,size_t local_name_off,const struct btf * targ_btf,size_t targ_name_off)1422ec6209c8SDaniel Müller static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off,
1423ec6209c8SDaniel Müller const struct btf *targ_btf, size_t targ_name_off)
1424ec6209c8SDaniel Müller {
1425ec6209c8SDaniel Müller const char *local_n, *targ_n;
1426ec6209c8SDaniel Müller size_t local_len, targ_len;
1427ec6209c8SDaniel Müller
1428ec6209c8SDaniel Müller local_n = btf__name_by_offset(local_btf, local_name_off);
1429ec6209c8SDaniel Müller targ_n = btf__name_by_offset(targ_btf, targ_name_off);
1430ec6209c8SDaniel Müller
1431ec6209c8SDaniel Müller if (str_is_empty(targ_n))
1432ec6209c8SDaniel Müller return str_is_empty(local_n);
1433ec6209c8SDaniel Müller
1434ec6209c8SDaniel Müller targ_len = bpf_core_essential_name_len(targ_n);
1435ec6209c8SDaniel Müller local_len = bpf_core_essential_name_len(local_n);
1436ec6209c8SDaniel Müller
1437ec6209c8SDaniel Müller return targ_len == local_len && strncmp(local_n, targ_n, local_len) == 0;
1438ec6209c8SDaniel Müller }
1439ec6209c8SDaniel Müller
bpf_core_enums_match(const struct btf * local_btf,const struct btf_type * local_t,const struct btf * targ_btf,const struct btf_type * targ_t)1440ec6209c8SDaniel Müller static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t,
1441ec6209c8SDaniel Müller const struct btf *targ_btf, const struct btf_type *targ_t)
1442ec6209c8SDaniel Müller {
1443ec6209c8SDaniel Müller __u16 local_vlen = btf_vlen(local_t);
1444ec6209c8SDaniel Müller __u16 targ_vlen = btf_vlen(targ_t);
1445ec6209c8SDaniel Müller int i, j;
1446ec6209c8SDaniel Müller
1447ec6209c8SDaniel Müller if (local_t->size != targ_t->size)
1448ec6209c8SDaniel Müller return 0;
1449ec6209c8SDaniel Müller
1450ec6209c8SDaniel Müller if (local_vlen > targ_vlen)
1451ec6209c8SDaniel Müller return 0;
1452ec6209c8SDaniel Müller
1453ec6209c8SDaniel Müller /* iterate over the local enum's variants and make sure each has
1454ec6209c8SDaniel Müller * a symbolic name correspondent in the target
1455ec6209c8SDaniel Müller */
1456ec6209c8SDaniel Müller for (i = 0; i < local_vlen; i++) {
1457ec6209c8SDaniel Müller bool matched = false;
1458ec6209c8SDaniel Müller __u32 local_n_off, targ_n_off;
1459ec6209c8SDaniel Müller
1460ec6209c8SDaniel Müller local_n_off = btf_is_enum(local_t) ? btf_enum(local_t)[i].name_off :
1461ec6209c8SDaniel Müller btf_enum64(local_t)[i].name_off;
1462ec6209c8SDaniel Müller
1463ec6209c8SDaniel Müller for (j = 0; j < targ_vlen; j++) {
1464ec6209c8SDaniel Müller targ_n_off = btf_is_enum(targ_t) ? btf_enum(targ_t)[j].name_off :
1465ec6209c8SDaniel Müller btf_enum64(targ_t)[j].name_off;
1466ec6209c8SDaniel Müller
1467ec6209c8SDaniel Müller if (bpf_core_names_match(local_btf, local_n_off, targ_btf, targ_n_off)) {
1468ec6209c8SDaniel Müller matched = true;
1469ec6209c8SDaniel Müller break;
1470ec6209c8SDaniel Müller }
1471ec6209c8SDaniel Müller }
1472ec6209c8SDaniel Müller
1473ec6209c8SDaniel Müller if (!matched)
1474ec6209c8SDaniel Müller return 0;
1475ec6209c8SDaniel Müller }
1476ec6209c8SDaniel Müller return 1;
1477ec6209c8SDaniel Müller }
1478ec6209c8SDaniel Müller
bpf_core_composites_match(const struct btf * local_btf,const struct btf_type * local_t,const struct btf * targ_btf,const struct btf_type * targ_t,bool behind_ptr,int level)1479ec6209c8SDaniel Müller static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t,
1480ec6209c8SDaniel Müller const struct btf *targ_btf, const struct btf_type *targ_t,
1481ec6209c8SDaniel Müller bool behind_ptr, int level)
1482ec6209c8SDaniel Müller {
1483ec6209c8SDaniel Müller const struct btf_member *local_m = btf_members(local_t);
1484ec6209c8SDaniel Müller __u16 local_vlen = btf_vlen(local_t);
1485ec6209c8SDaniel Müller __u16 targ_vlen = btf_vlen(targ_t);
1486ec6209c8SDaniel Müller int i, j, err;
1487ec6209c8SDaniel Müller
1488ec6209c8SDaniel Müller if (local_vlen > targ_vlen)
1489ec6209c8SDaniel Müller return 0;
1490ec6209c8SDaniel Müller
1491ec6209c8SDaniel Müller /* check that all local members have a match in the target */
1492ec6209c8SDaniel Müller for (i = 0; i < local_vlen; i++, local_m++) {
1493ec6209c8SDaniel Müller const struct btf_member *targ_m = btf_members(targ_t);
1494ec6209c8SDaniel Müller bool matched = false;
1495ec6209c8SDaniel Müller
1496ec6209c8SDaniel Müller for (j = 0; j < targ_vlen; j++, targ_m++) {
1497ec6209c8SDaniel Müller if (!bpf_core_names_match(local_btf, local_m->name_off,
1498ec6209c8SDaniel Müller targ_btf, targ_m->name_off))
1499ec6209c8SDaniel Müller continue;
1500ec6209c8SDaniel Müller
1501ec6209c8SDaniel Müller err = __bpf_core_types_match(local_btf, local_m->type, targ_btf,
1502ec6209c8SDaniel Müller targ_m->type, behind_ptr, level - 1);
150306cd4e9dSDaniel Müller if (err < 0)
150406cd4e9dSDaniel Müller return err;
1505ec6209c8SDaniel Müller if (err > 0) {
1506ec6209c8SDaniel Müller matched = true;
1507ec6209c8SDaniel Müller break;
1508ec6209c8SDaniel Müller }
1509ec6209c8SDaniel Müller }
1510ec6209c8SDaniel Müller
1511ec6209c8SDaniel Müller if (!matched)
1512ec6209c8SDaniel Müller return 0;
1513ec6209c8SDaniel Müller }
1514ec6209c8SDaniel Müller return 1;
1515ec6209c8SDaniel Müller }
1516ec6209c8SDaniel Müller
151706cd4e9dSDaniel Müller /* Check that two types "match". This function assumes that root types were
151806cd4e9dSDaniel Müller * already checked for name match.
1519ec6209c8SDaniel Müller *
1520ec6209c8SDaniel Müller * The matching relation is defined as follows:
1521ec6209c8SDaniel Müller * - modifiers and typedefs are stripped (and, hence, effectively ignored)
1522ec6209c8SDaniel Müller * - generally speaking types need to be of same kind (struct vs. struct, union
1523ec6209c8SDaniel Müller * vs. union, etc.)
1524ec6209c8SDaniel Müller * - exceptions are struct/union behind a pointer which could also match a
1525ec6209c8SDaniel Müller * forward declaration of a struct or union, respectively, and enum vs.
1526ec6209c8SDaniel Müller * enum64 (see below)
1527ec6209c8SDaniel Müller * Then, depending on type:
1528ec6209c8SDaniel Müller * - integers:
1529ec6209c8SDaniel Müller * - match if size and signedness match
1530ec6209c8SDaniel Müller * - arrays & pointers:
1531ec6209c8SDaniel Müller * - target types are recursively matched
1532ec6209c8SDaniel Müller * - structs & unions:
1533ec6209c8SDaniel Müller * - local members need to exist in target with the same name
1534ec6209c8SDaniel Müller * - for each member we recursively check match unless it is already behind a
1535ec6209c8SDaniel Müller * pointer, in which case we only check matching names and compatible kind
1536ec6209c8SDaniel Müller * - enums:
1537ec6209c8SDaniel Müller * - local variants have to have a match in target by symbolic name (but not
1538ec6209c8SDaniel Müller * numeric value)
1539ec6209c8SDaniel Müller * - size has to match (but enum may match enum64 and vice versa)
1540ec6209c8SDaniel Müller * - function pointers:
1541ec6209c8SDaniel Müller * - number and position of arguments in local type has to match target
1542ec6209c8SDaniel Müller * - for each argument and the return value we recursively check match
1543ec6209c8SDaniel Müller */
__bpf_core_types_match(const struct btf * local_btf,__u32 local_id,const struct btf * targ_btf,__u32 targ_id,bool behind_ptr,int level)1544ec6209c8SDaniel Müller int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf,
1545ec6209c8SDaniel Müller __u32 targ_id, bool behind_ptr, int level)
1546ec6209c8SDaniel Müller {
1547ec6209c8SDaniel Müller const struct btf_type *local_t, *targ_t;
1548ec6209c8SDaniel Müller int depth = 32; /* max recursion depth */
1549ec6209c8SDaniel Müller __u16 local_k, targ_k;
1550ec6209c8SDaniel Müller
1551ec6209c8SDaniel Müller if (level <= 0)
1552ec6209c8SDaniel Müller return -EINVAL;
1553ec6209c8SDaniel Müller
1554ec6209c8SDaniel Müller recur:
1555ec6209c8SDaniel Müller depth--;
1556ec6209c8SDaniel Müller if (depth < 0)
1557ec6209c8SDaniel Müller return -EINVAL;
1558ec6209c8SDaniel Müller
1559ec6209c8SDaniel Müller local_t = skip_mods_and_typedefs(local_btf, local_id, &local_id);
1560ec6209c8SDaniel Müller targ_t = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
1561ec6209c8SDaniel Müller if (!local_t || !targ_t)
1562ec6209c8SDaniel Müller return -EINVAL;
1563ec6209c8SDaniel Müller
156406cd4e9dSDaniel Müller /* While the name check happens after typedefs are skipped, root-level
156506cd4e9dSDaniel Müller * typedefs would still be name-matched as that's the contract with
156606cd4e9dSDaniel Müller * callers.
156706cd4e9dSDaniel Müller */
1568ec6209c8SDaniel Müller if (!bpf_core_names_match(local_btf, local_t->name_off, targ_btf, targ_t->name_off))
1569ec6209c8SDaniel Müller return 0;
1570ec6209c8SDaniel Müller
1571ec6209c8SDaniel Müller local_k = btf_kind(local_t);
1572ec6209c8SDaniel Müller targ_k = btf_kind(targ_t);
1573ec6209c8SDaniel Müller
1574ec6209c8SDaniel Müller switch (local_k) {
1575ec6209c8SDaniel Müller case BTF_KIND_UNKN:
1576ec6209c8SDaniel Müller return local_k == targ_k;
1577ec6209c8SDaniel Müller case BTF_KIND_FWD: {
1578ec6209c8SDaniel Müller bool local_f = BTF_INFO_KFLAG(local_t->info);
1579ec6209c8SDaniel Müller
1580ec6209c8SDaniel Müller if (behind_ptr) {
1581ec6209c8SDaniel Müller if (local_k == targ_k)
1582ec6209c8SDaniel Müller return local_f == BTF_INFO_KFLAG(targ_t->info);
1583ec6209c8SDaniel Müller
1584ec6209c8SDaniel Müller /* for forward declarations kflag dictates whether the
1585ec6209c8SDaniel Müller * target is a struct (0) or union (1)
1586ec6209c8SDaniel Müller */
1587ec6209c8SDaniel Müller return (targ_k == BTF_KIND_STRUCT && !local_f) ||
1588ec6209c8SDaniel Müller (targ_k == BTF_KIND_UNION && local_f);
1589ec6209c8SDaniel Müller } else {
1590ec6209c8SDaniel Müller if (local_k != targ_k)
1591ec6209c8SDaniel Müller return 0;
1592ec6209c8SDaniel Müller
1593ec6209c8SDaniel Müller /* match if the forward declaration is for the same kind */
1594ec6209c8SDaniel Müller return local_f == BTF_INFO_KFLAG(targ_t->info);
1595ec6209c8SDaniel Müller }
1596ec6209c8SDaniel Müller }
1597ec6209c8SDaniel Müller case BTF_KIND_ENUM:
1598ec6209c8SDaniel Müller case BTF_KIND_ENUM64:
1599ec6209c8SDaniel Müller if (!btf_is_any_enum(targ_t))
1600ec6209c8SDaniel Müller return 0;
1601ec6209c8SDaniel Müller
1602ec6209c8SDaniel Müller return bpf_core_enums_match(local_btf, local_t, targ_btf, targ_t);
1603ec6209c8SDaniel Müller case BTF_KIND_STRUCT:
1604ec6209c8SDaniel Müller case BTF_KIND_UNION:
1605ec6209c8SDaniel Müller if (behind_ptr) {
1606ec6209c8SDaniel Müller bool targ_f = BTF_INFO_KFLAG(targ_t->info);
1607ec6209c8SDaniel Müller
1608ec6209c8SDaniel Müller if (local_k == targ_k)
1609ec6209c8SDaniel Müller return 1;
1610ec6209c8SDaniel Müller
1611ec6209c8SDaniel Müller if (targ_k != BTF_KIND_FWD)
1612ec6209c8SDaniel Müller return 0;
1613ec6209c8SDaniel Müller
1614ec6209c8SDaniel Müller return (local_k == BTF_KIND_UNION) == targ_f;
1615ec6209c8SDaniel Müller } else {
1616ec6209c8SDaniel Müller if (local_k != targ_k)
1617ec6209c8SDaniel Müller return 0;
1618ec6209c8SDaniel Müller
1619ec6209c8SDaniel Müller return bpf_core_composites_match(local_btf, local_t, targ_btf, targ_t,
1620ec6209c8SDaniel Müller behind_ptr, level);
1621ec6209c8SDaniel Müller }
1622ec6209c8SDaniel Müller case BTF_KIND_INT: {
1623ec6209c8SDaniel Müller __u8 local_sgn;
1624ec6209c8SDaniel Müller __u8 targ_sgn;
1625ec6209c8SDaniel Müller
1626ec6209c8SDaniel Müller if (local_k != targ_k)
1627ec6209c8SDaniel Müller return 0;
1628ec6209c8SDaniel Müller
1629ec6209c8SDaniel Müller local_sgn = btf_int_encoding(local_t) & BTF_INT_SIGNED;
1630ec6209c8SDaniel Müller targ_sgn = btf_int_encoding(targ_t) & BTF_INT_SIGNED;
1631ec6209c8SDaniel Müller
1632ec6209c8SDaniel Müller return local_t->size == targ_t->size && local_sgn == targ_sgn;
1633ec6209c8SDaniel Müller }
1634ec6209c8SDaniel Müller case BTF_KIND_PTR:
1635ec6209c8SDaniel Müller if (local_k != targ_k)
1636ec6209c8SDaniel Müller return 0;
1637ec6209c8SDaniel Müller
1638ec6209c8SDaniel Müller behind_ptr = true;
1639ec6209c8SDaniel Müller
1640ec6209c8SDaniel Müller local_id = local_t->type;
1641ec6209c8SDaniel Müller targ_id = targ_t->type;
1642ec6209c8SDaniel Müller goto recur;
1643ec6209c8SDaniel Müller case BTF_KIND_ARRAY: {
1644ec6209c8SDaniel Müller const struct btf_array *local_array = btf_array(local_t);
1645ec6209c8SDaniel Müller const struct btf_array *targ_array = btf_array(targ_t);
1646ec6209c8SDaniel Müller
1647ec6209c8SDaniel Müller if (local_k != targ_k)
1648ec6209c8SDaniel Müller return 0;
1649ec6209c8SDaniel Müller
1650ec6209c8SDaniel Müller if (local_array->nelems != targ_array->nelems)
1651ec6209c8SDaniel Müller return 0;
1652ec6209c8SDaniel Müller
1653ec6209c8SDaniel Müller local_id = local_array->type;
1654ec6209c8SDaniel Müller targ_id = targ_array->type;
1655ec6209c8SDaniel Müller goto recur;
1656ec6209c8SDaniel Müller }
1657ec6209c8SDaniel Müller case BTF_KIND_FUNC_PROTO: {
1658ec6209c8SDaniel Müller struct btf_param *local_p = btf_params(local_t);
1659ec6209c8SDaniel Müller struct btf_param *targ_p = btf_params(targ_t);
1660ec6209c8SDaniel Müller __u16 local_vlen = btf_vlen(local_t);
1661ec6209c8SDaniel Müller __u16 targ_vlen = btf_vlen(targ_t);
1662ec6209c8SDaniel Müller int i, err;
1663ec6209c8SDaniel Müller
1664ec6209c8SDaniel Müller if (local_k != targ_k)
1665ec6209c8SDaniel Müller return 0;
1666ec6209c8SDaniel Müller
1667ec6209c8SDaniel Müller if (local_vlen != targ_vlen)
1668ec6209c8SDaniel Müller return 0;
1669ec6209c8SDaniel Müller
1670ec6209c8SDaniel Müller for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
1671ec6209c8SDaniel Müller err = __bpf_core_types_match(local_btf, local_p->type, targ_btf,
1672ec6209c8SDaniel Müller targ_p->type, behind_ptr, level - 1);
1673ec6209c8SDaniel Müller if (err <= 0)
1674ec6209c8SDaniel Müller return err;
1675ec6209c8SDaniel Müller }
1676ec6209c8SDaniel Müller
1677ec6209c8SDaniel Müller /* tail recurse for return type check */
1678ec6209c8SDaniel Müller local_id = local_t->type;
1679ec6209c8SDaniel Müller targ_id = targ_t->type;
1680ec6209c8SDaniel Müller goto recur;
1681ec6209c8SDaniel Müller }
1682ec6209c8SDaniel Müller default:
1683ec6209c8SDaniel Müller pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
1684ec6209c8SDaniel Müller btf_kind_str(local_t), local_id, targ_id);
1685ec6209c8SDaniel Müller return 0;
1686ec6209c8SDaniel Müller }
1687ec6209c8SDaniel Müller }
1688