11bc38b8fSAlexei Starovoitov // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 28a138aedSMartin KaFai Lau /* Copyright (c) 2018 Facebook */ 38a138aedSMartin KaFai Lau 43289959bSAndrii Nakryiko #include <byteswap.h> 5cdb2f920SArnaldo Carvalho de Melo #include <endian.h> 696408c43SYonghong Song #include <stdio.h> 78a138aedSMartin KaFai Lau #include <stdlib.h> 88a138aedSMartin KaFai Lau #include <string.h> 9e6c64855SAndrii Nakryiko #include <fcntl.h> 108a138aedSMartin KaFai Lau #include <unistd.h> 118a138aedSMartin KaFai Lau #include <errno.h> 12fb2426adSMartin KaFai Lau #include <sys/utsname.h> 13fb2426adSMartin KaFai Lau #include <sys/param.h> 14fb2426adSMartin KaFai Lau #include <sys/stat.h> 15fb2426adSMartin KaFai Lau #include <linux/kernel.h> 168a138aedSMartin KaFai Lau #include <linux/err.h> 178a138aedSMartin KaFai Lau #include <linux/btf.h> 18e6c64855SAndrii Nakryiko #include <gelf.h> 198a138aedSMartin KaFai Lau #include "btf.h" 208a138aedSMartin KaFai Lau #include "bpf.h" 218461ef8bSYonghong Song #include "libbpf.h" 22d72386feSAndrii Nakryiko #include "libbpf_internal.h" 232fc3fc0bSAndrii Nakryiko #include "hashmap.h" 2490d76d3eSAndrii Nakryiko #include "strset.h" 258a138aedSMartin KaFai Lau 26fb2426adSMartin KaFai Lau #define BTF_MAX_NR_TYPES 0x7fffffffU 27fb2426adSMartin KaFai Lau #define BTF_MAX_STR_OFFSET 0x7fffffffU 288a138aedSMartin KaFai Lau 298a138aedSMartin KaFai Lau static struct btf_type btf_void; 308a138aedSMartin KaFai Lau 318a138aedSMartin KaFai Lau struct btf { 323289959bSAndrii Nakryiko /* raw BTF data in native endianness */ 33b8604247SAndrii Nakryiko void *raw_data; 343289959bSAndrii Nakryiko /* raw BTF data in non-native endianness */ 353289959bSAndrii Nakryiko void *raw_data_swapped; 36b8604247SAndrii Nakryiko __u32 raw_size; 373289959bSAndrii Nakryiko /* whether target endianness differs from the native one */ 383289959bSAndrii Nakryiko bool swapped_endian; 39b8604247SAndrii Nakryiko 40b8604247SAndrii Nakryiko /* 419141f75aSAndrii Nakryiko * When BTF is loaded from an ELF or raw memory it is stored 429141f75aSAndrii Nakryiko * in a contiguous memory block. The hdr, type_data, and, strs_data 439141f75aSAndrii Nakryiko * point inside that memory region to their respective parts of BTF 449141f75aSAndrii Nakryiko * representation: 45b8604247SAndrii Nakryiko * 46b8604247SAndrii Nakryiko * +--------------------------------+ 47b8604247SAndrii Nakryiko * | Header | Types | Strings | 48b8604247SAndrii Nakryiko * +--------------------------------+ 49b8604247SAndrii Nakryiko * ^ ^ ^ 50b8604247SAndrii Nakryiko * | | | 51b8604247SAndrii Nakryiko * hdr | | 52b8604247SAndrii Nakryiko * types_data-+ | 53b8604247SAndrii Nakryiko * strs_data------------+ 54919d2b1dSAndrii Nakryiko * 55919d2b1dSAndrii Nakryiko * If BTF data is later modified, e.g., due to types added or 56919d2b1dSAndrii Nakryiko * removed, BTF deduplication performed, etc, this contiguous 57919d2b1dSAndrii Nakryiko * representation is broken up into three independently allocated 58919d2b1dSAndrii Nakryiko * memory regions to be able to modify them independently. 59919d2b1dSAndrii Nakryiko * raw_data is nulled out at that point, but can be later allocated 606a886de0SHengqi Chen * and cached again if user calls btf__raw_data(), at which point 61919d2b1dSAndrii Nakryiko * raw_data will contain a contiguous copy of header, types, and 62919d2b1dSAndrii Nakryiko * strings: 63919d2b1dSAndrii Nakryiko * 64919d2b1dSAndrii Nakryiko * +----------+ +---------+ +-----------+ 65919d2b1dSAndrii Nakryiko * | Header | | Types | | Strings | 66919d2b1dSAndrii Nakryiko * +----------+ +---------+ +-----------+ 67919d2b1dSAndrii Nakryiko * ^ ^ ^ 68919d2b1dSAndrii Nakryiko * | | | 69919d2b1dSAndrii Nakryiko * hdr | | 70919d2b1dSAndrii Nakryiko * types_data----+ | 7190d76d3eSAndrii Nakryiko * strset__data(strs_set)-----+ 72919d2b1dSAndrii Nakryiko * 73919d2b1dSAndrii Nakryiko * +----------+---------+-----------+ 74919d2b1dSAndrii Nakryiko * | Header | Types | Strings | 75919d2b1dSAndrii Nakryiko * raw_data----->+----------+---------+-----------+ 76b8604247SAndrii Nakryiko */ 778a138aedSMartin KaFai Lau struct btf_header *hdr; 78919d2b1dSAndrii Nakryiko 79b8604247SAndrii Nakryiko void *types_data; 80919d2b1dSAndrii Nakryiko size_t types_data_cap; /* used size stored in hdr->type_len */ 81b8604247SAndrii Nakryiko 82ba451366SAndrii Nakryiko /* type ID to `struct btf_type *` lookup index 83ba451366SAndrii Nakryiko * type_offs[0] corresponds to the first non-VOID type: 84ba451366SAndrii Nakryiko * - for base BTF it's type [1]; 85ba451366SAndrii Nakryiko * - for split BTF it's the first non-base BTF type. 86ba451366SAndrii Nakryiko */ 87740e69c3SAndrii Nakryiko __u32 *type_offs; 88192f5a1fSAndrii Nakryiko size_t type_offs_cap; 89ba451366SAndrii Nakryiko /* number of types in this BTF instance: 90ba451366SAndrii Nakryiko * - doesn't include special [0] void type; 91ba451366SAndrii Nakryiko * - for split BTF counts number of types added on top of base BTF. 92ba451366SAndrii Nakryiko */ 935b891af7SMartin KaFai Lau __u32 nr_types; 94ba451366SAndrii Nakryiko /* if not NULL, points to the base BTF on top of which the current 95ba451366SAndrii Nakryiko * split BTF is based 96ba451366SAndrii Nakryiko */ 97ba451366SAndrii Nakryiko struct btf *base_btf; 98ba451366SAndrii Nakryiko /* BTF type ID of the first type in this BTF instance: 99ba451366SAndrii Nakryiko * - for base BTF it's equal to 1; 100ba451366SAndrii Nakryiko * - for split BTF it's equal to biggest type ID of base BTF plus 1. 101ba451366SAndrii Nakryiko */ 102ba451366SAndrii Nakryiko int start_id; 103ba451366SAndrii Nakryiko /* logical string offset of this BTF instance: 104ba451366SAndrii Nakryiko * - for base BTF it's equal to 0; 105ba451366SAndrii Nakryiko * - for split BTF it's equal to total size of base BTF's string section size. 106ba451366SAndrii Nakryiko */ 107ba451366SAndrii Nakryiko int start_str_off; 108b8604247SAndrii Nakryiko 10990d76d3eSAndrii Nakryiko /* only one of strs_data or strs_set can be non-NULL, depending on 11090d76d3eSAndrii Nakryiko * whether BTF is in a modifiable state (strs_set is used) or not 11190d76d3eSAndrii Nakryiko * (strs_data points inside raw_data) 11290d76d3eSAndrii Nakryiko */ 113919d2b1dSAndrii Nakryiko void *strs_data; 11490d76d3eSAndrii Nakryiko /* a set of unique strings */ 11590d76d3eSAndrii Nakryiko struct strset *strs_set; 116919d2b1dSAndrii Nakryiko /* whether strings are already deduplicated */ 117919d2b1dSAndrii Nakryiko bool strs_deduped; 11888a82c2aSAndrii Nakryiko 119b8604247SAndrii Nakryiko /* BTF object FD, if loaded into kernel */ 1208a138aedSMartin KaFai Lau int fd; 121b8604247SAndrii Nakryiko 122b8604247SAndrii Nakryiko /* Pointer size (in bytes) for a target architecture of this BTF */ 12344ad23dfSAndrii Nakryiko int ptr_sz; 1248a138aedSMartin KaFai Lau }; 1258a138aedSMartin KaFai Lau 126d7f5b5e0SYonghong Song static inline __u64 ptr_to_u64(const void *ptr) 127d7f5b5e0SYonghong Song { 128d7f5b5e0SYonghong Song return (__u64) (unsigned long) ptr; 129d7f5b5e0SYonghong Song } 130d7f5b5e0SYonghong Song 131192f5a1fSAndrii Nakryiko /* Ensure given dynamically allocated memory region pointed to by *data* with 132192f5a1fSAndrii Nakryiko * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough 1339bbdfad8SDaniel Müller * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements 134192f5a1fSAndrii Nakryiko * are already used. At most *max_cnt* elements can be ever allocated. 135192f5a1fSAndrii Nakryiko * If necessary, memory is reallocated and all existing data is copied over, 136192f5a1fSAndrii Nakryiko * new pointer to the memory region is stored at *data, new memory region 137192f5a1fSAndrii Nakryiko * capacity (in number of elements) is stored in *cap. 138192f5a1fSAndrii Nakryiko * On success, memory pointer to the beginning of unused memory is returned. 139192f5a1fSAndrii Nakryiko * On error, NULL is returned. 140192f5a1fSAndrii Nakryiko */ 1413b029e06SAndrii Nakryiko void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 142192f5a1fSAndrii Nakryiko size_t cur_cnt, size_t max_cnt, size_t add_cnt) 1438a138aedSMartin KaFai Lau { 144192f5a1fSAndrii Nakryiko size_t new_cnt; 145192f5a1fSAndrii Nakryiko void *new_data; 1468a138aedSMartin KaFai Lau 147192f5a1fSAndrii Nakryiko if (cur_cnt + add_cnt <= *cap_cnt) 148192f5a1fSAndrii Nakryiko return *data + cur_cnt * elem_sz; 1498a138aedSMartin KaFai Lau 150192f5a1fSAndrii Nakryiko /* requested more than the set limit */ 151192f5a1fSAndrii Nakryiko if (cur_cnt + add_cnt > max_cnt) 152192f5a1fSAndrii Nakryiko return NULL; 1538a138aedSMartin KaFai Lau 154192f5a1fSAndrii Nakryiko new_cnt = *cap_cnt; 155192f5a1fSAndrii Nakryiko new_cnt += new_cnt / 4; /* expand by 25% */ 156192f5a1fSAndrii Nakryiko if (new_cnt < 16) /* but at least 16 elements */ 157192f5a1fSAndrii Nakryiko new_cnt = 16; 158192f5a1fSAndrii Nakryiko if (new_cnt > max_cnt) /* but not exceeding a set limit */ 159192f5a1fSAndrii Nakryiko new_cnt = max_cnt; 160192f5a1fSAndrii Nakryiko if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ 161192f5a1fSAndrii Nakryiko new_cnt = cur_cnt + add_cnt; 1628a138aedSMartin KaFai Lau 163192f5a1fSAndrii Nakryiko new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); 164192f5a1fSAndrii Nakryiko if (!new_data) 165192f5a1fSAndrii Nakryiko return NULL; 1668a138aedSMartin KaFai Lau 167192f5a1fSAndrii Nakryiko /* zero out newly allocated portion of memory */ 168192f5a1fSAndrii Nakryiko memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); 169192f5a1fSAndrii Nakryiko 170192f5a1fSAndrii Nakryiko *data = new_data; 171192f5a1fSAndrii Nakryiko *cap_cnt = new_cnt; 172192f5a1fSAndrii Nakryiko return new_data + cur_cnt * elem_sz; 1738a138aedSMartin KaFai Lau } 1748a138aedSMartin KaFai Lau 1759c6c5c48SAndrii Nakryiko /* Ensure given dynamically allocated memory region has enough allocated space 1769c6c5c48SAndrii Nakryiko * to accommodate *need_cnt* elements of size *elem_sz* bytes each 1779c6c5c48SAndrii Nakryiko */ 1783b029e06SAndrii Nakryiko int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) 1799c6c5c48SAndrii Nakryiko { 1809c6c5c48SAndrii Nakryiko void *p; 1819c6c5c48SAndrii Nakryiko 1829c6c5c48SAndrii Nakryiko if (need_cnt <= *cap_cnt) 1839c6c5c48SAndrii Nakryiko return 0; 1849c6c5c48SAndrii Nakryiko 1853b029e06SAndrii Nakryiko p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); 1869c6c5c48SAndrii Nakryiko if (!p) 1879c6c5c48SAndrii Nakryiko return -ENOMEM; 1889c6c5c48SAndrii Nakryiko 1899c6c5c48SAndrii Nakryiko return 0; 1909c6c5c48SAndrii Nakryiko } 1919c6c5c48SAndrii Nakryiko 1927ca61121SAndrii Nakryiko static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt) 1937ca61121SAndrii Nakryiko { 1947ca61121SAndrii Nakryiko return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), 1957ca61121SAndrii Nakryiko btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); 1967ca61121SAndrii Nakryiko } 1977ca61121SAndrii Nakryiko 198192f5a1fSAndrii Nakryiko static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) 199192f5a1fSAndrii Nakryiko { 200192f5a1fSAndrii Nakryiko __u32 *p; 2018a138aedSMartin KaFai Lau 2027ca61121SAndrii Nakryiko p = btf_add_type_offs_mem(btf, 1); 203192f5a1fSAndrii Nakryiko if (!p) 204192f5a1fSAndrii Nakryiko return -ENOMEM; 205192f5a1fSAndrii Nakryiko 206192f5a1fSAndrii Nakryiko *p = type_off; 2078a138aedSMartin KaFai Lau return 0; 2088a138aedSMartin KaFai Lau } 2098a138aedSMartin KaFai Lau 2103289959bSAndrii Nakryiko static void btf_bswap_hdr(struct btf_header *h) 2113289959bSAndrii Nakryiko { 2123289959bSAndrii Nakryiko h->magic = bswap_16(h->magic); 2133289959bSAndrii Nakryiko h->hdr_len = bswap_32(h->hdr_len); 2143289959bSAndrii Nakryiko h->type_off = bswap_32(h->type_off); 2153289959bSAndrii Nakryiko h->type_len = bswap_32(h->type_len); 2163289959bSAndrii Nakryiko h->str_off = bswap_32(h->str_off); 2173289959bSAndrii Nakryiko h->str_len = bswap_32(h->str_len); 2183289959bSAndrii Nakryiko } 2193289959bSAndrii Nakryiko 2208461ef8bSYonghong Song static int btf_parse_hdr(struct btf *btf) 2218a138aedSMartin KaFai Lau { 2223289959bSAndrii Nakryiko struct btf_header *hdr = btf->hdr; 2235b891af7SMartin KaFai Lau __u32 meta_left; 2248a138aedSMartin KaFai Lau 225b8604247SAndrii Nakryiko if (btf->raw_size < sizeof(struct btf_header)) { 2268461ef8bSYonghong Song pr_debug("BTF header not found\n"); 2278a138aedSMartin KaFai Lau return -EINVAL; 2288a138aedSMartin KaFai Lau } 2298a138aedSMartin KaFai Lau 2303289959bSAndrii Nakryiko if (hdr->magic == bswap_16(BTF_MAGIC)) { 2313289959bSAndrii Nakryiko btf->swapped_endian = true; 2323289959bSAndrii Nakryiko if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { 2333289959bSAndrii Nakryiko pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", 2343289959bSAndrii Nakryiko bswap_32(hdr->hdr_len)); 2353289959bSAndrii Nakryiko return -ENOTSUP; 2363289959bSAndrii Nakryiko } 2373289959bSAndrii Nakryiko btf_bswap_hdr(hdr); 2383289959bSAndrii Nakryiko } else if (hdr->magic != BTF_MAGIC) { 2398461ef8bSYonghong Song pr_debug("Invalid BTF magic: %x\n", hdr->magic); 2408a138aedSMartin KaFai Lau return -EINVAL; 2418a138aedSMartin KaFai Lau } 2428a138aedSMartin KaFai Lau 243c825f5feSAndrii Nakryiko if (btf->raw_size < hdr->hdr_len) { 244c825f5feSAndrii Nakryiko pr_debug("BTF header len %u larger than data size %u\n", 245c825f5feSAndrii Nakryiko hdr->hdr_len, btf->raw_size); 246c825f5feSAndrii Nakryiko return -EINVAL; 247c825f5feSAndrii Nakryiko } 248c825f5feSAndrii Nakryiko 249c825f5feSAndrii Nakryiko meta_left = btf->raw_size - hdr->hdr_len; 2505245dafeSAndrii Nakryiko if (meta_left < (long long)hdr->str_off + hdr->str_len) { 251d8123624SAndrii Nakryiko pr_debug("Invalid BTF total size: %u\n", btf->raw_size); 2528a138aedSMartin KaFai Lau return -EINVAL; 2538a138aedSMartin KaFai Lau } 2548a138aedSMartin KaFai Lau 2555245dafeSAndrii Nakryiko if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) { 256d8123624SAndrii Nakryiko pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", 257d8123624SAndrii Nakryiko hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len); 2588a138aedSMartin KaFai Lau return -EINVAL; 2598a138aedSMartin KaFai Lau } 2608a138aedSMartin KaFai Lau 261d8123624SAndrii Nakryiko if (hdr->type_off % 4) { 2628461ef8bSYonghong Song pr_debug("BTF type section is not aligned to 4 bytes\n"); 2638a138aedSMartin KaFai Lau return -EINVAL; 2648a138aedSMartin KaFai Lau } 2658a138aedSMartin KaFai Lau 2668a138aedSMartin KaFai Lau return 0; 2678a138aedSMartin KaFai Lau } 2688a138aedSMartin KaFai Lau 2698461ef8bSYonghong Song static int btf_parse_str_sec(struct btf *btf) 2708a138aedSMartin KaFai Lau { 2718a138aedSMartin KaFai Lau const struct btf_header *hdr = btf->hdr; 272b8604247SAndrii Nakryiko const char *start = btf->strs_data; 2738a138aedSMartin KaFai Lau const char *end = start + btf->hdr->str_len; 2748a138aedSMartin KaFai Lau 275ba451366SAndrii Nakryiko if (btf->base_btf && hdr->str_len == 0) 276ba451366SAndrii Nakryiko return 0; 277ba451366SAndrii Nakryiko if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { 2788461ef8bSYonghong Song pr_debug("Invalid BTF string section\n"); 2798a138aedSMartin KaFai Lau return -EINVAL; 2808a138aedSMartin KaFai Lau } 281ba451366SAndrii Nakryiko if (!btf->base_btf && start[0]) { 282ba451366SAndrii Nakryiko pr_debug("Invalid BTF string section\n"); 283ba451366SAndrii Nakryiko return -EINVAL; 284ba451366SAndrii Nakryiko } 2858a138aedSMartin KaFai Lau return 0; 2868a138aedSMartin KaFai Lau } 2878a138aedSMartin KaFai Lau 288740e69c3SAndrii Nakryiko static int btf_type_size(const struct btf_type *t) 28969eaab04SAndrii Nakryiko { 2903289959bSAndrii Nakryiko const int base_size = sizeof(struct btf_type); 291b03bc685SAndrii Nakryiko __u16 vlen = btf_vlen(t); 29269eaab04SAndrii Nakryiko 293b03bc685SAndrii Nakryiko switch (btf_kind(t)) { 29469eaab04SAndrii Nakryiko case BTF_KIND_FWD: 29569eaab04SAndrii Nakryiko case BTF_KIND_CONST: 29669eaab04SAndrii Nakryiko case BTF_KIND_VOLATILE: 29769eaab04SAndrii Nakryiko case BTF_KIND_RESTRICT: 29869eaab04SAndrii Nakryiko case BTF_KIND_PTR: 29969eaab04SAndrii Nakryiko case BTF_KIND_TYPEDEF: 30069eaab04SAndrii Nakryiko case BTF_KIND_FUNC: 30122541a9eSIlya Leoshkevich case BTF_KIND_FLOAT: 3022dc1e488SYonghong Song case BTF_KIND_TYPE_TAG: 30369eaab04SAndrii Nakryiko return base_size; 30469eaab04SAndrii Nakryiko case BTF_KIND_INT: 30569eaab04SAndrii Nakryiko return base_size + sizeof(__u32); 30669eaab04SAndrii Nakryiko case BTF_KIND_ENUM: 30769eaab04SAndrii Nakryiko return base_size + vlen * sizeof(struct btf_enum); 308dffbbdc2SYonghong Song case BTF_KIND_ENUM64: 309dffbbdc2SYonghong Song return base_size + vlen * sizeof(struct btf_enum64); 31069eaab04SAndrii Nakryiko case BTF_KIND_ARRAY: 31169eaab04SAndrii Nakryiko return base_size + sizeof(struct btf_array); 31269eaab04SAndrii Nakryiko case BTF_KIND_STRUCT: 31369eaab04SAndrii Nakryiko case BTF_KIND_UNION: 31469eaab04SAndrii Nakryiko return base_size + vlen * sizeof(struct btf_member); 31569eaab04SAndrii Nakryiko case BTF_KIND_FUNC_PROTO: 31669eaab04SAndrii Nakryiko return base_size + vlen * sizeof(struct btf_param); 3171713d68bSDaniel Borkmann case BTF_KIND_VAR: 3181713d68bSDaniel Borkmann return base_size + sizeof(struct btf_var); 3191713d68bSDaniel Borkmann case BTF_KIND_DATASEC: 3201713d68bSDaniel Borkmann return base_size + vlen * sizeof(struct btf_var_secinfo); 321223f903eSYonghong Song case BTF_KIND_DECL_TAG: 322223f903eSYonghong Song return base_size + sizeof(struct btf_decl_tag); 32369eaab04SAndrii Nakryiko default: 324b03bc685SAndrii Nakryiko pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 32569eaab04SAndrii Nakryiko return -EINVAL; 32669eaab04SAndrii Nakryiko } 32769eaab04SAndrii Nakryiko } 32869eaab04SAndrii Nakryiko 3293289959bSAndrii Nakryiko static void btf_bswap_type_base(struct btf_type *t) 3303289959bSAndrii Nakryiko { 3313289959bSAndrii Nakryiko t->name_off = bswap_32(t->name_off); 3323289959bSAndrii Nakryiko t->info = bswap_32(t->info); 3333289959bSAndrii Nakryiko t->type = bswap_32(t->type); 3343289959bSAndrii Nakryiko } 3353289959bSAndrii Nakryiko 3363289959bSAndrii Nakryiko static int btf_bswap_type_rest(struct btf_type *t) 3373289959bSAndrii Nakryiko { 3383289959bSAndrii Nakryiko struct btf_var_secinfo *v; 339dffbbdc2SYonghong Song struct btf_enum64 *e64; 3403289959bSAndrii Nakryiko struct btf_member *m; 3413289959bSAndrii Nakryiko struct btf_array *a; 3423289959bSAndrii Nakryiko struct btf_param *p; 3433289959bSAndrii Nakryiko struct btf_enum *e; 3443289959bSAndrii Nakryiko __u16 vlen = btf_vlen(t); 3453289959bSAndrii Nakryiko int i; 3463289959bSAndrii Nakryiko 3473289959bSAndrii Nakryiko switch (btf_kind(t)) { 3483289959bSAndrii Nakryiko case BTF_KIND_FWD: 3493289959bSAndrii Nakryiko case BTF_KIND_CONST: 3503289959bSAndrii Nakryiko case BTF_KIND_VOLATILE: 3513289959bSAndrii Nakryiko case BTF_KIND_RESTRICT: 3523289959bSAndrii Nakryiko case BTF_KIND_PTR: 3533289959bSAndrii Nakryiko case BTF_KIND_TYPEDEF: 3543289959bSAndrii Nakryiko case BTF_KIND_FUNC: 35522541a9eSIlya Leoshkevich case BTF_KIND_FLOAT: 3562dc1e488SYonghong Song case BTF_KIND_TYPE_TAG: 3573289959bSAndrii Nakryiko return 0; 3583289959bSAndrii Nakryiko case BTF_KIND_INT: 3593289959bSAndrii Nakryiko *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); 3603289959bSAndrii Nakryiko return 0; 3613289959bSAndrii Nakryiko case BTF_KIND_ENUM: 3623289959bSAndrii Nakryiko for (i = 0, e = btf_enum(t); i < vlen; i++, e++) { 3633289959bSAndrii Nakryiko e->name_off = bswap_32(e->name_off); 3643289959bSAndrii Nakryiko e->val = bswap_32(e->val); 3653289959bSAndrii Nakryiko } 3663289959bSAndrii Nakryiko return 0; 367dffbbdc2SYonghong Song case BTF_KIND_ENUM64: 368dffbbdc2SYonghong Song for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) { 369dffbbdc2SYonghong Song e64->name_off = bswap_32(e64->name_off); 370dffbbdc2SYonghong Song e64->val_lo32 = bswap_32(e64->val_lo32); 371dffbbdc2SYonghong Song e64->val_hi32 = bswap_32(e64->val_hi32); 372dffbbdc2SYonghong Song } 373dffbbdc2SYonghong Song return 0; 3743289959bSAndrii Nakryiko case BTF_KIND_ARRAY: 3753289959bSAndrii Nakryiko a = btf_array(t); 3763289959bSAndrii Nakryiko a->type = bswap_32(a->type); 3773289959bSAndrii Nakryiko a->index_type = bswap_32(a->index_type); 3783289959bSAndrii Nakryiko a->nelems = bswap_32(a->nelems); 3793289959bSAndrii Nakryiko return 0; 3803289959bSAndrii Nakryiko case BTF_KIND_STRUCT: 3813289959bSAndrii Nakryiko case BTF_KIND_UNION: 3823289959bSAndrii Nakryiko for (i = 0, m = btf_members(t); i < vlen; i++, m++) { 3833289959bSAndrii Nakryiko m->name_off = bswap_32(m->name_off); 3843289959bSAndrii Nakryiko m->type = bswap_32(m->type); 3853289959bSAndrii Nakryiko m->offset = bswap_32(m->offset); 3863289959bSAndrii Nakryiko } 3873289959bSAndrii Nakryiko return 0; 3883289959bSAndrii Nakryiko case BTF_KIND_FUNC_PROTO: 3893289959bSAndrii Nakryiko for (i = 0, p = btf_params(t); i < vlen; i++, p++) { 3903289959bSAndrii Nakryiko p->name_off = bswap_32(p->name_off); 3913289959bSAndrii Nakryiko p->type = bswap_32(p->type); 3923289959bSAndrii Nakryiko } 3933289959bSAndrii Nakryiko return 0; 3943289959bSAndrii Nakryiko case BTF_KIND_VAR: 3953289959bSAndrii Nakryiko btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); 3963289959bSAndrii Nakryiko return 0; 3973289959bSAndrii Nakryiko case BTF_KIND_DATASEC: 3983289959bSAndrii Nakryiko for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) { 3993289959bSAndrii Nakryiko v->type = bswap_32(v->type); 4003289959bSAndrii Nakryiko v->offset = bswap_32(v->offset); 4013289959bSAndrii Nakryiko v->size = bswap_32(v->size); 4023289959bSAndrii Nakryiko } 4033289959bSAndrii Nakryiko return 0; 404223f903eSYonghong Song case BTF_KIND_DECL_TAG: 405223f903eSYonghong Song btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); 4065b84bd10SYonghong Song return 0; 4073289959bSAndrii Nakryiko default: 4083289959bSAndrii Nakryiko pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); 4093289959bSAndrii Nakryiko return -EINVAL; 4103289959bSAndrii Nakryiko } 4113289959bSAndrii Nakryiko } 4123289959bSAndrii Nakryiko 4138461ef8bSYonghong Song static int btf_parse_type_sec(struct btf *btf) 4148a138aedSMartin KaFai Lau { 4158a138aedSMartin KaFai Lau struct btf_header *hdr = btf->hdr; 416b8604247SAndrii Nakryiko void *next_type = btf->types_data; 417740e69c3SAndrii Nakryiko void *end_type = next_type + hdr->type_len; 418ba451366SAndrii Nakryiko int err, type_size; 419740e69c3SAndrii Nakryiko 4203289959bSAndrii Nakryiko while (next_type + sizeof(struct btf_type) <= end_type) { 4213289959bSAndrii Nakryiko if (btf->swapped_endian) 4223289959bSAndrii Nakryiko btf_bswap_type_base(next_type); 423740e69c3SAndrii Nakryiko 424740e69c3SAndrii Nakryiko type_size = btf_type_size(next_type); 425740e69c3SAndrii Nakryiko if (type_size < 0) 426740e69c3SAndrii Nakryiko return type_size; 4273289959bSAndrii Nakryiko if (next_type + type_size > end_type) { 428ba451366SAndrii Nakryiko pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); 4293289959bSAndrii Nakryiko return -EINVAL; 4303289959bSAndrii Nakryiko } 4313289959bSAndrii Nakryiko 4323289959bSAndrii Nakryiko if (btf->swapped_endian && btf_bswap_type_rest(next_type)) 4333289959bSAndrii Nakryiko return -EINVAL; 4343289959bSAndrii Nakryiko 4353289959bSAndrii Nakryiko err = btf_add_type_idx_entry(btf, next_type - btf->types_data); 4363289959bSAndrii Nakryiko if (err) 4373289959bSAndrii Nakryiko return err; 438740e69c3SAndrii Nakryiko 439740e69c3SAndrii Nakryiko next_type += type_size; 440740e69c3SAndrii Nakryiko btf->nr_types++; 4418a138aedSMartin KaFai Lau } 4428a138aedSMartin KaFai Lau 4433289959bSAndrii Nakryiko if (next_type != end_type) { 4443289959bSAndrii Nakryiko pr_warn("BTF types data is malformed\n"); 4453289959bSAndrii Nakryiko return -EINVAL; 4463289959bSAndrii Nakryiko } 4473289959bSAndrii Nakryiko 4488a138aedSMartin KaFai Lau return 0; 4498a138aedSMartin KaFai Lau } 4508a138aedSMartin KaFai Lau 4516a886de0SHengqi Chen __u32 btf__type_cnt(const struct btf *btf) 4526a886de0SHengqi Chen { 4536a886de0SHengqi Chen return btf->start_id + btf->nr_types; 4546a886de0SHengqi Chen } 4556a886de0SHengqi Chen 4560cfdcd63SAndrii Nakryiko const struct btf *btf__base_btf(const struct btf *btf) 4570cfdcd63SAndrii Nakryiko { 4580cfdcd63SAndrii Nakryiko return btf->base_btf; 4590cfdcd63SAndrii Nakryiko } 4600cfdcd63SAndrii Nakryiko 461740e69c3SAndrii Nakryiko /* internal helper returning non-const pointer to a type */ 46274753e14SAlexei Starovoitov struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id) 463740e69c3SAndrii Nakryiko { 464740e69c3SAndrii Nakryiko if (type_id == 0) 465740e69c3SAndrii Nakryiko return &btf_void; 466ba451366SAndrii Nakryiko if (type_id < btf->start_id) 467ba451366SAndrii Nakryiko return btf_type_by_id(btf->base_btf, type_id); 468ba451366SAndrii Nakryiko return btf->types_data + btf->type_offs[type_id - btf->start_id]; 469740e69c3SAndrii Nakryiko } 470740e69c3SAndrii Nakryiko 47138d5d3b3SMartin KaFai Lau const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) 4728a138aedSMartin KaFai Lau { 473ba451366SAndrii Nakryiko if (type_id >= btf->start_id + btf->nr_types) 474e9fc3ce9SAndrii Nakryiko return errno = EINVAL, NULL; 475740e69c3SAndrii Nakryiko return btf_type_by_id((struct btf *)btf, type_id); 4768a138aedSMartin KaFai Lau } 4778a138aedSMartin KaFai Lau 47844ad23dfSAndrii Nakryiko static int determine_ptr_size(const struct btf *btf) 47944ad23dfSAndrii Nakryiko { 480610cd93bSDouglas Raillard static const char * const long_aliases[] = { 481610cd93bSDouglas Raillard "long", 482610cd93bSDouglas Raillard "long int", 483610cd93bSDouglas Raillard "int long", 484610cd93bSDouglas Raillard "unsigned long", 485610cd93bSDouglas Raillard "long unsigned", 486610cd93bSDouglas Raillard "unsigned long int", 487610cd93bSDouglas Raillard "unsigned int long", 488610cd93bSDouglas Raillard "long unsigned int", 489610cd93bSDouglas Raillard "long int unsigned", 490610cd93bSDouglas Raillard "int unsigned long", 491610cd93bSDouglas Raillard "int long unsigned", 492610cd93bSDouglas Raillard }; 49344ad23dfSAndrii Nakryiko const struct btf_type *t; 49444ad23dfSAndrii Nakryiko const char *name; 495610cd93bSDouglas Raillard int i, j, n; 49644ad23dfSAndrii Nakryiko 497ba451366SAndrii Nakryiko if (btf->base_btf && btf->base_btf->ptr_sz > 0) 498ba451366SAndrii Nakryiko return btf->base_btf->ptr_sz; 499ba451366SAndrii Nakryiko 5006a886de0SHengqi Chen n = btf__type_cnt(btf); 5016a886de0SHengqi Chen for (i = 1; i < n; i++) { 50244ad23dfSAndrii Nakryiko t = btf__type_by_id(btf, i); 50344ad23dfSAndrii Nakryiko if (!btf_is_int(t)) 50444ad23dfSAndrii Nakryiko continue; 50544ad23dfSAndrii Nakryiko 506610cd93bSDouglas Raillard if (t->size != 4 && t->size != 8) 507610cd93bSDouglas Raillard continue; 508610cd93bSDouglas Raillard 50944ad23dfSAndrii Nakryiko name = btf__name_by_offset(btf, t->name_off); 51044ad23dfSAndrii Nakryiko if (!name) 51144ad23dfSAndrii Nakryiko continue; 51244ad23dfSAndrii Nakryiko 513610cd93bSDouglas Raillard for (j = 0; j < ARRAY_SIZE(long_aliases); j++) { 514610cd93bSDouglas Raillard if (strcmp(name, long_aliases[j]) == 0) 51544ad23dfSAndrii Nakryiko return t->size; 51644ad23dfSAndrii Nakryiko } 51744ad23dfSAndrii Nakryiko } 51844ad23dfSAndrii Nakryiko 51944ad23dfSAndrii Nakryiko return -1; 52044ad23dfSAndrii Nakryiko } 52144ad23dfSAndrii Nakryiko 52244ad23dfSAndrii Nakryiko static size_t btf_ptr_sz(const struct btf *btf) 52344ad23dfSAndrii Nakryiko { 52444ad23dfSAndrii Nakryiko if (!btf->ptr_sz) 52544ad23dfSAndrii Nakryiko ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 52644ad23dfSAndrii Nakryiko return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz; 52744ad23dfSAndrii Nakryiko } 52844ad23dfSAndrii Nakryiko 52944ad23dfSAndrii Nakryiko /* Return pointer size this BTF instance assumes. The size is heuristically 53044ad23dfSAndrii Nakryiko * determined by looking for 'long' or 'unsigned long' integer type and 53144ad23dfSAndrii Nakryiko * recording its size in bytes. If BTF type information doesn't have any such 53244ad23dfSAndrii Nakryiko * type, this function returns 0. In the latter case, native architecture's 53344ad23dfSAndrii Nakryiko * pointer size is assumed, so will be either 4 or 8, depending on 53444ad23dfSAndrii Nakryiko * architecture that libbpf was compiled for. It's possible to override 53544ad23dfSAndrii Nakryiko * guessed value by using btf__set_pointer_size() API. 53644ad23dfSAndrii Nakryiko */ 53744ad23dfSAndrii Nakryiko size_t btf__pointer_size(const struct btf *btf) 53844ad23dfSAndrii Nakryiko { 53944ad23dfSAndrii Nakryiko if (!btf->ptr_sz) 54044ad23dfSAndrii Nakryiko ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf); 54144ad23dfSAndrii Nakryiko 54244ad23dfSAndrii Nakryiko if (btf->ptr_sz < 0) 54344ad23dfSAndrii Nakryiko /* not enough BTF type info to guess */ 54444ad23dfSAndrii Nakryiko return 0; 54544ad23dfSAndrii Nakryiko 54644ad23dfSAndrii Nakryiko return btf->ptr_sz; 54744ad23dfSAndrii Nakryiko } 54844ad23dfSAndrii Nakryiko 54944ad23dfSAndrii Nakryiko /* Override or set pointer size in bytes. Only values of 4 and 8 are 55044ad23dfSAndrii Nakryiko * supported. 55144ad23dfSAndrii Nakryiko */ 55244ad23dfSAndrii Nakryiko int btf__set_pointer_size(struct btf *btf, size_t ptr_sz) 55344ad23dfSAndrii Nakryiko { 55444ad23dfSAndrii Nakryiko if (ptr_sz != 4 && ptr_sz != 8) 555e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 55644ad23dfSAndrii Nakryiko btf->ptr_sz = ptr_sz; 55744ad23dfSAndrii Nakryiko return 0; 55844ad23dfSAndrii Nakryiko } 55944ad23dfSAndrii Nakryiko 5603289959bSAndrii Nakryiko static bool is_host_big_endian(void) 5613289959bSAndrii Nakryiko { 5623930198dSIlya Leoshkevich #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 5633289959bSAndrii Nakryiko return false; 5643930198dSIlya Leoshkevich #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 5653289959bSAndrii Nakryiko return true; 5663289959bSAndrii Nakryiko #else 5673289959bSAndrii Nakryiko # error "Unrecognized __BYTE_ORDER__" 5683289959bSAndrii Nakryiko #endif 5693289959bSAndrii Nakryiko } 5703289959bSAndrii Nakryiko 5713289959bSAndrii Nakryiko enum btf_endianness btf__endianness(const struct btf *btf) 5723289959bSAndrii Nakryiko { 5733289959bSAndrii Nakryiko if (is_host_big_endian()) 5743289959bSAndrii Nakryiko return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; 5753289959bSAndrii Nakryiko else 5763289959bSAndrii Nakryiko return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; 5773289959bSAndrii Nakryiko } 5783289959bSAndrii Nakryiko 5793289959bSAndrii Nakryiko int btf__set_endianness(struct btf *btf, enum btf_endianness endian) 5803289959bSAndrii Nakryiko { 5813289959bSAndrii Nakryiko if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) 582e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 5833289959bSAndrii Nakryiko 5843289959bSAndrii Nakryiko btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); 5853289959bSAndrii Nakryiko if (!btf->swapped_endian) { 5863289959bSAndrii Nakryiko free(btf->raw_data_swapped); 5873289959bSAndrii Nakryiko btf->raw_data_swapped = NULL; 5883289959bSAndrii Nakryiko } 5893289959bSAndrii Nakryiko return 0; 5903289959bSAndrii Nakryiko } 5913289959bSAndrii Nakryiko 5928a138aedSMartin KaFai Lau static bool btf_type_is_void(const struct btf_type *t) 5938a138aedSMartin KaFai Lau { 594b03bc685SAndrii Nakryiko return t == &btf_void || btf_is_fwd(t); 5958a138aedSMartin KaFai Lau } 5968a138aedSMartin KaFai Lau 5978a138aedSMartin KaFai Lau static bool btf_type_is_void_or_null(const struct btf_type *t) 5988a138aedSMartin KaFai Lau { 5998a138aedSMartin KaFai Lau return !t || btf_type_is_void(t); 6008a138aedSMartin KaFai Lau } 6018a138aedSMartin KaFai Lau 6028a138aedSMartin KaFai Lau #define MAX_RESOLVE_DEPTH 32 6038a138aedSMartin KaFai Lau 6045b891af7SMartin KaFai Lau __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) 6058a138aedSMartin KaFai Lau { 6068a138aedSMartin KaFai Lau const struct btf_array *array; 6078a138aedSMartin KaFai Lau const struct btf_type *t; 6085b891af7SMartin KaFai Lau __u32 nelems = 1; 6095b891af7SMartin KaFai Lau __s64 size = -1; 6108a138aedSMartin KaFai Lau int i; 6118a138aedSMartin KaFai Lau 61292b57121SOkash Khawaja t = btf__type_by_id(btf, type_id); 613e9fc3ce9SAndrii Nakryiko for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) { 614b03bc685SAndrii Nakryiko switch (btf_kind(t)) { 61569eaab04SAndrii Nakryiko case BTF_KIND_INT: 61669eaab04SAndrii Nakryiko case BTF_KIND_STRUCT: 61769eaab04SAndrii Nakryiko case BTF_KIND_UNION: 61869eaab04SAndrii Nakryiko case BTF_KIND_ENUM: 619dffbbdc2SYonghong Song case BTF_KIND_ENUM64: 6201713d68bSDaniel Borkmann case BTF_KIND_DATASEC: 62122541a9eSIlya Leoshkevich case BTF_KIND_FLOAT: 62269eaab04SAndrii Nakryiko size = t->size; 62369eaab04SAndrii Nakryiko goto done; 62469eaab04SAndrii Nakryiko case BTF_KIND_PTR: 62544ad23dfSAndrii Nakryiko size = btf_ptr_sz(btf); 62669eaab04SAndrii Nakryiko goto done; 6278a138aedSMartin KaFai Lau case BTF_KIND_TYPEDEF: 6288a138aedSMartin KaFai Lau case BTF_KIND_VOLATILE: 6298a138aedSMartin KaFai Lau case BTF_KIND_CONST: 6308a138aedSMartin KaFai Lau case BTF_KIND_RESTRICT: 6311713d68bSDaniel Borkmann case BTF_KIND_VAR: 632223f903eSYonghong Song case BTF_KIND_DECL_TAG: 63369a055d5SYonghong Song case BTF_KIND_TYPE_TAG: 6348a138aedSMartin KaFai Lau type_id = t->type; 6358a138aedSMartin KaFai Lau break; 6368a138aedSMartin KaFai Lau case BTF_KIND_ARRAY: 637b03bc685SAndrii Nakryiko array = btf_array(t); 6388a138aedSMartin KaFai Lau if (nelems && array->nelems > UINT32_MAX / nelems) 639e9fc3ce9SAndrii Nakryiko return libbpf_err(-E2BIG); 6408a138aedSMartin KaFai Lau nelems *= array->nelems; 6418a138aedSMartin KaFai Lau type_id = array->type; 6428a138aedSMartin KaFai Lau break; 6438a138aedSMartin KaFai Lau default: 644e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 6458a138aedSMartin KaFai Lau } 6468a138aedSMartin KaFai Lau 64792b57121SOkash Khawaja t = btf__type_by_id(btf, type_id); 6488a138aedSMartin KaFai Lau } 6498a138aedSMartin KaFai Lau 650994021a7SAndrii Nakryiko done: 6518a138aedSMartin KaFai Lau if (size < 0) 652e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 6538a138aedSMartin KaFai Lau if (nelems && size > UINT32_MAX / nelems) 654e9fc3ce9SAndrii Nakryiko return libbpf_err(-E2BIG); 6558a138aedSMartin KaFai Lau 6568a138aedSMartin KaFai Lau return nelems * size; 6578a138aedSMartin KaFai Lau } 6588a138aedSMartin KaFai Lau 6593d208f4cSAndrii Nakryiko int btf__align_of(const struct btf *btf, __u32 id) 6603d208f4cSAndrii Nakryiko { 6613d208f4cSAndrii Nakryiko const struct btf_type *t = btf__type_by_id(btf, id); 6623d208f4cSAndrii Nakryiko __u16 kind = btf_kind(t); 6633d208f4cSAndrii Nakryiko 6643d208f4cSAndrii Nakryiko switch (kind) { 6653d208f4cSAndrii Nakryiko case BTF_KIND_INT: 6663d208f4cSAndrii Nakryiko case BTF_KIND_ENUM: 667dffbbdc2SYonghong Song case BTF_KIND_ENUM64: 66822541a9eSIlya Leoshkevich case BTF_KIND_FLOAT: 66944ad23dfSAndrii Nakryiko return min(btf_ptr_sz(btf), (size_t)t->size); 6703d208f4cSAndrii Nakryiko case BTF_KIND_PTR: 67144ad23dfSAndrii Nakryiko return btf_ptr_sz(btf); 6723d208f4cSAndrii Nakryiko case BTF_KIND_TYPEDEF: 6733d208f4cSAndrii Nakryiko case BTF_KIND_VOLATILE: 6743d208f4cSAndrii Nakryiko case BTF_KIND_CONST: 6753d208f4cSAndrii Nakryiko case BTF_KIND_RESTRICT: 6762dc1e488SYonghong Song case BTF_KIND_TYPE_TAG: 6773d208f4cSAndrii Nakryiko return btf__align_of(btf, t->type); 6783d208f4cSAndrii Nakryiko case BTF_KIND_ARRAY: 6793d208f4cSAndrii Nakryiko return btf__align_of(btf, btf_array(t)->type); 6803d208f4cSAndrii Nakryiko case BTF_KIND_STRUCT: 6813d208f4cSAndrii Nakryiko case BTF_KIND_UNION: { 6823d208f4cSAndrii Nakryiko const struct btf_member *m = btf_members(t); 6833d208f4cSAndrii Nakryiko __u16 vlen = btf_vlen(t); 684a79ac2d1SPrashant Bhole int i, max_align = 1, align; 6853d208f4cSAndrii Nakryiko 6863d208f4cSAndrii Nakryiko for (i = 0; i < vlen; i++, m++) { 687a79ac2d1SPrashant Bhole align = btf__align_of(btf, m->type); 688a79ac2d1SPrashant Bhole if (align <= 0) 689e9fc3ce9SAndrii Nakryiko return libbpf_err(align); 690a79ac2d1SPrashant Bhole max_align = max(max_align, align); 6913d208f4cSAndrii Nakryiko } 6923d208f4cSAndrii Nakryiko 693a79ac2d1SPrashant Bhole return max_align; 6943d208f4cSAndrii Nakryiko } 6953d208f4cSAndrii Nakryiko default: 6963d208f4cSAndrii Nakryiko pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t)); 697e9fc3ce9SAndrii Nakryiko return errno = EINVAL, 0; 6983d208f4cSAndrii Nakryiko } 6993d208f4cSAndrii Nakryiko } 7003d208f4cSAndrii Nakryiko 70192b57121SOkash Khawaja int btf__resolve_type(const struct btf *btf, __u32 type_id) 70292b57121SOkash Khawaja { 70392b57121SOkash Khawaja const struct btf_type *t; 70492b57121SOkash Khawaja int depth = 0; 70592b57121SOkash Khawaja 70692b57121SOkash Khawaja t = btf__type_by_id(btf, type_id); 70792b57121SOkash Khawaja while (depth < MAX_RESOLVE_DEPTH && 70892b57121SOkash Khawaja !btf_type_is_void_or_null(t) && 709b03bc685SAndrii Nakryiko (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) { 71092b57121SOkash Khawaja type_id = t->type; 71192b57121SOkash Khawaja t = btf__type_by_id(btf, type_id); 71292b57121SOkash Khawaja depth++; 71392b57121SOkash Khawaja } 71492b57121SOkash Khawaja 71592b57121SOkash Khawaja if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t)) 716e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 71792b57121SOkash Khawaja 71892b57121SOkash Khawaja return type_id; 71992b57121SOkash Khawaja } 72092b57121SOkash Khawaja 7215b891af7SMartin KaFai Lau __s32 btf__find_by_name(const struct btf *btf, const char *type_name) 7228a138aedSMartin KaFai Lau { 7236a886de0SHengqi Chen __u32 i, nr_types = btf__type_cnt(btf); 7248a138aedSMartin KaFai Lau 7258a138aedSMartin KaFai Lau if (!strcmp(type_name, "void")) 7268a138aedSMartin KaFai Lau return 0; 7278a138aedSMartin KaFai Lau 7286a886de0SHengqi Chen for (i = 1; i < nr_types; i++) { 729740e69c3SAndrii Nakryiko const struct btf_type *t = btf__type_by_id(btf, i); 73092b57121SOkash Khawaja const char *name = btf__name_by_offset(btf, t->name_off); 7318a138aedSMartin KaFai Lau 7328a138aedSMartin KaFai Lau if (name && !strcmp(type_name, name)) 7338a138aedSMartin KaFai Lau return i; 7348a138aedSMartin KaFai Lau } 7358a138aedSMartin KaFai Lau 736e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOENT); 7378a138aedSMartin KaFai Lau } 7388a138aedSMartin KaFai Lau 7399dbe6015SKumar Kartikeya Dwivedi static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id, 7409dbe6015SKumar Kartikeya Dwivedi const char *type_name, __u32 kind) 7411442e287SAlexei Starovoitov { 7426a886de0SHengqi Chen __u32 i, nr_types = btf__type_cnt(btf); 7431442e287SAlexei Starovoitov 7441442e287SAlexei Starovoitov if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void")) 7451442e287SAlexei Starovoitov return 0; 7461442e287SAlexei Starovoitov 7476a886de0SHengqi Chen for (i = start_id; i < nr_types; i++) { 748740e69c3SAndrii Nakryiko const struct btf_type *t = btf__type_by_id(btf, i); 7491442e287SAlexei Starovoitov const char *name; 7501442e287SAlexei Starovoitov 7511442e287SAlexei Starovoitov if (btf_kind(t) != kind) 7521442e287SAlexei Starovoitov continue; 7531442e287SAlexei Starovoitov name = btf__name_by_offset(btf, t->name_off); 7541442e287SAlexei Starovoitov if (name && !strcmp(type_name, name)) 7551442e287SAlexei Starovoitov return i; 7561442e287SAlexei Starovoitov } 7571442e287SAlexei Starovoitov 758e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOENT); 7591442e287SAlexei Starovoitov } 7601442e287SAlexei Starovoitov 7619dbe6015SKumar Kartikeya Dwivedi __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, 7629dbe6015SKumar Kartikeya Dwivedi __u32 kind) 7639dbe6015SKumar Kartikeya Dwivedi { 7649dbe6015SKumar Kartikeya Dwivedi return btf_find_by_name_kind(btf, btf->start_id, type_name, kind); 7659dbe6015SKumar Kartikeya Dwivedi } 7669dbe6015SKumar Kartikeya Dwivedi 7679dbe6015SKumar Kartikeya Dwivedi __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, 7689dbe6015SKumar Kartikeya Dwivedi __u32 kind) 7699dbe6015SKumar Kartikeya Dwivedi { 7709dbe6015SKumar Kartikeya Dwivedi return btf_find_by_name_kind(btf, 1, type_name, kind); 7719dbe6015SKumar Kartikeya Dwivedi } 7729dbe6015SKumar Kartikeya Dwivedi 773919d2b1dSAndrii Nakryiko static bool btf_is_modifiable(const struct btf *btf) 774919d2b1dSAndrii Nakryiko { 775919d2b1dSAndrii Nakryiko return (void *)btf->hdr != btf->raw_data; 776919d2b1dSAndrii Nakryiko } 777919d2b1dSAndrii Nakryiko 7788a138aedSMartin KaFai Lau void btf__free(struct btf *btf) 7798a138aedSMartin KaFai Lau { 78050450fc7SAndrii Nakryiko if (IS_ERR_OR_NULL(btf)) 7818a138aedSMartin KaFai Lau return; 7828a138aedSMartin KaFai Lau 78381372e12SAndrii Nakryiko if (btf->fd >= 0) 7848a138aedSMartin KaFai Lau close(btf->fd); 7858a138aedSMartin KaFai Lau 786919d2b1dSAndrii Nakryiko if (btf_is_modifiable(btf)) { 787919d2b1dSAndrii Nakryiko /* if BTF was modified after loading, it will have a split 788919d2b1dSAndrii Nakryiko * in-memory representation for header, types, and strings 789919d2b1dSAndrii Nakryiko * sections, so we need to free all of them individually. It 790919d2b1dSAndrii Nakryiko * might still have a cached contiguous raw data present, 791919d2b1dSAndrii Nakryiko * which will be unconditionally freed below. 792919d2b1dSAndrii Nakryiko */ 793919d2b1dSAndrii Nakryiko free(btf->hdr); 794919d2b1dSAndrii Nakryiko free(btf->types_data); 79590d76d3eSAndrii Nakryiko strset__free(btf->strs_set); 796919d2b1dSAndrii Nakryiko } 797b8604247SAndrii Nakryiko free(btf->raw_data); 7983289959bSAndrii Nakryiko free(btf->raw_data_swapped); 799740e69c3SAndrii Nakryiko free(btf->type_offs); 8008a138aedSMartin KaFai Lau free(btf); 8018a138aedSMartin KaFai Lau } 8028a138aedSMartin KaFai Lau 803ba451366SAndrii Nakryiko static struct btf *btf_new_empty(struct btf *base_btf) 804a871b043SAndrii Nakryiko { 805a871b043SAndrii Nakryiko struct btf *btf; 806a871b043SAndrii Nakryiko 807a871b043SAndrii Nakryiko btf = calloc(1, sizeof(*btf)); 808a871b043SAndrii Nakryiko if (!btf) 809a871b043SAndrii Nakryiko return ERR_PTR(-ENOMEM); 8103289959bSAndrii Nakryiko 811ba451366SAndrii Nakryiko btf->nr_types = 0; 812ba451366SAndrii Nakryiko btf->start_id = 1; 813ba451366SAndrii Nakryiko btf->start_str_off = 0; 814a871b043SAndrii Nakryiko btf->fd = -1; 815a871b043SAndrii Nakryiko btf->ptr_sz = sizeof(void *); 8163289959bSAndrii Nakryiko btf->swapped_endian = false; 817a871b043SAndrii Nakryiko 818ba451366SAndrii Nakryiko if (base_btf) { 819ba451366SAndrii Nakryiko btf->base_btf = base_btf; 8206a886de0SHengqi Chen btf->start_id = btf__type_cnt(base_btf); 821ba451366SAndrii Nakryiko btf->start_str_off = base_btf->hdr->str_len; 822ba451366SAndrii Nakryiko } 823ba451366SAndrii Nakryiko 824a871b043SAndrii Nakryiko /* +1 for empty string at offset 0 */ 825ba451366SAndrii Nakryiko btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1); 826a871b043SAndrii Nakryiko btf->raw_data = calloc(1, btf->raw_size); 827a871b043SAndrii Nakryiko if (!btf->raw_data) { 828a871b043SAndrii Nakryiko free(btf); 829a871b043SAndrii Nakryiko return ERR_PTR(-ENOMEM); 830a871b043SAndrii Nakryiko } 831a871b043SAndrii Nakryiko 832a871b043SAndrii Nakryiko btf->hdr = btf->raw_data; 833a871b043SAndrii Nakryiko btf->hdr->hdr_len = sizeof(struct btf_header); 834a871b043SAndrii Nakryiko btf->hdr->magic = BTF_MAGIC; 835a871b043SAndrii Nakryiko btf->hdr->version = BTF_VERSION; 836a871b043SAndrii Nakryiko 837a871b043SAndrii Nakryiko btf->types_data = btf->raw_data + btf->hdr->hdr_len; 838a871b043SAndrii Nakryiko btf->strs_data = btf->raw_data + btf->hdr->hdr_len; 839ba451366SAndrii Nakryiko btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */ 840a871b043SAndrii Nakryiko 841a871b043SAndrii Nakryiko return btf; 842a871b043SAndrii Nakryiko } 843a871b043SAndrii Nakryiko 844ba451366SAndrii Nakryiko struct btf *btf__new_empty(void) 845ba451366SAndrii Nakryiko { 846e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_new_empty(NULL)); 847ba451366SAndrii Nakryiko } 848ba451366SAndrii Nakryiko 849ba451366SAndrii Nakryiko struct btf *btf__new_empty_split(struct btf *base_btf) 850ba451366SAndrii Nakryiko { 851e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_new_empty(base_btf)); 852ba451366SAndrii Nakryiko } 853ba451366SAndrii Nakryiko 854ba451366SAndrii Nakryiko static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf) 8558a138aedSMartin KaFai Lau { 8568a138aedSMartin KaFai Lau struct btf *btf; 8578a138aedSMartin KaFai Lau int err; 8588a138aedSMartin KaFai Lau 8598a138aedSMartin KaFai Lau btf = calloc(1, sizeof(struct btf)); 8608a138aedSMartin KaFai Lau if (!btf) 8618a138aedSMartin KaFai Lau return ERR_PTR(-ENOMEM); 8628a138aedSMartin KaFai Lau 863ba451366SAndrii Nakryiko btf->nr_types = 0; 864ba451366SAndrii Nakryiko btf->start_id = 1; 865ba451366SAndrii Nakryiko btf->start_str_off = 0; 866c34c338aSDaniel Xu btf->fd = -1; 867ba451366SAndrii Nakryiko 868ba451366SAndrii Nakryiko if (base_btf) { 869ba451366SAndrii Nakryiko btf->base_btf = base_btf; 8706a886de0SHengqi Chen btf->start_id = btf__type_cnt(base_btf); 871ba451366SAndrii Nakryiko btf->start_str_off = base_btf->hdr->str_len; 872ba451366SAndrii Nakryiko } 873ba451366SAndrii Nakryiko 874b8604247SAndrii Nakryiko btf->raw_data = malloc(size); 875b8604247SAndrii Nakryiko if (!btf->raw_data) { 8768a138aedSMartin KaFai Lau err = -ENOMEM; 8778a138aedSMartin KaFai Lau goto done; 8788a138aedSMartin KaFai Lau } 879b8604247SAndrii Nakryiko memcpy(btf->raw_data, data, size); 880b8604247SAndrii Nakryiko btf->raw_size = size; 8818a138aedSMartin KaFai Lau 882b8604247SAndrii Nakryiko btf->hdr = btf->raw_data; 8838461ef8bSYonghong Song err = btf_parse_hdr(btf); 8848a138aedSMartin KaFai Lau if (err) 8858a138aedSMartin KaFai Lau goto done; 8868a138aedSMartin KaFai Lau 887b8604247SAndrii Nakryiko btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; 888b8604247SAndrii Nakryiko btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; 8898a138aedSMartin KaFai Lau 890b8604247SAndrii Nakryiko err = btf_parse_str_sec(btf); 891b8604247SAndrii Nakryiko err = err ?: btf_parse_type_sec(btf); 8923289959bSAndrii Nakryiko if (err) 8933289959bSAndrii Nakryiko goto done; 8943289959bSAndrii Nakryiko 8958a138aedSMartin KaFai Lau done: 8968a138aedSMartin KaFai Lau if (err) { 8978a138aedSMartin KaFai Lau btf__free(btf); 8988a138aedSMartin KaFai Lau return ERR_PTR(err); 8998a138aedSMartin KaFai Lau } 9008a138aedSMartin KaFai Lau 9018a138aedSMartin KaFai Lau return btf; 9028a138aedSMartin KaFai Lau } 9038a138aedSMartin KaFai Lau 904ba451366SAndrii Nakryiko struct btf *btf__new(const void *data, __u32 size) 905ba451366SAndrii Nakryiko { 906e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_new(data, size, NULL)); 907ba451366SAndrii Nakryiko } 908ba451366SAndrii Nakryiko 909ba451366SAndrii Nakryiko static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, 910ba451366SAndrii Nakryiko struct btf_ext **btf_ext) 911e6c64855SAndrii Nakryiko { 912e6c64855SAndrii Nakryiko Elf_Data *btf_data = NULL, *btf_ext_data = NULL; 913e6c64855SAndrii Nakryiko int err = 0, fd = -1, idx = 0; 914e6c64855SAndrii Nakryiko struct btf *btf = NULL; 915e6c64855SAndrii Nakryiko Elf_Scn *scn = NULL; 916e6c64855SAndrii Nakryiko Elf *elf = NULL; 917e6c64855SAndrii Nakryiko GElf_Ehdr ehdr; 9186095d5a2SJiri Olsa size_t shstrndx; 919e6c64855SAndrii Nakryiko 920e6c64855SAndrii Nakryiko if (elf_version(EV_CURRENT) == EV_NONE) { 921be18010eSKefeng Wang pr_warn("failed to init libelf for %s\n", path); 922e6c64855SAndrii Nakryiko return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 923e6c64855SAndrii Nakryiko } 924e6c64855SAndrii Nakryiko 92592274e24SKumar Kartikeya Dwivedi fd = open(path, O_RDONLY | O_CLOEXEC); 926e6c64855SAndrii Nakryiko if (fd < 0) { 927e6c64855SAndrii Nakryiko err = -errno; 928be18010eSKefeng Wang pr_warn("failed to open %s: %s\n", path, strerror(errno)); 929e6c64855SAndrii Nakryiko return ERR_PTR(err); 930e6c64855SAndrii Nakryiko } 931e6c64855SAndrii Nakryiko 932e6c64855SAndrii Nakryiko err = -LIBBPF_ERRNO__FORMAT; 933e6c64855SAndrii Nakryiko 934e6c64855SAndrii Nakryiko elf = elf_begin(fd, ELF_C_READ, NULL); 935e6c64855SAndrii Nakryiko if (!elf) { 936be18010eSKefeng Wang pr_warn("failed to open %s as ELF file\n", path); 937e6c64855SAndrii Nakryiko goto done; 938e6c64855SAndrii Nakryiko } 939e6c64855SAndrii Nakryiko if (!gelf_getehdr(elf, &ehdr)) { 940be18010eSKefeng Wang pr_warn("failed to get EHDR from %s\n", path); 941e6c64855SAndrii Nakryiko goto done; 942e6c64855SAndrii Nakryiko } 9436095d5a2SJiri Olsa 9446095d5a2SJiri Olsa if (elf_getshdrstrndx(elf, &shstrndx)) { 9456095d5a2SJiri Olsa pr_warn("failed to get section names section index for %s\n", 9466095d5a2SJiri Olsa path); 9476095d5a2SJiri Olsa goto done; 9486095d5a2SJiri Olsa } 9496095d5a2SJiri Olsa 9506095d5a2SJiri Olsa if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) { 951be18010eSKefeng Wang pr_warn("failed to get e_shstrndx from %s\n", path); 952e6c64855SAndrii Nakryiko goto done; 953e6c64855SAndrii Nakryiko } 954e6c64855SAndrii Nakryiko 955e6c64855SAndrii Nakryiko while ((scn = elf_nextscn(elf, scn)) != NULL) { 956e6c64855SAndrii Nakryiko GElf_Shdr sh; 957e6c64855SAndrii Nakryiko char *name; 958e6c64855SAndrii Nakryiko 959e6c64855SAndrii Nakryiko idx++; 960e6c64855SAndrii Nakryiko if (gelf_getshdr(scn, &sh) != &sh) { 961be18010eSKefeng Wang pr_warn("failed to get section(%d) header from %s\n", 962e6c64855SAndrii Nakryiko idx, path); 963e6c64855SAndrii Nakryiko goto done; 964e6c64855SAndrii Nakryiko } 9656095d5a2SJiri Olsa name = elf_strptr(elf, shstrndx, sh.sh_name); 966e6c64855SAndrii Nakryiko if (!name) { 967be18010eSKefeng Wang pr_warn("failed to get section(%d) name from %s\n", 968e6c64855SAndrii Nakryiko idx, path); 969e6c64855SAndrii Nakryiko goto done; 970e6c64855SAndrii Nakryiko } 971e6c64855SAndrii Nakryiko if (strcmp(name, BTF_ELF_SEC) == 0) { 972e6c64855SAndrii Nakryiko btf_data = elf_getdata(scn, 0); 973e6c64855SAndrii Nakryiko if (!btf_data) { 974be18010eSKefeng Wang pr_warn("failed to get section(%d, %s) data from %s\n", 975e6c64855SAndrii Nakryiko idx, name, path); 976e6c64855SAndrii Nakryiko goto done; 977e6c64855SAndrii Nakryiko } 978e6c64855SAndrii Nakryiko continue; 979e6c64855SAndrii Nakryiko } else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) { 980e6c64855SAndrii Nakryiko btf_ext_data = elf_getdata(scn, 0); 981e6c64855SAndrii Nakryiko if (!btf_ext_data) { 982be18010eSKefeng Wang pr_warn("failed to get section(%d, %s) data from %s\n", 983e6c64855SAndrii Nakryiko idx, name, path); 984e6c64855SAndrii Nakryiko goto done; 985e6c64855SAndrii Nakryiko } 986e6c64855SAndrii Nakryiko continue; 987e6c64855SAndrii Nakryiko } 988e6c64855SAndrii Nakryiko } 989e6c64855SAndrii Nakryiko 990e6c64855SAndrii Nakryiko err = 0; 991e6c64855SAndrii Nakryiko 992e6c64855SAndrii Nakryiko if (!btf_data) { 993e6c64855SAndrii Nakryiko err = -ENOENT; 994e6c64855SAndrii Nakryiko goto done; 995e6c64855SAndrii Nakryiko } 996ba451366SAndrii Nakryiko btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf); 997e9fc3ce9SAndrii Nakryiko err = libbpf_get_error(btf); 998e9fc3ce9SAndrii Nakryiko if (err) 999e6c64855SAndrii Nakryiko goto done; 1000e6c64855SAndrii Nakryiko 100144ad23dfSAndrii Nakryiko switch (gelf_getclass(elf)) { 100244ad23dfSAndrii Nakryiko case ELFCLASS32: 100344ad23dfSAndrii Nakryiko btf__set_pointer_size(btf, 4); 100444ad23dfSAndrii Nakryiko break; 100544ad23dfSAndrii Nakryiko case ELFCLASS64: 100644ad23dfSAndrii Nakryiko btf__set_pointer_size(btf, 8); 100744ad23dfSAndrii Nakryiko break; 100844ad23dfSAndrii Nakryiko default: 100944ad23dfSAndrii Nakryiko pr_warn("failed to get ELF class (bitness) for %s\n", path); 101044ad23dfSAndrii Nakryiko break; 101144ad23dfSAndrii Nakryiko } 101244ad23dfSAndrii Nakryiko 1013e6c64855SAndrii Nakryiko if (btf_ext && btf_ext_data) { 1014e9fc3ce9SAndrii Nakryiko *btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 1015e9fc3ce9SAndrii Nakryiko err = libbpf_get_error(*btf_ext); 1016e9fc3ce9SAndrii Nakryiko if (err) 1017e6c64855SAndrii Nakryiko goto done; 1018e6c64855SAndrii Nakryiko } else if (btf_ext) { 1019e6c64855SAndrii Nakryiko *btf_ext = NULL; 1020e6c64855SAndrii Nakryiko } 1021e6c64855SAndrii Nakryiko done: 1022e6c64855SAndrii Nakryiko if (elf) 1023e6c64855SAndrii Nakryiko elf_end(elf); 1024e6c64855SAndrii Nakryiko close(fd); 1025e6c64855SAndrii Nakryiko 1026e9fc3ce9SAndrii Nakryiko if (!err) 1027e6c64855SAndrii Nakryiko return btf; 1028e9fc3ce9SAndrii Nakryiko 1029e9fc3ce9SAndrii Nakryiko if (btf_ext) 1030e9fc3ce9SAndrii Nakryiko btf_ext__free(*btf_ext); 1031e6c64855SAndrii Nakryiko btf__free(btf); 1032e9fc3ce9SAndrii Nakryiko 1033e6c64855SAndrii Nakryiko return ERR_PTR(err); 1034e6c64855SAndrii Nakryiko } 1035e6c64855SAndrii Nakryiko 1036ba451366SAndrii Nakryiko struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) 1037ba451366SAndrii Nakryiko { 1038e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext)); 1039ba451366SAndrii Nakryiko } 1040ba451366SAndrii Nakryiko 1041ba451366SAndrii Nakryiko struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf) 1042ba451366SAndrii Nakryiko { 1043e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_parse_elf(path, base_btf, NULL)); 1044ba451366SAndrii Nakryiko } 1045ba451366SAndrii Nakryiko 1046ba451366SAndrii Nakryiko static struct btf *btf_parse_raw(const char *path, struct btf *base_btf) 104794a1feddSAndrii Nakryiko { 1048932ac54aSDaniel T. Lee struct btf *btf = NULL; 104994a1feddSAndrii Nakryiko void *data = NULL; 105094a1feddSAndrii Nakryiko FILE *f = NULL; 105194a1feddSAndrii Nakryiko __u16 magic; 105294a1feddSAndrii Nakryiko int err = 0; 105394a1feddSAndrii Nakryiko long sz; 105494a1feddSAndrii Nakryiko 105594a1feddSAndrii Nakryiko f = fopen(path, "rb"); 105694a1feddSAndrii Nakryiko if (!f) { 105794a1feddSAndrii Nakryiko err = -errno; 105894a1feddSAndrii Nakryiko goto err_out; 105994a1feddSAndrii Nakryiko } 106094a1feddSAndrii Nakryiko 106194a1feddSAndrii Nakryiko /* check BTF magic */ 106294a1feddSAndrii Nakryiko if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) { 106394a1feddSAndrii Nakryiko err = -EIO; 106494a1feddSAndrii Nakryiko goto err_out; 106594a1feddSAndrii Nakryiko } 10663289959bSAndrii Nakryiko if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) { 106794a1feddSAndrii Nakryiko /* definitely not a raw BTF */ 106894a1feddSAndrii Nakryiko err = -EPROTO; 106994a1feddSAndrii Nakryiko goto err_out; 107094a1feddSAndrii Nakryiko } 107194a1feddSAndrii Nakryiko 107294a1feddSAndrii Nakryiko /* get file size */ 107394a1feddSAndrii Nakryiko if (fseek(f, 0, SEEK_END)) { 107494a1feddSAndrii Nakryiko err = -errno; 107594a1feddSAndrii Nakryiko goto err_out; 107694a1feddSAndrii Nakryiko } 107794a1feddSAndrii Nakryiko sz = ftell(f); 107894a1feddSAndrii Nakryiko if (sz < 0) { 107994a1feddSAndrii Nakryiko err = -errno; 108094a1feddSAndrii Nakryiko goto err_out; 108194a1feddSAndrii Nakryiko } 108294a1feddSAndrii Nakryiko /* rewind to the start */ 108394a1feddSAndrii Nakryiko if (fseek(f, 0, SEEK_SET)) { 108494a1feddSAndrii Nakryiko err = -errno; 108594a1feddSAndrii Nakryiko goto err_out; 108694a1feddSAndrii Nakryiko } 108794a1feddSAndrii Nakryiko 108894a1feddSAndrii Nakryiko /* pre-alloc memory and read all of BTF data */ 108994a1feddSAndrii Nakryiko data = malloc(sz); 109094a1feddSAndrii Nakryiko if (!data) { 109194a1feddSAndrii Nakryiko err = -ENOMEM; 109294a1feddSAndrii Nakryiko goto err_out; 109394a1feddSAndrii Nakryiko } 109494a1feddSAndrii Nakryiko if (fread(data, 1, sz, f) < sz) { 109594a1feddSAndrii Nakryiko err = -EIO; 109694a1feddSAndrii Nakryiko goto err_out; 109794a1feddSAndrii Nakryiko } 109894a1feddSAndrii Nakryiko 109994a1feddSAndrii Nakryiko /* finally parse BTF data */ 1100ba451366SAndrii Nakryiko btf = btf_new(data, sz, base_btf); 110194a1feddSAndrii Nakryiko 110294a1feddSAndrii Nakryiko err_out: 110394a1feddSAndrii Nakryiko free(data); 110494a1feddSAndrii Nakryiko if (f) 110594a1feddSAndrii Nakryiko fclose(f); 110694a1feddSAndrii Nakryiko return err ? ERR_PTR(err) : btf; 110794a1feddSAndrii Nakryiko } 110894a1feddSAndrii Nakryiko 1109ba451366SAndrii Nakryiko struct btf *btf__parse_raw(const char *path) 1110ba451366SAndrii Nakryiko { 1111e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_parse_raw(path, NULL)); 1112ba451366SAndrii Nakryiko } 1113ba451366SAndrii Nakryiko 1114ba451366SAndrii Nakryiko struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf) 1115ba451366SAndrii Nakryiko { 1116e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_parse_raw(path, base_btf)); 1117ba451366SAndrii Nakryiko } 1118ba451366SAndrii Nakryiko 1119ba451366SAndrii Nakryiko static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext) 112094a1feddSAndrii Nakryiko { 112194a1feddSAndrii Nakryiko struct btf *btf; 1122e9fc3ce9SAndrii Nakryiko int err; 112394a1feddSAndrii Nakryiko 112494a1feddSAndrii Nakryiko if (btf_ext) 112594a1feddSAndrii Nakryiko *btf_ext = NULL; 112694a1feddSAndrii Nakryiko 1127ba451366SAndrii Nakryiko btf = btf_parse_raw(path, base_btf); 1128e9fc3ce9SAndrii Nakryiko err = libbpf_get_error(btf); 1129e9fc3ce9SAndrii Nakryiko if (!err) 113094a1feddSAndrii Nakryiko return btf; 1131e9fc3ce9SAndrii Nakryiko if (err != -EPROTO) 1132e9fc3ce9SAndrii Nakryiko return ERR_PTR(err); 1133ba451366SAndrii Nakryiko return btf_parse_elf(path, base_btf, btf_ext); 1134ba451366SAndrii Nakryiko } 1135ba451366SAndrii Nakryiko 1136ba451366SAndrii Nakryiko struct btf *btf__parse(const char *path, struct btf_ext **btf_ext) 1137ba451366SAndrii Nakryiko { 1138e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_parse(path, NULL, btf_ext)); 1139ba451366SAndrii Nakryiko } 1140ba451366SAndrii Nakryiko 1141ba451366SAndrii Nakryiko struct btf *btf__parse_split(const char *path, struct btf *base_btf) 1142ba451366SAndrii Nakryiko { 1143e9fc3ce9SAndrii Nakryiko return libbpf_ptr(btf_parse(path, base_btf, NULL)); 114494a1feddSAndrii Nakryiko } 114594a1feddSAndrii Nakryiko 11463289959bSAndrii Nakryiko static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); 11473289959bSAndrii Nakryiko 11481a190d1eSAndrii Nakryiko int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level) 1149d29d87f7SAndrii Nakryiko { 11501a190d1eSAndrii Nakryiko LIBBPF_OPTS(bpf_btf_load_opts, opts); 11511a190d1eSAndrii Nakryiko __u32 buf_sz = 0, raw_size; 11521a190d1eSAndrii Nakryiko char *buf = NULL, *tmp; 11533289959bSAndrii Nakryiko void *raw_data; 1154d29d87f7SAndrii Nakryiko int err = 0; 1155d29d87f7SAndrii Nakryiko 1156d29d87f7SAndrii Nakryiko if (btf->fd >= 0) 1157e9fc3ce9SAndrii Nakryiko return libbpf_err(-EEXIST); 11581a190d1eSAndrii Nakryiko if (log_sz && !log_buf) 11591a190d1eSAndrii Nakryiko return libbpf_err(-EINVAL); 1160d29d87f7SAndrii Nakryiko 11611a190d1eSAndrii Nakryiko /* cache native raw data representation */ 11623289959bSAndrii Nakryiko raw_data = btf_get_raw_data(btf, &raw_size, false); 1163b8604247SAndrii Nakryiko if (!raw_data) { 1164b8604247SAndrii Nakryiko err = -ENOMEM; 1165b8604247SAndrii Nakryiko goto done; 1166b8604247SAndrii Nakryiko } 11673289959bSAndrii Nakryiko btf->raw_size = raw_size; 11683289959bSAndrii Nakryiko btf->raw_data = raw_data; 1169b8604247SAndrii Nakryiko 11701a190d1eSAndrii Nakryiko retry_load: 11711a190d1eSAndrii Nakryiko /* if log_level is 0, we won't provide log_buf/log_size to the kernel, 11721a190d1eSAndrii Nakryiko * initially. Only if BTF loading fails, we bump log_level to 1 and 11731a190d1eSAndrii Nakryiko * retry, using either auto-allocated or custom log_buf. This way 11741a190d1eSAndrii Nakryiko * non-NULL custom log_buf provides a buffer just in case, but hopes 11751a190d1eSAndrii Nakryiko * for successful load and no need for log_buf. 11761a190d1eSAndrii Nakryiko */ 11771a190d1eSAndrii Nakryiko if (log_level) { 11781a190d1eSAndrii Nakryiko /* if caller didn't provide custom log_buf, we'll keep 11791a190d1eSAndrii Nakryiko * allocating our own progressively bigger buffers for BTF 11801a190d1eSAndrii Nakryiko * verification log 11811a190d1eSAndrii Nakryiko */ 11821a190d1eSAndrii Nakryiko if (!log_buf) { 11831a190d1eSAndrii Nakryiko buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2); 11841a190d1eSAndrii Nakryiko tmp = realloc(buf, buf_sz); 11851a190d1eSAndrii Nakryiko if (!tmp) { 11861a190d1eSAndrii Nakryiko err = -ENOMEM; 11871a190d1eSAndrii Nakryiko goto done; 11881a190d1eSAndrii Nakryiko } 11891a190d1eSAndrii Nakryiko buf = tmp; 11901a190d1eSAndrii Nakryiko buf[0] = '\0'; 11918395f320SStanislav Fomichev } 11928395f320SStanislav Fomichev 11931a190d1eSAndrii Nakryiko opts.log_buf = log_buf ? log_buf : buf; 11941a190d1eSAndrii Nakryiko opts.log_size = log_buf ? log_sz : buf_sz; 11951a190d1eSAndrii Nakryiko opts.log_level = log_level; 11961a190d1eSAndrii Nakryiko } 11971a190d1eSAndrii Nakryiko 11981a190d1eSAndrii Nakryiko btf->fd = bpf_btf_load(raw_data, raw_size, &opts); 11991a190d1eSAndrii Nakryiko if (btf->fd < 0) { 12001a190d1eSAndrii Nakryiko /* time to turn on verbose mode and try again */ 12011a190d1eSAndrii Nakryiko if (log_level == 0) { 12021a190d1eSAndrii Nakryiko log_level = 1; 12031a190d1eSAndrii Nakryiko goto retry_load; 12041a190d1eSAndrii Nakryiko } 12051a190d1eSAndrii Nakryiko /* only retry if caller didn't provide custom log_buf, but 12061a190d1eSAndrii Nakryiko * make sure we can never overflow buf_sz 12071a190d1eSAndrii Nakryiko */ 12081a190d1eSAndrii Nakryiko if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2) 12091a190d1eSAndrii Nakryiko goto retry_load; 12101a190d1eSAndrii Nakryiko 1211d29d87f7SAndrii Nakryiko err = -errno; 12121a190d1eSAndrii Nakryiko pr_warn("BTF loading error: %d\n", err); 12131a190d1eSAndrii Nakryiko /* don't print out contents of custom log_buf */ 12141a190d1eSAndrii Nakryiko if (!log_buf && buf[0]) 12151a190d1eSAndrii Nakryiko pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); 1216d29d87f7SAndrii Nakryiko } 1217d29d87f7SAndrii Nakryiko 1218d29d87f7SAndrii Nakryiko done: 12191a190d1eSAndrii Nakryiko free(buf); 1220e9fc3ce9SAndrii Nakryiko return libbpf_err(err); 1221d29d87f7SAndrii Nakryiko } 12221a190d1eSAndrii Nakryiko 12231a190d1eSAndrii Nakryiko int btf__load_into_kernel(struct btf *btf) 12241a190d1eSAndrii Nakryiko { 12251a190d1eSAndrii Nakryiko return btf_load_into_kernel(btf, NULL, 0, 0); 12261a190d1eSAndrii Nakryiko } 12271a190d1eSAndrii Nakryiko 12288a138aedSMartin KaFai Lau int btf__fd(const struct btf *btf) 12298a138aedSMartin KaFai Lau { 12308a138aedSMartin KaFai Lau return btf->fd; 12318a138aedSMartin KaFai Lau } 123292b57121SOkash Khawaja 123381372e12SAndrii Nakryiko void btf__set_fd(struct btf *btf, int fd) 123481372e12SAndrii Nakryiko { 123581372e12SAndrii Nakryiko btf->fd = fd; 123681372e12SAndrii Nakryiko } 123781372e12SAndrii Nakryiko 123890d76d3eSAndrii Nakryiko static const void *btf_strs_data(const struct btf *btf) 123990d76d3eSAndrii Nakryiko { 124090d76d3eSAndrii Nakryiko return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); 124190d76d3eSAndrii Nakryiko } 124290d76d3eSAndrii Nakryiko 12433289959bSAndrii Nakryiko static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) 12443289959bSAndrii Nakryiko { 12453289959bSAndrii Nakryiko struct btf_header *hdr = btf->hdr; 12463289959bSAndrii Nakryiko struct btf_type *t; 12473289959bSAndrii Nakryiko void *data, *p; 12483289959bSAndrii Nakryiko __u32 data_sz; 12493289959bSAndrii Nakryiko int i; 12503289959bSAndrii Nakryiko 12513289959bSAndrii Nakryiko data = swap_endian ? btf->raw_data_swapped : btf->raw_data; 12523289959bSAndrii Nakryiko if (data) { 12533289959bSAndrii Nakryiko *size = btf->raw_size; 12543289959bSAndrii Nakryiko return data; 12553289959bSAndrii Nakryiko } 12563289959bSAndrii Nakryiko 12573289959bSAndrii Nakryiko data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; 12583289959bSAndrii Nakryiko data = calloc(1, data_sz); 12593289959bSAndrii Nakryiko if (!data) 12603289959bSAndrii Nakryiko return NULL; 12613289959bSAndrii Nakryiko p = data; 12623289959bSAndrii Nakryiko 12633289959bSAndrii Nakryiko memcpy(p, hdr, hdr->hdr_len); 12643289959bSAndrii Nakryiko if (swap_endian) 12653289959bSAndrii Nakryiko btf_bswap_hdr(p); 12663289959bSAndrii Nakryiko p += hdr->hdr_len; 12673289959bSAndrii Nakryiko 12683289959bSAndrii Nakryiko memcpy(p, btf->types_data, hdr->type_len); 12693289959bSAndrii Nakryiko if (swap_endian) { 1270ba451366SAndrii Nakryiko for (i = 0; i < btf->nr_types; i++) { 12713289959bSAndrii Nakryiko t = p + btf->type_offs[i]; 12723289959bSAndrii Nakryiko /* btf_bswap_type_rest() relies on native t->info, so 12733289959bSAndrii Nakryiko * we swap base type info after we swapped all the 12743289959bSAndrii Nakryiko * additional information 12753289959bSAndrii Nakryiko */ 12763289959bSAndrii Nakryiko if (btf_bswap_type_rest(t)) 12773289959bSAndrii Nakryiko goto err_out; 12783289959bSAndrii Nakryiko btf_bswap_type_base(t); 12793289959bSAndrii Nakryiko } 12803289959bSAndrii Nakryiko } 12813289959bSAndrii Nakryiko p += hdr->type_len; 12823289959bSAndrii Nakryiko 128390d76d3eSAndrii Nakryiko memcpy(p, btf_strs_data(btf), hdr->str_len); 12843289959bSAndrii Nakryiko p += hdr->str_len; 12853289959bSAndrii Nakryiko 12863289959bSAndrii Nakryiko *size = data_sz; 12873289959bSAndrii Nakryiko return data; 12883289959bSAndrii Nakryiko err_out: 12893289959bSAndrii Nakryiko free(data); 12903289959bSAndrii Nakryiko return NULL; 12913289959bSAndrii Nakryiko } 12923289959bSAndrii Nakryiko 12936a886de0SHengqi Chen const void *btf__raw_data(const struct btf *btf_ro, __u32 *size) 129402c87446SAndrii Nakryiko { 1295919d2b1dSAndrii Nakryiko struct btf *btf = (struct btf *)btf_ro; 12963289959bSAndrii Nakryiko __u32 data_sz; 1297919d2b1dSAndrii Nakryiko void *data; 1298919d2b1dSAndrii Nakryiko 12993289959bSAndrii Nakryiko data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); 13003289959bSAndrii Nakryiko if (!data) 13016a886de0SHengqi Chen return errno = ENOMEM, NULL; 1302919d2b1dSAndrii Nakryiko 13033289959bSAndrii Nakryiko btf->raw_size = data_sz; 13043289959bSAndrii Nakryiko if (btf->swapped_endian) 13053289959bSAndrii Nakryiko btf->raw_data_swapped = data; 13063289959bSAndrii Nakryiko else 13073289959bSAndrii Nakryiko btf->raw_data = data; 13083289959bSAndrii Nakryiko *size = data_sz; 13093289959bSAndrii Nakryiko return data; 131002c87446SAndrii Nakryiko } 131102c87446SAndrii Nakryiko 13126a886de0SHengqi Chen __attribute__((alias("btf__raw_data"))) 13136a886de0SHengqi Chen const void *btf__get_raw_data(const struct btf *btf, __u32 *size); 13146a886de0SHengqi Chen 1315f86ed050SAndrii Nakryiko const char *btf__str_by_offset(const struct btf *btf, __u32 offset) 131692b57121SOkash Khawaja { 1317ba451366SAndrii Nakryiko if (offset < btf->start_str_off) 1318ba451366SAndrii Nakryiko return btf__str_by_offset(btf->base_btf, offset); 1319ba451366SAndrii Nakryiko else if (offset - btf->start_str_off < btf->hdr->str_len) 132090d76d3eSAndrii Nakryiko return btf_strs_data(btf) + (offset - btf->start_str_off); 132192b57121SOkash Khawaja else 1322e9fc3ce9SAndrii Nakryiko return errno = EINVAL, NULL; 132392b57121SOkash Khawaja } 13242993e051SYonghong Song 1325f86ed050SAndrii Nakryiko const char *btf__name_by_offset(const struct btf *btf, __u32 offset) 1326f86ed050SAndrii Nakryiko { 1327f86ed050SAndrii Nakryiko return btf__str_by_offset(btf, offset); 1328f86ed050SAndrii Nakryiko } 1329f86ed050SAndrii Nakryiko 1330a19f93cfSAndrii Nakryiko struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) 1331d7f5b5e0SYonghong Song { 1332a19f93cfSAndrii Nakryiko struct bpf_btf_info btf_info; 1333d7f5b5e0SYonghong Song __u32 len = sizeof(btf_info); 1334d7f5b5e0SYonghong Song __u32 last_size; 1335a19f93cfSAndrii Nakryiko struct btf *btf; 1336d7f5b5e0SYonghong Song void *ptr; 1337d7f5b5e0SYonghong Song int err; 1338d7f5b5e0SYonghong Song 1339d7f5b5e0SYonghong Song /* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so 1340d7f5b5e0SYonghong Song * let's start with a sane default - 4KiB here - and resize it only if 1341d7f5b5e0SYonghong Song * bpf_obj_get_info_by_fd() needs a bigger buffer. 1342d7f5b5e0SYonghong Song */ 1343a19f93cfSAndrii Nakryiko last_size = 4096; 1344d7f5b5e0SYonghong Song ptr = malloc(last_size); 1345a19f93cfSAndrii Nakryiko if (!ptr) 1346a19f93cfSAndrii Nakryiko return ERR_PTR(-ENOMEM); 1347d7f5b5e0SYonghong Song 1348a19f93cfSAndrii Nakryiko memset(&btf_info, 0, sizeof(btf_info)); 1349d7f5b5e0SYonghong Song btf_info.btf = ptr_to_u64(ptr); 1350a19f93cfSAndrii Nakryiko btf_info.btf_size = last_size; 1351d7f5b5e0SYonghong Song err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); 1352d7f5b5e0SYonghong Song 1353d7f5b5e0SYonghong Song if (!err && btf_info.btf_size > last_size) { 1354d7f5b5e0SYonghong Song void *temp_ptr; 1355d7f5b5e0SYonghong Song 1356d7f5b5e0SYonghong Song last_size = btf_info.btf_size; 1357d7f5b5e0SYonghong Song temp_ptr = realloc(ptr, last_size); 1358d7f5b5e0SYonghong Song if (!temp_ptr) { 1359a19f93cfSAndrii Nakryiko btf = ERR_PTR(-ENOMEM); 1360d7f5b5e0SYonghong Song goto exit_free; 1361d7f5b5e0SYonghong Song } 1362d7f5b5e0SYonghong Song ptr = temp_ptr; 1363a19f93cfSAndrii Nakryiko 1364a19f93cfSAndrii Nakryiko len = sizeof(btf_info); 1365a19f93cfSAndrii Nakryiko memset(&btf_info, 0, sizeof(btf_info)); 1366d7f5b5e0SYonghong Song btf_info.btf = ptr_to_u64(ptr); 1367a19f93cfSAndrii Nakryiko btf_info.btf_size = last_size; 1368a19f93cfSAndrii Nakryiko 1369d7f5b5e0SYonghong Song err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); 1370d7f5b5e0SYonghong Song } 1371d7f5b5e0SYonghong Song 1372d7f5b5e0SYonghong Song if (err || btf_info.btf_size > last_size) { 1373a19f93cfSAndrii Nakryiko btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG); 1374d7f5b5e0SYonghong Song goto exit_free; 1375d7f5b5e0SYonghong Song } 1376d7f5b5e0SYonghong Song 1377a19f93cfSAndrii Nakryiko btf = btf_new(ptr, btf_info.btf_size, base_btf); 1378d7f5b5e0SYonghong Song 1379d7f5b5e0SYonghong Song exit_free: 1380d7f5b5e0SYonghong Song free(ptr); 1381a19f93cfSAndrii Nakryiko return btf; 1382a19f93cfSAndrii Nakryiko } 1383d7f5b5e0SYonghong Song 138461fc51b1SQuentin Monnet struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf) 13856cc93e2fSQuentin Monnet { 13866cc93e2fSQuentin Monnet struct btf *btf; 13876cc93e2fSQuentin Monnet int btf_fd; 13886cc93e2fSQuentin Monnet 13896cc93e2fSQuentin Monnet btf_fd = bpf_btf_get_fd_by_id(id); 13906cc93e2fSQuentin Monnet if (btf_fd < 0) 13916cc93e2fSQuentin Monnet return libbpf_err_ptr(-errno); 13926cc93e2fSQuentin Monnet 139361fc51b1SQuentin Monnet btf = btf_get_from_fd(btf_fd, base_btf); 13946cc93e2fSQuentin Monnet close(btf_fd); 13956cc93e2fSQuentin Monnet 13966cc93e2fSQuentin Monnet return libbpf_ptr(btf); 13976cc93e2fSQuentin Monnet } 13986cc93e2fSQuentin Monnet 139961fc51b1SQuentin Monnet struct btf *btf__load_from_kernel_by_id(__u32 id) 140061fc51b1SQuentin Monnet { 140161fc51b1SQuentin Monnet return btf__load_from_kernel_by_id_split(id, NULL); 140261fc51b1SQuentin Monnet } 140361fc51b1SQuentin Monnet 14043289959bSAndrii Nakryiko static void btf_invalidate_raw_data(struct btf *btf) 14053289959bSAndrii Nakryiko { 14063289959bSAndrii Nakryiko if (btf->raw_data) { 14073289959bSAndrii Nakryiko free(btf->raw_data); 14083289959bSAndrii Nakryiko btf->raw_data = NULL; 14093289959bSAndrii Nakryiko } 14103289959bSAndrii Nakryiko if (btf->raw_data_swapped) { 14113289959bSAndrii Nakryiko free(btf->raw_data_swapped); 14123289959bSAndrii Nakryiko btf->raw_data_swapped = NULL; 14133289959bSAndrii Nakryiko } 14143289959bSAndrii Nakryiko } 14153289959bSAndrii Nakryiko 1416919d2b1dSAndrii Nakryiko /* Ensure BTF is ready to be modified (by splitting into a three memory 1417919d2b1dSAndrii Nakryiko * regions for header, types, and strings). Also invalidate cached 1418919d2b1dSAndrii Nakryiko * raw_data, if any. 1419919d2b1dSAndrii Nakryiko */ 1420919d2b1dSAndrii Nakryiko static int btf_ensure_modifiable(struct btf *btf) 1421919d2b1dSAndrii Nakryiko { 142290d76d3eSAndrii Nakryiko void *hdr, *types; 142390d76d3eSAndrii Nakryiko struct strset *set = NULL; 142490d76d3eSAndrii Nakryiko int err = -ENOMEM; 1425919d2b1dSAndrii Nakryiko 1426919d2b1dSAndrii Nakryiko if (btf_is_modifiable(btf)) { 1427919d2b1dSAndrii Nakryiko /* any BTF modification invalidates raw_data */ 14283289959bSAndrii Nakryiko btf_invalidate_raw_data(btf); 1429919d2b1dSAndrii Nakryiko return 0; 1430919d2b1dSAndrii Nakryiko } 1431919d2b1dSAndrii Nakryiko 1432919d2b1dSAndrii Nakryiko /* split raw data into three memory regions */ 1433919d2b1dSAndrii Nakryiko hdr = malloc(btf->hdr->hdr_len); 1434919d2b1dSAndrii Nakryiko types = malloc(btf->hdr->type_len); 143590d76d3eSAndrii Nakryiko if (!hdr || !types) 1436919d2b1dSAndrii Nakryiko goto err_out; 1437919d2b1dSAndrii Nakryiko 1438919d2b1dSAndrii Nakryiko memcpy(hdr, btf->hdr, btf->hdr->hdr_len); 1439919d2b1dSAndrii Nakryiko memcpy(types, btf->types_data, btf->hdr->type_len); 144088a82c2aSAndrii Nakryiko 1441919d2b1dSAndrii Nakryiko /* build lookup index for all strings */ 144290d76d3eSAndrii Nakryiko set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); 144390d76d3eSAndrii Nakryiko if (IS_ERR(set)) { 144490d76d3eSAndrii Nakryiko err = PTR_ERR(set); 1445919d2b1dSAndrii Nakryiko goto err_out; 1446919d2b1dSAndrii Nakryiko } 1447919d2b1dSAndrii Nakryiko 1448919d2b1dSAndrii Nakryiko /* only when everything was successful, update internal state */ 1449919d2b1dSAndrii Nakryiko btf->hdr = hdr; 1450919d2b1dSAndrii Nakryiko btf->types_data = types; 1451919d2b1dSAndrii Nakryiko btf->types_data_cap = btf->hdr->type_len; 145290d76d3eSAndrii Nakryiko btf->strs_data = NULL; 145390d76d3eSAndrii Nakryiko btf->strs_set = set; 1454919d2b1dSAndrii Nakryiko /* if BTF was created from scratch, all strings are guaranteed to be 1455919d2b1dSAndrii Nakryiko * unique and deduplicated 1456919d2b1dSAndrii Nakryiko */ 1457ba451366SAndrii Nakryiko if (btf->hdr->str_len == 0) 1458ba451366SAndrii Nakryiko btf->strs_deduped = true; 1459ba451366SAndrii Nakryiko if (!btf->base_btf && btf->hdr->str_len == 1) 1460ba451366SAndrii Nakryiko btf->strs_deduped = true; 1461919d2b1dSAndrii Nakryiko 1462919d2b1dSAndrii Nakryiko /* invalidate raw_data representation */ 14633289959bSAndrii Nakryiko btf_invalidate_raw_data(btf); 1464919d2b1dSAndrii Nakryiko 1465919d2b1dSAndrii Nakryiko return 0; 1466919d2b1dSAndrii Nakryiko 1467919d2b1dSAndrii Nakryiko err_out: 146890d76d3eSAndrii Nakryiko strset__free(set); 1469919d2b1dSAndrii Nakryiko free(hdr); 1470919d2b1dSAndrii Nakryiko free(types); 147190d76d3eSAndrii Nakryiko return err; 1472919d2b1dSAndrii Nakryiko } 1473919d2b1dSAndrii Nakryiko 1474919d2b1dSAndrii Nakryiko /* Find an offset in BTF string section that corresponds to a given string *s*. 1475919d2b1dSAndrii Nakryiko * Returns: 1476919d2b1dSAndrii Nakryiko * - >0 offset into string section, if string is found; 1477919d2b1dSAndrii Nakryiko * - -ENOENT, if string is not in the string section; 1478919d2b1dSAndrii Nakryiko * - <0, on any other error. 1479919d2b1dSAndrii Nakryiko */ 1480919d2b1dSAndrii Nakryiko int btf__find_str(struct btf *btf, const char *s) 1481919d2b1dSAndrii Nakryiko { 148290d76d3eSAndrii Nakryiko int off; 1483919d2b1dSAndrii Nakryiko 1484ba451366SAndrii Nakryiko if (btf->base_btf) { 148590d76d3eSAndrii Nakryiko off = btf__find_str(btf->base_btf, s); 148690d76d3eSAndrii Nakryiko if (off != -ENOENT) 148790d76d3eSAndrii Nakryiko return off; 1488ba451366SAndrii Nakryiko } 1489ba451366SAndrii Nakryiko 1490919d2b1dSAndrii Nakryiko /* BTF needs to be in a modifiable state to build string lookup index */ 1491919d2b1dSAndrii Nakryiko if (btf_ensure_modifiable(btf)) 1492e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 1493919d2b1dSAndrii Nakryiko 149490d76d3eSAndrii Nakryiko off = strset__find_str(btf->strs_set, s); 149590d76d3eSAndrii Nakryiko if (off < 0) 1496e9fc3ce9SAndrii Nakryiko return libbpf_err(off); 1497919d2b1dSAndrii Nakryiko 149890d76d3eSAndrii Nakryiko return btf->start_str_off + off; 1499919d2b1dSAndrii Nakryiko } 1500919d2b1dSAndrii Nakryiko 1501919d2b1dSAndrii Nakryiko /* Add a string s to the BTF string section. 1502919d2b1dSAndrii Nakryiko * Returns: 1503919d2b1dSAndrii Nakryiko * - > 0 offset into string section, on success; 1504919d2b1dSAndrii Nakryiko * - < 0, on error. 1505919d2b1dSAndrii Nakryiko */ 1506919d2b1dSAndrii Nakryiko int btf__add_str(struct btf *btf, const char *s) 1507919d2b1dSAndrii Nakryiko { 150890d76d3eSAndrii Nakryiko int off; 1509919d2b1dSAndrii Nakryiko 1510ba451366SAndrii Nakryiko if (btf->base_btf) { 151190d76d3eSAndrii Nakryiko off = btf__find_str(btf->base_btf, s); 151290d76d3eSAndrii Nakryiko if (off != -ENOENT) 151390d76d3eSAndrii Nakryiko return off; 1514ba451366SAndrii Nakryiko } 1515ba451366SAndrii Nakryiko 1516919d2b1dSAndrii Nakryiko if (btf_ensure_modifiable(btf)) 1517e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 1518919d2b1dSAndrii Nakryiko 151990d76d3eSAndrii Nakryiko off = strset__add_str(btf->strs_set, s); 152090d76d3eSAndrii Nakryiko if (off < 0) 1521e9fc3ce9SAndrii Nakryiko return libbpf_err(off); 1522919d2b1dSAndrii Nakryiko 152390d76d3eSAndrii Nakryiko btf->hdr->str_len = strset__data_size(btf->strs_set); 1524919d2b1dSAndrii Nakryiko 152590d76d3eSAndrii Nakryiko return btf->start_str_off + off; 1526919d2b1dSAndrii Nakryiko } 1527919d2b1dSAndrii Nakryiko 15284a3b33f8SAndrii Nakryiko static void *btf_add_type_mem(struct btf *btf, size_t add_sz) 15294a3b33f8SAndrii Nakryiko { 15303b029e06SAndrii Nakryiko return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, 15314a3b33f8SAndrii Nakryiko btf->hdr->type_len, UINT_MAX, add_sz); 15324a3b33f8SAndrii Nakryiko } 15334a3b33f8SAndrii Nakryiko 15344a3b33f8SAndrii Nakryiko static void btf_type_inc_vlen(struct btf_type *t) 15354a3b33f8SAndrii Nakryiko { 15364a3b33f8SAndrii Nakryiko t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); 15374a3b33f8SAndrii Nakryiko } 15384a3b33f8SAndrii Nakryiko 1539c81ed6d8SAndrii Nakryiko static int btf_commit_type(struct btf *btf, int data_sz) 1540c81ed6d8SAndrii Nakryiko { 1541c81ed6d8SAndrii Nakryiko int err; 1542c81ed6d8SAndrii Nakryiko 1543c81ed6d8SAndrii Nakryiko err = btf_add_type_idx_entry(btf, btf->hdr->type_len); 1544c81ed6d8SAndrii Nakryiko if (err) 1545e9fc3ce9SAndrii Nakryiko return libbpf_err(err); 1546c81ed6d8SAndrii Nakryiko 1547c81ed6d8SAndrii Nakryiko btf->hdr->type_len += data_sz; 1548c81ed6d8SAndrii Nakryiko btf->hdr->str_off += data_sz; 1549c81ed6d8SAndrii Nakryiko btf->nr_types++; 1550ba451366SAndrii Nakryiko return btf->start_id + btf->nr_types - 1; 1551c81ed6d8SAndrii Nakryiko } 1552c81ed6d8SAndrii Nakryiko 15539af44bc5SAndrii Nakryiko struct btf_pipe { 15549af44bc5SAndrii Nakryiko const struct btf *src; 15559af44bc5SAndrii Nakryiko struct btf *dst; 1556d81283d2SKui-Feng Lee struct hashmap *str_off_map; /* map string offsets from src to dst */ 15579af44bc5SAndrii Nakryiko }; 15589af44bc5SAndrii Nakryiko 15599af44bc5SAndrii Nakryiko static int btf_rewrite_str(__u32 *str_off, void *ctx) 15609af44bc5SAndrii Nakryiko { 15619af44bc5SAndrii Nakryiko struct btf_pipe *p = ctx; 1562c302378bSEduard Zingerman long mapped_off; 1563d81283d2SKui-Feng Lee int off, err; 15649af44bc5SAndrii Nakryiko 15659af44bc5SAndrii Nakryiko if (!*str_off) /* nothing to do for empty strings */ 15669af44bc5SAndrii Nakryiko return 0; 15679af44bc5SAndrii Nakryiko 1568d81283d2SKui-Feng Lee if (p->str_off_map && 1569c302378bSEduard Zingerman hashmap__find(p->str_off_map, *str_off, &mapped_off)) { 1570c302378bSEduard Zingerman *str_off = mapped_off; 1571d81283d2SKui-Feng Lee return 0; 1572d81283d2SKui-Feng Lee } 1573d81283d2SKui-Feng Lee 15749af44bc5SAndrii Nakryiko off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); 15759af44bc5SAndrii Nakryiko if (off < 0) 15769af44bc5SAndrii Nakryiko return off; 15779af44bc5SAndrii Nakryiko 1578d81283d2SKui-Feng Lee /* Remember string mapping from src to dst. It avoids 1579d81283d2SKui-Feng Lee * performing expensive string comparisons. 1580d81283d2SKui-Feng Lee */ 1581d81283d2SKui-Feng Lee if (p->str_off_map) { 1582c302378bSEduard Zingerman err = hashmap__append(p->str_off_map, *str_off, off); 1583d81283d2SKui-Feng Lee if (err) 1584d81283d2SKui-Feng Lee return err; 1585d81283d2SKui-Feng Lee } 1586d81283d2SKui-Feng Lee 15879af44bc5SAndrii Nakryiko *str_off = off; 15889af44bc5SAndrii Nakryiko return 0; 15899af44bc5SAndrii Nakryiko } 15909af44bc5SAndrii Nakryiko 15919af44bc5SAndrii Nakryiko int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type) 15929af44bc5SAndrii Nakryiko { 15939af44bc5SAndrii Nakryiko struct btf_pipe p = { .src = src_btf, .dst = btf }; 15949af44bc5SAndrii Nakryiko struct btf_type *t; 15959af44bc5SAndrii Nakryiko int sz, err; 15969af44bc5SAndrii Nakryiko 15979af44bc5SAndrii Nakryiko sz = btf_type_size(src_type); 15989af44bc5SAndrii Nakryiko if (sz < 0) 1599e9fc3ce9SAndrii Nakryiko return libbpf_err(sz); 16009af44bc5SAndrii Nakryiko 16019af44bc5SAndrii Nakryiko /* deconstruct BTF, if necessary, and invalidate raw_data */ 16029af44bc5SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 1603e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 16049af44bc5SAndrii Nakryiko 16059af44bc5SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 16069af44bc5SAndrii Nakryiko if (!t) 1607e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 16089af44bc5SAndrii Nakryiko 16099af44bc5SAndrii Nakryiko memcpy(t, src_type, sz); 16109af44bc5SAndrii Nakryiko 16119af44bc5SAndrii Nakryiko err = btf_type_visit_str_offs(t, btf_rewrite_str, &p); 16129af44bc5SAndrii Nakryiko if (err) 1613e9fc3ce9SAndrii Nakryiko return libbpf_err(err); 16149af44bc5SAndrii Nakryiko 16159af44bc5SAndrii Nakryiko return btf_commit_type(btf, sz); 16169af44bc5SAndrii Nakryiko } 16179af44bc5SAndrii Nakryiko 16187ca61121SAndrii Nakryiko static int btf_rewrite_type_ids(__u32 *type_id, void *ctx) 16197ca61121SAndrii Nakryiko { 16207ca61121SAndrii Nakryiko struct btf *btf = ctx; 16217ca61121SAndrii Nakryiko 16227ca61121SAndrii Nakryiko if (!*type_id) /* nothing to do for VOID references */ 16237ca61121SAndrii Nakryiko return 0; 16247ca61121SAndrii Nakryiko 16257ca61121SAndrii Nakryiko /* we haven't updated btf's type count yet, so 16267ca61121SAndrii Nakryiko * btf->start_id + btf->nr_types - 1 is the type ID offset we should 16277ca61121SAndrii Nakryiko * add to all newly added BTF types 16287ca61121SAndrii Nakryiko */ 16297ca61121SAndrii Nakryiko *type_id += btf->start_id + btf->nr_types - 1; 16307ca61121SAndrii Nakryiko return 0; 16317ca61121SAndrii Nakryiko } 16327ca61121SAndrii Nakryiko 1633c302378bSEduard Zingerman static size_t btf_dedup_identity_hash_fn(long key, void *ctx); 1634c302378bSEduard Zingerman static bool btf_dedup_equal_fn(long k1, long k2, void *ctx); 1635d81283d2SKui-Feng Lee 16367ca61121SAndrii Nakryiko int btf__add_btf(struct btf *btf, const struct btf *src_btf) 16377ca61121SAndrii Nakryiko { 16387ca61121SAndrii Nakryiko struct btf_pipe p = { .src = src_btf, .dst = btf }; 16397ca61121SAndrii Nakryiko int data_sz, sz, cnt, i, err, old_strs_len; 16407ca61121SAndrii Nakryiko __u32 *off; 16417ca61121SAndrii Nakryiko void *t; 16427ca61121SAndrii Nakryiko 16437ca61121SAndrii Nakryiko /* appending split BTF isn't supported yet */ 16447ca61121SAndrii Nakryiko if (src_btf->base_btf) 16457ca61121SAndrii Nakryiko return libbpf_err(-ENOTSUP); 16467ca61121SAndrii Nakryiko 16477ca61121SAndrii Nakryiko /* deconstruct BTF, if necessary, and invalidate raw_data */ 16487ca61121SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 16497ca61121SAndrii Nakryiko return libbpf_err(-ENOMEM); 16507ca61121SAndrii Nakryiko 16517ca61121SAndrii Nakryiko /* remember original strings section size if we have to roll back 16527ca61121SAndrii Nakryiko * partial strings section changes 16537ca61121SAndrii Nakryiko */ 16547ca61121SAndrii Nakryiko old_strs_len = btf->hdr->str_len; 16557ca61121SAndrii Nakryiko 16567ca61121SAndrii Nakryiko data_sz = src_btf->hdr->type_len; 16576a886de0SHengqi Chen cnt = btf__type_cnt(src_btf) - 1; 16587ca61121SAndrii Nakryiko 16597ca61121SAndrii Nakryiko /* pre-allocate enough memory for new types */ 16607ca61121SAndrii Nakryiko t = btf_add_type_mem(btf, data_sz); 16617ca61121SAndrii Nakryiko if (!t) 16627ca61121SAndrii Nakryiko return libbpf_err(-ENOMEM); 16637ca61121SAndrii Nakryiko 16647ca61121SAndrii Nakryiko /* pre-allocate enough memory for type offset index for new types */ 16657ca61121SAndrii Nakryiko off = btf_add_type_offs_mem(btf, cnt); 16667ca61121SAndrii Nakryiko if (!off) 16677ca61121SAndrii Nakryiko return libbpf_err(-ENOMEM); 16687ca61121SAndrii Nakryiko 1669d81283d2SKui-Feng Lee /* Map the string offsets from src_btf to the offsets from btf to improve performance */ 1670d81283d2SKui-Feng Lee p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 1671d81283d2SKui-Feng Lee if (IS_ERR(p.str_off_map)) 1672d81283d2SKui-Feng Lee return libbpf_err(-ENOMEM); 1673d81283d2SKui-Feng Lee 16747ca61121SAndrii Nakryiko /* bulk copy types data for all types from src_btf */ 16757ca61121SAndrii Nakryiko memcpy(t, src_btf->types_data, data_sz); 16767ca61121SAndrii Nakryiko 16777ca61121SAndrii Nakryiko for (i = 0; i < cnt; i++) { 16787ca61121SAndrii Nakryiko sz = btf_type_size(t); 16797ca61121SAndrii Nakryiko if (sz < 0) { 16807ca61121SAndrii Nakryiko /* unlikely, has to be corrupted src_btf */ 16817ca61121SAndrii Nakryiko err = sz; 16827ca61121SAndrii Nakryiko goto err_out; 16837ca61121SAndrii Nakryiko } 16847ca61121SAndrii Nakryiko 16857ca61121SAndrii Nakryiko /* fill out type ID to type offset mapping for lookups by type ID */ 16867ca61121SAndrii Nakryiko *off = t - btf->types_data; 16877ca61121SAndrii Nakryiko 16887ca61121SAndrii Nakryiko /* add, dedup, and remap strings referenced by this BTF type */ 16897ca61121SAndrii Nakryiko err = btf_type_visit_str_offs(t, btf_rewrite_str, &p); 16907ca61121SAndrii Nakryiko if (err) 16917ca61121SAndrii Nakryiko goto err_out; 16927ca61121SAndrii Nakryiko 16937ca61121SAndrii Nakryiko /* remap all type IDs referenced from this BTF type */ 16947ca61121SAndrii Nakryiko err = btf_type_visit_type_ids(t, btf_rewrite_type_ids, btf); 16957ca61121SAndrii Nakryiko if (err) 16967ca61121SAndrii Nakryiko goto err_out; 16977ca61121SAndrii Nakryiko 16987ca61121SAndrii Nakryiko /* go to next type data and type offset index entry */ 16997ca61121SAndrii Nakryiko t += sz; 17007ca61121SAndrii Nakryiko off++; 17017ca61121SAndrii Nakryiko } 17027ca61121SAndrii Nakryiko 17037ca61121SAndrii Nakryiko /* Up until now any of the copied type data was effectively invisible, 17047ca61121SAndrii Nakryiko * so if we exited early before this point due to error, BTF would be 17057ca61121SAndrii Nakryiko * effectively unmodified. There would be extra internal memory 17067ca61121SAndrii Nakryiko * pre-allocated, but it would not be available for querying. But now 17077ca61121SAndrii Nakryiko * that we've copied and rewritten all the data successfully, we can 17087ca61121SAndrii Nakryiko * update type count and various internal offsets and sizes to 17097ca61121SAndrii Nakryiko * "commit" the changes and made them visible to the outside world. 17107ca61121SAndrii Nakryiko */ 17117ca61121SAndrii Nakryiko btf->hdr->type_len += data_sz; 17127ca61121SAndrii Nakryiko btf->hdr->str_off += data_sz; 17137ca61121SAndrii Nakryiko btf->nr_types += cnt; 17147ca61121SAndrii Nakryiko 1715d81283d2SKui-Feng Lee hashmap__free(p.str_off_map); 1716d81283d2SKui-Feng Lee 17177ca61121SAndrii Nakryiko /* return type ID of the first added BTF type */ 17187ca61121SAndrii Nakryiko return btf->start_id + btf->nr_types - cnt; 17197ca61121SAndrii Nakryiko err_out: 17207ca61121SAndrii Nakryiko /* zero out preallocated memory as if it was just allocated with 17217ca61121SAndrii Nakryiko * libbpf_add_mem() 17227ca61121SAndrii Nakryiko */ 17237ca61121SAndrii Nakryiko memset(btf->types_data + btf->hdr->type_len, 0, data_sz); 17247ca61121SAndrii Nakryiko memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len); 17257ca61121SAndrii Nakryiko 17267ca61121SAndrii Nakryiko /* and now restore original strings section size; types data size 17277ca61121SAndrii Nakryiko * wasn't modified, so doesn't need restoring, see big comment above */ 17287ca61121SAndrii Nakryiko btf->hdr->str_len = old_strs_len; 17297ca61121SAndrii Nakryiko 1730d81283d2SKui-Feng Lee hashmap__free(p.str_off_map); 1731d81283d2SKui-Feng Lee 17327ca61121SAndrii Nakryiko return libbpf_err(err); 17337ca61121SAndrii Nakryiko } 17347ca61121SAndrii Nakryiko 17354a3b33f8SAndrii Nakryiko /* 17364a3b33f8SAndrii Nakryiko * Append new BTF_KIND_INT type with: 17374a3b33f8SAndrii Nakryiko * - *name* - non-empty, non-NULL type name; 17384a3b33f8SAndrii Nakryiko * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes; 17394a3b33f8SAndrii Nakryiko * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL. 17404a3b33f8SAndrii Nakryiko * Returns: 17414a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 17424a3b33f8SAndrii Nakryiko * - <0, on error. 17434a3b33f8SAndrii Nakryiko */ 17444a3b33f8SAndrii Nakryiko int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding) 17454a3b33f8SAndrii Nakryiko { 17464a3b33f8SAndrii Nakryiko struct btf_type *t; 1747c81ed6d8SAndrii Nakryiko int sz, name_off; 17484a3b33f8SAndrii Nakryiko 17494a3b33f8SAndrii Nakryiko /* non-empty name */ 17504a3b33f8SAndrii Nakryiko if (!name || !name[0]) 1751e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 17524a3b33f8SAndrii Nakryiko /* byte_sz must be power of 2 */ 17534a3b33f8SAndrii Nakryiko if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) 1754e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 17554a3b33f8SAndrii Nakryiko if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL)) 1756e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 17574a3b33f8SAndrii Nakryiko 17584a3b33f8SAndrii Nakryiko /* deconstruct BTF, if necessary, and invalidate raw_data */ 17594a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 1760e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 17614a3b33f8SAndrii Nakryiko 17624a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type) + sizeof(int); 17634a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 17644a3b33f8SAndrii Nakryiko if (!t) 1765e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 17664a3b33f8SAndrii Nakryiko 17674a3b33f8SAndrii Nakryiko /* if something goes wrong later, we might end up with an extra string, 17684a3b33f8SAndrii Nakryiko * but that shouldn't be a problem, because BTF can't be constructed 17694a3b33f8SAndrii Nakryiko * completely anyway and will most probably be just discarded 17704a3b33f8SAndrii Nakryiko */ 17714a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 17724a3b33f8SAndrii Nakryiko if (name_off < 0) 17734a3b33f8SAndrii Nakryiko return name_off; 17744a3b33f8SAndrii Nakryiko 17754a3b33f8SAndrii Nakryiko t->name_off = name_off; 17764a3b33f8SAndrii Nakryiko t->info = btf_type_info(BTF_KIND_INT, 0, 0); 17774a3b33f8SAndrii Nakryiko t->size = byte_sz; 17784a3b33f8SAndrii Nakryiko /* set INT info, we don't allow setting legacy bit offset/size */ 17794a3b33f8SAndrii Nakryiko *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8); 17804a3b33f8SAndrii Nakryiko 1781c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 17824a3b33f8SAndrii Nakryiko } 17834a3b33f8SAndrii Nakryiko 178422541a9eSIlya Leoshkevich /* 178522541a9eSIlya Leoshkevich * Append new BTF_KIND_FLOAT type with: 178622541a9eSIlya Leoshkevich * - *name* - non-empty, non-NULL type name; 178722541a9eSIlya Leoshkevich * - *sz* - size of the type, in bytes; 178822541a9eSIlya Leoshkevich * Returns: 178922541a9eSIlya Leoshkevich * - >0, type ID of newly added BTF type; 179022541a9eSIlya Leoshkevich * - <0, on error. 179122541a9eSIlya Leoshkevich */ 179222541a9eSIlya Leoshkevich int btf__add_float(struct btf *btf, const char *name, size_t byte_sz) 179322541a9eSIlya Leoshkevich { 179422541a9eSIlya Leoshkevich struct btf_type *t; 179522541a9eSIlya Leoshkevich int sz, name_off; 179622541a9eSIlya Leoshkevich 179722541a9eSIlya Leoshkevich /* non-empty name */ 179822541a9eSIlya Leoshkevich if (!name || !name[0]) 1799e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 180022541a9eSIlya Leoshkevich 180122541a9eSIlya Leoshkevich /* byte_sz must be one of the explicitly allowed values */ 180222541a9eSIlya Leoshkevich if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 && 180322541a9eSIlya Leoshkevich byte_sz != 16) 1804e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 180522541a9eSIlya Leoshkevich 180622541a9eSIlya Leoshkevich if (btf_ensure_modifiable(btf)) 1807e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 180822541a9eSIlya Leoshkevich 180922541a9eSIlya Leoshkevich sz = sizeof(struct btf_type); 181022541a9eSIlya Leoshkevich t = btf_add_type_mem(btf, sz); 181122541a9eSIlya Leoshkevich if (!t) 1812e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 181322541a9eSIlya Leoshkevich 181422541a9eSIlya Leoshkevich name_off = btf__add_str(btf, name); 181522541a9eSIlya Leoshkevich if (name_off < 0) 181622541a9eSIlya Leoshkevich return name_off; 181722541a9eSIlya Leoshkevich 181822541a9eSIlya Leoshkevich t->name_off = name_off; 181922541a9eSIlya Leoshkevich t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0); 182022541a9eSIlya Leoshkevich t->size = byte_sz; 182122541a9eSIlya Leoshkevich 182222541a9eSIlya Leoshkevich return btf_commit_type(btf, sz); 182322541a9eSIlya Leoshkevich } 182422541a9eSIlya Leoshkevich 18254a3b33f8SAndrii Nakryiko /* it's completely legal to append BTF types with type IDs pointing forward to 18264a3b33f8SAndrii Nakryiko * types that haven't been appended yet, so we only make sure that id looks 18274a3b33f8SAndrii Nakryiko * sane, we can't guarantee that ID will always be valid 18284a3b33f8SAndrii Nakryiko */ 18294a3b33f8SAndrii Nakryiko static int validate_type_id(int id) 18304a3b33f8SAndrii Nakryiko { 18314a3b33f8SAndrii Nakryiko if (id < 0 || id > BTF_MAX_NR_TYPES) 18324a3b33f8SAndrii Nakryiko return -EINVAL; 18334a3b33f8SAndrii Nakryiko return 0; 18344a3b33f8SAndrii Nakryiko } 18354a3b33f8SAndrii Nakryiko 18364a3b33f8SAndrii Nakryiko /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */ 18374a3b33f8SAndrii Nakryiko static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id) 18384a3b33f8SAndrii Nakryiko { 18394a3b33f8SAndrii Nakryiko struct btf_type *t; 1840c81ed6d8SAndrii Nakryiko int sz, name_off = 0; 18414a3b33f8SAndrii Nakryiko 18424a3b33f8SAndrii Nakryiko if (validate_type_id(ref_type_id)) 1843e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 18444a3b33f8SAndrii Nakryiko 18454a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 1846e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 18474a3b33f8SAndrii Nakryiko 18484a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type); 18494a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 18504a3b33f8SAndrii Nakryiko if (!t) 1851e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 18524a3b33f8SAndrii Nakryiko 18534a3b33f8SAndrii Nakryiko if (name && name[0]) { 18544a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 18554a3b33f8SAndrii Nakryiko if (name_off < 0) 18564a3b33f8SAndrii Nakryiko return name_off; 18574a3b33f8SAndrii Nakryiko } 18584a3b33f8SAndrii Nakryiko 18594a3b33f8SAndrii Nakryiko t->name_off = name_off; 18604a3b33f8SAndrii Nakryiko t->info = btf_type_info(kind, 0, 0); 18614a3b33f8SAndrii Nakryiko t->type = ref_type_id; 18624a3b33f8SAndrii Nakryiko 1863c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 18644a3b33f8SAndrii Nakryiko } 18654a3b33f8SAndrii Nakryiko 18664a3b33f8SAndrii Nakryiko /* 18674a3b33f8SAndrii Nakryiko * Append new BTF_KIND_PTR type with: 18684a3b33f8SAndrii Nakryiko * - *ref_type_id* - referenced type ID, it might not exist yet; 18694a3b33f8SAndrii Nakryiko * Returns: 18704a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 18714a3b33f8SAndrii Nakryiko * - <0, on error. 18724a3b33f8SAndrii Nakryiko */ 18734a3b33f8SAndrii Nakryiko int btf__add_ptr(struct btf *btf, int ref_type_id) 18744a3b33f8SAndrii Nakryiko { 18754a3b33f8SAndrii Nakryiko return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id); 18764a3b33f8SAndrii Nakryiko } 18774a3b33f8SAndrii Nakryiko 18784a3b33f8SAndrii Nakryiko /* 18794a3b33f8SAndrii Nakryiko * Append new BTF_KIND_ARRAY type with: 18804a3b33f8SAndrii Nakryiko * - *index_type_id* - type ID of the type describing array index; 18814a3b33f8SAndrii Nakryiko * - *elem_type_id* - type ID of the type describing array element; 18824a3b33f8SAndrii Nakryiko * - *nr_elems* - the size of the array; 18834a3b33f8SAndrii Nakryiko * Returns: 18844a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 18854a3b33f8SAndrii Nakryiko * - <0, on error. 18864a3b33f8SAndrii Nakryiko */ 18874a3b33f8SAndrii Nakryiko int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems) 18884a3b33f8SAndrii Nakryiko { 18894a3b33f8SAndrii Nakryiko struct btf_type *t; 18904a3b33f8SAndrii Nakryiko struct btf_array *a; 1891c81ed6d8SAndrii Nakryiko int sz; 18924a3b33f8SAndrii Nakryiko 18934a3b33f8SAndrii Nakryiko if (validate_type_id(index_type_id) || validate_type_id(elem_type_id)) 1894e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 18954a3b33f8SAndrii Nakryiko 18964a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 1897e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 18984a3b33f8SAndrii Nakryiko 18994a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type) + sizeof(struct btf_array); 19004a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 19014a3b33f8SAndrii Nakryiko if (!t) 1902e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 19034a3b33f8SAndrii Nakryiko 19044a3b33f8SAndrii Nakryiko t->name_off = 0; 19054a3b33f8SAndrii Nakryiko t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); 19064a3b33f8SAndrii Nakryiko t->size = 0; 19074a3b33f8SAndrii Nakryiko 19084a3b33f8SAndrii Nakryiko a = btf_array(t); 19094a3b33f8SAndrii Nakryiko a->type = elem_type_id; 19104a3b33f8SAndrii Nakryiko a->index_type = index_type_id; 19114a3b33f8SAndrii Nakryiko a->nelems = nr_elems; 19124a3b33f8SAndrii Nakryiko 1913c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 19144a3b33f8SAndrii Nakryiko } 19154a3b33f8SAndrii Nakryiko 19164a3b33f8SAndrii Nakryiko /* generic STRUCT/UNION append function */ 19174a3b33f8SAndrii Nakryiko static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz) 19184a3b33f8SAndrii Nakryiko { 19194a3b33f8SAndrii Nakryiko struct btf_type *t; 1920c81ed6d8SAndrii Nakryiko int sz, name_off = 0; 19214a3b33f8SAndrii Nakryiko 19224a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 1923e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 19244a3b33f8SAndrii Nakryiko 19254a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type); 19264a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 19274a3b33f8SAndrii Nakryiko if (!t) 1928e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 19294a3b33f8SAndrii Nakryiko 19304a3b33f8SAndrii Nakryiko if (name && name[0]) { 19314a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 19324a3b33f8SAndrii Nakryiko if (name_off < 0) 19334a3b33f8SAndrii Nakryiko return name_off; 19344a3b33f8SAndrii Nakryiko } 19354a3b33f8SAndrii Nakryiko 19364a3b33f8SAndrii Nakryiko /* start out with vlen=0 and no kflag; this will be adjusted when 19374a3b33f8SAndrii Nakryiko * adding each member 19384a3b33f8SAndrii Nakryiko */ 19394a3b33f8SAndrii Nakryiko t->name_off = name_off; 19404a3b33f8SAndrii Nakryiko t->info = btf_type_info(kind, 0, 0); 19414a3b33f8SAndrii Nakryiko t->size = bytes_sz; 19424a3b33f8SAndrii Nakryiko 1943c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 19444a3b33f8SAndrii Nakryiko } 19454a3b33f8SAndrii Nakryiko 19464a3b33f8SAndrii Nakryiko /* 19474a3b33f8SAndrii Nakryiko * Append new BTF_KIND_STRUCT type with: 19484a3b33f8SAndrii Nakryiko * - *name* - name of the struct, can be NULL or empty for anonymous structs; 19494a3b33f8SAndrii Nakryiko * - *byte_sz* - size of the struct, in bytes; 19504a3b33f8SAndrii Nakryiko * 19514a3b33f8SAndrii Nakryiko * Struct initially has no fields in it. Fields can be added by 19524a3b33f8SAndrii Nakryiko * btf__add_field() right after btf__add_struct() succeeds. 19534a3b33f8SAndrii Nakryiko * 19544a3b33f8SAndrii Nakryiko * Returns: 19554a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 19564a3b33f8SAndrii Nakryiko * - <0, on error. 19574a3b33f8SAndrii Nakryiko */ 19584a3b33f8SAndrii Nakryiko int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz) 19594a3b33f8SAndrii Nakryiko { 19604a3b33f8SAndrii Nakryiko return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz); 19614a3b33f8SAndrii Nakryiko } 19624a3b33f8SAndrii Nakryiko 19634a3b33f8SAndrii Nakryiko /* 19644a3b33f8SAndrii Nakryiko * Append new BTF_KIND_UNION type with: 19654a3b33f8SAndrii Nakryiko * - *name* - name of the union, can be NULL or empty for anonymous union; 19664a3b33f8SAndrii Nakryiko * - *byte_sz* - size of the union, in bytes; 19674a3b33f8SAndrii Nakryiko * 19684a3b33f8SAndrii Nakryiko * Union initially has no fields in it. Fields can be added by 19694a3b33f8SAndrii Nakryiko * btf__add_field() right after btf__add_union() succeeds. All fields 19704a3b33f8SAndrii Nakryiko * should have *bit_offset* of 0. 19714a3b33f8SAndrii Nakryiko * 19724a3b33f8SAndrii Nakryiko * Returns: 19734a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 19744a3b33f8SAndrii Nakryiko * - <0, on error. 19754a3b33f8SAndrii Nakryiko */ 19764a3b33f8SAndrii Nakryiko int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz) 19774a3b33f8SAndrii Nakryiko { 19784a3b33f8SAndrii Nakryiko return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz); 19794a3b33f8SAndrii Nakryiko } 19804a3b33f8SAndrii Nakryiko 1981c81ed6d8SAndrii Nakryiko static struct btf_type *btf_last_type(struct btf *btf) 1982c81ed6d8SAndrii Nakryiko { 19836a886de0SHengqi Chen return btf_type_by_id(btf, btf__type_cnt(btf) - 1); 1984c81ed6d8SAndrii Nakryiko } 1985c81ed6d8SAndrii Nakryiko 19864a3b33f8SAndrii Nakryiko /* 19874a3b33f8SAndrii Nakryiko * Append new field for the current STRUCT/UNION type with: 19884a3b33f8SAndrii Nakryiko * - *name* - name of the field, can be NULL or empty for anonymous field; 19894a3b33f8SAndrii Nakryiko * - *type_id* - type ID for the type describing field type; 19904a3b33f8SAndrii Nakryiko * - *bit_offset* - bit offset of the start of the field within struct/union; 19914a3b33f8SAndrii Nakryiko * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields; 19924a3b33f8SAndrii Nakryiko * Returns: 19934a3b33f8SAndrii Nakryiko * - 0, on success; 19944a3b33f8SAndrii Nakryiko * - <0, on error. 19954a3b33f8SAndrii Nakryiko */ 19964a3b33f8SAndrii Nakryiko int btf__add_field(struct btf *btf, const char *name, int type_id, 19974a3b33f8SAndrii Nakryiko __u32 bit_offset, __u32 bit_size) 19984a3b33f8SAndrii Nakryiko { 19994a3b33f8SAndrii Nakryiko struct btf_type *t; 20004a3b33f8SAndrii Nakryiko struct btf_member *m; 20014a3b33f8SAndrii Nakryiko bool is_bitfield; 20024a3b33f8SAndrii Nakryiko int sz, name_off = 0; 20034a3b33f8SAndrii Nakryiko 20044a3b33f8SAndrii Nakryiko /* last type should be union/struct */ 20054a3b33f8SAndrii Nakryiko if (btf->nr_types == 0) 2006e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 2007c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 20084a3b33f8SAndrii Nakryiko if (!btf_is_composite(t)) 2009e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 20104a3b33f8SAndrii Nakryiko 20114a3b33f8SAndrii Nakryiko if (validate_type_id(type_id)) 2012e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 20134a3b33f8SAndrii Nakryiko /* best-effort bit field offset/size enforcement */ 20144a3b33f8SAndrii Nakryiko is_bitfield = bit_size || (bit_offset % 8 != 0); 20154a3b33f8SAndrii Nakryiko if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff)) 2016e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 20174a3b33f8SAndrii Nakryiko 20184a3b33f8SAndrii Nakryiko /* only offset 0 is allowed for unions */ 20194a3b33f8SAndrii Nakryiko if (btf_is_union(t) && bit_offset) 2020e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 20214a3b33f8SAndrii Nakryiko 20224a3b33f8SAndrii Nakryiko /* decompose and invalidate raw data */ 20234a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2024e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 20254a3b33f8SAndrii Nakryiko 20264a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_member); 20274a3b33f8SAndrii Nakryiko m = btf_add_type_mem(btf, sz); 20284a3b33f8SAndrii Nakryiko if (!m) 2029e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 20304a3b33f8SAndrii Nakryiko 20314a3b33f8SAndrii Nakryiko if (name && name[0]) { 20324a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 20334a3b33f8SAndrii Nakryiko if (name_off < 0) 20344a3b33f8SAndrii Nakryiko return name_off; 20354a3b33f8SAndrii Nakryiko } 20364a3b33f8SAndrii Nakryiko 20374a3b33f8SAndrii Nakryiko m->name_off = name_off; 20384a3b33f8SAndrii Nakryiko m->type = type_id; 20394a3b33f8SAndrii Nakryiko m->offset = bit_offset | (bit_size << 24); 20404a3b33f8SAndrii Nakryiko 20414a3b33f8SAndrii Nakryiko /* btf_add_type_mem can invalidate t pointer */ 2042c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 20434a3b33f8SAndrii Nakryiko /* update parent type's vlen and kflag */ 20444a3b33f8SAndrii Nakryiko t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); 20454a3b33f8SAndrii Nakryiko 20464a3b33f8SAndrii Nakryiko btf->hdr->type_len += sz; 20474a3b33f8SAndrii Nakryiko btf->hdr->str_off += sz; 20484a3b33f8SAndrii Nakryiko return 0; 20494a3b33f8SAndrii Nakryiko } 20504a3b33f8SAndrii Nakryiko 20518479aa75SYonghong Song static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz, 20528479aa75SYonghong Song bool is_signed, __u8 kind) 20534a3b33f8SAndrii Nakryiko { 20544a3b33f8SAndrii Nakryiko struct btf_type *t; 2055c81ed6d8SAndrii Nakryiko int sz, name_off = 0; 20564a3b33f8SAndrii Nakryiko 20574a3b33f8SAndrii Nakryiko /* byte_sz must be power of 2 */ 20584a3b33f8SAndrii Nakryiko if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) 2059e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 20604a3b33f8SAndrii Nakryiko 20614a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2062e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 20634a3b33f8SAndrii Nakryiko 20644a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type); 20654a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 20664a3b33f8SAndrii Nakryiko if (!t) 2067e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 20684a3b33f8SAndrii Nakryiko 20694a3b33f8SAndrii Nakryiko if (name && name[0]) { 20704a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 20714a3b33f8SAndrii Nakryiko if (name_off < 0) 20724a3b33f8SAndrii Nakryiko return name_off; 20734a3b33f8SAndrii Nakryiko } 20744a3b33f8SAndrii Nakryiko 20754a3b33f8SAndrii Nakryiko /* start out with vlen=0; it will be adjusted when adding enum values */ 20764a3b33f8SAndrii Nakryiko t->name_off = name_off; 20778479aa75SYonghong Song t->info = btf_type_info(kind, 0, is_signed); 20784a3b33f8SAndrii Nakryiko t->size = byte_sz; 20794a3b33f8SAndrii Nakryiko 2080c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 20814a3b33f8SAndrii Nakryiko } 20824a3b33f8SAndrii Nakryiko 20834a3b33f8SAndrii Nakryiko /* 20848479aa75SYonghong Song * Append new BTF_KIND_ENUM type with: 20858479aa75SYonghong Song * - *name* - name of the enum, can be NULL or empty for anonymous enums; 20868479aa75SYonghong Song * - *byte_sz* - size of the enum, in bytes. 20878479aa75SYonghong Song * 20888479aa75SYonghong Song * Enum initially has no enum values in it (and corresponds to enum forward 20898479aa75SYonghong Song * declaration). Enumerator values can be added by btf__add_enum_value() 20908479aa75SYonghong Song * immediately after btf__add_enum() succeeds. 20918479aa75SYonghong Song * 20928479aa75SYonghong Song * Returns: 20938479aa75SYonghong Song * - >0, type ID of newly added BTF type; 20948479aa75SYonghong Song * - <0, on error. 20958479aa75SYonghong Song */ 20968479aa75SYonghong Song int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) 20978479aa75SYonghong Song { 2098dffbbdc2SYonghong Song /* 2099dffbbdc2SYonghong Song * set the signedness to be unsigned, it will change to signed 2100dffbbdc2SYonghong Song * if any later enumerator is negative. 2101dffbbdc2SYonghong Song */ 21028479aa75SYonghong Song return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM); 21038479aa75SYonghong Song } 21048479aa75SYonghong Song 21058479aa75SYonghong Song /* 21064a3b33f8SAndrii Nakryiko * Append new enum value for the current ENUM type with: 21074a3b33f8SAndrii Nakryiko * - *name* - name of the enumerator value, can't be NULL or empty; 21084a3b33f8SAndrii Nakryiko * - *value* - integer value corresponding to enum value *name*; 21094a3b33f8SAndrii Nakryiko * Returns: 21104a3b33f8SAndrii Nakryiko * - 0, on success; 21114a3b33f8SAndrii Nakryiko * - <0, on error. 21124a3b33f8SAndrii Nakryiko */ 21134a3b33f8SAndrii Nakryiko int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) 21144a3b33f8SAndrii Nakryiko { 21154a3b33f8SAndrii Nakryiko struct btf_type *t; 21164a3b33f8SAndrii Nakryiko struct btf_enum *v; 21174a3b33f8SAndrii Nakryiko int sz, name_off; 21184a3b33f8SAndrii Nakryiko 21194a3b33f8SAndrii Nakryiko /* last type should be BTF_KIND_ENUM */ 21204a3b33f8SAndrii Nakryiko if (btf->nr_types == 0) 2121e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 2122c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 21234a3b33f8SAndrii Nakryiko if (!btf_is_enum(t)) 2124e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 21254a3b33f8SAndrii Nakryiko 21264a3b33f8SAndrii Nakryiko /* non-empty name */ 21274a3b33f8SAndrii Nakryiko if (!name || !name[0]) 2128e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 21294a3b33f8SAndrii Nakryiko if (value < INT_MIN || value > UINT_MAX) 2130e9fc3ce9SAndrii Nakryiko return libbpf_err(-E2BIG); 21314a3b33f8SAndrii Nakryiko 21324a3b33f8SAndrii Nakryiko /* decompose and invalidate raw data */ 21334a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2134e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 21354a3b33f8SAndrii Nakryiko 21364a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_enum); 21374a3b33f8SAndrii Nakryiko v = btf_add_type_mem(btf, sz); 21384a3b33f8SAndrii Nakryiko if (!v) 2139e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 21404a3b33f8SAndrii Nakryiko 21414a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 21424a3b33f8SAndrii Nakryiko if (name_off < 0) 21434a3b33f8SAndrii Nakryiko return name_off; 21444a3b33f8SAndrii Nakryiko 21454a3b33f8SAndrii Nakryiko v->name_off = name_off; 21464a3b33f8SAndrii Nakryiko v->val = value; 21474a3b33f8SAndrii Nakryiko 21484a3b33f8SAndrii Nakryiko /* update parent type's vlen */ 2149c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 21504a3b33f8SAndrii Nakryiko btf_type_inc_vlen(t); 21514a3b33f8SAndrii Nakryiko 2152dffbbdc2SYonghong Song /* if negative value, set signedness to signed */ 2153dffbbdc2SYonghong Song if (value < 0) 2154dffbbdc2SYonghong Song t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); 2155dffbbdc2SYonghong Song 2156dffbbdc2SYonghong Song btf->hdr->type_len += sz; 2157dffbbdc2SYonghong Song btf->hdr->str_off += sz; 2158dffbbdc2SYonghong Song return 0; 2159dffbbdc2SYonghong Song } 2160dffbbdc2SYonghong Song 2161dffbbdc2SYonghong Song /* 2162dffbbdc2SYonghong Song * Append new BTF_KIND_ENUM64 type with: 2163dffbbdc2SYonghong Song * - *name* - name of the enum, can be NULL or empty for anonymous enums; 2164dffbbdc2SYonghong Song * - *byte_sz* - size of the enum, in bytes. 2165dffbbdc2SYonghong Song * - *is_signed* - whether the enum values are signed or not; 2166dffbbdc2SYonghong Song * 2167dffbbdc2SYonghong Song * Enum initially has no enum values in it (and corresponds to enum forward 2168dffbbdc2SYonghong Song * declaration). Enumerator values can be added by btf__add_enum64_value() 2169dffbbdc2SYonghong Song * immediately after btf__add_enum64() succeeds. 2170dffbbdc2SYonghong Song * 2171dffbbdc2SYonghong Song * Returns: 2172dffbbdc2SYonghong Song * - >0, type ID of newly added BTF type; 2173dffbbdc2SYonghong Song * - <0, on error. 2174dffbbdc2SYonghong Song */ 2175dffbbdc2SYonghong Song int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz, 2176dffbbdc2SYonghong Song bool is_signed) 2177dffbbdc2SYonghong Song { 2178dffbbdc2SYonghong Song return btf_add_enum_common(btf, name, byte_sz, is_signed, 2179dffbbdc2SYonghong Song BTF_KIND_ENUM64); 2180dffbbdc2SYonghong Song } 2181dffbbdc2SYonghong Song 2182dffbbdc2SYonghong Song /* 2183dffbbdc2SYonghong Song * Append new enum value for the current ENUM64 type with: 2184dffbbdc2SYonghong Song * - *name* - name of the enumerator value, can't be NULL or empty; 2185dffbbdc2SYonghong Song * - *value* - integer value corresponding to enum value *name*; 2186dffbbdc2SYonghong Song * Returns: 2187dffbbdc2SYonghong Song * - 0, on success; 2188dffbbdc2SYonghong Song * - <0, on error. 2189dffbbdc2SYonghong Song */ 2190dffbbdc2SYonghong Song int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) 2191dffbbdc2SYonghong Song { 2192dffbbdc2SYonghong Song struct btf_enum64 *v; 2193dffbbdc2SYonghong Song struct btf_type *t; 2194dffbbdc2SYonghong Song int sz, name_off; 2195dffbbdc2SYonghong Song 2196dffbbdc2SYonghong Song /* last type should be BTF_KIND_ENUM64 */ 2197dffbbdc2SYonghong Song if (btf->nr_types == 0) 2198dffbbdc2SYonghong Song return libbpf_err(-EINVAL); 2199dffbbdc2SYonghong Song t = btf_last_type(btf); 2200dffbbdc2SYonghong Song if (!btf_is_enum64(t)) 2201dffbbdc2SYonghong Song return libbpf_err(-EINVAL); 2202dffbbdc2SYonghong Song 2203dffbbdc2SYonghong Song /* non-empty name */ 2204dffbbdc2SYonghong Song if (!name || !name[0]) 2205dffbbdc2SYonghong Song return libbpf_err(-EINVAL); 2206dffbbdc2SYonghong Song 2207dffbbdc2SYonghong Song /* decompose and invalidate raw data */ 2208dffbbdc2SYonghong Song if (btf_ensure_modifiable(btf)) 2209dffbbdc2SYonghong Song return libbpf_err(-ENOMEM); 2210dffbbdc2SYonghong Song 2211dffbbdc2SYonghong Song sz = sizeof(struct btf_enum64); 2212dffbbdc2SYonghong Song v = btf_add_type_mem(btf, sz); 2213dffbbdc2SYonghong Song if (!v) 2214dffbbdc2SYonghong Song return libbpf_err(-ENOMEM); 2215dffbbdc2SYonghong Song 2216dffbbdc2SYonghong Song name_off = btf__add_str(btf, name); 2217dffbbdc2SYonghong Song if (name_off < 0) 2218dffbbdc2SYonghong Song return name_off; 2219dffbbdc2SYonghong Song 2220dffbbdc2SYonghong Song v->name_off = name_off; 2221dffbbdc2SYonghong Song v->val_lo32 = (__u32)value; 2222dffbbdc2SYonghong Song v->val_hi32 = value >> 32; 2223dffbbdc2SYonghong Song 2224dffbbdc2SYonghong Song /* update parent type's vlen */ 2225dffbbdc2SYonghong Song t = btf_last_type(btf); 2226dffbbdc2SYonghong Song btf_type_inc_vlen(t); 2227dffbbdc2SYonghong Song 22284a3b33f8SAndrii Nakryiko btf->hdr->type_len += sz; 22294a3b33f8SAndrii Nakryiko btf->hdr->str_off += sz; 22304a3b33f8SAndrii Nakryiko return 0; 22314a3b33f8SAndrii Nakryiko } 22324a3b33f8SAndrii Nakryiko 22334a3b33f8SAndrii Nakryiko /* 22344a3b33f8SAndrii Nakryiko * Append new BTF_KIND_FWD type with: 22354a3b33f8SAndrii Nakryiko * - *name*, non-empty/non-NULL name; 22364a3b33f8SAndrii Nakryiko * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT, 22374a3b33f8SAndrii Nakryiko * BTF_FWD_UNION, or BTF_FWD_ENUM; 22384a3b33f8SAndrii Nakryiko * Returns: 22394a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 22404a3b33f8SAndrii Nakryiko * - <0, on error. 22414a3b33f8SAndrii Nakryiko */ 22424a3b33f8SAndrii Nakryiko int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind) 22434a3b33f8SAndrii Nakryiko { 22444a3b33f8SAndrii Nakryiko if (!name || !name[0]) 2245e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 22464a3b33f8SAndrii Nakryiko 22474a3b33f8SAndrii Nakryiko switch (fwd_kind) { 22484a3b33f8SAndrii Nakryiko case BTF_FWD_STRUCT: 22494a3b33f8SAndrii Nakryiko case BTF_FWD_UNION: { 22504a3b33f8SAndrii Nakryiko struct btf_type *t; 22514a3b33f8SAndrii Nakryiko int id; 22524a3b33f8SAndrii Nakryiko 22534a3b33f8SAndrii Nakryiko id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0); 22544a3b33f8SAndrii Nakryiko if (id <= 0) 22554a3b33f8SAndrii Nakryiko return id; 22564a3b33f8SAndrii Nakryiko t = btf_type_by_id(btf, id); 22574a3b33f8SAndrii Nakryiko t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); 22584a3b33f8SAndrii Nakryiko return id; 22594a3b33f8SAndrii Nakryiko } 22604a3b33f8SAndrii Nakryiko case BTF_FWD_ENUM: 22614a3b33f8SAndrii Nakryiko /* enum forward in BTF currently is just an enum with no enum 22624a3b33f8SAndrii Nakryiko * values; we also assume a standard 4-byte size for it 22634a3b33f8SAndrii Nakryiko */ 22644a3b33f8SAndrii Nakryiko return btf__add_enum(btf, name, sizeof(int)); 22654a3b33f8SAndrii Nakryiko default: 2266e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 22674a3b33f8SAndrii Nakryiko } 22684a3b33f8SAndrii Nakryiko } 22694a3b33f8SAndrii Nakryiko 22704a3b33f8SAndrii Nakryiko /* 22714a3b33f8SAndrii Nakryiko * Append new BTF_KING_TYPEDEF type with: 22724a3b33f8SAndrii Nakryiko * - *name*, non-empty/non-NULL name; 22734a3b33f8SAndrii Nakryiko * - *ref_type_id* - referenced type ID, it might not exist yet; 22744a3b33f8SAndrii Nakryiko * Returns: 22754a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 22764a3b33f8SAndrii Nakryiko * - <0, on error. 22774a3b33f8SAndrii Nakryiko */ 22784a3b33f8SAndrii Nakryiko int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id) 22794a3b33f8SAndrii Nakryiko { 22804a3b33f8SAndrii Nakryiko if (!name || !name[0]) 2281e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 22824a3b33f8SAndrii Nakryiko 22834a3b33f8SAndrii Nakryiko return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id); 22844a3b33f8SAndrii Nakryiko } 22854a3b33f8SAndrii Nakryiko 22864a3b33f8SAndrii Nakryiko /* 22874a3b33f8SAndrii Nakryiko * Append new BTF_KIND_VOLATILE type with: 22884a3b33f8SAndrii Nakryiko * - *ref_type_id* - referenced type ID, it might not exist yet; 22894a3b33f8SAndrii Nakryiko * Returns: 22904a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 22914a3b33f8SAndrii Nakryiko * - <0, on error. 22924a3b33f8SAndrii Nakryiko */ 22934a3b33f8SAndrii Nakryiko int btf__add_volatile(struct btf *btf, int ref_type_id) 22944a3b33f8SAndrii Nakryiko { 22954a3b33f8SAndrii Nakryiko return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id); 22964a3b33f8SAndrii Nakryiko } 22974a3b33f8SAndrii Nakryiko 22984a3b33f8SAndrii Nakryiko /* 22994a3b33f8SAndrii Nakryiko * Append new BTF_KIND_CONST type with: 23004a3b33f8SAndrii Nakryiko * - *ref_type_id* - referenced type ID, it might not exist yet; 23014a3b33f8SAndrii Nakryiko * Returns: 23024a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 23034a3b33f8SAndrii Nakryiko * - <0, on error. 23044a3b33f8SAndrii Nakryiko */ 23054a3b33f8SAndrii Nakryiko int btf__add_const(struct btf *btf, int ref_type_id) 23064a3b33f8SAndrii Nakryiko { 23074a3b33f8SAndrii Nakryiko return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id); 23084a3b33f8SAndrii Nakryiko } 23094a3b33f8SAndrii Nakryiko 23104a3b33f8SAndrii Nakryiko /* 23114a3b33f8SAndrii Nakryiko * Append new BTF_KIND_RESTRICT type with: 23124a3b33f8SAndrii Nakryiko * - *ref_type_id* - referenced type ID, it might not exist yet; 23134a3b33f8SAndrii Nakryiko * Returns: 23144a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 23154a3b33f8SAndrii Nakryiko * - <0, on error. 23164a3b33f8SAndrii Nakryiko */ 23174a3b33f8SAndrii Nakryiko int btf__add_restrict(struct btf *btf, int ref_type_id) 23184a3b33f8SAndrii Nakryiko { 23194a3b33f8SAndrii Nakryiko return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id); 23204a3b33f8SAndrii Nakryiko } 23214a3b33f8SAndrii Nakryiko 23224a3b33f8SAndrii Nakryiko /* 23232dc1e488SYonghong Song * Append new BTF_KIND_TYPE_TAG type with: 23242dc1e488SYonghong Song * - *value*, non-empty/non-NULL tag value; 23252dc1e488SYonghong Song * - *ref_type_id* - referenced type ID, it might not exist yet; 23262dc1e488SYonghong Song * Returns: 23272dc1e488SYonghong Song * - >0, type ID of newly added BTF type; 23282dc1e488SYonghong Song * - <0, on error. 23292dc1e488SYonghong Song */ 23302dc1e488SYonghong Song int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id) 23312dc1e488SYonghong Song { 23322dc1e488SYonghong Song if (!value|| !value[0]) 23332dc1e488SYonghong Song return libbpf_err(-EINVAL); 23342dc1e488SYonghong Song 23352dc1e488SYonghong Song return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id); 23362dc1e488SYonghong Song } 23372dc1e488SYonghong Song 23382dc1e488SYonghong Song /* 23394a3b33f8SAndrii Nakryiko * Append new BTF_KIND_FUNC type with: 23404a3b33f8SAndrii Nakryiko * - *name*, non-empty/non-NULL name; 23414a3b33f8SAndrii Nakryiko * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet; 23424a3b33f8SAndrii Nakryiko * Returns: 23434a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 23444a3b33f8SAndrii Nakryiko * - <0, on error. 23454a3b33f8SAndrii Nakryiko */ 23464a3b33f8SAndrii Nakryiko int btf__add_func(struct btf *btf, const char *name, 23474a3b33f8SAndrii Nakryiko enum btf_func_linkage linkage, int proto_type_id) 23484a3b33f8SAndrii Nakryiko { 23494a3b33f8SAndrii Nakryiko int id; 23504a3b33f8SAndrii Nakryiko 23514a3b33f8SAndrii Nakryiko if (!name || !name[0]) 2352e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 23534a3b33f8SAndrii Nakryiko if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL && 23544a3b33f8SAndrii Nakryiko linkage != BTF_FUNC_EXTERN) 2355e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 23564a3b33f8SAndrii Nakryiko 23574a3b33f8SAndrii Nakryiko id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id); 23584a3b33f8SAndrii Nakryiko if (id > 0) { 23594a3b33f8SAndrii Nakryiko struct btf_type *t = btf_type_by_id(btf, id); 23604a3b33f8SAndrii Nakryiko 23614a3b33f8SAndrii Nakryiko t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); 23624a3b33f8SAndrii Nakryiko } 2363e9fc3ce9SAndrii Nakryiko return libbpf_err(id); 23644a3b33f8SAndrii Nakryiko } 23654a3b33f8SAndrii Nakryiko 23664a3b33f8SAndrii Nakryiko /* 23674a3b33f8SAndrii Nakryiko * Append new BTF_KIND_FUNC_PROTO with: 23684a3b33f8SAndrii Nakryiko * - *ret_type_id* - type ID for return result of a function. 23694a3b33f8SAndrii Nakryiko * 23704a3b33f8SAndrii Nakryiko * Function prototype initially has no arguments, but they can be added by 23714a3b33f8SAndrii Nakryiko * btf__add_func_param() one by one, immediately after 23724a3b33f8SAndrii Nakryiko * btf__add_func_proto() succeeded. 23734a3b33f8SAndrii Nakryiko * 23744a3b33f8SAndrii Nakryiko * Returns: 23754a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 23764a3b33f8SAndrii Nakryiko * - <0, on error. 23774a3b33f8SAndrii Nakryiko */ 23784a3b33f8SAndrii Nakryiko int btf__add_func_proto(struct btf *btf, int ret_type_id) 23794a3b33f8SAndrii Nakryiko { 23804a3b33f8SAndrii Nakryiko struct btf_type *t; 2381c81ed6d8SAndrii Nakryiko int sz; 23824a3b33f8SAndrii Nakryiko 23834a3b33f8SAndrii Nakryiko if (validate_type_id(ret_type_id)) 2384e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 23854a3b33f8SAndrii Nakryiko 23864a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2387e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 23884a3b33f8SAndrii Nakryiko 23894a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type); 23904a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 23914a3b33f8SAndrii Nakryiko if (!t) 2392e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 23934a3b33f8SAndrii Nakryiko 23944a3b33f8SAndrii Nakryiko /* start out with vlen=0; this will be adjusted when adding enum 23954a3b33f8SAndrii Nakryiko * values, if necessary 23964a3b33f8SAndrii Nakryiko */ 23974a3b33f8SAndrii Nakryiko t->name_off = 0; 23984a3b33f8SAndrii Nakryiko t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); 23994a3b33f8SAndrii Nakryiko t->type = ret_type_id; 24004a3b33f8SAndrii Nakryiko 2401c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 24024a3b33f8SAndrii Nakryiko } 24034a3b33f8SAndrii Nakryiko 24044a3b33f8SAndrii Nakryiko /* 24054a3b33f8SAndrii Nakryiko * Append new function parameter for current FUNC_PROTO type with: 24064a3b33f8SAndrii Nakryiko * - *name* - parameter name, can be NULL or empty; 24074a3b33f8SAndrii Nakryiko * - *type_id* - type ID describing the type of the parameter. 24084a3b33f8SAndrii Nakryiko * Returns: 24094a3b33f8SAndrii Nakryiko * - 0, on success; 24104a3b33f8SAndrii Nakryiko * - <0, on error. 24114a3b33f8SAndrii Nakryiko */ 24124a3b33f8SAndrii Nakryiko int btf__add_func_param(struct btf *btf, const char *name, int type_id) 24134a3b33f8SAndrii Nakryiko { 24144a3b33f8SAndrii Nakryiko struct btf_type *t; 24154a3b33f8SAndrii Nakryiko struct btf_param *p; 24164a3b33f8SAndrii Nakryiko int sz, name_off = 0; 24174a3b33f8SAndrii Nakryiko 24184a3b33f8SAndrii Nakryiko if (validate_type_id(type_id)) 2419e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 24204a3b33f8SAndrii Nakryiko 24214a3b33f8SAndrii Nakryiko /* last type should be BTF_KIND_FUNC_PROTO */ 24224a3b33f8SAndrii Nakryiko if (btf->nr_types == 0) 2423e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 2424c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 24254a3b33f8SAndrii Nakryiko if (!btf_is_func_proto(t)) 2426e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 24274a3b33f8SAndrii Nakryiko 24284a3b33f8SAndrii Nakryiko /* decompose and invalidate raw data */ 24294a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2430e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 24314a3b33f8SAndrii Nakryiko 24324a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_param); 24334a3b33f8SAndrii Nakryiko p = btf_add_type_mem(btf, sz); 24344a3b33f8SAndrii Nakryiko if (!p) 2435e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 24364a3b33f8SAndrii Nakryiko 24374a3b33f8SAndrii Nakryiko if (name && name[0]) { 24384a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 24394a3b33f8SAndrii Nakryiko if (name_off < 0) 24404a3b33f8SAndrii Nakryiko return name_off; 24414a3b33f8SAndrii Nakryiko } 24424a3b33f8SAndrii Nakryiko 24434a3b33f8SAndrii Nakryiko p->name_off = name_off; 24444a3b33f8SAndrii Nakryiko p->type = type_id; 24454a3b33f8SAndrii Nakryiko 24464a3b33f8SAndrii Nakryiko /* update parent type's vlen */ 2447c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 24484a3b33f8SAndrii Nakryiko btf_type_inc_vlen(t); 24494a3b33f8SAndrii Nakryiko 24504a3b33f8SAndrii Nakryiko btf->hdr->type_len += sz; 24514a3b33f8SAndrii Nakryiko btf->hdr->str_off += sz; 24524a3b33f8SAndrii Nakryiko return 0; 24534a3b33f8SAndrii Nakryiko } 24544a3b33f8SAndrii Nakryiko 24554a3b33f8SAndrii Nakryiko /* 24564a3b33f8SAndrii Nakryiko * Append new BTF_KIND_VAR type with: 24574a3b33f8SAndrii Nakryiko * - *name* - non-empty/non-NULL name; 24584a3b33f8SAndrii Nakryiko * - *linkage* - variable linkage, one of BTF_VAR_STATIC, 24594a3b33f8SAndrii Nakryiko * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN; 24604a3b33f8SAndrii Nakryiko * - *type_id* - type ID of the type describing the type of the variable. 24614a3b33f8SAndrii Nakryiko * Returns: 24624a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 24634a3b33f8SAndrii Nakryiko * - <0, on error. 24644a3b33f8SAndrii Nakryiko */ 24654a3b33f8SAndrii Nakryiko int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) 24664a3b33f8SAndrii Nakryiko { 24674a3b33f8SAndrii Nakryiko struct btf_type *t; 24684a3b33f8SAndrii Nakryiko struct btf_var *v; 2469c81ed6d8SAndrii Nakryiko int sz, name_off; 24704a3b33f8SAndrii Nakryiko 24714a3b33f8SAndrii Nakryiko /* non-empty name */ 24724a3b33f8SAndrii Nakryiko if (!name || !name[0]) 2473e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 24744a3b33f8SAndrii Nakryiko if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED && 24754a3b33f8SAndrii Nakryiko linkage != BTF_VAR_GLOBAL_EXTERN) 2476e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 24774a3b33f8SAndrii Nakryiko if (validate_type_id(type_id)) 2478e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 24794a3b33f8SAndrii Nakryiko 24804a3b33f8SAndrii Nakryiko /* deconstruct BTF, if necessary, and invalidate raw_data */ 24814a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2482e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 24834a3b33f8SAndrii Nakryiko 24844a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type) + sizeof(struct btf_var); 24854a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 24864a3b33f8SAndrii Nakryiko if (!t) 2487e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 24884a3b33f8SAndrii Nakryiko 24894a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 24904a3b33f8SAndrii Nakryiko if (name_off < 0) 24914a3b33f8SAndrii Nakryiko return name_off; 24924a3b33f8SAndrii Nakryiko 24934a3b33f8SAndrii Nakryiko t->name_off = name_off; 24944a3b33f8SAndrii Nakryiko t->info = btf_type_info(BTF_KIND_VAR, 0, 0); 24954a3b33f8SAndrii Nakryiko t->type = type_id; 24964a3b33f8SAndrii Nakryiko 24974a3b33f8SAndrii Nakryiko v = btf_var(t); 24984a3b33f8SAndrii Nakryiko v->linkage = linkage; 24994a3b33f8SAndrii Nakryiko 2500c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 25014a3b33f8SAndrii Nakryiko } 25024a3b33f8SAndrii Nakryiko 25034a3b33f8SAndrii Nakryiko /* 25044a3b33f8SAndrii Nakryiko * Append new BTF_KIND_DATASEC type with: 25054a3b33f8SAndrii Nakryiko * - *name* - non-empty/non-NULL name; 25064a3b33f8SAndrii Nakryiko * - *byte_sz* - data section size, in bytes. 25074a3b33f8SAndrii Nakryiko * 25084a3b33f8SAndrii Nakryiko * Data section is initially empty. Variables info can be added with 25094a3b33f8SAndrii Nakryiko * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds. 25104a3b33f8SAndrii Nakryiko * 25114a3b33f8SAndrii Nakryiko * Returns: 25124a3b33f8SAndrii Nakryiko * - >0, type ID of newly added BTF type; 25134a3b33f8SAndrii Nakryiko * - <0, on error. 25144a3b33f8SAndrii Nakryiko */ 25154a3b33f8SAndrii Nakryiko int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz) 25164a3b33f8SAndrii Nakryiko { 25174a3b33f8SAndrii Nakryiko struct btf_type *t; 2518c81ed6d8SAndrii Nakryiko int sz, name_off; 25194a3b33f8SAndrii Nakryiko 25204a3b33f8SAndrii Nakryiko /* non-empty name */ 25214a3b33f8SAndrii Nakryiko if (!name || !name[0]) 2522e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 25234a3b33f8SAndrii Nakryiko 25244a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2525e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 25264a3b33f8SAndrii Nakryiko 25274a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_type); 25284a3b33f8SAndrii Nakryiko t = btf_add_type_mem(btf, sz); 25294a3b33f8SAndrii Nakryiko if (!t) 2530e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 25314a3b33f8SAndrii Nakryiko 25324a3b33f8SAndrii Nakryiko name_off = btf__add_str(btf, name); 25334a3b33f8SAndrii Nakryiko if (name_off < 0) 25344a3b33f8SAndrii Nakryiko return name_off; 25354a3b33f8SAndrii Nakryiko 25364a3b33f8SAndrii Nakryiko /* start with vlen=0, which will be update as var_secinfos are added */ 25374a3b33f8SAndrii Nakryiko t->name_off = name_off; 25384a3b33f8SAndrii Nakryiko t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); 25394a3b33f8SAndrii Nakryiko t->size = byte_sz; 25404a3b33f8SAndrii Nakryiko 2541c81ed6d8SAndrii Nakryiko return btf_commit_type(btf, sz); 25424a3b33f8SAndrii Nakryiko } 25434a3b33f8SAndrii Nakryiko 25444a3b33f8SAndrii Nakryiko /* 25454a3b33f8SAndrii Nakryiko * Append new data section variable information entry for current DATASEC type: 25464a3b33f8SAndrii Nakryiko * - *var_type_id* - type ID, describing type of the variable; 25474a3b33f8SAndrii Nakryiko * - *offset* - variable offset within data section, in bytes; 25484a3b33f8SAndrii Nakryiko * - *byte_sz* - variable size, in bytes. 25494a3b33f8SAndrii Nakryiko * 25504a3b33f8SAndrii Nakryiko * Returns: 25514a3b33f8SAndrii Nakryiko * - 0, on success; 25524a3b33f8SAndrii Nakryiko * - <0, on error. 25534a3b33f8SAndrii Nakryiko */ 25544a3b33f8SAndrii Nakryiko int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz) 25554a3b33f8SAndrii Nakryiko { 25564a3b33f8SAndrii Nakryiko struct btf_type *t; 25574a3b33f8SAndrii Nakryiko struct btf_var_secinfo *v; 25584a3b33f8SAndrii Nakryiko int sz; 25594a3b33f8SAndrii Nakryiko 25604a3b33f8SAndrii Nakryiko /* last type should be BTF_KIND_DATASEC */ 25614a3b33f8SAndrii Nakryiko if (btf->nr_types == 0) 2562e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 2563c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 25644a3b33f8SAndrii Nakryiko if (!btf_is_datasec(t)) 2565e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 25664a3b33f8SAndrii Nakryiko 25674a3b33f8SAndrii Nakryiko if (validate_type_id(var_type_id)) 2568e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 25694a3b33f8SAndrii Nakryiko 25704a3b33f8SAndrii Nakryiko /* decompose and invalidate raw data */ 25714a3b33f8SAndrii Nakryiko if (btf_ensure_modifiable(btf)) 2572e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 25734a3b33f8SAndrii Nakryiko 25744a3b33f8SAndrii Nakryiko sz = sizeof(struct btf_var_secinfo); 25754a3b33f8SAndrii Nakryiko v = btf_add_type_mem(btf, sz); 25764a3b33f8SAndrii Nakryiko if (!v) 2577e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM); 25784a3b33f8SAndrii Nakryiko 25794a3b33f8SAndrii Nakryiko v->type = var_type_id; 25804a3b33f8SAndrii Nakryiko v->offset = offset; 25814a3b33f8SAndrii Nakryiko v->size = byte_sz; 25824a3b33f8SAndrii Nakryiko 25834a3b33f8SAndrii Nakryiko /* update parent type's vlen */ 2584c81ed6d8SAndrii Nakryiko t = btf_last_type(btf); 25854a3b33f8SAndrii Nakryiko btf_type_inc_vlen(t); 25864a3b33f8SAndrii Nakryiko 25874a3b33f8SAndrii Nakryiko btf->hdr->type_len += sz; 25884a3b33f8SAndrii Nakryiko btf->hdr->str_off += sz; 25894a3b33f8SAndrii Nakryiko return 0; 25904a3b33f8SAndrii Nakryiko } 25914a3b33f8SAndrii Nakryiko 25925b84bd10SYonghong Song /* 2593223f903eSYonghong Song * Append new BTF_KIND_DECL_TAG type with: 25945b84bd10SYonghong Song * - *value* - non-empty/non-NULL string; 25955b84bd10SYonghong Song * - *ref_type_id* - referenced type ID, it might not exist yet; 25965b84bd10SYonghong Song * - *component_idx* - -1 for tagging reference type, otherwise struct/union 25975b84bd10SYonghong Song * member or function argument index; 25985b84bd10SYonghong Song * Returns: 25995b84bd10SYonghong Song * - >0, type ID of newly added BTF type; 26005b84bd10SYonghong Song * - <0, on error. 26015b84bd10SYonghong Song */ 2602223f903eSYonghong Song int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, 26035b84bd10SYonghong Song int component_idx) 26045b84bd10SYonghong Song { 26055b84bd10SYonghong Song struct btf_type *t; 26065b84bd10SYonghong Song int sz, value_off; 26075b84bd10SYonghong Song 26085b84bd10SYonghong Song if (!value || !value[0] || component_idx < -1) 26095b84bd10SYonghong Song return libbpf_err(-EINVAL); 26105b84bd10SYonghong Song 26115b84bd10SYonghong Song if (validate_type_id(ref_type_id)) 26125b84bd10SYonghong Song return libbpf_err(-EINVAL); 26135b84bd10SYonghong Song 26145b84bd10SYonghong Song if (btf_ensure_modifiable(btf)) 26155b84bd10SYonghong Song return libbpf_err(-ENOMEM); 26165b84bd10SYonghong Song 2617223f903eSYonghong Song sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag); 26185b84bd10SYonghong Song t = btf_add_type_mem(btf, sz); 26195b84bd10SYonghong Song if (!t) 26205b84bd10SYonghong Song return libbpf_err(-ENOMEM); 26215b84bd10SYonghong Song 26225b84bd10SYonghong Song value_off = btf__add_str(btf, value); 26235b84bd10SYonghong Song if (value_off < 0) 26245b84bd10SYonghong Song return value_off; 26255b84bd10SYonghong Song 26265b84bd10SYonghong Song t->name_off = value_off; 2627223f903eSYonghong Song t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false); 26285b84bd10SYonghong Song t->type = ref_type_id; 2629223f903eSYonghong Song btf_decl_tag(t)->component_idx = component_idx; 26305b84bd10SYonghong Song 26315b84bd10SYonghong Song return btf_commit_type(btf, sz); 26325b84bd10SYonghong Song } 26335b84bd10SYonghong Song 2634ae4ab4b4SAndrii Nakryiko struct btf_ext_sec_setup_param { 26353d650141SMartin KaFai Lau __u32 off; 26363d650141SMartin KaFai Lau __u32 len; 26373d650141SMartin KaFai Lau __u32 min_rec_size; 26383d650141SMartin KaFai Lau struct btf_ext_info *ext_info; 26393d650141SMartin KaFai Lau const char *desc; 26403d650141SMartin KaFai Lau }; 26413d650141SMartin KaFai Lau 2642ae4ab4b4SAndrii Nakryiko static int btf_ext_setup_info(struct btf_ext *btf_ext, 2643ae4ab4b4SAndrii Nakryiko struct btf_ext_sec_setup_param *ext_sec) 26442993e051SYonghong Song { 26453d650141SMartin KaFai Lau const struct btf_ext_info_sec *sinfo; 26463d650141SMartin KaFai Lau struct btf_ext_info *ext_info; 2647f0187f0bSMartin KaFai Lau __u32 info_left, record_size; 264811d5daa8SAndrii Nakryiko size_t sec_cnt = 0; 2649f0187f0bSMartin KaFai Lau /* The start of the info sec (including the __u32 record_size). */ 2650ae4ab4b4SAndrii Nakryiko void *info; 2651f0187f0bSMartin KaFai Lau 26524cedc0daSAndrii Nakryiko if (ext_sec->len == 0) 26534cedc0daSAndrii Nakryiko return 0; 26544cedc0daSAndrii Nakryiko 26553d650141SMartin KaFai Lau if (ext_sec->off & 0x03) { 26568461ef8bSYonghong Song pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n", 26573d650141SMartin KaFai Lau ext_sec->desc); 2658f0187f0bSMartin KaFai Lau return -EINVAL; 2659f0187f0bSMartin KaFai Lau } 2660f0187f0bSMartin KaFai Lau 2661ae4ab4b4SAndrii Nakryiko info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; 2662ae4ab4b4SAndrii Nakryiko info_left = ext_sec->len; 2663ae4ab4b4SAndrii Nakryiko 2664ae4ab4b4SAndrii Nakryiko if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { 26658461ef8bSYonghong Song pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", 26663d650141SMartin KaFai Lau ext_sec->desc, ext_sec->off, ext_sec->len); 2667f0187f0bSMartin KaFai Lau return -EINVAL; 2668f0187f0bSMartin KaFai Lau } 2669f0187f0bSMartin KaFai Lau 26703d650141SMartin KaFai Lau /* At least a record size */ 2671f0187f0bSMartin KaFai Lau if (info_left < sizeof(__u32)) { 26728461ef8bSYonghong Song pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); 26732993e051SYonghong Song return -EINVAL; 26742993e051SYonghong Song } 26752993e051SYonghong Song 2676f0187f0bSMartin KaFai Lau /* The record size needs to meet the minimum standard */ 2677f0187f0bSMartin KaFai Lau record_size = *(__u32 *)info; 26783d650141SMartin KaFai Lau if (record_size < ext_sec->min_rec_size || 2679f0187f0bSMartin KaFai Lau record_size & 0x03) { 26808461ef8bSYonghong Song pr_debug("%s section in .BTF.ext has invalid record size %u\n", 26813d650141SMartin KaFai Lau ext_sec->desc, record_size); 26822993e051SYonghong Song return -EINVAL; 26832993e051SYonghong Song } 26842993e051SYonghong Song 2685f0187f0bSMartin KaFai Lau sinfo = info + sizeof(__u32); 2686f0187f0bSMartin KaFai Lau info_left -= sizeof(__u32); 26872993e051SYonghong Song 26883d650141SMartin KaFai Lau /* If no records, return failure now so .BTF.ext won't be used. */ 2689f0187f0bSMartin KaFai Lau if (!info_left) { 26908461ef8bSYonghong Song pr_debug("%s section in .BTF.ext has no records", ext_sec->desc); 26912993e051SYonghong Song return -EINVAL; 26922993e051SYonghong Song } 26932993e051SYonghong Song 2694f0187f0bSMartin KaFai Lau while (info_left) { 26953d650141SMartin KaFai Lau unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 2696f0187f0bSMartin KaFai Lau __u64 total_record_size; 2697f0187f0bSMartin KaFai Lau __u32 num_records; 2698f0187f0bSMartin KaFai Lau 2699f0187f0bSMartin KaFai Lau if (info_left < sec_hdrlen) { 27008461ef8bSYonghong Song pr_debug("%s section header is not found in .BTF.ext\n", 27013d650141SMartin KaFai Lau ext_sec->desc); 27022993e051SYonghong Song return -EINVAL; 27032993e051SYonghong Song } 27042993e051SYonghong Song 27053d650141SMartin KaFai Lau num_records = sinfo->num_info; 27062993e051SYonghong Song if (num_records == 0) { 27078461ef8bSYonghong Song pr_debug("%s section has incorrect num_records in .BTF.ext\n", 27083d650141SMartin KaFai Lau ext_sec->desc); 27092993e051SYonghong Song return -EINVAL; 27102993e051SYonghong Song } 27112993e051SYonghong Song 271211d5daa8SAndrii Nakryiko total_record_size = sec_hdrlen + (__u64)num_records * record_size; 2713f0187f0bSMartin KaFai Lau if (info_left < total_record_size) { 27148461ef8bSYonghong Song pr_debug("%s section has incorrect num_records in .BTF.ext\n", 27153d650141SMartin KaFai Lau ext_sec->desc); 27162993e051SYonghong Song return -EINVAL; 27172993e051SYonghong Song } 27182993e051SYonghong Song 2719f0187f0bSMartin KaFai Lau info_left -= total_record_size; 27202993e051SYonghong Song sinfo = (void *)sinfo + total_record_size; 272111d5daa8SAndrii Nakryiko sec_cnt++; 27222993e051SYonghong Song } 27232993e051SYonghong Song 27243d650141SMartin KaFai Lau ext_info = ext_sec->ext_info; 27253d650141SMartin KaFai Lau ext_info->len = ext_sec->len - sizeof(__u32); 27263d650141SMartin KaFai Lau ext_info->rec_size = record_size; 2727ae4ab4b4SAndrii Nakryiko ext_info->info = info + sizeof(__u32); 272811d5daa8SAndrii Nakryiko ext_info->sec_cnt = sec_cnt; 2729f0187f0bSMartin KaFai Lau 27302993e051SYonghong Song return 0; 27312993e051SYonghong Song } 27322993e051SYonghong Song 2733ae4ab4b4SAndrii Nakryiko static int btf_ext_setup_func_info(struct btf_ext *btf_ext) 27343d650141SMartin KaFai Lau { 2735ae4ab4b4SAndrii Nakryiko struct btf_ext_sec_setup_param param = { 2736ae4ab4b4SAndrii Nakryiko .off = btf_ext->hdr->func_info_off, 2737ae4ab4b4SAndrii Nakryiko .len = btf_ext->hdr->func_info_len, 27383d650141SMartin KaFai Lau .min_rec_size = sizeof(struct bpf_func_info_min), 27393d650141SMartin KaFai Lau .ext_info = &btf_ext->func_info, 27403d650141SMartin KaFai Lau .desc = "func_info" 27413d650141SMartin KaFai Lau }; 27423d650141SMartin KaFai Lau 2743ae4ab4b4SAndrii Nakryiko return btf_ext_setup_info(btf_ext, ¶m); 27443d650141SMartin KaFai Lau } 27453d650141SMartin KaFai Lau 2746ae4ab4b4SAndrii Nakryiko static int btf_ext_setup_line_info(struct btf_ext *btf_ext) 27473d650141SMartin KaFai Lau { 2748ae4ab4b4SAndrii Nakryiko struct btf_ext_sec_setup_param param = { 2749ae4ab4b4SAndrii Nakryiko .off = btf_ext->hdr->line_info_off, 2750ae4ab4b4SAndrii Nakryiko .len = btf_ext->hdr->line_info_len, 27513d650141SMartin KaFai Lau .min_rec_size = sizeof(struct bpf_line_info_min), 27523d650141SMartin KaFai Lau .ext_info = &btf_ext->line_info, 27533d650141SMartin KaFai Lau .desc = "line_info", 27543d650141SMartin KaFai Lau }; 27553d650141SMartin KaFai Lau 2756ae4ab4b4SAndrii Nakryiko return btf_ext_setup_info(btf_ext, ¶m); 27573d650141SMartin KaFai Lau } 27583d650141SMartin KaFai Lau 275928b93c64SAndrii Nakryiko static int btf_ext_setup_core_relos(struct btf_ext *btf_ext) 27604cedc0daSAndrii Nakryiko { 27614cedc0daSAndrii Nakryiko struct btf_ext_sec_setup_param param = { 276228b93c64SAndrii Nakryiko .off = btf_ext->hdr->core_relo_off, 276328b93c64SAndrii Nakryiko .len = btf_ext->hdr->core_relo_len, 276428b93c64SAndrii Nakryiko .min_rec_size = sizeof(struct bpf_core_relo), 276528b93c64SAndrii Nakryiko .ext_info = &btf_ext->core_relo_info, 276628b93c64SAndrii Nakryiko .desc = "core_relo", 27674cedc0daSAndrii Nakryiko }; 27684cedc0daSAndrii Nakryiko 27694cedc0daSAndrii Nakryiko return btf_ext_setup_info(btf_ext, ¶m); 27704cedc0daSAndrii Nakryiko } 27714cedc0daSAndrii Nakryiko 27728461ef8bSYonghong Song static int btf_ext_parse_hdr(__u8 *data, __u32 data_size) 27732993e051SYonghong Song { 27742993e051SYonghong Song const struct btf_ext_header *hdr = (struct btf_ext_header *)data; 27752993e051SYonghong Song 27764cedc0daSAndrii Nakryiko if (data_size < offsetofend(struct btf_ext_header, hdr_len) || 27772993e051SYonghong Song data_size < hdr->hdr_len) { 27788461ef8bSYonghong Song pr_debug("BTF.ext header not found"); 27792993e051SYonghong Song return -EINVAL; 27802993e051SYonghong Song } 27812993e051SYonghong Song 27823289959bSAndrii Nakryiko if (hdr->magic == bswap_16(BTF_MAGIC)) { 27833289959bSAndrii Nakryiko pr_warn("BTF.ext in non-native endianness is not supported\n"); 27843289959bSAndrii Nakryiko return -ENOTSUP; 27853289959bSAndrii Nakryiko } else if (hdr->magic != BTF_MAGIC) { 27868461ef8bSYonghong Song pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); 27872993e051SYonghong Song return -EINVAL; 27882993e051SYonghong Song } 27892993e051SYonghong Song 27902993e051SYonghong Song if (hdr->version != BTF_VERSION) { 27918461ef8bSYonghong Song pr_debug("Unsupported BTF.ext version:%u\n", hdr->version); 27922993e051SYonghong Song return -ENOTSUP; 27932993e051SYonghong Song } 27942993e051SYonghong Song 27952993e051SYonghong Song if (hdr->flags) { 27968461ef8bSYonghong Song pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags); 27972993e051SYonghong Song return -ENOTSUP; 27982993e051SYonghong Song } 27992993e051SYonghong Song 2800f0187f0bSMartin KaFai Lau if (data_size == hdr->hdr_len) { 28018461ef8bSYonghong Song pr_debug("BTF.ext has no data\n"); 28022993e051SYonghong Song return -EINVAL; 28032993e051SYonghong Song } 28042993e051SYonghong Song 2805f0187f0bSMartin KaFai Lau return 0; 28062993e051SYonghong Song } 28072993e051SYonghong Song 28082993e051SYonghong Song void btf_ext__free(struct btf_ext *btf_ext) 28092993e051SYonghong Song { 281050450fc7SAndrii Nakryiko if (IS_ERR_OR_NULL(btf_ext)) 28112993e051SYonghong Song return; 281211d5daa8SAndrii Nakryiko free(btf_ext->func_info.sec_idxs); 281311d5daa8SAndrii Nakryiko free(btf_ext->line_info.sec_idxs); 281411d5daa8SAndrii Nakryiko free(btf_ext->core_relo_info.sec_idxs); 2815ae4ab4b4SAndrii Nakryiko free(btf_ext->data); 28162993e051SYonghong Song free(btf_ext); 28172993e051SYonghong Song } 28182993e051SYonghong Song 2819401891a9SAndrii Nakryiko struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) 28202993e051SYonghong Song { 28212993e051SYonghong Song struct btf_ext *btf_ext; 28222993e051SYonghong Song int err; 28232993e051SYonghong Song 28242993e051SYonghong Song btf_ext = calloc(1, sizeof(struct btf_ext)); 28252993e051SYonghong Song if (!btf_ext) 2826e9fc3ce9SAndrii Nakryiko return libbpf_err_ptr(-ENOMEM); 28272993e051SYonghong Song 2828ae4ab4b4SAndrii Nakryiko btf_ext->data_size = size; 2829ae4ab4b4SAndrii Nakryiko btf_ext->data = malloc(size); 2830ae4ab4b4SAndrii Nakryiko if (!btf_ext->data) { 2831ae4ab4b4SAndrii Nakryiko err = -ENOMEM; 2832ae4ab4b4SAndrii Nakryiko goto done; 28332993e051SYonghong Song } 2834ae4ab4b4SAndrii Nakryiko memcpy(btf_ext->data, data, size); 28352993e051SYonghong Song 2836401891a9SAndrii Nakryiko err = btf_ext_parse_hdr(btf_ext->data, size); 2837401891a9SAndrii Nakryiko if (err) 2838401891a9SAndrii Nakryiko goto done; 2839401891a9SAndrii Nakryiko 2840e9fc3ce9SAndrii Nakryiko if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) { 2841e9fc3ce9SAndrii Nakryiko err = -EINVAL; 28424cedc0daSAndrii Nakryiko goto done; 2843e9fc3ce9SAndrii Nakryiko } 2844e9fc3ce9SAndrii Nakryiko 2845ae4ab4b4SAndrii Nakryiko err = btf_ext_setup_func_info(btf_ext); 2846ae4ab4b4SAndrii Nakryiko if (err) 2847ae4ab4b4SAndrii Nakryiko goto done; 2848ae4ab4b4SAndrii Nakryiko 2849ae4ab4b4SAndrii Nakryiko err = btf_ext_setup_line_info(btf_ext); 2850ae4ab4b4SAndrii Nakryiko if (err) 2851ae4ab4b4SAndrii Nakryiko goto done; 2852ae4ab4b4SAndrii Nakryiko 2853e93f3999SYuntao Wang if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) 2854e93f3999SYuntao Wang goto done; /* skip core relos parsing */ 2855e9fc3ce9SAndrii Nakryiko 285628b93c64SAndrii Nakryiko err = btf_ext_setup_core_relos(btf_ext); 28574cedc0daSAndrii Nakryiko if (err) 28584cedc0daSAndrii Nakryiko goto done; 28594cedc0daSAndrii Nakryiko 2860ae4ab4b4SAndrii Nakryiko done: 28613d650141SMartin KaFai Lau if (err) { 28623d650141SMartin KaFai Lau btf_ext__free(btf_ext); 2863e9fc3ce9SAndrii Nakryiko return libbpf_err_ptr(err); 28643d650141SMartin KaFai Lau } 28653d650141SMartin KaFai Lau 28662993e051SYonghong Song return btf_ext; 28672993e051SYonghong Song } 28682993e051SYonghong Song 2869ae4ab4b4SAndrii Nakryiko const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size) 2870ae4ab4b4SAndrii Nakryiko { 2871ae4ab4b4SAndrii Nakryiko *size = btf_ext->data_size; 2872ae4ab4b4SAndrii Nakryiko return btf_ext->data; 2873ae4ab4b4SAndrii Nakryiko } 2874ae4ab4b4SAndrii Nakryiko 2875d5caef5bSAndrii Nakryiko struct btf_dedup; 2876d5caef5bSAndrii Nakryiko 2877957d350aSAndrii Nakryiko static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts); 2878d5caef5bSAndrii Nakryiko static void btf_dedup_free(struct btf_dedup *d); 2879f86524efSAndrii Nakryiko static int btf_dedup_prep(struct btf_dedup *d); 2880d5caef5bSAndrii Nakryiko static int btf_dedup_strings(struct btf_dedup *d); 2881d5caef5bSAndrii Nakryiko static int btf_dedup_prim_types(struct btf_dedup *d); 2882d5caef5bSAndrii Nakryiko static int btf_dedup_struct_types(struct btf_dedup *d); 2883d5caef5bSAndrii Nakryiko static int btf_dedup_ref_types(struct btf_dedup *d); 2884*082108fdSEduard Zingerman static int btf_dedup_resolve_fwds(struct btf_dedup *d); 2885d5caef5bSAndrii Nakryiko static int btf_dedup_compact_types(struct btf_dedup *d); 2886d5caef5bSAndrii Nakryiko static int btf_dedup_remap_types(struct btf_dedup *d); 2887d5caef5bSAndrii Nakryiko 2888d5caef5bSAndrii Nakryiko /* 2889d5caef5bSAndrii Nakryiko * Deduplicate BTF types and strings. 2890d5caef5bSAndrii Nakryiko * 2891d5caef5bSAndrii Nakryiko * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF 2892d5caef5bSAndrii Nakryiko * section with all BTF type descriptors and string data. It overwrites that 2893d5caef5bSAndrii Nakryiko * memory in-place with deduplicated types and strings without any loss of 2894d5caef5bSAndrii Nakryiko * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section 2895d5caef5bSAndrii Nakryiko * is provided, all the strings referenced from .BTF.ext section are honored 2896d5caef5bSAndrii Nakryiko * and updated to point to the right offsets after deduplication. 2897d5caef5bSAndrii Nakryiko * 2898d5caef5bSAndrii Nakryiko * If function returns with error, type/string data might be garbled and should 2899d5caef5bSAndrii Nakryiko * be discarded. 2900d5caef5bSAndrii Nakryiko * 2901d5caef5bSAndrii Nakryiko * More verbose and detailed description of both problem btf_dedup is solving, 2902d5caef5bSAndrii Nakryiko * as well as solution could be found at: 2903d5caef5bSAndrii Nakryiko * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html 2904d5caef5bSAndrii Nakryiko * 2905d5caef5bSAndrii Nakryiko * Problem description and justification 2906d5caef5bSAndrii Nakryiko * ===================================== 2907d5caef5bSAndrii Nakryiko * 2908d5caef5bSAndrii Nakryiko * BTF type information is typically emitted either as a result of conversion 2909d5caef5bSAndrii Nakryiko * from DWARF to BTF or directly by compiler. In both cases, each compilation 2910d5caef5bSAndrii Nakryiko * unit contains information about a subset of all the types that are used 2911d5caef5bSAndrii Nakryiko * in an application. These subsets are frequently overlapping and contain a lot 2912d5caef5bSAndrii Nakryiko * of duplicated information when later concatenated together into a single 2913d5caef5bSAndrii Nakryiko * binary. This algorithm ensures that each unique type is represented by single 2914d5caef5bSAndrii Nakryiko * BTF type descriptor, greatly reducing resulting size of BTF data. 2915d5caef5bSAndrii Nakryiko * 2916d5caef5bSAndrii Nakryiko * Compilation unit isolation and subsequent duplication of data is not the only 2917d5caef5bSAndrii Nakryiko * problem. The same type hierarchy (e.g., struct and all the type that struct 2918d5caef5bSAndrii Nakryiko * references) in different compilation units can be represented in BTF to 2919d5caef5bSAndrii Nakryiko * various degrees of completeness (or, rather, incompleteness) due to 2920d5caef5bSAndrii Nakryiko * struct/union forward declarations. 2921d5caef5bSAndrii Nakryiko * 2922d5caef5bSAndrii Nakryiko * Let's take a look at an example, that we'll use to better understand the 2923d5caef5bSAndrii Nakryiko * problem (and solution). Suppose we have two compilation units, each using 2924d5caef5bSAndrii Nakryiko * same `struct S`, but each of them having incomplete type information about 2925d5caef5bSAndrii Nakryiko * struct's fields: 2926d5caef5bSAndrii Nakryiko * 2927d5caef5bSAndrii Nakryiko * // CU #1: 2928d5caef5bSAndrii Nakryiko * struct S; 2929d5caef5bSAndrii Nakryiko * struct A { 2930d5caef5bSAndrii Nakryiko * int a; 2931d5caef5bSAndrii Nakryiko * struct A* self; 2932d5caef5bSAndrii Nakryiko * struct S* parent; 2933d5caef5bSAndrii Nakryiko * }; 2934d5caef5bSAndrii Nakryiko * struct B; 2935d5caef5bSAndrii Nakryiko * struct S { 2936d5caef5bSAndrii Nakryiko * struct A* a_ptr; 2937d5caef5bSAndrii Nakryiko * struct B* b_ptr; 2938d5caef5bSAndrii Nakryiko * }; 2939d5caef5bSAndrii Nakryiko * 2940d5caef5bSAndrii Nakryiko * // CU #2: 2941d5caef5bSAndrii Nakryiko * struct S; 2942d5caef5bSAndrii Nakryiko * struct A; 2943d5caef5bSAndrii Nakryiko * struct B { 2944d5caef5bSAndrii Nakryiko * int b; 2945d5caef5bSAndrii Nakryiko * struct B* self; 2946d5caef5bSAndrii Nakryiko * struct S* parent; 2947d5caef5bSAndrii Nakryiko * }; 2948d5caef5bSAndrii Nakryiko * struct S { 2949d5caef5bSAndrii Nakryiko * struct A* a_ptr; 2950d5caef5bSAndrii Nakryiko * struct B* b_ptr; 2951d5caef5bSAndrii Nakryiko * }; 2952d5caef5bSAndrii Nakryiko * 2953d5caef5bSAndrii Nakryiko * In case of CU #1, BTF data will know only that `struct B` exist (but no 2954d5caef5bSAndrii Nakryiko * more), but will know the complete type information about `struct A`. While 2955d5caef5bSAndrii Nakryiko * for CU #2, it will know full type information about `struct B`, but will 2956d5caef5bSAndrii Nakryiko * only know about forward declaration of `struct A` (in BTF terms, it will 2957d5caef5bSAndrii Nakryiko * have `BTF_KIND_FWD` type descriptor with name `B`). 2958d5caef5bSAndrii Nakryiko * 2959d5caef5bSAndrii Nakryiko * This compilation unit isolation means that it's possible that there is no 2960d5caef5bSAndrii Nakryiko * single CU with complete type information describing structs `S`, `A`, and 2961d5caef5bSAndrii Nakryiko * `B`. Also, we might get tons of duplicated and redundant type information. 2962d5caef5bSAndrii Nakryiko * 2963d5caef5bSAndrii Nakryiko * Additional complication we need to keep in mind comes from the fact that 2964d5caef5bSAndrii Nakryiko * types, in general, can form graphs containing cycles, not just DAGs. 2965d5caef5bSAndrii Nakryiko * 2966d5caef5bSAndrii Nakryiko * While algorithm does deduplication, it also merges and resolves type 2967d5caef5bSAndrii Nakryiko * information (unless disabled throught `struct btf_opts`), whenever possible. 2968d5caef5bSAndrii Nakryiko * E.g., in the example above with two compilation units having partial type 2969d5caef5bSAndrii Nakryiko * information for structs `A` and `B`, the output of algorithm will emit 2970d5caef5bSAndrii Nakryiko * a single copy of each BTF type that describes structs `A`, `B`, and `S` 2971d5caef5bSAndrii Nakryiko * (as well as type information for `int` and pointers), as if they were defined 2972d5caef5bSAndrii Nakryiko * in a single compilation unit as: 2973d5caef5bSAndrii Nakryiko * 2974d5caef5bSAndrii Nakryiko * struct A { 2975d5caef5bSAndrii Nakryiko * int a; 2976d5caef5bSAndrii Nakryiko * struct A* self; 2977d5caef5bSAndrii Nakryiko * struct S* parent; 2978d5caef5bSAndrii Nakryiko * }; 2979d5caef5bSAndrii Nakryiko * struct B { 2980d5caef5bSAndrii Nakryiko * int b; 2981d5caef5bSAndrii Nakryiko * struct B* self; 2982d5caef5bSAndrii Nakryiko * struct S* parent; 2983d5caef5bSAndrii Nakryiko * }; 2984d5caef5bSAndrii Nakryiko * struct S { 2985d5caef5bSAndrii Nakryiko * struct A* a_ptr; 2986d5caef5bSAndrii Nakryiko * struct B* b_ptr; 2987d5caef5bSAndrii Nakryiko * }; 2988d5caef5bSAndrii Nakryiko * 2989d5caef5bSAndrii Nakryiko * Algorithm summary 2990d5caef5bSAndrii Nakryiko * ================= 2991d5caef5bSAndrii Nakryiko * 2992*082108fdSEduard Zingerman * Algorithm completes its work in 7 separate passes: 2993d5caef5bSAndrii Nakryiko * 2994d5caef5bSAndrii Nakryiko * 1. Strings deduplication. 2995d5caef5bSAndrii Nakryiko * 2. Primitive types deduplication (int, enum, fwd). 2996d5caef5bSAndrii Nakryiko * 3. Struct/union types deduplication. 2997*082108fdSEduard Zingerman * 4. Resolve unambiguous forward declarations. 2998*082108fdSEduard Zingerman * 5. Reference types deduplication (pointers, typedefs, arrays, funcs, func 2999d5caef5bSAndrii Nakryiko * protos, and const/volatile/restrict modifiers). 3000*082108fdSEduard Zingerman * 6. Types compaction. 3001*082108fdSEduard Zingerman * 7. Types remapping. 3002d5caef5bSAndrii Nakryiko * 3003d5caef5bSAndrii Nakryiko * Algorithm determines canonical type descriptor, which is a single 3004d5caef5bSAndrii Nakryiko * representative type for each truly unique type. This canonical type is the 3005d5caef5bSAndrii Nakryiko * one that will go into final deduplicated BTF type information. For 3006d5caef5bSAndrii Nakryiko * struct/unions, it is also the type that algorithm will merge additional type 3007d5caef5bSAndrii Nakryiko * information into (while resolving FWDs), as it discovers it from data in 3008d5caef5bSAndrii Nakryiko * other CUs. Each input BTF type eventually gets either mapped to itself, if 3009d5caef5bSAndrii Nakryiko * that type is canonical, or to some other type, if that type is equivalent 3010d5caef5bSAndrii Nakryiko * and was chosen as canonical representative. This mapping is stored in 3011d5caef5bSAndrii Nakryiko * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that 3012d5caef5bSAndrii Nakryiko * FWD type got resolved to. 3013d5caef5bSAndrii Nakryiko * 3014d5caef5bSAndrii Nakryiko * To facilitate fast discovery of canonical types, we also maintain canonical 3015d5caef5bSAndrii Nakryiko * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash 3016d5caef5bSAndrii Nakryiko * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types 3017d5caef5bSAndrii Nakryiko * that match that signature. With sufficiently good choice of type signature 3018d5caef5bSAndrii Nakryiko * hashing function, we can limit number of canonical types for each unique type 3019d5caef5bSAndrii Nakryiko * signature to a very small number, allowing to find canonical type for any 3020d5caef5bSAndrii Nakryiko * duplicated type very quickly. 3021d5caef5bSAndrii Nakryiko * 3022d5caef5bSAndrii Nakryiko * Struct/union deduplication is the most critical part and algorithm for 3023d5caef5bSAndrii Nakryiko * deduplicating structs/unions is described in greater details in comments for 3024d5caef5bSAndrii Nakryiko * `btf_dedup_is_equiv` function. 3025d5caef5bSAndrii Nakryiko */ 3026aaf6886dSAndrii Nakryiko int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) 3027d5caef5bSAndrii Nakryiko { 3028957d350aSAndrii Nakryiko struct btf_dedup *d; 3029d5caef5bSAndrii Nakryiko int err; 3030d5caef5bSAndrii Nakryiko 3031957d350aSAndrii Nakryiko if (!OPTS_VALID(opts, btf_dedup_opts)) 3032957d350aSAndrii Nakryiko return libbpf_err(-EINVAL); 3033957d350aSAndrii Nakryiko 3034957d350aSAndrii Nakryiko d = btf_dedup_new(btf, opts); 3035d5caef5bSAndrii Nakryiko if (IS_ERR(d)) { 3036d5caef5bSAndrii Nakryiko pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d)); 3037e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL); 3038d5caef5bSAndrii Nakryiko } 3039d5caef5bSAndrii Nakryiko 30401000298cSMauricio Vásquez if (btf_ensure_modifiable(btf)) { 30411000298cSMauricio Vásquez err = -ENOMEM; 30421000298cSMauricio Vásquez goto done; 30431000298cSMauricio Vásquez } 3044919d2b1dSAndrii Nakryiko 3045f86524efSAndrii Nakryiko err = btf_dedup_prep(d); 3046f86524efSAndrii Nakryiko if (err) { 3047f86524efSAndrii Nakryiko pr_debug("btf_dedup_prep failed:%d\n", err); 3048f86524efSAndrii Nakryiko goto done; 3049f86524efSAndrii Nakryiko } 3050d5caef5bSAndrii Nakryiko err = btf_dedup_strings(d); 3051d5caef5bSAndrii Nakryiko if (err < 0) { 3052d5caef5bSAndrii Nakryiko pr_debug("btf_dedup_strings failed:%d\n", err); 3053d5caef5bSAndrii Nakryiko goto done; 3054d5caef5bSAndrii Nakryiko } 3055d5caef5bSAndrii Nakryiko err = btf_dedup_prim_types(d); 3056d5caef5bSAndrii Nakryiko if (err < 0) { 3057d5caef5bSAndrii Nakryiko pr_debug("btf_dedup_prim_types failed:%d\n", err); 3058d5caef5bSAndrii Nakryiko goto done; 3059d5caef5bSAndrii Nakryiko } 3060d5caef5bSAndrii Nakryiko err = btf_dedup_struct_types(d); 3061d5caef5bSAndrii Nakryiko if (err < 0) { 3062d5caef5bSAndrii Nakryiko pr_debug("btf_dedup_struct_types failed:%d\n", err); 3063d5caef5bSAndrii Nakryiko goto done; 3064d5caef5bSAndrii Nakryiko } 3065*082108fdSEduard Zingerman err = btf_dedup_resolve_fwds(d); 3066*082108fdSEduard Zingerman if (err < 0) { 3067*082108fdSEduard Zingerman pr_debug("btf_dedup_resolve_fwds failed:%d\n", err); 3068*082108fdSEduard Zingerman goto done; 3069*082108fdSEduard Zingerman } 3070d5caef5bSAndrii Nakryiko err = btf_dedup_ref_types(d); 3071d5caef5bSAndrii Nakryiko if (err < 0) { 3072d5caef5bSAndrii Nakryiko pr_debug("btf_dedup_ref_types failed:%d\n", err); 3073d5caef5bSAndrii Nakryiko goto done; 3074d5caef5bSAndrii Nakryiko } 3075d5caef5bSAndrii Nakryiko err = btf_dedup_compact_types(d); 3076d5caef5bSAndrii Nakryiko if (err < 0) { 3077d5caef5bSAndrii Nakryiko pr_debug("btf_dedup_compact_types failed:%d\n", err); 3078d5caef5bSAndrii Nakryiko goto done; 3079d5caef5bSAndrii Nakryiko } 3080d5caef5bSAndrii Nakryiko err = btf_dedup_remap_types(d); 3081d5caef5bSAndrii Nakryiko if (err < 0) { 3082d5caef5bSAndrii Nakryiko pr_debug("btf_dedup_remap_types failed:%d\n", err); 3083d5caef5bSAndrii Nakryiko goto done; 3084d5caef5bSAndrii Nakryiko } 3085d5caef5bSAndrii Nakryiko 3086d5caef5bSAndrii Nakryiko done: 3087d5caef5bSAndrii Nakryiko btf_dedup_free(d); 3088e9fc3ce9SAndrii Nakryiko return libbpf_err(err); 3089d5caef5bSAndrii Nakryiko } 3090d5caef5bSAndrii Nakryiko 3091d5caef5bSAndrii Nakryiko #define BTF_UNPROCESSED_ID ((__u32)-1) 3092d5caef5bSAndrii Nakryiko #define BTF_IN_PROGRESS_ID ((__u32)-2) 3093d5caef5bSAndrii Nakryiko 3094d5caef5bSAndrii Nakryiko struct btf_dedup { 3095d5caef5bSAndrii Nakryiko /* .BTF section to be deduped in-place */ 3096d5caef5bSAndrii Nakryiko struct btf *btf; 3097d5caef5bSAndrii Nakryiko /* 3098d5caef5bSAndrii Nakryiko * Optional .BTF.ext section. When provided, any strings referenced 3099d5caef5bSAndrii Nakryiko * from it will be taken into account when deduping strings 3100d5caef5bSAndrii Nakryiko */ 3101d5caef5bSAndrii Nakryiko struct btf_ext *btf_ext; 3102d5caef5bSAndrii Nakryiko /* 3103d5caef5bSAndrii Nakryiko * This is a map from any type's signature hash to a list of possible 3104d5caef5bSAndrii Nakryiko * canonical representative type candidates. Hash collisions are 3105d5caef5bSAndrii Nakryiko * ignored, so even types of various kinds can share same list of 3106d5caef5bSAndrii Nakryiko * candidates, which is fine because we rely on subsequent 3107d5caef5bSAndrii Nakryiko * btf_xxx_equal() checks to authoritatively verify type equality. 3108d5caef5bSAndrii Nakryiko */ 31092fc3fc0bSAndrii Nakryiko struct hashmap *dedup_table; 3110d5caef5bSAndrii Nakryiko /* Canonical types map */ 3111d5caef5bSAndrii Nakryiko __u32 *map; 3112d5caef5bSAndrii Nakryiko /* Hypothetical mapping, used during type graph equivalence checks */ 3113d5caef5bSAndrii Nakryiko __u32 *hypot_map; 3114d5caef5bSAndrii Nakryiko __u32 *hypot_list; 3115d5caef5bSAndrii Nakryiko size_t hypot_cnt; 3116d5caef5bSAndrii Nakryiko size_t hypot_cap; 3117f86524efSAndrii Nakryiko /* Whether hypothetical mapping, if successful, would need to adjust 3118f86524efSAndrii Nakryiko * already canonicalized types (due to a new forward declaration to 3119f86524efSAndrii Nakryiko * concrete type resolution). In such case, during split BTF dedup 3120f86524efSAndrii Nakryiko * candidate type would still be considered as different, because base 3121f86524efSAndrii Nakryiko * BTF is considered to be immutable. 3122f86524efSAndrii Nakryiko */ 3123f86524efSAndrii Nakryiko bool hypot_adjust_canon; 3124d5caef5bSAndrii Nakryiko /* Various option modifying behavior of algorithm */ 3125d5caef5bSAndrii Nakryiko struct btf_dedup_opts opts; 312688a82c2aSAndrii Nakryiko /* temporary strings deduplication state */ 312790d76d3eSAndrii Nakryiko struct strset *strs_set; 3128d5caef5bSAndrii Nakryiko }; 3129d5caef5bSAndrii Nakryiko 31302fc3fc0bSAndrii Nakryiko static long hash_combine(long h, long value) 3131d5caef5bSAndrii Nakryiko { 31322fc3fc0bSAndrii Nakryiko return h * 31 + value; 3133d5caef5bSAndrii Nakryiko } 3134d5caef5bSAndrii Nakryiko 31352fc3fc0bSAndrii Nakryiko #define for_each_dedup_cand(d, node, hash) \ 3136c302378bSEduard Zingerman hashmap__for_each_key_entry(d->dedup_table, node, hash) 3137d5caef5bSAndrii Nakryiko 31382fc3fc0bSAndrii Nakryiko static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id) 3139d5caef5bSAndrii Nakryiko { 3140c302378bSEduard Zingerman return hashmap__append(d->dedup_table, hash, type_id); 3141d5caef5bSAndrii Nakryiko } 3142d5caef5bSAndrii Nakryiko 3143d5caef5bSAndrii Nakryiko static int btf_dedup_hypot_map_add(struct btf_dedup *d, 3144d5caef5bSAndrii Nakryiko __u32 from_id, __u32 to_id) 3145d5caef5bSAndrii Nakryiko { 3146d5caef5bSAndrii Nakryiko if (d->hypot_cnt == d->hypot_cap) { 3147d5caef5bSAndrii Nakryiko __u32 *new_list; 3148d5caef5bSAndrii Nakryiko 3149fb2426adSMartin KaFai Lau d->hypot_cap += max((size_t)16, d->hypot_cap / 2); 3150029258d7SAndrii Nakryiko new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32)); 3151d5caef5bSAndrii Nakryiko if (!new_list) 3152d5caef5bSAndrii Nakryiko return -ENOMEM; 3153d5caef5bSAndrii Nakryiko d->hypot_list = new_list; 3154d5caef5bSAndrii Nakryiko } 3155d5caef5bSAndrii Nakryiko d->hypot_list[d->hypot_cnt++] = from_id; 3156d5caef5bSAndrii Nakryiko d->hypot_map[from_id] = to_id; 3157d5caef5bSAndrii Nakryiko return 0; 3158d5caef5bSAndrii Nakryiko } 3159d5caef5bSAndrii Nakryiko 3160d5caef5bSAndrii Nakryiko static void btf_dedup_clear_hypot_map(struct btf_dedup *d) 3161d5caef5bSAndrii Nakryiko { 3162d5caef5bSAndrii Nakryiko int i; 3163d5caef5bSAndrii Nakryiko 3164d5caef5bSAndrii Nakryiko for (i = 0; i < d->hypot_cnt; i++) 3165d5caef5bSAndrii Nakryiko d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID; 3166d5caef5bSAndrii Nakryiko d->hypot_cnt = 0; 3167f86524efSAndrii Nakryiko d->hypot_adjust_canon = false; 3168d5caef5bSAndrii Nakryiko } 3169d5caef5bSAndrii Nakryiko 3170d5caef5bSAndrii Nakryiko static void btf_dedup_free(struct btf_dedup *d) 3171d5caef5bSAndrii Nakryiko { 31722fc3fc0bSAndrii Nakryiko hashmap__free(d->dedup_table); 31732fc3fc0bSAndrii Nakryiko d->dedup_table = NULL; 3174d5caef5bSAndrii Nakryiko 3175d5caef5bSAndrii Nakryiko free(d->map); 3176d5caef5bSAndrii Nakryiko d->map = NULL; 3177d5caef5bSAndrii Nakryiko 3178d5caef5bSAndrii Nakryiko free(d->hypot_map); 3179d5caef5bSAndrii Nakryiko d->hypot_map = NULL; 3180d5caef5bSAndrii Nakryiko 3181d5caef5bSAndrii Nakryiko free(d->hypot_list); 3182d5caef5bSAndrii Nakryiko d->hypot_list = NULL; 3183d5caef5bSAndrii Nakryiko 3184d5caef5bSAndrii Nakryiko free(d); 3185d5caef5bSAndrii Nakryiko } 3186d5caef5bSAndrii Nakryiko 3187c302378bSEduard Zingerman static size_t btf_dedup_identity_hash_fn(long key, void *ctx) 318851edf5f6SAndrii Nakryiko { 3189c302378bSEduard Zingerman return key; 319051edf5f6SAndrii Nakryiko } 319151edf5f6SAndrii Nakryiko 3192c302378bSEduard Zingerman static size_t btf_dedup_collision_hash_fn(long key, void *ctx) 31932fc3fc0bSAndrii Nakryiko { 31942fc3fc0bSAndrii Nakryiko return 0; 31952fc3fc0bSAndrii Nakryiko } 31962fc3fc0bSAndrii Nakryiko 3197c302378bSEduard Zingerman static bool btf_dedup_equal_fn(long k1, long k2, void *ctx) 31982fc3fc0bSAndrii Nakryiko { 31992fc3fc0bSAndrii Nakryiko return k1 == k2; 32002fc3fc0bSAndrii Nakryiko } 320151edf5f6SAndrii Nakryiko 3202957d350aSAndrii Nakryiko static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts) 3203d5caef5bSAndrii Nakryiko { 3204d5caef5bSAndrii Nakryiko struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup)); 32052fc3fc0bSAndrii Nakryiko hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn; 3206f86524efSAndrii Nakryiko int i, err = 0, type_cnt; 3207d5caef5bSAndrii Nakryiko 3208d5caef5bSAndrii Nakryiko if (!d) 3209d5caef5bSAndrii Nakryiko return ERR_PTR(-ENOMEM); 3210d5caef5bSAndrii Nakryiko 3211957d350aSAndrii Nakryiko if (OPTS_GET(opts, force_collisions, false)) 32122fc3fc0bSAndrii Nakryiko hash_fn = btf_dedup_collision_hash_fn; 321351edf5f6SAndrii Nakryiko 3214d5caef5bSAndrii Nakryiko d->btf = btf; 3215957d350aSAndrii Nakryiko d->btf_ext = OPTS_GET(opts, btf_ext, NULL); 3216d5caef5bSAndrii Nakryiko 32172fc3fc0bSAndrii Nakryiko d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); 32182fc3fc0bSAndrii Nakryiko if (IS_ERR(d->dedup_table)) { 32192fc3fc0bSAndrii Nakryiko err = PTR_ERR(d->dedup_table); 32202fc3fc0bSAndrii Nakryiko d->dedup_table = NULL; 3221d5caef5bSAndrii Nakryiko goto done; 3222d5caef5bSAndrii Nakryiko } 3223d5caef5bSAndrii Nakryiko 32246a886de0SHengqi Chen type_cnt = btf__type_cnt(btf); 3225f86524efSAndrii Nakryiko d->map = malloc(sizeof(__u32) * type_cnt); 3226d5caef5bSAndrii Nakryiko if (!d->map) { 3227d5caef5bSAndrii Nakryiko err = -ENOMEM; 3228d5caef5bSAndrii Nakryiko goto done; 3229d5caef5bSAndrii Nakryiko } 3230d5caef5bSAndrii Nakryiko /* special BTF "void" type is made canonical immediately */ 3231d5caef5bSAndrii Nakryiko d->map[0] = 0; 3232f86524efSAndrii Nakryiko for (i = 1; i < type_cnt; i++) { 3233740e69c3SAndrii Nakryiko struct btf_type *t = btf_type_by_id(d->btf, i); 3234189cf5a4SAndrii Nakryiko 3235189cf5a4SAndrii Nakryiko /* VAR and DATASEC are never deduped and are self-canonical */ 3236b03bc685SAndrii Nakryiko if (btf_is_var(t) || btf_is_datasec(t)) 3237189cf5a4SAndrii Nakryiko d->map[i] = i; 3238189cf5a4SAndrii Nakryiko else 3239d5caef5bSAndrii Nakryiko d->map[i] = BTF_UNPROCESSED_ID; 3240189cf5a4SAndrii Nakryiko } 3241d5caef5bSAndrii Nakryiko 3242f86524efSAndrii Nakryiko d->hypot_map = malloc(sizeof(__u32) * type_cnt); 3243d5caef5bSAndrii Nakryiko if (!d->hypot_map) { 3244d5caef5bSAndrii Nakryiko err = -ENOMEM; 3245d5caef5bSAndrii Nakryiko goto done; 3246d5caef5bSAndrii Nakryiko } 3247f86524efSAndrii Nakryiko for (i = 0; i < type_cnt; i++) 3248d5caef5bSAndrii Nakryiko d->hypot_map[i] = BTF_UNPROCESSED_ID; 3249d5caef5bSAndrii Nakryiko 3250d5caef5bSAndrii Nakryiko done: 3251d5caef5bSAndrii Nakryiko if (err) { 3252d5caef5bSAndrii Nakryiko btf_dedup_free(d); 3253d5caef5bSAndrii Nakryiko return ERR_PTR(err); 3254d5caef5bSAndrii Nakryiko } 3255d5caef5bSAndrii Nakryiko 3256d5caef5bSAndrii Nakryiko return d; 3257d5caef5bSAndrii Nakryiko } 3258d5caef5bSAndrii Nakryiko 3259d5caef5bSAndrii Nakryiko /* 3260d5caef5bSAndrii Nakryiko * Iterate over all possible places in .BTF and .BTF.ext that can reference 3261d5caef5bSAndrii Nakryiko * string and pass pointer to it to a provided callback `fn`. 3262d5caef5bSAndrii Nakryiko */ 3263f36e99a4SAndrii Nakryiko static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx) 3264d5caef5bSAndrii Nakryiko { 3265f36e99a4SAndrii Nakryiko int i, r; 3266d5caef5bSAndrii Nakryiko 3267f86524efSAndrii Nakryiko for (i = 0; i < d->btf->nr_types; i++) { 3268f36e99a4SAndrii Nakryiko struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 3269f36e99a4SAndrii Nakryiko 3270f36e99a4SAndrii Nakryiko r = btf_type_visit_str_offs(t, fn, ctx); 3271d5caef5bSAndrii Nakryiko if (r) 3272d5caef5bSAndrii Nakryiko return r; 3273d5caef5bSAndrii Nakryiko } 3274d5caef5bSAndrii Nakryiko 3275d5caef5bSAndrii Nakryiko if (!d->btf_ext) 3276d5caef5bSAndrii Nakryiko return 0; 3277d5caef5bSAndrii Nakryiko 3278f36e99a4SAndrii Nakryiko r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); 3279d5caef5bSAndrii Nakryiko if (r) 3280d5caef5bSAndrii Nakryiko return r; 3281d5caef5bSAndrii Nakryiko 3282d5caef5bSAndrii Nakryiko return 0; 3283d5caef5bSAndrii Nakryiko } 3284d5caef5bSAndrii Nakryiko 328588a82c2aSAndrii Nakryiko static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx) 3286d5caef5bSAndrii Nakryiko { 328788a82c2aSAndrii Nakryiko struct btf_dedup *d = ctx; 3288f86524efSAndrii Nakryiko __u32 str_off = *str_off_ptr; 328988a82c2aSAndrii Nakryiko const char *s; 329090d76d3eSAndrii Nakryiko int off, err; 3291d5caef5bSAndrii Nakryiko 3292f86524efSAndrii Nakryiko /* don't touch empty string or string in main BTF */ 3293f86524efSAndrii Nakryiko if (str_off == 0 || str_off < d->btf->start_str_off) 3294d5caef5bSAndrii Nakryiko return 0; 3295d5caef5bSAndrii Nakryiko 3296f86524efSAndrii Nakryiko s = btf__str_by_offset(d->btf, str_off); 3297f86524efSAndrii Nakryiko if (d->btf->base_btf) { 3298f86524efSAndrii Nakryiko err = btf__find_str(d->btf->base_btf, s); 3299f86524efSAndrii Nakryiko if (err >= 0) { 3300f86524efSAndrii Nakryiko *str_off_ptr = err; 3301f86524efSAndrii Nakryiko return 0; 3302f86524efSAndrii Nakryiko } 3303f86524efSAndrii Nakryiko if (err != -ENOENT) 3304f86524efSAndrii Nakryiko return err; 3305f86524efSAndrii Nakryiko } 3306f86524efSAndrii Nakryiko 330790d76d3eSAndrii Nakryiko off = strset__add_str(d->strs_set, s); 330890d76d3eSAndrii Nakryiko if (off < 0) 330990d76d3eSAndrii Nakryiko return off; 331088a82c2aSAndrii Nakryiko 331190d76d3eSAndrii Nakryiko *str_off_ptr = d->btf->start_str_off + off; 3312d5caef5bSAndrii Nakryiko return 0; 3313d5caef5bSAndrii Nakryiko } 3314d5caef5bSAndrii Nakryiko 3315d5caef5bSAndrii Nakryiko /* 3316d5caef5bSAndrii Nakryiko * Dedup string and filter out those that are not referenced from either .BTF 3317d5caef5bSAndrii Nakryiko * or .BTF.ext (if provided) sections. 3318d5caef5bSAndrii Nakryiko * 3319d5caef5bSAndrii Nakryiko * This is done by building index of all strings in BTF's string section, 3320d5caef5bSAndrii Nakryiko * then iterating over all entities that can reference strings (e.g., type 3321d5caef5bSAndrii Nakryiko * names, struct field names, .BTF.ext line info, etc) and marking corresponding 3322d5caef5bSAndrii Nakryiko * strings as used. After that all used strings are deduped and compacted into 3323d5caef5bSAndrii Nakryiko * sequential blob of memory and new offsets are calculated. Then all the string 3324d5caef5bSAndrii Nakryiko * references are iterated again and rewritten using new offsets. 3325d5caef5bSAndrii Nakryiko */ 3326d5caef5bSAndrii Nakryiko static int btf_dedup_strings(struct btf_dedup *d) 3327d5caef5bSAndrii Nakryiko { 332888a82c2aSAndrii Nakryiko int err; 3329d5caef5bSAndrii Nakryiko 3330919d2b1dSAndrii Nakryiko if (d->btf->strs_deduped) 3331919d2b1dSAndrii Nakryiko return 0; 3332919d2b1dSAndrii Nakryiko 333390d76d3eSAndrii Nakryiko d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); 333490d76d3eSAndrii Nakryiko if (IS_ERR(d->strs_set)) { 333590d76d3eSAndrii Nakryiko err = PTR_ERR(d->strs_set); 333688a82c2aSAndrii Nakryiko goto err_out; 3337d5caef5bSAndrii Nakryiko } 3338d5caef5bSAndrii Nakryiko 3339f86524efSAndrii Nakryiko if (!d->btf->base_btf) { 334088a82c2aSAndrii Nakryiko /* insert empty string; we won't be looking it up during strings 334188a82c2aSAndrii Nakryiko * dedup, but it's good to have it for generic BTF string lookups 334288a82c2aSAndrii Nakryiko */ 334390d76d3eSAndrii Nakryiko err = strset__add_str(d->strs_set, ""); 334490d76d3eSAndrii Nakryiko if (err < 0) 334588a82c2aSAndrii Nakryiko goto err_out; 3346f86524efSAndrii Nakryiko } 3347d5caef5bSAndrii Nakryiko 3348d5caef5bSAndrii Nakryiko /* remap string offsets */ 334988a82c2aSAndrii Nakryiko err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d); 3350d5caef5bSAndrii Nakryiko if (err) 335188a82c2aSAndrii Nakryiko goto err_out; 3352d5caef5bSAndrii Nakryiko 335388a82c2aSAndrii Nakryiko /* replace BTF string data and hash with deduped ones */ 335490d76d3eSAndrii Nakryiko strset__free(d->btf->strs_set); 335590d76d3eSAndrii Nakryiko d->btf->hdr->str_len = strset__data_size(d->strs_set); 335690d76d3eSAndrii Nakryiko d->btf->strs_set = d->strs_set; 335790d76d3eSAndrii Nakryiko d->strs_set = NULL; 3358919d2b1dSAndrii Nakryiko d->btf->strs_deduped = true; 335988a82c2aSAndrii Nakryiko return 0; 3360d5caef5bSAndrii Nakryiko 336188a82c2aSAndrii Nakryiko err_out: 336290d76d3eSAndrii Nakryiko strset__free(d->strs_set); 336390d76d3eSAndrii Nakryiko d->strs_set = NULL; 336488a82c2aSAndrii Nakryiko 3365d5caef5bSAndrii Nakryiko return err; 3366d5caef5bSAndrii Nakryiko } 3367d5caef5bSAndrii Nakryiko 33682fc3fc0bSAndrii Nakryiko static long btf_hash_common(struct btf_type *t) 3369d5caef5bSAndrii Nakryiko { 33702fc3fc0bSAndrii Nakryiko long h; 3371d5caef5bSAndrii Nakryiko 3372d5caef5bSAndrii Nakryiko h = hash_combine(0, t->name_off); 3373d5caef5bSAndrii Nakryiko h = hash_combine(h, t->info); 3374d5caef5bSAndrii Nakryiko h = hash_combine(h, t->size); 3375d5caef5bSAndrii Nakryiko return h; 3376d5caef5bSAndrii Nakryiko } 3377d5caef5bSAndrii Nakryiko 3378d5caef5bSAndrii Nakryiko static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2) 3379d5caef5bSAndrii Nakryiko { 3380d5caef5bSAndrii Nakryiko return t1->name_off == t2->name_off && 3381d5caef5bSAndrii Nakryiko t1->info == t2->info && 3382d5caef5bSAndrii Nakryiko t1->size == t2->size; 3383d5caef5bSAndrii Nakryiko } 3384d5caef5bSAndrii Nakryiko 338530025e8bSYonghong Song /* Calculate type signature hash of INT or TAG. */ 3386223f903eSYonghong Song static long btf_hash_int_decl_tag(struct btf_type *t) 3387d5caef5bSAndrii Nakryiko { 3388d5caef5bSAndrii Nakryiko __u32 info = *(__u32 *)(t + 1); 33892fc3fc0bSAndrii Nakryiko long h; 3390d5caef5bSAndrii Nakryiko 3391d5caef5bSAndrii Nakryiko h = btf_hash_common(t); 3392d5caef5bSAndrii Nakryiko h = hash_combine(h, info); 3393d5caef5bSAndrii Nakryiko return h; 3394d5caef5bSAndrii Nakryiko } 3395d5caef5bSAndrii Nakryiko 339630025e8bSYonghong Song /* Check structural equality of two INTs or TAGs. */ 339730025e8bSYonghong Song static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2) 3398d5caef5bSAndrii Nakryiko { 3399d5caef5bSAndrii Nakryiko __u32 info1, info2; 3400d5caef5bSAndrii Nakryiko 3401d5caef5bSAndrii Nakryiko if (!btf_equal_common(t1, t2)) 3402d5caef5bSAndrii Nakryiko return false; 3403d5caef5bSAndrii Nakryiko info1 = *(__u32 *)(t1 + 1); 3404d5caef5bSAndrii Nakryiko info2 = *(__u32 *)(t2 + 1); 3405d5caef5bSAndrii Nakryiko return info1 == info2; 3406d5caef5bSAndrii Nakryiko } 3407d5caef5bSAndrii Nakryiko 34082ef20263SYonghong Song /* Calculate type signature hash of ENUM/ENUM64. */ 34092fc3fc0bSAndrii Nakryiko static long btf_hash_enum(struct btf_type *t) 3410d5caef5bSAndrii Nakryiko { 34112fc3fc0bSAndrii Nakryiko long h; 3412d5caef5bSAndrii Nakryiko 3413de048b6eSEduard Zingerman /* don't hash vlen, enum members and size to support enum fwd resolving */ 34149768095bSAndrii Nakryiko h = hash_combine(0, t->name_off); 3415d5caef5bSAndrii Nakryiko return h; 3416d5caef5bSAndrii Nakryiko } 3417d5caef5bSAndrii Nakryiko 3418de048b6eSEduard Zingerman static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2) 3419d5caef5bSAndrii Nakryiko { 3420b03bc685SAndrii Nakryiko const struct btf_enum *m1, *m2; 3421d5caef5bSAndrii Nakryiko __u16 vlen; 3422d5caef5bSAndrii Nakryiko int i; 3423d5caef5bSAndrii Nakryiko 3424b03bc685SAndrii Nakryiko vlen = btf_vlen(t1); 3425b03bc685SAndrii Nakryiko m1 = btf_enum(t1); 3426b03bc685SAndrii Nakryiko m2 = btf_enum(t2); 3427d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 3428d5caef5bSAndrii Nakryiko if (m1->name_off != m2->name_off || m1->val != m2->val) 3429d5caef5bSAndrii Nakryiko return false; 3430d5caef5bSAndrii Nakryiko m1++; 3431d5caef5bSAndrii Nakryiko m2++; 3432d5caef5bSAndrii Nakryiko } 3433d5caef5bSAndrii Nakryiko return true; 3434d5caef5bSAndrii Nakryiko } 3435d5caef5bSAndrii Nakryiko 3436de048b6eSEduard Zingerman static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2) 34372ef20263SYonghong Song { 34382ef20263SYonghong Song const struct btf_enum64 *m1, *m2; 34392ef20263SYonghong Song __u16 vlen; 34402ef20263SYonghong Song int i; 34412ef20263SYonghong Song 34422ef20263SYonghong Song vlen = btf_vlen(t1); 34432ef20263SYonghong Song m1 = btf_enum64(t1); 34442ef20263SYonghong Song m2 = btf_enum64(t2); 34452ef20263SYonghong Song for (i = 0; i < vlen; i++) { 34462ef20263SYonghong Song if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || 34472ef20263SYonghong Song m1->val_hi32 != m2->val_hi32) 34482ef20263SYonghong Song return false; 34492ef20263SYonghong Song m1++; 34502ef20263SYonghong Song m2++; 34512ef20263SYonghong Song } 34522ef20263SYonghong Song return true; 34532ef20263SYonghong Song } 34542ef20263SYonghong Song 3455de048b6eSEduard Zingerman /* Check structural equality of two ENUMs or ENUM64s. */ 3456de048b6eSEduard Zingerman static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2) 3457de048b6eSEduard Zingerman { 3458de048b6eSEduard Zingerman if (!btf_equal_common(t1, t2)) 3459de048b6eSEduard Zingerman return false; 3460de048b6eSEduard Zingerman 3461de048b6eSEduard Zingerman /* t1 & t2 kinds are identical because of btf_equal_common */ 3462de048b6eSEduard Zingerman if (btf_kind(t1) == BTF_KIND_ENUM) 3463de048b6eSEduard Zingerman return btf_equal_enum_members(t1, t2); 3464de048b6eSEduard Zingerman else 3465de048b6eSEduard Zingerman return btf_equal_enum64_members(t1, t2); 3466de048b6eSEduard Zingerman } 3467de048b6eSEduard Zingerman 34689768095bSAndrii Nakryiko static inline bool btf_is_enum_fwd(struct btf_type *t) 34699768095bSAndrii Nakryiko { 34702ef20263SYonghong Song return btf_is_any_enum(t) && btf_vlen(t) == 0; 34719768095bSAndrii Nakryiko } 34729768095bSAndrii Nakryiko 34739768095bSAndrii Nakryiko static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) 34749768095bSAndrii Nakryiko { 34759768095bSAndrii Nakryiko if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2)) 34769768095bSAndrii Nakryiko return btf_equal_enum(t1, t2); 3477de048b6eSEduard Zingerman /* At this point either t1 or t2 or both are forward declarations, thus: 3478de048b6eSEduard Zingerman * - skip comparing vlen because it is zero for forward declarations; 3479de048b6eSEduard Zingerman * - skip comparing size to allow enum forward declarations 3480de048b6eSEduard Zingerman * to be compatible with enum64 full declarations; 3481de048b6eSEduard Zingerman * - skip comparing kind for the same reason. 3482de048b6eSEduard Zingerman */ 34839768095bSAndrii Nakryiko return t1->name_off == t2->name_off && 3484de048b6eSEduard Zingerman btf_is_any_enum(t1) && btf_is_any_enum(t2); 34852ef20263SYonghong Song } 34862ef20263SYonghong Song 3487d5caef5bSAndrii Nakryiko /* 3488d5caef5bSAndrii Nakryiko * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, 3489d5caef5bSAndrii Nakryiko * as referenced type IDs equivalence is established separately during type 3490d5caef5bSAndrii Nakryiko * graph equivalence check algorithm. 3491d5caef5bSAndrii Nakryiko */ 34922fc3fc0bSAndrii Nakryiko static long btf_hash_struct(struct btf_type *t) 3493d5caef5bSAndrii Nakryiko { 3494b03bc685SAndrii Nakryiko const struct btf_member *member = btf_members(t); 3495b03bc685SAndrii Nakryiko __u32 vlen = btf_vlen(t); 34962fc3fc0bSAndrii Nakryiko long h = btf_hash_common(t); 3497d5caef5bSAndrii Nakryiko int i; 3498d5caef5bSAndrii Nakryiko 3499d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 3500d5caef5bSAndrii Nakryiko h = hash_combine(h, member->name_off); 3501d5caef5bSAndrii Nakryiko h = hash_combine(h, member->offset); 3502d5caef5bSAndrii Nakryiko /* no hashing of referenced type ID, it can be unresolved yet */ 3503d5caef5bSAndrii Nakryiko member++; 3504d5caef5bSAndrii Nakryiko } 3505d5caef5bSAndrii Nakryiko return h; 3506d5caef5bSAndrii Nakryiko } 3507d5caef5bSAndrii Nakryiko 3508d5caef5bSAndrii Nakryiko /* 3509efdd3eb8SAndrii Nakryiko * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced 3510efdd3eb8SAndrii Nakryiko * type IDs. This check is performed during type graph equivalence check and 3511d5caef5bSAndrii Nakryiko * referenced types equivalence is checked separately. 3512d5caef5bSAndrii Nakryiko */ 351391097fbeSAndrii Nakryiko static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2) 3514d5caef5bSAndrii Nakryiko { 3515b03bc685SAndrii Nakryiko const struct btf_member *m1, *m2; 3516d5caef5bSAndrii Nakryiko __u16 vlen; 3517d5caef5bSAndrii Nakryiko int i; 3518d5caef5bSAndrii Nakryiko 3519d5caef5bSAndrii Nakryiko if (!btf_equal_common(t1, t2)) 3520d5caef5bSAndrii Nakryiko return false; 3521d5caef5bSAndrii Nakryiko 3522b03bc685SAndrii Nakryiko vlen = btf_vlen(t1); 3523b03bc685SAndrii Nakryiko m1 = btf_members(t1); 3524b03bc685SAndrii Nakryiko m2 = btf_members(t2); 3525d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 3526d5caef5bSAndrii Nakryiko if (m1->name_off != m2->name_off || m1->offset != m2->offset) 3527d5caef5bSAndrii Nakryiko return false; 3528d5caef5bSAndrii Nakryiko m1++; 3529d5caef5bSAndrii Nakryiko m2++; 3530d5caef5bSAndrii Nakryiko } 3531d5caef5bSAndrii Nakryiko return true; 3532d5caef5bSAndrii Nakryiko } 3533d5caef5bSAndrii Nakryiko 3534d5caef5bSAndrii Nakryiko /* 3535d5caef5bSAndrii Nakryiko * Calculate type signature hash of ARRAY, including referenced type IDs, 3536d5caef5bSAndrii Nakryiko * under assumption that they were already resolved to canonical type IDs and 3537d5caef5bSAndrii Nakryiko * are not going to change. 3538d5caef5bSAndrii Nakryiko */ 35392fc3fc0bSAndrii Nakryiko static long btf_hash_array(struct btf_type *t) 3540d5caef5bSAndrii Nakryiko { 3541b03bc685SAndrii Nakryiko const struct btf_array *info = btf_array(t); 35422fc3fc0bSAndrii Nakryiko long h = btf_hash_common(t); 3543d5caef5bSAndrii Nakryiko 3544d5caef5bSAndrii Nakryiko h = hash_combine(h, info->type); 3545d5caef5bSAndrii Nakryiko h = hash_combine(h, info->index_type); 3546d5caef5bSAndrii Nakryiko h = hash_combine(h, info->nelems); 3547d5caef5bSAndrii Nakryiko return h; 3548d5caef5bSAndrii Nakryiko } 3549d5caef5bSAndrii Nakryiko 3550d5caef5bSAndrii Nakryiko /* 3551d5caef5bSAndrii Nakryiko * Check exact equality of two ARRAYs, taking into account referenced 3552d5caef5bSAndrii Nakryiko * type IDs, under assumption that they were already resolved to canonical 3553d5caef5bSAndrii Nakryiko * type IDs and are not going to change. 3554d5caef5bSAndrii Nakryiko * This function is called during reference types deduplication to compare 3555d5caef5bSAndrii Nakryiko * ARRAY to potential canonical representative. 3556d5caef5bSAndrii Nakryiko */ 3557d5caef5bSAndrii Nakryiko static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2) 3558d5caef5bSAndrii Nakryiko { 3559b03bc685SAndrii Nakryiko const struct btf_array *info1, *info2; 3560d5caef5bSAndrii Nakryiko 3561d5caef5bSAndrii Nakryiko if (!btf_equal_common(t1, t2)) 3562d5caef5bSAndrii Nakryiko return false; 3563d5caef5bSAndrii Nakryiko 3564b03bc685SAndrii Nakryiko info1 = btf_array(t1); 3565b03bc685SAndrii Nakryiko info2 = btf_array(t2); 3566d5caef5bSAndrii Nakryiko return info1->type == info2->type && 3567d5caef5bSAndrii Nakryiko info1->index_type == info2->index_type && 3568d5caef5bSAndrii Nakryiko info1->nelems == info2->nelems; 3569d5caef5bSAndrii Nakryiko } 3570d5caef5bSAndrii Nakryiko 3571d5caef5bSAndrii Nakryiko /* 3572d5caef5bSAndrii Nakryiko * Check structural compatibility of two ARRAYs, ignoring referenced type 3573d5caef5bSAndrii Nakryiko * IDs. This check is performed during type graph equivalence check and 3574d5caef5bSAndrii Nakryiko * referenced types equivalence is checked separately. 3575d5caef5bSAndrii Nakryiko */ 3576d5caef5bSAndrii Nakryiko static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2) 3577d5caef5bSAndrii Nakryiko { 3578d5caef5bSAndrii Nakryiko if (!btf_equal_common(t1, t2)) 3579d5caef5bSAndrii Nakryiko return false; 3580d5caef5bSAndrii Nakryiko 3581b03bc685SAndrii Nakryiko return btf_array(t1)->nelems == btf_array(t2)->nelems; 3582d5caef5bSAndrii Nakryiko } 3583d5caef5bSAndrii Nakryiko 3584d5caef5bSAndrii Nakryiko /* 3585d5caef5bSAndrii Nakryiko * Calculate type signature hash of FUNC_PROTO, including referenced type IDs, 3586d5caef5bSAndrii Nakryiko * under assumption that they were already resolved to canonical type IDs and 3587d5caef5bSAndrii Nakryiko * are not going to change. 3588d5caef5bSAndrii Nakryiko */ 35892fc3fc0bSAndrii Nakryiko static long btf_hash_fnproto(struct btf_type *t) 3590d5caef5bSAndrii Nakryiko { 3591b03bc685SAndrii Nakryiko const struct btf_param *member = btf_params(t); 3592b03bc685SAndrii Nakryiko __u16 vlen = btf_vlen(t); 35932fc3fc0bSAndrii Nakryiko long h = btf_hash_common(t); 3594d5caef5bSAndrii Nakryiko int i; 3595d5caef5bSAndrii Nakryiko 3596d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 3597d5caef5bSAndrii Nakryiko h = hash_combine(h, member->name_off); 3598d5caef5bSAndrii Nakryiko h = hash_combine(h, member->type); 3599d5caef5bSAndrii Nakryiko member++; 3600d5caef5bSAndrii Nakryiko } 3601d5caef5bSAndrii Nakryiko return h; 3602d5caef5bSAndrii Nakryiko } 3603d5caef5bSAndrii Nakryiko 3604d5caef5bSAndrii Nakryiko /* 3605d5caef5bSAndrii Nakryiko * Check exact equality of two FUNC_PROTOs, taking into account referenced 3606d5caef5bSAndrii Nakryiko * type IDs, under assumption that they were already resolved to canonical 3607d5caef5bSAndrii Nakryiko * type IDs and are not going to change. 3608d5caef5bSAndrii Nakryiko * This function is called during reference types deduplication to compare 3609d5caef5bSAndrii Nakryiko * FUNC_PROTO to potential canonical representative. 3610d5caef5bSAndrii Nakryiko */ 36112fc3fc0bSAndrii Nakryiko static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) 3612d5caef5bSAndrii Nakryiko { 3613b03bc685SAndrii Nakryiko const struct btf_param *m1, *m2; 3614d5caef5bSAndrii Nakryiko __u16 vlen; 3615d5caef5bSAndrii Nakryiko int i; 3616d5caef5bSAndrii Nakryiko 3617d5caef5bSAndrii Nakryiko if (!btf_equal_common(t1, t2)) 3618d5caef5bSAndrii Nakryiko return false; 3619d5caef5bSAndrii Nakryiko 3620b03bc685SAndrii Nakryiko vlen = btf_vlen(t1); 3621b03bc685SAndrii Nakryiko m1 = btf_params(t1); 3622b03bc685SAndrii Nakryiko m2 = btf_params(t2); 3623d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 3624d5caef5bSAndrii Nakryiko if (m1->name_off != m2->name_off || m1->type != m2->type) 3625d5caef5bSAndrii Nakryiko return false; 3626d5caef5bSAndrii Nakryiko m1++; 3627d5caef5bSAndrii Nakryiko m2++; 3628d5caef5bSAndrii Nakryiko } 3629d5caef5bSAndrii Nakryiko return true; 3630d5caef5bSAndrii Nakryiko } 3631d5caef5bSAndrii Nakryiko 3632d5caef5bSAndrii Nakryiko /* 3633d5caef5bSAndrii Nakryiko * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type 3634d5caef5bSAndrii Nakryiko * IDs. This check is performed during type graph equivalence check and 3635d5caef5bSAndrii Nakryiko * referenced types equivalence is checked separately. 3636d5caef5bSAndrii Nakryiko */ 36372fc3fc0bSAndrii Nakryiko static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) 3638d5caef5bSAndrii Nakryiko { 3639b03bc685SAndrii Nakryiko const struct btf_param *m1, *m2; 3640d5caef5bSAndrii Nakryiko __u16 vlen; 3641d5caef5bSAndrii Nakryiko int i; 3642d5caef5bSAndrii Nakryiko 3643d5caef5bSAndrii Nakryiko /* skip return type ID */ 3644d5caef5bSAndrii Nakryiko if (t1->name_off != t2->name_off || t1->info != t2->info) 3645d5caef5bSAndrii Nakryiko return false; 3646d5caef5bSAndrii Nakryiko 3647b03bc685SAndrii Nakryiko vlen = btf_vlen(t1); 3648b03bc685SAndrii Nakryiko m1 = btf_params(t1); 3649b03bc685SAndrii Nakryiko m2 = btf_params(t2); 3650d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 3651d5caef5bSAndrii Nakryiko if (m1->name_off != m2->name_off) 3652d5caef5bSAndrii Nakryiko return false; 3653d5caef5bSAndrii Nakryiko m1++; 3654d5caef5bSAndrii Nakryiko m2++; 3655d5caef5bSAndrii Nakryiko } 3656d5caef5bSAndrii Nakryiko return true; 3657d5caef5bSAndrii Nakryiko } 3658d5caef5bSAndrii Nakryiko 3659f86524efSAndrii Nakryiko /* Prepare split BTF for deduplication by calculating hashes of base BTF's 3660f86524efSAndrii Nakryiko * types and initializing the rest of the state (canonical type mapping) for 3661f86524efSAndrii Nakryiko * the fixed base BTF part. 3662f86524efSAndrii Nakryiko */ 3663f86524efSAndrii Nakryiko static int btf_dedup_prep(struct btf_dedup *d) 3664f86524efSAndrii Nakryiko { 3665f86524efSAndrii Nakryiko struct btf_type *t; 3666f86524efSAndrii Nakryiko int type_id; 3667f86524efSAndrii Nakryiko long h; 3668f86524efSAndrii Nakryiko 3669f86524efSAndrii Nakryiko if (!d->btf->base_btf) 3670f86524efSAndrii Nakryiko return 0; 3671f86524efSAndrii Nakryiko 3672f86524efSAndrii Nakryiko for (type_id = 1; type_id < d->btf->start_id; type_id++) { 3673f86524efSAndrii Nakryiko t = btf_type_by_id(d->btf, type_id); 3674f86524efSAndrii Nakryiko 3675f86524efSAndrii Nakryiko /* all base BTF types are self-canonical by definition */ 3676f86524efSAndrii Nakryiko d->map[type_id] = type_id; 3677f86524efSAndrii Nakryiko 3678f86524efSAndrii Nakryiko switch (btf_kind(t)) { 3679f86524efSAndrii Nakryiko case BTF_KIND_VAR: 3680f86524efSAndrii Nakryiko case BTF_KIND_DATASEC: 3681f86524efSAndrii Nakryiko /* VAR and DATASEC are never hash/deduplicated */ 3682f86524efSAndrii Nakryiko continue; 3683f86524efSAndrii Nakryiko case BTF_KIND_CONST: 3684f86524efSAndrii Nakryiko case BTF_KIND_VOLATILE: 3685f86524efSAndrii Nakryiko case BTF_KIND_RESTRICT: 3686f86524efSAndrii Nakryiko case BTF_KIND_PTR: 3687f86524efSAndrii Nakryiko case BTF_KIND_FWD: 3688f86524efSAndrii Nakryiko case BTF_KIND_TYPEDEF: 3689f86524efSAndrii Nakryiko case BTF_KIND_FUNC: 369022541a9eSIlya Leoshkevich case BTF_KIND_FLOAT: 36912dc1e488SYonghong Song case BTF_KIND_TYPE_TAG: 3692f86524efSAndrii Nakryiko h = btf_hash_common(t); 3693f86524efSAndrii Nakryiko break; 3694f86524efSAndrii Nakryiko case BTF_KIND_INT: 3695223f903eSYonghong Song case BTF_KIND_DECL_TAG: 3696223f903eSYonghong Song h = btf_hash_int_decl_tag(t); 3697f86524efSAndrii Nakryiko break; 3698f86524efSAndrii Nakryiko case BTF_KIND_ENUM: 36992ef20263SYonghong Song case BTF_KIND_ENUM64: 3700f86524efSAndrii Nakryiko h = btf_hash_enum(t); 3701f86524efSAndrii Nakryiko break; 3702f86524efSAndrii Nakryiko case BTF_KIND_STRUCT: 3703f86524efSAndrii Nakryiko case BTF_KIND_UNION: 3704f86524efSAndrii Nakryiko h = btf_hash_struct(t); 3705f86524efSAndrii Nakryiko break; 3706f86524efSAndrii Nakryiko case BTF_KIND_ARRAY: 3707f86524efSAndrii Nakryiko h = btf_hash_array(t); 3708f86524efSAndrii Nakryiko break; 3709f86524efSAndrii Nakryiko case BTF_KIND_FUNC_PROTO: 3710f86524efSAndrii Nakryiko h = btf_hash_fnproto(t); 3711f86524efSAndrii Nakryiko break; 3712f86524efSAndrii Nakryiko default: 3713f86524efSAndrii Nakryiko pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id); 3714f86524efSAndrii Nakryiko return -EINVAL; 3715f86524efSAndrii Nakryiko } 3716f86524efSAndrii Nakryiko if (btf_dedup_table_add(d, h, type_id)) 3717f86524efSAndrii Nakryiko return -ENOMEM; 3718f86524efSAndrii Nakryiko } 3719f86524efSAndrii Nakryiko 3720f86524efSAndrii Nakryiko return 0; 3721f86524efSAndrii Nakryiko } 3722f86524efSAndrii Nakryiko 3723d5caef5bSAndrii Nakryiko /* 3724d5caef5bSAndrii Nakryiko * Deduplicate primitive types, that can't reference other types, by calculating 3725d5caef5bSAndrii Nakryiko * their type signature hash and comparing them with any possible canonical 3726d5caef5bSAndrii Nakryiko * candidate. If no canonical candidate matches, type itself is marked as 3727d5caef5bSAndrii Nakryiko * canonical and is added into `btf_dedup->dedup_table` as another candidate. 3728d5caef5bSAndrii Nakryiko */ 3729d5caef5bSAndrii Nakryiko static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) 3730d5caef5bSAndrii Nakryiko { 3731740e69c3SAndrii Nakryiko struct btf_type *t = btf_type_by_id(d->btf, type_id); 37322fc3fc0bSAndrii Nakryiko struct hashmap_entry *hash_entry; 3733d5caef5bSAndrii Nakryiko struct btf_type *cand; 3734d5caef5bSAndrii Nakryiko /* if we don't find equivalent type, then we are canonical */ 3735d5caef5bSAndrii Nakryiko __u32 new_id = type_id; 37362fc3fc0bSAndrii Nakryiko __u32 cand_id; 37372fc3fc0bSAndrii Nakryiko long h; 3738d5caef5bSAndrii Nakryiko 3739b03bc685SAndrii Nakryiko switch (btf_kind(t)) { 3740d5caef5bSAndrii Nakryiko case BTF_KIND_CONST: 3741d5caef5bSAndrii Nakryiko case BTF_KIND_VOLATILE: 3742d5caef5bSAndrii Nakryiko case BTF_KIND_RESTRICT: 3743d5caef5bSAndrii Nakryiko case BTF_KIND_PTR: 3744d5caef5bSAndrii Nakryiko case BTF_KIND_TYPEDEF: 3745d5caef5bSAndrii Nakryiko case BTF_KIND_ARRAY: 3746d5caef5bSAndrii Nakryiko case BTF_KIND_STRUCT: 3747d5caef5bSAndrii Nakryiko case BTF_KIND_UNION: 3748d5caef5bSAndrii Nakryiko case BTF_KIND_FUNC: 3749d5caef5bSAndrii Nakryiko case BTF_KIND_FUNC_PROTO: 3750189cf5a4SAndrii Nakryiko case BTF_KIND_VAR: 3751189cf5a4SAndrii Nakryiko case BTF_KIND_DATASEC: 3752223f903eSYonghong Song case BTF_KIND_DECL_TAG: 37532dc1e488SYonghong Song case BTF_KIND_TYPE_TAG: 3754d5caef5bSAndrii Nakryiko return 0; 3755d5caef5bSAndrii Nakryiko 3756d5caef5bSAndrii Nakryiko case BTF_KIND_INT: 3757223f903eSYonghong Song h = btf_hash_int_decl_tag(t); 37582fc3fc0bSAndrii Nakryiko for_each_dedup_cand(d, hash_entry, h) { 3759c302378bSEduard Zingerman cand_id = hash_entry->value; 3760740e69c3SAndrii Nakryiko cand = btf_type_by_id(d->btf, cand_id); 376130025e8bSYonghong Song if (btf_equal_int_tag(t, cand)) { 37622fc3fc0bSAndrii Nakryiko new_id = cand_id; 3763d5caef5bSAndrii Nakryiko break; 3764d5caef5bSAndrii Nakryiko } 3765d5caef5bSAndrii Nakryiko } 3766d5caef5bSAndrii Nakryiko break; 3767d5caef5bSAndrii Nakryiko 3768d5caef5bSAndrii Nakryiko case BTF_KIND_ENUM: 3769de048b6eSEduard Zingerman case BTF_KIND_ENUM64: 3770d5caef5bSAndrii Nakryiko h = btf_hash_enum(t); 37712fc3fc0bSAndrii Nakryiko for_each_dedup_cand(d, hash_entry, h) { 3772c302378bSEduard Zingerman cand_id = hash_entry->value; 3773740e69c3SAndrii Nakryiko cand = btf_type_by_id(d->btf, cand_id); 3774d5caef5bSAndrii Nakryiko if (btf_equal_enum(t, cand)) { 37752fc3fc0bSAndrii Nakryiko new_id = cand_id; 3776d5caef5bSAndrii Nakryiko break; 3777d5caef5bSAndrii Nakryiko } 37789768095bSAndrii Nakryiko if (btf_compat_enum(t, cand)) { 37799768095bSAndrii Nakryiko if (btf_is_enum_fwd(t)) { 37809768095bSAndrii Nakryiko /* resolve fwd to full enum */ 37812fc3fc0bSAndrii Nakryiko new_id = cand_id; 37829768095bSAndrii Nakryiko break; 37839768095bSAndrii Nakryiko } 37849768095bSAndrii Nakryiko /* resolve canonical enum fwd to full enum */ 37852fc3fc0bSAndrii Nakryiko d->map[cand_id] = type_id; 37869768095bSAndrii Nakryiko } 3787d5caef5bSAndrii Nakryiko } 3788d5caef5bSAndrii Nakryiko break; 3789d5caef5bSAndrii Nakryiko 3790d5caef5bSAndrii Nakryiko case BTF_KIND_FWD: 379122541a9eSIlya Leoshkevich case BTF_KIND_FLOAT: 3792d5caef5bSAndrii Nakryiko h = btf_hash_common(t); 37932fc3fc0bSAndrii Nakryiko for_each_dedup_cand(d, hash_entry, h) { 3794c302378bSEduard Zingerman cand_id = hash_entry->value; 3795740e69c3SAndrii Nakryiko cand = btf_type_by_id(d->btf, cand_id); 3796d5caef5bSAndrii Nakryiko if (btf_equal_common(t, cand)) { 37972fc3fc0bSAndrii Nakryiko new_id = cand_id; 3798d5caef5bSAndrii Nakryiko break; 3799d5caef5bSAndrii Nakryiko } 3800d5caef5bSAndrii Nakryiko } 3801d5caef5bSAndrii Nakryiko break; 3802d5caef5bSAndrii Nakryiko 3803d5caef5bSAndrii Nakryiko default: 3804d5caef5bSAndrii Nakryiko return -EINVAL; 3805d5caef5bSAndrii Nakryiko } 3806d5caef5bSAndrii Nakryiko 3807d5caef5bSAndrii Nakryiko d->map[type_id] = new_id; 3808d5caef5bSAndrii Nakryiko if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 3809d5caef5bSAndrii Nakryiko return -ENOMEM; 3810d5caef5bSAndrii Nakryiko 3811d5caef5bSAndrii Nakryiko return 0; 3812d5caef5bSAndrii Nakryiko } 3813d5caef5bSAndrii Nakryiko 3814d5caef5bSAndrii Nakryiko static int btf_dedup_prim_types(struct btf_dedup *d) 3815d5caef5bSAndrii Nakryiko { 3816d5caef5bSAndrii Nakryiko int i, err; 3817d5caef5bSAndrii Nakryiko 3818f86524efSAndrii Nakryiko for (i = 0; i < d->btf->nr_types; i++) { 3819f86524efSAndrii Nakryiko err = btf_dedup_prim_type(d, d->btf->start_id + i); 3820d5caef5bSAndrii Nakryiko if (err) 3821d5caef5bSAndrii Nakryiko return err; 3822d5caef5bSAndrii Nakryiko } 3823d5caef5bSAndrii Nakryiko return 0; 3824d5caef5bSAndrii Nakryiko } 3825d5caef5bSAndrii Nakryiko 3826d5caef5bSAndrii Nakryiko /* 3827d5caef5bSAndrii Nakryiko * Check whether type is already mapped into canonical one (could be to itself). 3828d5caef5bSAndrii Nakryiko */ 3829d5caef5bSAndrii Nakryiko static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id) 3830d5caef5bSAndrii Nakryiko { 38315aab392cSAndrii Nakryiko return d->map[type_id] <= BTF_MAX_NR_TYPES; 3832d5caef5bSAndrii Nakryiko } 3833d5caef5bSAndrii Nakryiko 3834d5caef5bSAndrii Nakryiko /* 3835d5caef5bSAndrii Nakryiko * Resolve type ID into its canonical type ID, if any; otherwise return original 3836d5caef5bSAndrii Nakryiko * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow 3837d5caef5bSAndrii Nakryiko * STRUCT/UNION link and resolve it into canonical type ID as well. 3838d5caef5bSAndrii Nakryiko */ 3839d5caef5bSAndrii Nakryiko static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id) 3840d5caef5bSAndrii Nakryiko { 3841d5caef5bSAndrii Nakryiko while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 3842d5caef5bSAndrii Nakryiko type_id = d->map[type_id]; 3843d5caef5bSAndrii Nakryiko return type_id; 3844d5caef5bSAndrii Nakryiko } 3845d5caef5bSAndrii Nakryiko 3846d5caef5bSAndrii Nakryiko /* 3847d5caef5bSAndrii Nakryiko * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original 3848d5caef5bSAndrii Nakryiko * type ID. 3849d5caef5bSAndrii Nakryiko */ 3850d5caef5bSAndrii Nakryiko static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id) 3851d5caef5bSAndrii Nakryiko { 3852d5caef5bSAndrii Nakryiko __u32 orig_type_id = type_id; 3853d5caef5bSAndrii Nakryiko 3854740e69c3SAndrii Nakryiko if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 3855d5caef5bSAndrii Nakryiko return type_id; 3856d5caef5bSAndrii Nakryiko 3857d5caef5bSAndrii Nakryiko while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) 3858d5caef5bSAndrii Nakryiko type_id = d->map[type_id]; 3859d5caef5bSAndrii Nakryiko 3860740e69c3SAndrii Nakryiko if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) 3861d5caef5bSAndrii Nakryiko return type_id; 3862d5caef5bSAndrii Nakryiko 3863d5caef5bSAndrii Nakryiko return orig_type_id; 3864d5caef5bSAndrii Nakryiko } 3865d5caef5bSAndrii Nakryiko 3866d5caef5bSAndrii Nakryiko 3867d5caef5bSAndrii Nakryiko static inline __u16 btf_fwd_kind(struct btf_type *t) 3868d5caef5bSAndrii Nakryiko { 3869b03bc685SAndrii Nakryiko return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT; 3870d5caef5bSAndrii Nakryiko } 3871d5caef5bSAndrii Nakryiko 38726b6e6b1dSAndrii Nakryiko /* Check if given two types are identical ARRAY definitions */ 3873f3c51fe0SAlan Maguire static bool btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2) 38746b6e6b1dSAndrii Nakryiko { 38756b6e6b1dSAndrii Nakryiko struct btf_type *t1, *t2; 38766b6e6b1dSAndrii Nakryiko 38776b6e6b1dSAndrii Nakryiko t1 = btf_type_by_id(d->btf, id1); 38786b6e6b1dSAndrii Nakryiko t2 = btf_type_by_id(d->btf, id2); 38796b6e6b1dSAndrii Nakryiko if (!btf_is_array(t1) || !btf_is_array(t2)) 3880f3c51fe0SAlan Maguire return false; 38816b6e6b1dSAndrii Nakryiko 38826b6e6b1dSAndrii Nakryiko return btf_equal_array(t1, t2); 38836b6e6b1dSAndrii Nakryiko } 38846b6e6b1dSAndrii Nakryiko 3885efdd3eb8SAndrii Nakryiko /* Check if given two types are identical STRUCT/UNION definitions */ 3886efdd3eb8SAndrii Nakryiko static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2) 3887efdd3eb8SAndrii Nakryiko { 3888efdd3eb8SAndrii Nakryiko const struct btf_member *m1, *m2; 3889efdd3eb8SAndrii Nakryiko struct btf_type *t1, *t2; 3890efdd3eb8SAndrii Nakryiko int n, i; 3891efdd3eb8SAndrii Nakryiko 3892efdd3eb8SAndrii Nakryiko t1 = btf_type_by_id(d->btf, id1); 3893efdd3eb8SAndrii Nakryiko t2 = btf_type_by_id(d->btf, id2); 3894efdd3eb8SAndrii Nakryiko 3895efdd3eb8SAndrii Nakryiko if (!btf_is_composite(t1) || btf_kind(t1) != btf_kind(t2)) 3896efdd3eb8SAndrii Nakryiko return false; 3897efdd3eb8SAndrii Nakryiko 3898efdd3eb8SAndrii Nakryiko if (!btf_shallow_equal_struct(t1, t2)) 3899efdd3eb8SAndrii Nakryiko return false; 3900efdd3eb8SAndrii Nakryiko 3901efdd3eb8SAndrii Nakryiko m1 = btf_members(t1); 3902efdd3eb8SAndrii Nakryiko m2 = btf_members(t2); 3903efdd3eb8SAndrii Nakryiko for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) { 3904f3c51fe0SAlan Maguire if (m1->type != m2->type && 3905f3c51fe0SAlan Maguire !btf_dedup_identical_arrays(d, m1->type, m2->type) && 3906f3c51fe0SAlan Maguire !btf_dedup_identical_structs(d, m1->type, m2->type)) 3907efdd3eb8SAndrii Nakryiko return false; 3908efdd3eb8SAndrii Nakryiko } 3909efdd3eb8SAndrii Nakryiko return true; 3910efdd3eb8SAndrii Nakryiko } 3911efdd3eb8SAndrii Nakryiko 3912d5caef5bSAndrii Nakryiko /* 3913d5caef5bSAndrii Nakryiko * Check equivalence of BTF type graph formed by candidate struct/union (we'll 3914d5caef5bSAndrii Nakryiko * call it "candidate graph" in this description for brevity) to a type graph 3915d5caef5bSAndrii Nakryiko * formed by (potential) canonical struct/union ("canonical graph" for brevity 3916d5caef5bSAndrii Nakryiko * here, though keep in mind that not all types in canonical graph are 3917d5caef5bSAndrii Nakryiko * necessarily canonical representatives themselves, some of them might be 3918d5caef5bSAndrii Nakryiko * duplicates or its uniqueness might not have been established yet). 3919d5caef5bSAndrii Nakryiko * Returns: 3920d5caef5bSAndrii Nakryiko * - >0, if type graphs are equivalent; 3921d5caef5bSAndrii Nakryiko * - 0, if not equivalent; 3922d5caef5bSAndrii Nakryiko * - <0, on error. 3923d5caef5bSAndrii Nakryiko * 3924d5caef5bSAndrii Nakryiko * Algorithm performs side-by-side DFS traversal of both type graphs and checks 3925d5caef5bSAndrii Nakryiko * equivalence of BTF types at each step. If at any point BTF types in candidate 3926d5caef5bSAndrii Nakryiko * and canonical graphs are not compatible structurally, whole graphs are 3927d5caef5bSAndrii Nakryiko * incompatible. If types are structurally equivalent (i.e., all information 3928d5caef5bSAndrii Nakryiko * except referenced type IDs is exactly the same), a mapping from `canon_id` to 3929d5caef5bSAndrii Nakryiko * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`). 3930d5caef5bSAndrii Nakryiko * If a type references other types, then those referenced types are checked 3931d5caef5bSAndrii Nakryiko * for equivalence recursively. 3932d5caef5bSAndrii Nakryiko * 3933d5caef5bSAndrii Nakryiko * During DFS traversal, if we find that for current `canon_id` type we 3934d5caef5bSAndrii Nakryiko * already have some mapping in hypothetical map, we check for two possible 3935d5caef5bSAndrii Nakryiko * situations: 3936d5caef5bSAndrii Nakryiko * - `canon_id` is mapped to exactly the same type as `cand_id`. This will 3937d5caef5bSAndrii Nakryiko * happen when type graphs have cycles. In this case we assume those two 3938d5caef5bSAndrii Nakryiko * types are equivalent. 3939d5caef5bSAndrii Nakryiko * - `canon_id` is mapped to different type. This is contradiction in our 3940d5caef5bSAndrii Nakryiko * hypothetical mapping, because same graph in canonical graph corresponds 3941d5caef5bSAndrii Nakryiko * to two different types in candidate graph, which for equivalent type 3942d5caef5bSAndrii Nakryiko * graphs shouldn't happen. This condition terminates equivalence check 3943d5caef5bSAndrii Nakryiko * with negative result. 3944d5caef5bSAndrii Nakryiko * 3945d5caef5bSAndrii Nakryiko * If type graphs traversal exhausts types to check and find no contradiction, 3946d5caef5bSAndrii Nakryiko * then type graphs are equivalent. 3947d5caef5bSAndrii Nakryiko * 3948d5caef5bSAndrii Nakryiko * When checking types for equivalence, there is one special case: FWD types. 3949d5caef5bSAndrii Nakryiko * If FWD type resolution is allowed and one of the types (either from canonical 3950d5caef5bSAndrii Nakryiko * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind 3951d5caef5bSAndrii Nakryiko * flag) and their names match, hypothetical mapping is updated to point from 3952d5caef5bSAndrii Nakryiko * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully, 3953d5caef5bSAndrii Nakryiko * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently. 3954d5caef5bSAndrii Nakryiko * 3955d5caef5bSAndrii Nakryiko * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution, 3956d5caef5bSAndrii Nakryiko * if there are two exactly named (or anonymous) structs/unions that are 3957d5caef5bSAndrii Nakryiko * compatible structurally, one of which has FWD field, while other is concrete 3958d5caef5bSAndrii Nakryiko * STRUCT/UNION, but according to C sources they are different structs/unions 3959d5caef5bSAndrii Nakryiko * that are referencing different types with the same name. This is extremely 3960d5caef5bSAndrii Nakryiko * unlikely to happen, but btf_dedup API allows to disable FWD resolution if 3961d5caef5bSAndrii Nakryiko * this logic is causing problems. 3962d5caef5bSAndrii Nakryiko * 3963d5caef5bSAndrii Nakryiko * Doing FWD resolution means that both candidate and/or canonical graphs can 3964d5caef5bSAndrii Nakryiko * consists of portions of the graph that come from multiple compilation units. 3965d5caef5bSAndrii Nakryiko * This is due to the fact that types within single compilation unit are always 3966d5caef5bSAndrii Nakryiko * deduplicated and FWDs are already resolved, if referenced struct/union 3967d5caef5bSAndrii Nakryiko * definiton is available. So, if we had unresolved FWD and found corresponding 3968d5caef5bSAndrii Nakryiko * STRUCT/UNION, they will be from different compilation units. This 3969d5caef5bSAndrii Nakryiko * consequently means that when we "link" FWD to corresponding STRUCT/UNION, 3970d5caef5bSAndrii Nakryiko * type graph will likely have at least two different BTF types that describe 3971d5caef5bSAndrii Nakryiko * same type (e.g., most probably there will be two different BTF types for the 3972d5caef5bSAndrii Nakryiko * same 'int' primitive type) and could even have "overlapping" parts of type 3973d5caef5bSAndrii Nakryiko * graph that describe same subset of types. 3974d5caef5bSAndrii Nakryiko * 3975d5caef5bSAndrii Nakryiko * This in turn means that our assumption that each type in canonical graph 3976d5caef5bSAndrii Nakryiko * must correspond to exactly one type in candidate graph might not hold 3977d5caef5bSAndrii Nakryiko * anymore and will make it harder to detect contradictions using hypothetical 3978d5caef5bSAndrii Nakryiko * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION 3979d5caef5bSAndrii Nakryiko * resolution only in canonical graph. FWDs in candidate graphs are never 3980d5caef5bSAndrii Nakryiko * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs 3981d5caef5bSAndrii Nakryiko * that can occur: 3982d5caef5bSAndrii Nakryiko * - Both types in canonical and candidate graphs are FWDs. If they are 3983d5caef5bSAndrii Nakryiko * structurally equivalent, then they can either be both resolved to the 3984d5caef5bSAndrii Nakryiko * same STRUCT/UNION or not resolved at all. In both cases they are 3985d5caef5bSAndrii Nakryiko * equivalent and there is no need to resolve FWD on candidate side. 3986d5caef5bSAndrii Nakryiko * - Both types in canonical and candidate graphs are concrete STRUCT/UNION, 3987d5caef5bSAndrii Nakryiko * so nothing to resolve as well, algorithm will check equivalence anyway. 3988d5caef5bSAndrii Nakryiko * - Type in canonical graph is FWD, while type in candidate is concrete 3989d5caef5bSAndrii Nakryiko * STRUCT/UNION. In this case candidate graph comes from single compilation 3990d5caef5bSAndrii Nakryiko * unit, so there is exactly one BTF type for each unique C type. After 3991d5caef5bSAndrii Nakryiko * resolving FWD into STRUCT/UNION, there might be more than one BTF type 3992d5caef5bSAndrii Nakryiko * in canonical graph mapping to single BTF type in candidate graph, but 3993d5caef5bSAndrii Nakryiko * because hypothetical mapping maps from canonical to candidate types, it's 3994d5caef5bSAndrii Nakryiko * alright, and we still maintain the property of having single `canon_id` 3995d5caef5bSAndrii Nakryiko * mapping to single `cand_id` (there could be two different `canon_id` 3996d5caef5bSAndrii Nakryiko * mapped to the same `cand_id`, but it's not contradictory). 3997d5caef5bSAndrii Nakryiko * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate 3998d5caef5bSAndrii Nakryiko * graph is FWD. In this case we are just going to check compatibility of 3999d5caef5bSAndrii Nakryiko * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll 4000d5caef5bSAndrii Nakryiko * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to 4001d5caef5bSAndrii Nakryiko * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs 4002d5caef5bSAndrii Nakryiko * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from 4003d5caef5bSAndrii Nakryiko * canonical graph. 4004d5caef5bSAndrii Nakryiko */ 4005d5caef5bSAndrii Nakryiko static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, 4006d5caef5bSAndrii Nakryiko __u32 canon_id) 4007d5caef5bSAndrii Nakryiko { 4008d5caef5bSAndrii Nakryiko struct btf_type *cand_type; 4009d5caef5bSAndrii Nakryiko struct btf_type *canon_type; 4010d5caef5bSAndrii Nakryiko __u32 hypot_type_id; 4011d5caef5bSAndrii Nakryiko __u16 cand_kind; 4012d5caef5bSAndrii Nakryiko __u16 canon_kind; 4013d5caef5bSAndrii Nakryiko int i, eq; 4014d5caef5bSAndrii Nakryiko 4015d5caef5bSAndrii Nakryiko /* if both resolve to the same canonical, they must be equivalent */ 4016d5caef5bSAndrii Nakryiko if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id)) 4017d5caef5bSAndrii Nakryiko return 1; 4018d5caef5bSAndrii Nakryiko 4019d5caef5bSAndrii Nakryiko canon_id = resolve_fwd_id(d, canon_id); 4020d5caef5bSAndrii Nakryiko 4021d5caef5bSAndrii Nakryiko hypot_type_id = d->hypot_map[canon_id]; 40226b6e6b1dSAndrii Nakryiko if (hypot_type_id <= BTF_MAX_NR_TYPES) { 4023efdd3eb8SAndrii Nakryiko if (hypot_type_id == cand_id) 4024efdd3eb8SAndrii Nakryiko return 1; 40256b6e6b1dSAndrii Nakryiko /* In some cases compiler will generate different DWARF types 40266b6e6b1dSAndrii Nakryiko * for *identical* array type definitions and use them for 40276b6e6b1dSAndrii Nakryiko * different fields within the *same* struct. This breaks type 40286b6e6b1dSAndrii Nakryiko * equivalence check, which makes an assumption that candidate 40296b6e6b1dSAndrii Nakryiko * types sub-graph has a consistent and deduped-by-compiler 40306b6e6b1dSAndrii Nakryiko * types within a single CU. So work around that by explicitly 40316b6e6b1dSAndrii Nakryiko * allowing identical array types here. 40326b6e6b1dSAndrii Nakryiko */ 4033efdd3eb8SAndrii Nakryiko if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id)) 4034efdd3eb8SAndrii Nakryiko return 1; 4035efdd3eb8SAndrii Nakryiko /* It turns out that similar situation can happen with 4036efdd3eb8SAndrii Nakryiko * struct/union sometimes, sigh... Handle the case where 4037efdd3eb8SAndrii Nakryiko * structs/unions are exactly the same, down to the referenced 4038efdd3eb8SAndrii Nakryiko * type IDs. Anything more complicated (e.g., if referenced 4039efdd3eb8SAndrii Nakryiko * types are different, but equivalent) is *way more* 4040efdd3eb8SAndrii Nakryiko * complicated and requires a many-to-many equivalence mapping. 4041efdd3eb8SAndrii Nakryiko */ 4042efdd3eb8SAndrii Nakryiko if (btf_dedup_identical_structs(d, hypot_type_id, cand_id)) 4043efdd3eb8SAndrii Nakryiko return 1; 4044efdd3eb8SAndrii Nakryiko return 0; 40456b6e6b1dSAndrii Nakryiko } 4046d5caef5bSAndrii Nakryiko 4047d5caef5bSAndrii Nakryiko if (btf_dedup_hypot_map_add(d, canon_id, cand_id)) 4048d5caef5bSAndrii Nakryiko return -ENOMEM; 4049d5caef5bSAndrii Nakryiko 4050740e69c3SAndrii Nakryiko cand_type = btf_type_by_id(d->btf, cand_id); 4051740e69c3SAndrii Nakryiko canon_type = btf_type_by_id(d->btf, canon_id); 4052b03bc685SAndrii Nakryiko cand_kind = btf_kind(cand_type); 4053b03bc685SAndrii Nakryiko canon_kind = btf_kind(canon_type); 4054d5caef5bSAndrii Nakryiko 4055d5caef5bSAndrii Nakryiko if (cand_type->name_off != canon_type->name_off) 4056d5caef5bSAndrii Nakryiko return 0; 4057d5caef5bSAndrii Nakryiko 4058d5caef5bSAndrii Nakryiko /* FWD <--> STRUCT/UNION equivalence check, if enabled */ 4059957d350aSAndrii Nakryiko if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD) 4060d5caef5bSAndrii Nakryiko && cand_kind != canon_kind) { 4061d5caef5bSAndrii Nakryiko __u16 real_kind; 4062d5caef5bSAndrii Nakryiko __u16 fwd_kind; 4063d5caef5bSAndrii Nakryiko 4064d5caef5bSAndrii Nakryiko if (cand_kind == BTF_KIND_FWD) { 4065d5caef5bSAndrii Nakryiko real_kind = canon_kind; 4066d5caef5bSAndrii Nakryiko fwd_kind = btf_fwd_kind(cand_type); 4067d5caef5bSAndrii Nakryiko } else { 4068d5caef5bSAndrii Nakryiko real_kind = cand_kind; 4069d5caef5bSAndrii Nakryiko fwd_kind = btf_fwd_kind(canon_type); 4070f86524efSAndrii Nakryiko /* we'd need to resolve base FWD to STRUCT/UNION */ 4071f86524efSAndrii Nakryiko if (fwd_kind == real_kind && canon_id < d->btf->start_id) 4072f86524efSAndrii Nakryiko d->hypot_adjust_canon = true; 4073d5caef5bSAndrii Nakryiko } 4074d5caef5bSAndrii Nakryiko return fwd_kind == real_kind; 4075d5caef5bSAndrii Nakryiko } 4076d5caef5bSAndrii Nakryiko 40779ec71c1cSAndrii Nakryiko if (cand_kind != canon_kind) 40789ec71c1cSAndrii Nakryiko return 0; 40799ec71c1cSAndrii Nakryiko 4080d5caef5bSAndrii Nakryiko switch (cand_kind) { 4081d5caef5bSAndrii Nakryiko case BTF_KIND_INT: 408230025e8bSYonghong Song return btf_equal_int_tag(cand_type, canon_type); 4083d5caef5bSAndrii Nakryiko 4084d5caef5bSAndrii Nakryiko case BTF_KIND_ENUM: 40852ef20263SYonghong Song case BTF_KIND_ENUM64: 4086de048b6eSEduard Zingerman return btf_compat_enum(cand_type, canon_type); 40872ef20263SYonghong Song 4088d5caef5bSAndrii Nakryiko case BTF_KIND_FWD: 408922541a9eSIlya Leoshkevich case BTF_KIND_FLOAT: 4090d5caef5bSAndrii Nakryiko return btf_equal_common(cand_type, canon_type); 4091d5caef5bSAndrii Nakryiko 4092d5caef5bSAndrii Nakryiko case BTF_KIND_CONST: 4093d5caef5bSAndrii Nakryiko case BTF_KIND_VOLATILE: 4094d5caef5bSAndrii Nakryiko case BTF_KIND_RESTRICT: 4095d5caef5bSAndrii Nakryiko case BTF_KIND_PTR: 4096d5caef5bSAndrii Nakryiko case BTF_KIND_TYPEDEF: 4097d5caef5bSAndrii Nakryiko case BTF_KIND_FUNC: 409869a055d5SYonghong Song case BTF_KIND_TYPE_TAG: 40999768095bSAndrii Nakryiko if (cand_type->info != canon_type->info) 41009768095bSAndrii Nakryiko return 0; 4101d5caef5bSAndrii Nakryiko return btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4102d5caef5bSAndrii Nakryiko 4103d5caef5bSAndrii Nakryiko case BTF_KIND_ARRAY: { 4104b03bc685SAndrii Nakryiko const struct btf_array *cand_arr, *canon_arr; 4105d5caef5bSAndrii Nakryiko 4106d5caef5bSAndrii Nakryiko if (!btf_compat_array(cand_type, canon_type)) 4107d5caef5bSAndrii Nakryiko return 0; 4108b03bc685SAndrii Nakryiko cand_arr = btf_array(cand_type); 4109b03bc685SAndrii Nakryiko canon_arr = btf_array(canon_type); 4110f86524efSAndrii Nakryiko eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type); 4111d5caef5bSAndrii Nakryiko if (eq <= 0) 4112d5caef5bSAndrii Nakryiko return eq; 4113d5caef5bSAndrii Nakryiko return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type); 4114d5caef5bSAndrii Nakryiko } 4115d5caef5bSAndrii Nakryiko 4116d5caef5bSAndrii Nakryiko case BTF_KIND_STRUCT: 4117d5caef5bSAndrii Nakryiko case BTF_KIND_UNION: { 4118b03bc685SAndrii Nakryiko const struct btf_member *cand_m, *canon_m; 4119d5caef5bSAndrii Nakryiko __u16 vlen; 4120d5caef5bSAndrii Nakryiko 412191097fbeSAndrii Nakryiko if (!btf_shallow_equal_struct(cand_type, canon_type)) 4122d5caef5bSAndrii Nakryiko return 0; 4123b03bc685SAndrii Nakryiko vlen = btf_vlen(cand_type); 4124b03bc685SAndrii Nakryiko cand_m = btf_members(cand_type); 4125b03bc685SAndrii Nakryiko canon_m = btf_members(canon_type); 4126d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 4127d5caef5bSAndrii Nakryiko eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); 4128d5caef5bSAndrii Nakryiko if (eq <= 0) 4129d5caef5bSAndrii Nakryiko return eq; 4130d5caef5bSAndrii Nakryiko cand_m++; 4131d5caef5bSAndrii Nakryiko canon_m++; 4132d5caef5bSAndrii Nakryiko } 4133d5caef5bSAndrii Nakryiko 4134d5caef5bSAndrii Nakryiko return 1; 4135d5caef5bSAndrii Nakryiko } 4136d5caef5bSAndrii Nakryiko 4137d5caef5bSAndrii Nakryiko case BTF_KIND_FUNC_PROTO: { 4138b03bc685SAndrii Nakryiko const struct btf_param *cand_p, *canon_p; 4139d5caef5bSAndrii Nakryiko __u16 vlen; 4140d5caef5bSAndrii Nakryiko 4141d5caef5bSAndrii Nakryiko if (!btf_compat_fnproto(cand_type, canon_type)) 4142d5caef5bSAndrii Nakryiko return 0; 4143d5caef5bSAndrii Nakryiko eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type); 4144d5caef5bSAndrii Nakryiko if (eq <= 0) 4145d5caef5bSAndrii Nakryiko return eq; 4146b03bc685SAndrii Nakryiko vlen = btf_vlen(cand_type); 4147b03bc685SAndrii Nakryiko cand_p = btf_params(cand_type); 4148b03bc685SAndrii Nakryiko canon_p = btf_params(canon_type); 4149d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 4150d5caef5bSAndrii Nakryiko eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type); 4151d5caef5bSAndrii Nakryiko if (eq <= 0) 4152d5caef5bSAndrii Nakryiko return eq; 4153d5caef5bSAndrii Nakryiko cand_p++; 4154d5caef5bSAndrii Nakryiko canon_p++; 4155d5caef5bSAndrii Nakryiko } 4156d5caef5bSAndrii Nakryiko return 1; 4157d5caef5bSAndrii Nakryiko } 4158d5caef5bSAndrii Nakryiko 4159d5caef5bSAndrii Nakryiko default: 4160d5caef5bSAndrii Nakryiko return -EINVAL; 4161d5caef5bSAndrii Nakryiko } 4162d5caef5bSAndrii Nakryiko return 0; 4163d5caef5bSAndrii Nakryiko } 4164d5caef5bSAndrii Nakryiko 4165d5caef5bSAndrii Nakryiko /* 4166d5caef5bSAndrii Nakryiko * Use hypothetical mapping, produced by successful type graph equivalence 4167d5caef5bSAndrii Nakryiko * check, to augment existing struct/union canonical mapping, where possible. 4168d5caef5bSAndrii Nakryiko * 4169d5caef5bSAndrii Nakryiko * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record 4170d5caef5bSAndrii Nakryiko * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional: 4171d5caef5bSAndrii Nakryiko * it doesn't matter if FWD type was part of canonical graph or candidate one, 4172d5caef5bSAndrii Nakryiko * we are recording the mapping anyway. As opposed to carefulness required 4173d5caef5bSAndrii Nakryiko * for struct/union correspondence mapping (described below), for FWD resolution 4174d5caef5bSAndrii Nakryiko * it's not important, as by the time that FWD type (reference type) will be 4175d5caef5bSAndrii Nakryiko * deduplicated all structs/unions will be deduped already anyway. 4176d5caef5bSAndrii Nakryiko * 4177d5caef5bSAndrii Nakryiko * Recording STRUCT/UNION mapping is purely a performance optimization and is 4178d5caef5bSAndrii Nakryiko * not required for correctness. It needs to be done carefully to ensure that 4179d5caef5bSAndrii Nakryiko * struct/union from candidate's type graph is not mapped into corresponding 4180d5caef5bSAndrii Nakryiko * struct/union from canonical type graph that itself hasn't been resolved into 4181d5caef5bSAndrii Nakryiko * canonical representative. The only guarantee we have is that canonical 4182d5caef5bSAndrii Nakryiko * struct/union was determined as canonical and that won't change. But any 4183d5caef5bSAndrii Nakryiko * types referenced through that struct/union fields could have been not yet 4184d5caef5bSAndrii Nakryiko * resolved, so in case like that it's too early to establish any kind of 4185d5caef5bSAndrii Nakryiko * correspondence between structs/unions. 4186d5caef5bSAndrii Nakryiko * 4187d5caef5bSAndrii Nakryiko * No canonical correspondence is derived for primitive types (they are already 4188d5caef5bSAndrii Nakryiko * deduplicated completely already anyway) or reference types (they rely on 4189d5caef5bSAndrii Nakryiko * stability of struct/union canonical relationship for equivalence checks). 4190d5caef5bSAndrii Nakryiko */ 4191d5caef5bSAndrii Nakryiko static void btf_dedup_merge_hypot_map(struct btf_dedup *d) 4192d5caef5bSAndrii Nakryiko { 4193f86524efSAndrii Nakryiko __u32 canon_type_id, targ_type_id; 4194d5caef5bSAndrii Nakryiko __u16 t_kind, c_kind; 4195d5caef5bSAndrii Nakryiko __u32 t_id, c_id; 4196d5caef5bSAndrii Nakryiko int i; 4197d5caef5bSAndrii Nakryiko 4198d5caef5bSAndrii Nakryiko for (i = 0; i < d->hypot_cnt; i++) { 4199f86524efSAndrii Nakryiko canon_type_id = d->hypot_list[i]; 4200f86524efSAndrii Nakryiko targ_type_id = d->hypot_map[canon_type_id]; 4201d5caef5bSAndrii Nakryiko t_id = resolve_type_id(d, targ_type_id); 4202f86524efSAndrii Nakryiko c_id = resolve_type_id(d, canon_type_id); 4203740e69c3SAndrii Nakryiko t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); 4204740e69c3SAndrii Nakryiko c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); 4205d5caef5bSAndrii Nakryiko /* 4206d5caef5bSAndrii Nakryiko * Resolve FWD into STRUCT/UNION. 4207d5caef5bSAndrii Nakryiko * It's ok to resolve FWD into STRUCT/UNION that's not yet 4208d5caef5bSAndrii Nakryiko * mapped to canonical representative (as opposed to 4209d5caef5bSAndrii Nakryiko * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because 4210d5caef5bSAndrii Nakryiko * eventually that struct is going to be mapped and all resolved 4211d5caef5bSAndrii Nakryiko * FWDs will automatically resolve to correct canonical 4212d5caef5bSAndrii Nakryiko * representative. This will happen before ref type deduping, 4213d5caef5bSAndrii Nakryiko * which critically depends on stability of these mapping. This 4214d5caef5bSAndrii Nakryiko * stability is not a requirement for STRUCT/UNION equivalence 4215d5caef5bSAndrii Nakryiko * checks, though. 4216d5caef5bSAndrii Nakryiko */ 4217f86524efSAndrii Nakryiko 4218f86524efSAndrii Nakryiko /* if it's the split BTF case, we still need to point base FWD 4219f86524efSAndrii Nakryiko * to STRUCT/UNION in a split BTF, because FWDs from split BTF 4220f86524efSAndrii Nakryiko * will be resolved against base FWD. If we don't point base 4221f86524efSAndrii Nakryiko * canonical FWD to the resolved STRUCT/UNION, then all the 4222f86524efSAndrii Nakryiko * FWDs in split BTF won't be correctly resolved to a proper 4223f86524efSAndrii Nakryiko * STRUCT/UNION. 4224f86524efSAndrii Nakryiko */ 4225d5caef5bSAndrii Nakryiko if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD) 4226d5caef5bSAndrii Nakryiko d->map[c_id] = t_id; 4227f86524efSAndrii Nakryiko 4228f86524efSAndrii Nakryiko /* if graph equivalence determined that we'd need to adjust 4229f86524efSAndrii Nakryiko * base canonical types, then we need to only point base FWDs 4230f86524efSAndrii Nakryiko * to STRUCTs/UNIONs and do no more modifications. For all 4231f86524efSAndrii Nakryiko * other purposes the type graphs were not equivalent. 4232f86524efSAndrii Nakryiko */ 4233f86524efSAndrii Nakryiko if (d->hypot_adjust_canon) 4234f86524efSAndrii Nakryiko continue; 4235f86524efSAndrii Nakryiko 4236f86524efSAndrii Nakryiko if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD) 4237d5caef5bSAndrii Nakryiko d->map[t_id] = c_id; 4238d5caef5bSAndrii Nakryiko 4239d5caef5bSAndrii Nakryiko if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) && 4240d5caef5bSAndrii Nakryiko c_kind != BTF_KIND_FWD && 4241d5caef5bSAndrii Nakryiko is_type_mapped(d, c_id) && 4242d5caef5bSAndrii Nakryiko !is_type_mapped(d, t_id)) { 4243d5caef5bSAndrii Nakryiko /* 4244d5caef5bSAndrii Nakryiko * as a perf optimization, we can map struct/union 4245d5caef5bSAndrii Nakryiko * that's part of type graph we just verified for 4246d5caef5bSAndrii Nakryiko * equivalence. We can do that for struct/union that has 4247d5caef5bSAndrii Nakryiko * canonical representative only, though. 4248d5caef5bSAndrii Nakryiko */ 4249d5caef5bSAndrii Nakryiko d->map[t_id] = c_id; 4250d5caef5bSAndrii Nakryiko } 4251d5caef5bSAndrii Nakryiko } 4252d5caef5bSAndrii Nakryiko } 4253d5caef5bSAndrii Nakryiko 4254d5caef5bSAndrii Nakryiko /* 4255d5caef5bSAndrii Nakryiko * Deduplicate struct/union types. 4256d5caef5bSAndrii Nakryiko * 4257d5caef5bSAndrii Nakryiko * For each struct/union type its type signature hash is calculated, taking 4258d5caef5bSAndrii Nakryiko * into account type's name, size, number, order and names of fields, but 4259d5caef5bSAndrii Nakryiko * ignoring type ID's referenced from fields, because they might not be deduped 4260d5caef5bSAndrii Nakryiko * completely until after reference types deduplication phase. This type hash 4261d5caef5bSAndrii Nakryiko * is used to iterate over all potential canonical types, sharing same hash. 4262d5caef5bSAndrii Nakryiko * For each canonical candidate we check whether type graphs that they form 4263d5caef5bSAndrii Nakryiko * (through referenced types in fields and so on) are equivalent using algorithm 4264d5caef5bSAndrii Nakryiko * implemented in `btf_dedup_is_equiv`. If such equivalence is found and 4265d5caef5bSAndrii Nakryiko * BTF_KIND_FWD resolution is allowed, then hypothetical mapping 4266d5caef5bSAndrii Nakryiko * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence 4267d5caef5bSAndrii Nakryiko * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to 4268d5caef5bSAndrii Nakryiko * potentially map other structs/unions to their canonical representatives, 4269d5caef5bSAndrii Nakryiko * if such relationship hasn't yet been established. This speeds up algorithm 4270d5caef5bSAndrii Nakryiko * by eliminating some of the duplicate work. 4271d5caef5bSAndrii Nakryiko * 4272d5caef5bSAndrii Nakryiko * If no matching canonical representative was found, struct/union is marked 4273d5caef5bSAndrii Nakryiko * as canonical for itself and is added into btf_dedup->dedup_table hash map 4274d5caef5bSAndrii Nakryiko * for further look ups. 4275d5caef5bSAndrii Nakryiko */ 4276d5caef5bSAndrii Nakryiko static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) 4277d5caef5bSAndrii Nakryiko { 427891097fbeSAndrii Nakryiko struct btf_type *cand_type, *t; 42792fc3fc0bSAndrii Nakryiko struct hashmap_entry *hash_entry; 4280d5caef5bSAndrii Nakryiko /* if we don't find equivalent type, then we are canonical */ 4281d5caef5bSAndrii Nakryiko __u32 new_id = type_id; 4282d5caef5bSAndrii Nakryiko __u16 kind; 42832fc3fc0bSAndrii Nakryiko long h; 4284d5caef5bSAndrii Nakryiko 4285d5caef5bSAndrii Nakryiko /* already deduped or is in process of deduping (loop detected) */ 42865aab392cSAndrii Nakryiko if (d->map[type_id] <= BTF_MAX_NR_TYPES) 4287d5caef5bSAndrii Nakryiko return 0; 4288d5caef5bSAndrii Nakryiko 4289740e69c3SAndrii Nakryiko t = btf_type_by_id(d->btf, type_id); 4290b03bc685SAndrii Nakryiko kind = btf_kind(t); 4291d5caef5bSAndrii Nakryiko 4292d5caef5bSAndrii Nakryiko if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) 4293d5caef5bSAndrii Nakryiko return 0; 4294d5caef5bSAndrii Nakryiko 4295d5caef5bSAndrii Nakryiko h = btf_hash_struct(t); 42962fc3fc0bSAndrii Nakryiko for_each_dedup_cand(d, hash_entry, h) { 4297c302378bSEduard Zingerman __u32 cand_id = hash_entry->value; 4298d5caef5bSAndrii Nakryiko int eq; 4299d5caef5bSAndrii Nakryiko 430091097fbeSAndrii Nakryiko /* 430191097fbeSAndrii Nakryiko * Even though btf_dedup_is_equiv() checks for 430291097fbeSAndrii Nakryiko * btf_shallow_equal_struct() internally when checking two 430391097fbeSAndrii Nakryiko * structs (unions) for equivalence, we need to guard here 430491097fbeSAndrii Nakryiko * from picking matching FWD type as a dedup candidate. 430591097fbeSAndrii Nakryiko * This can happen due to hash collision. In such case just 430691097fbeSAndrii Nakryiko * relying on btf_dedup_is_equiv() would lead to potentially 430791097fbeSAndrii Nakryiko * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because 430891097fbeSAndrii Nakryiko * FWD and compatible STRUCT/UNION are considered equivalent. 430991097fbeSAndrii Nakryiko */ 4310740e69c3SAndrii Nakryiko cand_type = btf_type_by_id(d->btf, cand_id); 431191097fbeSAndrii Nakryiko if (!btf_shallow_equal_struct(t, cand_type)) 431291097fbeSAndrii Nakryiko continue; 431391097fbeSAndrii Nakryiko 4314d5caef5bSAndrii Nakryiko btf_dedup_clear_hypot_map(d); 43152fc3fc0bSAndrii Nakryiko eq = btf_dedup_is_equiv(d, type_id, cand_id); 4316d5caef5bSAndrii Nakryiko if (eq < 0) 4317d5caef5bSAndrii Nakryiko return eq; 4318d5caef5bSAndrii Nakryiko if (!eq) 4319d5caef5bSAndrii Nakryiko continue; 4320d5caef5bSAndrii Nakryiko btf_dedup_merge_hypot_map(d); 4321f86524efSAndrii Nakryiko if (d->hypot_adjust_canon) /* not really equivalent */ 4322f86524efSAndrii Nakryiko continue; 4323f86524efSAndrii Nakryiko new_id = cand_id; 4324d5caef5bSAndrii Nakryiko break; 4325d5caef5bSAndrii Nakryiko } 4326d5caef5bSAndrii Nakryiko 4327d5caef5bSAndrii Nakryiko d->map[type_id] = new_id; 4328d5caef5bSAndrii Nakryiko if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4329d5caef5bSAndrii Nakryiko return -ENOMEM; 4330d5caef5bSAndrii Nakryiko 4331d5caef5bSAndrii Nakryiko return 0; 4332d5caef5bSAndrii Nakryiko } 4333d5caef5bSAndrii Nakryiko 4334d5caef5bSAndrii Nakryiko static int btf_dedup_struct_types(struct btf_dedup *d) 4335d5caef5bSAndrii Nakryiko { 4336d5caef5bSAndrii Nakryiko int i, err; 4337d5caef5bSAndrii Nakryiko 4338f86524efSAndrii Nakryiko for (i = 0; i < d->btf->nr_types; i++) { 4339f86524efSAndrii Nakryiko err = btf_dedup_struct_type(d, d->btf->start_id + i); 4340d5caef5bSAndrii Nakryiko if (err) 4341d5caef5bSAndrii Nakryiko return err; 4342d5caef5bSAndrii Nakryiko } 4343d5caef5bSAndrii Nakryiko return 0; 4344d5caef5bSAndrii Nakryiko } 4345d5caef5bSAndrii Nakryiko 4346d5caef5bSAndrii Nakryiko /* 4347d5caef5bSAndrii Nakryiko * Deduplicate reference type. 4348d5caef5bSAndrii Nakryiko * 4349d5caef5bSAndrii Nakryiko * Once all primitive and struct/union types got deduplicated, we can easily 4350d5caef5bSAndrii Nakryiko * deduplicate all other (reference) BTF types. This is done in two steps: 4351d5caef5bSAndrii Nakryiko * 4352d5caef5bSAndrii Nakryiko * 1. Resolve all referenced type IDs into their canonical type IDs. This 4353d5caef5bSAndrii Nakryiko * resolution can be done either immediately for primitive or struct/union types 4354d5caef5bSAndrii Nakryiko * (because they were deduped in previous two phases) or recursively for 4355d5caef5bSAndrii Nakryiko * reference types. Recursion will always terminate at either primitive or 4356d5caef5bSAndrii Nakryiko * struct/union type, at which point we can "unwind" chain of reference types 4357d5caef5bSAndrii Nakryiko * one by one. There is no danger of encountering cycles because in C type 4358d5caef5bSAndrii Nakryiko * system the only way to form type cycle is through struct/union, so any chain 4359d5caef5bSAndrii Nakryiko * of reference types, even those taking part in a type cycle, will inevitably 4360d5caef5bSAndrii Nakryiko * reach struct/union at some point. 4361d5caef5bSAndrii Nakryiko * 4362d5caef5bSAndrii Nakryiko * 2. Once all referenced type IDs are resolved into canonical ones, BTF type 4363d5caef5bSAndrii Nakryiko * becomes "stable", in the sense that no further deduplication will cause 4364d5caef5bSAndrii Nakryiko * any changes to it. With that, it's now possible to calculate type's signature 4365d5caef5bSAndrii Nakryiko * hash (this time taking into account referenced type IDs) and loop over all 4366d5caef5bSAndrii Nakryiko * potential canonical representatives. If no match was found, current type 4367d5caef5bSAndrii Nakryiko * will become canonical representative of itself and will be added into 4368d5caef5bSAndrii Nakryiko * btf_dedup->dedup_table as another possible canonical representative. 4369d5caef5bSAndrii Nakryiko */ 4370d5caef5bSAndrii Nakryiko static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) 4371d5caef5bSAndrii Nakryiko { 43722fc3fc0bSAndrii Nakryiko struct hashmap_entry *hash_entry; 43732fc3fc0bSAndrii Nakryiko __u32 new_id = type_id, cand_id; 4374d5caef5bSAndrii Nakryiko struct btf_type *t, *cand; 4375d5caef5bSAndrii Nakryiko /* if we don't find equivalent type, then we are representative type */ 43763d8669e6SDan Carpenter int ref_type_id; 43772fc3fc0bSAndrii Nakryiko long h; 4378d5caef5bSAndrii Nakryiko 4379d5caef5bSAndrii Nakryiko if (d->map[type_id] == BTF_IN_PROGRESS_ID) 4380d5caef5bSAndrii Nakryiko return -ELOOP; 43815aab392cSAndrii Nakryiko if (d->map[type_id] <= BTF_MAX_NR_TYPES) 4382d5caef5bSAndrii Nakryiko return resolve_type_id(d, type_id); 4383d5caef5bSAndrii Nakryiko 4384740e69c3SAndrii Nakryiko t = btf_type_by_id(d->btf, type_id); 4385d5caef5bSAndrii Nakryiko d->map[type_id] = BTF_IN_PROGRESS_ID; 4386d5caef5bSAndrii Nakryiko 4387b03bc685SAndrii Nakryiko switch (btf_kind(t)) { 4388d5caef5bSAndrii Nakryiko case BTF_KIND_CONST: 4389d5caef5bSAndrii Nakryiko case BTF_KIND_VOLATILE: 4390d5caef5bSAndrii Nakryiko case BTF_KIND_RESTRICT: 4391d5caef5bSAndrii Nakryiko case BTF_KIND_PTR: 4392d5caef5bSAndrii Nakryiko case BTF_KIND_TYPEDEF: 4393d5caef5bSAndrii Nakryiko case BTF_KIND_FUNC: 43942dc1e488SYonghong Song case BTF_KIND_TYPE_TAG: 4395d5caef5bSAndrii Nakryiko ref_type_id = btf_dedup_ref_type(d, t->type); 4396d5caef5bSAndrii Nakryiko if (ref_type_id < 0) 4397d5caef5bSAndrii Nakryiko return ref_type_id; 4398d5caef5bSAndrii Nakryiko t->type = ref_type_id; 4399d5caef5bSAndrii Nakryiko 4400d5caef5bSAndrii Nakryiko h = btf_hash_common(t); 44012fc3fc0bSAndrii Nakryiko for_each_dedup_cand(d, hash_entry, h) { 4402c302378bSEduard Zingerman cand_id = hash_entry->value; 4403740e69c3SAndrii Nakryiko cand = btf_type_by_id(d->btf, cand_id); 4404d5caef5bSAndrii Nakryiko if (btf_equal_common(t, cand)) { 44052fc3fc0bSAndrii Nakryiko new_id = cand_id; 4406d5caef5bSAndrii Nakryiko break; 4407d5caef5bSAndrii Nakryiko } 4408d5caef5bSAndrii Nakryiko } 4409d5caef5bSAndrii Nakryiko break; 4410d5caef5bSAndrii Nakryiko 4411223f903eSYonghong Song case BTF_KIND_DECL_TAG: 44125b84bd10SYonghong Song ref_type_id = btf_dedup_ref_type(d, t->type); 44135b84bd10SYonghong Song if (ref_type_id < 0) 44145b84bd10SYonghong Song return ref_type_id; 44155b84bd10SYonghong Song t->type = ref_type_id; 44165b84bd10SYonghong Song 4417223f903eSYonghong Song h = btf_hash_int_decl_tag(t); 44185b84bd10SYonghong Song for_each_dedup_cand(d, hash_entry, h) { 4419c302378bSEduard Zingerman cand_id = hash_entry->value; 44205b84bd10SYonghong Song cand = btf_type_by_id(d->btf, cand_id); 44215b84bd10SYonghong Song if (btf_equal_int_tag(t, cand)) { 44225b84bd10SYonghong Song new_id = cand_id; 44235b84bd10SYonghong Song break; 44245b84bd10SYonghong Song } 44255b84bd10SYonghong Song } 44265b84bd10SYonghong Song break; 44275b84bd10SYonghong Song 4428d5caef5bSAndrii Nakryiko case BTF_KIND_ARRAY: { 4429b03bc685SAndrii Nakryiko struct btf_array *info = btf_array(t); 4430d5caef5bSAndrii Nakryiko 4431d5caef5bSAndrii Nakryiko ref_type_id = btf_dedup_ref_type(d, info->type); 4432d5caef5bSAndrii Nakryiko if (ref_type_id < 0) 4433d5caef5bSAndrii Nakryiko return ref_type_id; 4434d5caef5bSAndrii Nakryiko info->type = ref_type_id; 4435d5caef5bSAndrii Nakryiko 4436d5caef5bSAndrii Nakryiko ref_type_id = btf_dedup_ref_type(d, info->index_type); 4437d5caef5bSAndrii Nakryiko if (ref_type_id < 0) 4438d5caef5bSAndrii Nakryiko return ref_type_id; 4439d5caef5bSAndrii Nakryiko info->index_type = ref_type_id; 4440d5caef5bSAndrii Nakryiko 4441d5caef5bSAndrii Nakryiko h = btf_hash_array(t); 44422fc3fc0bSAndrii Nakryiko for_each_dedup_cand(d, hash_entry, h) { 4443c302378bSEduard Zingerman cand_id = hash_entry->value; 4444740e69c3SAndrii Nakryiko cand = btf_type_by_id(d->btf, cand_id); 4445d5caef5bSAndrii Nakryiko if (btf_equal_array(t, cand)) { 44462fc3fc0bSAndrii Nakryiko new_id = cand_id; 4447d5caef5bSAndrii Nakryiko break; 4448d5caef5bSAndrii Nakryiko } 4449d5caef5bSAndrii Nakryiko } 4450d5caef5bSAndrii Nakryiko break; 4451d5caef5bSAndrii Nakryiko } 4452d5caef5bSAndrii Nakryiko 4453d5caef5bSAndrii Nakryiko case BTF_KIND_FUNC_PROTO: { 4454d5caef5bSAndrii Nakryiko struct btf_param *param; 4455d5caef5bSAndrii Nakryiko __u16 vlen; 4456d5caef5bSAndrii Nakryiko int i; 4457d5caef5bSAndrii Nakryiko 4458d5caef5bSAndrii Nakryiko ref_type_id = btf_dedup_ref_type(d, t->type); 4459d5caef5bSAndrii Nakryiko if (ref_type_id < 0) 4460d5caef5bSAndrii Nakryiko return ref_type_id; 4461d5caef5bSAndrii Nakryiko t->type = ref_type_id; 4462d5caef5bSAndrii Nakryiko 4463b03bc685SAndrii Nakryiko vlen = btf_vlen(t); 4464b03bc685SAndrii Nakryiko param = btf_params(t); 4465d5caef5bSAndrii Nakryiko for (i = 0; i < vlen; i++) { 4466d5caef5bSAndrii Nakryiko ref_type_id = btf_dedup_ref_type(d, param->type); 4467d5caef5bSAndrii Nakryiko if (ref_type_id < 0) 4468d5caef5bSAndrii Nakryiko return ref_type_id; 4469d5caef5bSAndrii Nakryiko param->type = ref_type_id; 4470d5caef5bSAndrii Nakryiko param++; 4471d5caef5bSAndrii Nakryiko } 4472d5caef5bSAndrii Nakryiko 4473d5caef5bSAndrii Nakryiko h = btf_hash_fnproto(t); 44742fc3fc0bSAndrii Nakryiko for_each_dedup_cand(d, hash_entry, h) { 4475c302378bSEduard Zingerman cand_id = hash_entry->value; 4476740e69c3SAndrii Nakryiko cand = btf_type_by_id(d->btf, cand_id); 4477d5caef5bSAndrii Nakryiko if (btf_equal_fnproto(t, cand)) { 44782fc3fc0bSAndrii Nakryiko new_id = cand_id; 4479d5caef5bSAndrii Nakryiko break; 4480d5caef5bSAndrii Nakryiko } 4481d5caef5bSAndrii Nakryiko } 4482d5caef5bSAndrii Nakryiko break; 4483d5caef5bSAndrii Nakryiko } 4484d5caef5bSAndrii Nakryiko 4485d5caef5bSAndrii Nakryiko default: 4486d5caef5bSAndrii Nakryiko return -EINVAL; 4487d5caef5bSAndrii Nakryiko } 4488d5caef5bSAndrii Nakryiko 4489d5caef5bSAndrii Nakryiko d->map[type_id] = new_id; 4490d5caef5bSAndrii Nakryiko if (type_id == new_id && btf_dedup_table_add(d, h, type_id)) 4491d5caef5bSAndrii Nakryiko return -ENOMEM; 4492d5caef5bSAndrii Nakryiko 4493d5caef5bSAndrii Nakryiko return new_id; 4494d5caef5bSAndrii Nakryiko } 4495d5caef5bSAndrii Nakryiko 4496d5caef5bSAndrii Nakryiko static int btf_dedup_ref_types(struct btf_dedup *d) 4497d5caef5bSAndrii Nakryiko { 4498d5caef5bSAndrii Nakryiko int i, err; 4499d5caef5bSAndrii Nakryiko 4500f86524efSAndrii Nakryiko for (i = 0; i < d->btf->nr_types; i++) { 4501f86524efSAndrii Nakryiko err = btf_dedup_ref_type(d, d->btf->start_id + i); 4502d5caef5bSAndrii Nakryiko if (err < 0) 4503d5caef5bSAndrii Nakryiko return err; 4504d5caef5bSAndrii Nakryiko } 45052fc3fc0bSAndrii Nakryiko /* we won't need d->dedup_table anymore */ 45062fc3fc0bSAndrii Nakryiko hashmap__free(d->dedup_table); 45072fc3fc0bSAndrii Nakryiko d->dedup_table = NULL; 4508d5caef5bSAndrii Nakryiko return 0; 4509d5caef5bSAndrii Nakryiko } 4510d5caef5bSAndrii Nakryiko 4511d5caef5bSAndrii Nakryiko /* 4512*082108fdSEduard Zingerman * Collect a map from type names to type ids for all canonical structs 4513*082108fdSEduard Zingerman * and unions. If the same name is shared by several canonical types 4514*082108fdSEduard Zingerman * use a special value 0 to indicate this fact. 4515*082108fdSEduard Zingerman */ 4516*082108fdSEduard Zingerman static int btf_dedup_fill_unique_names_map(struct btf_dedup *d, struct hashmap *names_map) 4517*082108fdSEduard Zingerman { 4518*082108fdSEduard Zingerman __u32 nr_types = btf__type_cnt(d->btf); 4519*082108fdSEduard Zingerman struct btf_type *t; 4520*082108fdSEduard Zingerman __u32 type_id; 4521*082108fdSEduard Zingerman __u16 kind; 4522*082108fdSEduard Zingerman int err; 4523*082108fdSEduard Zingerman 4524*082108fdSEduard Zingerman /* 4525*082108fdSEduard Zingerman * Iterate over base and split module ids in order to get all 4526*082108fdSEduard Zingerman * available structs in the map. 4527*082108fdSEduard Zingerman */ 4528*082108fdSEduard Zingerman for (type_id = 1; type_id < nr_types; ++type_id) { 4529*082108fdSEduard Zingerman t = btf_type_by_id(d->btf, type_id); 4530*082108fdSEduard Zingerman kind = btf_kind(t); 4531*082108fdSEduard Zingerman 4532*082108fdSEduard Zingerman if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) 4533*082108fdSEduard Zingerman continue; 4534*082108fdSEduard Zingerman 4535*082108fdSEduard Zingerman /* Skip non-canonical types */ 4536*082108fdSEduard Zingerman if (type_id != d->map[type_id]) 4537*082108fdSEduard Zingerman continue; 4538*082108fdSEduard Zingerman 4539*082108fdSEduard Zingerman err = hashmap__add(names_map, t->name_off, type_id); 4540*082108fdSEduard Zingerman if (err == -EEXIST) 4541*082108fdSEduard Zingerman err = hashmap__set(names_map, t->name_off, 0, NULL, NULL); 4542*082108fdSEduard Zingerman 4543*082108fdSEduard Zingerman if (err) 4544*082108fdSEduard Zingerman return err; 4545*082108fdSEduard Zingerman } 4546*082108fdSEduard Zingerman 4547*082108fdSEduard Zingerman return 0; 4548*082108fdSEduard Zingerman } 4549*082108fdSEduard Zingerman 4550*082108fdSEduard Zingerman static int btf_dedup_resolve_fwd(struct btf_dedup *d, struct hashmap *names_map, __u32 type_id) 4551*082108fdSEduard Zingerman { 4552*082108fdSEduard Zingerman struct btf_type *t = btf_type_by_id(d->btf, type_id); 4553*082108fdSEduard Zingerman enum btf_fwd_kind fwd_kind = btf_kflag(t); 4554*082108fdSEduard Zingerman __u16 cand_kind, kind = btf_kind(t); 4555*082108fdSEduard Zingerman struct btf_type *cand_t; 4556*082108fdSEduard Zingerman uintptr_t cand_id; 4557*082108fdSEduard Zingerman 4558*082108fdSEduard Zingerman if (kind != BTF_KIND_FWD) 4559*082108fdSEduard Zingerman return 0; 4560*082108fdSEduard Zingerman 4561*082108fdSEduard Zingerman /* Skip if this FWD already has a mapping */ 4562*082108fdSEduard Zingerman if (type_id != d->map[type_id]) 4563*082108fdSEduard Zingerman return 0; 4564*082108fdSEduard Zingerman 4565*082108fdSEduard Zingerman if (!hashmap__find(names_map, t->name_off, &cand_id)) 4566*082108fdSEduard Zingerman return 0; 4567*082108fdSEduard Zingerman 4568*082108fdSEduard Zingerman /* Zero is a special value indicating that name is not unique */ 4569*082108fdSEduard Zingerman if (!cand_id) 4570*082108fdSEduard Zingerman return 0; 4571*082108fdSEduard Zingerman 4572*082108fdSEduard Zingerman cand_t = btf_type_by_id(d->btf, cand_id); 4573*082108fdSEduard Zingerman cand_kind = btf_kind(cand_t); 4574*082108fdSEduard Zingerman if ((cand_kind == BTF_KIND_STRUCT && fwd_kind != BTF_FWD_STRUCT) || 4575*082108fdSEduard Zingerman (cand_kind == BTF_KIND_UNION && fwd_kind != BTF_FWD_UNION)) 4576*082108fdSEduard Zingerman return 0; 4577*082108fdSEduard Zingerman 4578*082108fdSEduard Zingerman d->map[type_id] = cand_id; 4579*082108fdSEduard Zingerman 4580*082108fdSEduard Zingerman return 0; 4581*082108fdSEduard Zingerman } 4582*082108fdSEduard Zingerman 4583*082108fdSEduard Zingerman /* 4584*082108fdSEduard Zingerman * Resolve unambiguous forward declarations. 4585*082108fdSEduard Zingerman * 4586*082108fdSEduard Zingerman * The lion's share of all FWD declarations is resolved during 4587*082108fdSEduard Zingerman * `btf_dedup_struct_types` phase when different type graphs are 4588*082108fdSEduard Zingerman * compared against each other. However, if in some compilation unit a 4589*082108fdSEduard Zingerman * FWD declaration is not a part of a type graph compared against 4590*082108fdSEduard Zingerman * another type graph that declaration's canonical type would not be 4591*082108fdSEduard Zingerman * changed. Example: 4592*082108fdSEduard Zingerman * 4593*082108fdSEduard Zingerman * CU #1: 4594*082108fdSEduard Zingerman * 4595*082108fdSEduard Zingerman * struct foo; 4596*082108fdSEduard Zingerman * struct foo *some_global; 4597*082108fdSEduard Zingerman * 4598*082108fdSEduard Zingerman * CU #2: 4599*082108fdSEduard Zingerman * 4600*082108fdSEduard Zingerman * struct foo { int u; }; 4601*082108fdSEduard Zingerman * struct foo *another_global; 4602*082108fdSEduard Zingerman * 4603*082108fdSEduard Zingerman * After `btf_dedup_struct_types` the BTF looks as follows: 4604*082108fdSEduard Zingerman * 4605*082108fdSEduard Zingerman * [1] STRUCT 'foo' size=4 vlen=1 ... 4606*082108fdSEduard Zingerman * [2] INT 'int' size=4 ... 4607*082108fdSEduard Zingerman * [3] PTR '(anon)' type_id=1 4608*082108fdSEduard Zingerman * [4] FWD 'foo' fwd_kind=struct 4609*082108fdSEduard Zingerman * [5] PTR '(anon)' type_id=4 4610*082108fdSEduard Zingerman * 4611*082108fdSEduard Zingerman * This pass assumes that such FWD declarations should be mapped to 4612*082108fdSEduard Zingerman * structs or unions with identical name in case if the name is not 4613*082108fdSEduard Zingerman * ambiguous. 4614*082108fdSEduard Zingerman */ 4615*082108fdSEduard Zingerman static int btf_dedup_resolve_fwds(struct btf_dedup *d) 4616*082108fdSEduard Zingerman { 4617*082108fdSEduard Zingerman int i, err; 4618*082108fdSEduard Zingerman struct hashmap *names_map; 4619*082108fdSEduard Zingerman 4620*082108fdSEduard Zingerman names_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); 4621*082108fdSEduard Zingerman if (IS_ERR(names_map)) 4622*082108fdSEduard Zingerman return PTR_ERR(names_map); 4623*082108fdSEduard Zingerman 4624*082108fdSEduard Zingerman err = btf_dedup_fill_unique_names_map(d, names_map); 4625*082108fdSEduard Zingerman if (err < 0) 4626*082108fdSEduard Zingerman goto exit; 4627*082108fdSEduard Zingerman 4628*082108fdSEduard Zingerman for (i = 0; i < d->btf->nr_types; i++) { 4629*082108fdSEduard Zingerman err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i); 4630*082108fdSEduard Zingerman if (err < 0) 4631*082108fdSEduard Zingerman break; 4632*082108fdSEduard Zingerman } 4633*082108fdSEduard Zingerman 4634*082108fdSEduard Zingerman exit: 4635*082108fdSEduard Zingerman hashmap__free(names_map); 4636*082108fdSEduard Zingerman return err; 4637*082108fdSEduard Zingerman } 4638*082108fdSEduard Zingerman 4639*082108fdSEduard Zingerman /* 4640d5caef5bSAndrii Nakryiko * Compact types. 4641d5caef5bSAndrii Nakryiko * 4642d5caef5bSAndrii Nakryiko * After we established for each type its corresponding canonical representative 4643d5caef5bSAndrii Nakryiko * type, we now can eliminate types that are not canonical and leave only 4644d5caef5bSAndrii Nakryiko * canonical ones layed out sequentially in memory by copying them over 4645d5caef5bSAndrii Nakryiko * duplicates. During compaction btf_dedup->hypot_map array is reused to store 4646d5caef5bSAndrii Nakryiko * a map from original type ID to a new compacted type ID, which will be used 4647d5caef5bSAndrii Nakryiko * during next phase to "fix up" type IDs, referenced from struct/union and 4648d5caef5bSAndrii Nakryiko * reference types. 4649d5caef5bSAndrii Nakryiko */ 4650d5caef5bSAndrii Nakryiko static int btf_dedup_compact_types(struct btf_dedup *d) 4651d5caef5bSAndrii Nakryiko { 4652740e69c3SAndrii Nakryiko __u32 *new_offs; 4653f86524efSAndrii Nakryiko __u32 next_type_id = d->btf->start_id; 4654f86524efSAndrii Nakryiko const struct btf_type *t; 4655740e69c3SAndrii Nakryiko void *p; 4656f86524efSAndrii Nakryiko int i, id, len; 4657d5caef5bSAndrii Nakryiko 4658d5caef5bSAndrii Nakryiko /* we are going to reuse hypot_map to store compaction remapping */ 4659d5caef5bSAndrii Nakryiko d->hypot_map[0] = 0; 4660f86524efSAndrii Nakryiko /* base BTF types are not renumbered */ 4661f86524efSAndrii Nakryiko for (id = 1; id < d->btf->start_id; id++) 4662f86524efSAndrii Nakryiko d->hypot_map[id] = id; 4663f86524efSAndrii Nakryiko for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) 4664f86524efSAndrii Nakryiko d->hypot_map[id] = BTF_UNPROCESSED_ID; 4665d5caef5bSAndrii Nakryiko 4666740e69c3SAndrii Nakryiko p = d->btf->types_data; 4667d5caef5bSAndrii Nakryiko 4668f86524efSAndrii Nakryiko for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) { 4669f86524efSAndrii Nakryiko if (d->map[id] != id) 4670d5caef5bSAndrii Nakryiko continue; 4671d5caef5bSAndrii Nakryiko 4672f86524efSAndrii Nakryiko t = btf__type_by_id(d->btf, id); 4673f86524efSAndrii Nakryiko len = btf_type_size(t); 4674d5caef5bSAndrii Nakryiko if (len < 0) 4675d5caef5bSAndrii Nakryiko return len; 4676d5caef5bSAndrii Nakryiko 4677f86524efSAndrii Nakryiko memmove(p, t, len); 4678f86524efSAndrii Nakryiko d->hypot_map[id] = next_type_id; 4679f86524efSAndrii Nakryiko d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data; 4680d5caef5bSAndrii Nakryiko p += len; 4681d5caef5bSAndrii Nakryiko next_type_id++; 4682d5caef5bSAndrii Nakryiko } 4683d5caef5bSAndrii Nakryiko 4684d5caef5bSAndrii Nakryiko /* shrink struct btf's internal types index and update btf_header */ 4685f86524efSAndrii Nakryiko d->btf->nr_types = next_type_id - d->btf->start_id; 4686ba451366SAndrii Nakryiko d->btf->type_offs_cap = d->btf->nr_types; 4687740e69c3SAndrii Nakryiko d->btf->hdr->type_len = p - d->btf->types_data; 4688740e69c3SAndrii Nakryiko new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, 4689740e69c3SAndrii Nakryiko sizeof(*new_offs)); 4690f86524efSAndrii Nakryiko if (d->btf->type_offs_cap && !new_offs) 4691d5caef5bSAndrii Nakryiko return -ENOMEM; 4692740e69c3SAndrii Nakryiko d->btf->type_offs = new_offs; 4693919d2b1dSAndrii Nakryiko d->btf->hdr->str_off = d->btf->hdr->type_len; 4694b8604247SAndrii Nakryiko d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; 4695d5caef5bSAndrii Nakryiko return 0; 4696d5caef5bSAndrii Nakryiko } 4697d5caef5bSAndrii Nakryiko 4698d5caef5bSAndrii Nakryiko /* 4699d5caef5bSAndrii Nakryiko * Figure out final (deduplicated and compacted) type ID for provided original 4700d5caef5bSAndrii Nakryiko * `type_id` by first resolving it into corresponding canonical type ID and 4701d5caef5bSAndrii Nakryiko * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map, 4702d5caef5bSAndrii Nakryiko * which is populated during compaction phase. 4703d5caef5bSAndrii Nakryiko */ 4704f36e99a4SAndrii Nakryiko static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx) 4705d5caef5bSAndrii Nakryiko { 4706f36e99a4SAndrii Nakryiko struct btf_dedup *d = ctx; 4707d5caef5bSAndrii Nakryiko __u32 resolved_type_id, new_type_id; 4708d5caef5bSAndrii Nakryiko 4709f36e99a4SAndrii Nakryiko resolved_type_id = resolve_type_id(d, *type_id); 4710d5caef5bSAndrii Nakryiko new_type_id = d->hypot_map[resolved_type_id]; 47115aab392cSAndrii Nakryiko if (new_type_id > BTF_MAX_NR_TYPES) 4712d5caef5bSAndrii Nakryiko return -EINVAL; 4713f36e99a4SAndrii Nakryiko 4714f36e99a4SAndrii Nakryiko *type_id = new_type_id; 4715f36e99a4SAndrii Nakryiko return 0; 4716d5caef5bSAndrii Nakryiko } 4717d5caef5bSAndrii Nakryiko 4718d5caef5bSAndrii Nakryiko /* 4719d5caef5bSAndrii Nakryiko * Remap referenced type IDs into deduped type IDs. 4720d5caef5bSAndrii Nakryiko * 4721d5caef5bSAndrii Nakryiko * After BTF types are deduplicated and compacted, their final type IDs may 4722d5caef5bSAndrii Nakryiko * differ from original ones. The map from original to a corresponding 4723d5caef5bSAndrii Nakryiko * deduped type ID is stored in btf_dedup->hypot_map and is populated during 4724d5caef5bSAndrii Nakryiko * compaction phase. During remapping phase we are rewriting all type IDs 4725d5caef5bSAndrii Nakryiko * referenced from any BTF type (e.g., struct fields, func proto args, etc) to 4726d5caef5bSAndrii Nakryiko * their final deduped type IDs. 4727d5caef5bSAndrii Nakryiko */ 4728d5caef5bSAndrii Nakryiko static int btf_dedup_remap_types(struct btf_dedup *d) 4729d5caef5bSAndrii Nakryiko { 4730d5caef5bSAndrii Nakryiko int i, r; 4731d5caef5bSAndrii Nakryiko 4732f86524efSAndrii Nakryiko for (i = 0; i < d->btf->nr_types; i++) { 4733f36e99a4SAndrii Nakryiko struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 4734f36e99a4SAndrii Nakryiko 4735f36e99a4SAndrii Nakryiko r = btf_type_visit_type_ids(t, btf_dedup_remap_type_id, d); 4736f36e99a4SAndrii Nakryiko if (r) 4737d5caef5bSAndrii Nakryiko return r; 4738d5caef5bSAndrii Nakryiko } 4739f36e99a4SAndrii Nakryiko 4740f36e99a4SAndrii Nakryiko if (!d->btf_ext) 4741f36e99a4SAndrii Nakryiko return 0; 4742f36e99a4SAndrii Nakryiko 4743f36e99a4SAndrii Nakryiko r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); 4744f36e99a4SAndrii Nakryiko if (r) 4745f36e99a4SAndrii Nakryiko return r; 4746f36e99a4SAndrii Nakryiko 4747d5caef5bSAndrii Nakryiko return 0; 4748d5caef5bSAndrii Nakryiko } 4749fb2426adSMartin KaFai Lau 4750fb2426adSMartin KaFai Lau /* 4751fb2426adSMartin KaFai Lau * Probe few well-known locations for vmlinux kernel image and try to load BTF 4752fb2426adSMartin KaFai Lau * data out of it to use for target BTF. 4753fb2426adSMartin KaFai Lau */ 4754a710eed3SHengqi Chen struct btf *btf__load_vmlinux_btf(void) 4755fb2426adSMartin KaFai Lau { 475601f2e36cSTao Chen const char *locations[] = { 4757fb2426adSMartin KaFai Lau /* try canonical vmlinux BTF through sysfs first */ 475801f2e36cSTao Chen "/sys/kernel/btf/vmlinux", 475901f2e36cSTao Chen /* fall back to trying to find vmlinux on disk otherwise */ 476001f2e36cSTao Chen "/boot/vmlinux-%1$s", 476101f2e36cSTao Chen "/lib/modules/%1$s/vmlinux-%1$s", 476201f2e36cSTao Chen "/lib/modules/%1$s/build/vmlinux", 476301f2e36cSTao Chen "/usr/lib/modules/%1$s/kernel/vmlinux", 476401f2e36cSTao Chen "/usr/lib/debug/boot/vmlinux-%1$s", 476501f2e36cSTao Chen "/usr/lib/debug/boot/vmlinux-%1$s.debug", 476601f2e36cSTao Chen "/usr/lib/debug/lib/modules/%1$s/vmlinux", 4767fb2426adSMartin KaFai Lau }; 4768fb2426adSMartin KaFai Lau char path[PATH_MAX + 1]; 4769fb2426adSMartin KaFai Lau struct utsname buf; 4770fb2426adSMartin KaFai Lau struct btf *btf; 4771e9fc3ce9SAndrii Nakryiko int i, err; 4772fb2426adSMartin KaFai Lau 4773fb2426adSMartin KaFai Lau uname(&buf); 4774fb2426adSMartin KaFai Lau 4775fb2426adSMartin KaFai Lau for (i = 0; i < ARRAY_SIZE(locations); i++) { 477601f2e36cSTao Chen snprintf(path, PATH_MAX, locations[i], buf.release); 4777fb2426adSMartin KaFai Lau 47786a4ab886SJon Doron if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS)) 4779fb2426adSMartin KaFai Lau continue; 4780fb2426adSMartin KaFai Lau 478101f2e36cSTao Chen btf = btf__parse(path, NULL); 4782e9fc3ce9SAndrii Nakryiko err = libbpf_get_error(btf); 4783e9fc3ce9SAndrii Nakryiko pr_debug("loading kernel BTF '%s': %d\n", path, err); 4784e9fc3ce9SAndrii Nakryiko if (err) 4785fb2426adSMartin KaFai Lau continue; 4786fb2426adSMartin KaFai Lau 4787fb2426adSMartin KaFai Lau return btf; 4788fb2426adSMartin KaFai Lau } 4789fb2426adSMartin KaFai Lau 4790fb2426adSMartin KaFai Lau pr_warn("failed to find valid kernel BTF\n"); 4791e9fc3ce9SAndrii Nakryiko return libbpf_err_ptr(-ESRCH); 4792fb2426adSMartin KaFai Lau } 4793f36e99a4SAndrii Nakryiko 4794a710eed3SHengqi Chen struct btf *libbpf_find_kernel_btf(void) __attribute__((alias("btf__load_vmlinux_btf"))); 4795a710eed3SHengqi Chen 4796a710eed3SHengqi Chen struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf) 4797a710eed3SHengqi Chen { 4798a710eed3SHengqi Chen char path[80]; 4799a710eed3SHengqi Chen 4800a710eed3SHengqi Chen snprintf(path, sizeof(path), "/sys/kernel/btf/%s", module_name); 4801a710eed3SHengqi Chen return btf__parse_split(path, vmlinux_btf); 4802a710eed3SHengqi Chen } 4803a710eed3SHengqi Chen 4804f36e99a4SAndrii Nakryiko int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx) 4805f36e99a4SAndrii Nakryiko { 4806f36e99a4SAndrii Nakryiko int i, n, err; 4807f36e99a4SAndrii Nakryiko 4808f36e99a4SAndrii Nakryiko switch (btf_kind(t)) { 4809f36e99a4SAndrii Nakryiko case BTF_KIND_INT: 4810f36e99a4SAndrii Nakryiko case BTF_KIND_FLOAT: 4811f36e99a4SAndrii Nakryiko case BTF_KIND_ENUM: 4812dffbbdc2SYonghong Song case BTF_KIND_ENUM64: 4813f36e99a4SAndrii Nakryiko return 0; 4814f36e99a4SAndrii Nakryiko 4815f36e99a4SAndrii Nakryiko case BTF_KIND_FWD: 4816f36e99a4SAndrii Nakryiko case BTF_KIND_CONST: 4817f36e99a4SAndrii Nakryiko case BTF_KIND_VOLATILE: 4818f36e99a4SAndrii Nakryiko case BTF_KIND_RESTRICT: 4819f36e99a4SAndrii Nakryiko case BTF_KIND_PTR: 4820f36e99a4SAndrii Nakryiko case BTF_KIND_TYPEDEF: 4821f36e99a4SAndrii Nakryiko case BTF_KIND_FUNC: 4822f36e99a4SAndrii Nakryiko case BTF_KIND_VAR: 4823223f903eSYonghong Song case BTF_KIND_DECL_TAG: 48242dc1e488SYonghong Song case BTF_KIND_TYPE_TAG: 4825f36e99a4SAndrii Nakryiko return visit(&t->type, ctx); 4826f36e99a4SAndrii Nakryiko 4827f36e99a4SAndrii Nakryiko case BTF_KIND_ARRAY: { 4828f36e99a4SAndrii Nakryiko struct btf_array *a = btf_array(t); 4829f36e99a4SAndrii Nakryiko 4830f36e99a4SAndrii Nakryiko err = visit(&a->type, ctx); 4831f36e99a4SAndrii Nakryiko err = err ?: visit(&a->index_type, ctx); 4832f36e99a4SAndrii Nakryiko return err; 4833f36e99a4SAndrii Nakryiko } 4834f36e99a4SAndrii Nakryiko 4835f36e99a4SAndrii Nakryiko case BTF_KIND_STRUCT: 4836f36e99a4SAndrii Nakryiko case BTF_KIND_UNION: { 4837f36e99a4SAndrii Nakryiko struct btf_member *m = btf_members(t); 4838f36e99a4SAndrii Nakryiko 4839f36e99a4SAndrii Nakryiko for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4840f36e99a4SAndrii Nakryiko err = visit(&m->type, ctx); 4841f36e99a4SAndrii Nakryiko if (err) 4842f36e99a4SAndrii Nakryiko return err; 4843f36e99a4SAndrii Nakryiko } 4844f36e99a4SAndrii Nakryiko return 0; 4845f36e99a4SAndrii Nakryiko } 4846f36e99a4SAndrii Nakryiko 4847f36e99a4SAndrii Nakryiko case BTF_KIND_FUNC_PROTO: { 4848f36e99a4SAndrii Nakryiko struct btf_param *m = btf_params(t); 4849f36e99a4SAndrii Nakryiko 4850f36e99a4SAndrii Nakryiko err = visit(&t->type, ctx); 4851f36e99a4SAndrii Nakryiko if (err) 4852f36e99a4SAndrii Nakryiko return err; 4853f36e99a4SAndrii Nakryiko for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4854f36e99a4SAndrii Nakryiko err = visit(&m->type, ctx); 4855f36e99a4SAndrii Nakryiko if (err) 4856f36e99a4SAndrii Nakryiko return err; 4857f36e99a4SAndrii Nakryiko } 4858f36e99a4SAndrii Nakryiko return 0; 4859f36e99a4SAndrii Nakryiko } 4860f36e99a4SAndrii Nakryiko 4861f36e99a4SAndrii Nakryiko case BTF_KIND_DATASEC: { 4862f36e99a4SAndrii Nakryiko struct btf_var_secinfo *m = btf_var_secinfos(t); 4863f36e99a4SAndrii Nakryiko 4864f36e99a4SAndrii Nakryiko for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4865f36e99a4SAndrii Nakryiko err = visit(&m->type, ctx); 4866f36e99a4SAndrii Nakryiko if (err) 4867f36e99a4SAndrii Nakryiko return err; 4868f36e99a4SAndrii Nakryiko } 4869f36e99a4SAndrii Nakryiko return 0; 4870f36e99a4SAndrii Nakryiko } 4871f36e99a4SAndrii Nakryiko 4872f36e99a4SAndrii Nakryiko default: 4873f36e99a4SAndrii Nakryiko return -EINVAL; 4874f36e99a4SAndrii Nakryiko } 4875f36e99a4SAndrii Nakryiko } 4876f36e99a4SAndrii Nakryiko 4877f36e99a4SAndrii Nakryiko int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx) 4878f36e99a4SAndrii Nakryiko { 4879f36e99a4SAndrii Nakryiko int i, n, err; 4880f36e99a4SAndrii Nakryiko 4881f36e99a4SAndrii Nakryiko err = visit(&t->name_off, ctx); 4882f36e99a4SAndrii Nakryiko if (err) 4883f36e99a4SAndrii Nakryiko return err; 4884f36e99a4SAndrii Nakryiko 4885f36e99a4SAndrii Nakryiko switch (btf_kind(t)) { 4886f36e99a4SAndrii Nakryiko case BTF_KIND_STRUCT: 4887f36e99a4SAndrii Nakryiko case BTF_KIND_UNION: { 4888f36e99a4SAndrii Nakryiko struct btf_member *m = btf_members(t); 4889f36e99a4SAndrii Nakryiko 4890f36e99a4SAndrii Nakryiko for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4891f36e99a4SAndrii Nakryiko err = visit(&m->name_off, ctx); 4892f36e99a4SAndrii Nakryiko if (err) 4893f36e99a4SAndrii Nakryiko return err; 4894f36e99a4SAndrii Nakryiko } 4895f36e99a4SAndrii Nakryiko break; 4896f36e99a4SAndrii Nakryiko } 4897f36e99a4SAndrii Nakryiko case BTF_KIND_ENUM: { 4898f36e99a4SAndrii Nakryiko struct btf_enum *m = btf_enum(t); 4899f36e99a4SAndrii Nakryiko 4900f36e99a4SAndrii Nakryiko for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4901f36e99a4SAndrii Nakryiko err = visit(&m->name_off, ctx); 4902f36e99a4SAndrii Nakryiko if (err) 4903f36e99a4SAndrii Nakryiko return err; 4904f36e99a4SAndrii Nakryiko } 4905f36e99a4SAndrii Nakryiko break; 4906f36e99a4SAndrii Nakryiko } 4907dffbbdc2SYonghong Song case BTF_KIND_ENUM64: { 4908dffbbdc2SYonghong Song struct btf_enum64 *m = btf_enum64(t); 4909dffbbdc2SYonghong Song 4910dffbbdc2SYonghong Song for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4911dffbbdc2SYonghong Song err = visit(&m->name_off, ctx); 4912dffbbdc2SYonghong Song if (err) 4913dffbbdc2SYonghong Song return err; 4914dffbbdc2SYonghong Song } 4915dffbbdc2SYonghong Song break; 4916dffbbdc2SYonghong Song } 4917f36e99a4SAndrii Nakryiko case BTF_KIND_FUNC_PROTO: { 4918f36e99a4SAndrii Nakryiko struct btf_param *m = btf_params(t); 4919f36e99a4SAndrii Nakryiko 4920f36e99a4SAndrii Nakryiko for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4921f36e99a4SAndrii Nakryiko err = visit(&m->name_off, ctx); 4922f36e99a4SAndrii Nakryiko if (err) 4923f36e99a4SAndrii Nakryiko return err; 4924f36e99a4SAndrii Nakryiko } 4925f36e99a4SAndrii Nakryiko break; 4926f36e99a4SAndrii Nakryiko } 4927f36e99a4SAndrii Nakryiko default: 4928f36e99a4SAndrii Nakryiko break; 4929f36e99a4SAndrii Nakryiko } 4930f36e99a4SAndrii Nakryiko 4931f36e99a4SAndrii Nakryiko return 0; 4932f36e99a4SAndrii Nakryiko } 4933f36e99a4SAndrii Nakryiko 4934f36e99a4SAndrii Nakryiko int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx) 4935f36e99a4SAndrii Nakryiko { 4936f36e99a4SAndrii Nakryiko const struct btf_ext_info *seg; 4937f36e99a4SAndrii Nakryiko struct btf_ext_info_sec *sec; 4938f36e99a4SAndrii Nakryiko int i, err; 4939f36e99a4SAndrii Nakryiko 4940f36e99a4SAndrii Nakryiko seg = &btf_ext->func_info; 4941f36e99a4SAndrii Nakryiko for_each_btf_ext_sec(seg, sec) { 4942f36e99a4SAndrii Nakryiko struct bpf_func_info_min *rec; 4943f36e99a4SAndrii Nakryiko 4944f36e99a4SAndrii Nakryiko for_each_btf_ext_rec(seg, sec, i, rec) { 4945f36e99a4SAndrii Nakryiko err = visit(&rec->type_id, ctx); 4946f36e99a4SAndrii Nakryiko if (err < 0) 4947f36e99a4SAndrii Nakryiko return err; 4948f36e99a4SAndrii Nakryiko } 4949f36e99a4SAndrii Nakryiko } 4950f36e99a4SAndrii Nakryiko 4951f36e99a4SAndrii Nakryiko seg = &btf_ext->core_relo_info; 4952f36e99a4SAndrii Nakryiko for_each_btf_ext_sec(seg, sec) { 4953f36e99a4SAndrii Nakryiko struct bpf_core_relo *rec; 4954f36e99a4SAndrii Nakryiko 4955f36e99a4SAndrii Nakryiko for_each_btf_ext_rec(seg, sec, i, rec) { 4956f36e99a4SAndrii Nakryiko err = visit(&rec->type_id, ctx); 4957f36e99a4SAndrii Nakryiko if (err < 0) 4958f36e99a4SAndrii Nakryiko return err; 4959f36e99a4SAndrii Nakryiko } 4960f36e99a4SAndrii Nakryiko } 4961f36e99a4SAndrii Nakryiko 4962f36e99a4SAndrii Nakryiko return 0; 4963f36e99a4SAndrii Nakryiko } 4964f36e99a4SAndrii Nakryiko 4965f36e99a4SAndrii Nakryiko int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx) 4966f36e99a4SAndrii Nakryiko { 4967f36e99a4SAndrii Nakryiko const struct btf_ext_info *seg; 4968f36e99a4SAndrii Nakryiko struct btf_ext_info_sec *sec; 4969f36e99a4SAndrii Nakryiko int i, err; 4970f36e99a4SAndrii Nakryiko 4971f36e99a4SAndrii Nakryiko seg = &btf_ext->func_info; 4972f36e99a4SAndrii Nakryiko for_each_btf_ext_sec(seg, sec) { 4973f36e99a4SAndrii Nakryiko err = visit(&sec->sec_name_off, ctx); 4974f36e99a4SAndrii Nakryiko if (err) 4975f36e99a4SAndrii Nakryiko return err; 4976f36e99a4SAndrii Nakryiko } 4977f36e99a4SAndrii Nakryiko 4978f36e99a4SAndrii Nakryiko seg = &btf_ext->line_info; 4979f36e99a4SAndrii Nakryiko for_each_btf_ext_sec(seg, sec) { 4980f36e99a4SAndrii Nakryiko struct bpf_line_info_min *rec; 4981f36e99a4SAndrii Nakryiko 4982f36e99a4SAndrii Nakryiko err = visit(&sec->sec_name_off, ctx); 4983f36e99a4SAndrii Nakryiko if (err) 4984f36e99a4SAndrii Nakryiko return err; 4985f36e99a4SAndrii Nakryiko 4986f36e99a4SAndrii Nakryiko for_each_btf_ext_rec(seg, sec, i, rec) { 4987f36e99a4SAndrii Nakryiko err = visit(&rec->file_name_off, ctx); 4988f36e99a4SAndrii Nakryiko if (err) 4989f36e99a4SAndrii Nakryiko return err; 4990f36e99a4SAndrii Nakryiko err = visit(&rec->line_off, ctx); 4991f36e99a4SAndrii Nakryiko if (err) 4992f36e99a4SAndrii Nakryiko return err; 4993f36e99a4SAndrii Nakryiko } 4994f36e99a4SAndrii Nakryiko } 4995f36e99a4SAndrii Nakryiko 4996f36e99a4SAndrii Nakryiko seg = &btf_ext->core_relo_info; 4997f36e99a4SAndrii Nakryiko for_each_btf_ext_sec(seg, sec) { 4998f36e99a4SAndrii Nakryiko struct bpf_core_relo *rec; 4999f36e99a4SAndrii Nakryiko 5000f36e99a4SAndrii Nakryiko err = visit(&sec->sec_name_off, ctx); 5001f36e99a4SAndrii Nakryiko if (err) 5002f36e99a4SAndrii Nakryiko return err; 5003f36e99a4SAndrii Nakryiko 5004f36e99a4SAndrii Nakryiko for_each_btf_ext_rec(seg, sec, i, rec) { 5005f36e99a4SAndrii Nakryiko err = visit(&rec->access_str_off, ctx); 5006f36e99a4SAndrii Nakryiko if (err) 5007f36e99a4SAndrii Nakryiko return err; 5008f36e99a4SAndrii Nakryiko } 5009f36e99a4SAndrii Nakryiko } 5010f36e99a4SAndrii Nakryiko 5011f36e99a4SAndrii Nakryiko return 0; 5012f36e99a4SAndrii Nakryiko } 5013