1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 3 /* 4 * Internal libbpf helpers. 5 * 6 * Copyright (c) 2019 Facebook 7 */ 8 9 #ifndef __LIBBPF_LIBBPF_INTERNAL_H 10 #define __LIBBPF_LIBBPF_INTERNAL_H 11 12 #include <stdlib.h> 13 #include <limits.h> 14 #include <errno.h> 15 #include <linux/err.h> 16 #include "libbpf_legacy.h" 17 #include "relo_core.h" 18 19 /* make sure libbpf doesn't use kernel-only integer typedefs */ 20 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 21 22 /* prevent accidental re-addition of reallocarray() */ 23 #pragma GCC poison reallocarray 24 25 #include "libbpf.h" 26 #include "btf.h" 27 28 #ifndef EM_BPF 29 #define EM_BPF 247 30 #endif 31 32 #ifndef R_BPF_64_64 33 #define R_BPF_64_64 1 34 #endif 35 #ifndef R_BPF_64_ABS64 36 #define R_BPF_64_ABS64 2 37 #endif 38 #ifndef R_BPF_64_ABS32 39 #define R_BPF_64_ABS32 3 40 #endif 41 #ifndef R_BPF_64_32 42 #define R_BPF_64_32 10 43 #endif 44 45 #ifndef SHT_LLVM_ADDRSIG 46 #define SHT_LLVM_ADDRSIG 0x6FFF4C03 47 #endif 48 49 /* if libelf is old and doesn't support mmap(), fall back to read() */ 50 #ifndef ELF_C_READ_MMAP 51 #define ELF_C_READ_MMAP ELF_C_READ 52 #endif 53 54 /* Older libelf all end up in this expression, for both 32 and 64 bit */ 55 #ifndef GELF_ST_VISIBILITY 56 #define GELF_ST_VISIBILITY(o) ((o) & 0x03) 57 #endif 58 59 #define BTF_INFO_ENC(kind, kind_flag, vlen) \ 60 ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) 61 #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) 62 #define BTF_INT_ENC(encoding, bits_offset, nr_bits) \ 63 ((encoding) << 24 | (bits_offset) << 16 | (nr_bits)) 64 #define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \ 65 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \ 66 BTF_INT_ENC(encoding, bits_offset, bits) 67 #define BTF_MEMBER_ENC(name, type, bits_offset) (name), (type), (bits_offset) 68 #define BTF_PARAM_ENC(name, type) (name), (type) 69 #define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size) 70 #define BTF_TYPE_FLOAT_ENC(name, sz) \ 71 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz) 72 #define BTF_TYPE_TAG_ENC(value, type, component_idx) \ 73 BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TAG, 0, 0), type), (component_idx) 74 75 #ifndef likely 76 #define likely(x) __builtin_expect(!!(x), 1) 77 #endif 78 #ifndef unlikely 79 #define unlikely(x) __builtin_expect(!!(x), 0) 80 #endif 81 #ifndef min 82 # define min(x, y) ((x) < (y) ? (x) : (y)) 83 #endif 84 #ifndef max 85 # define max(x, y) ((x) < (y) ? (y) : (x)) 86 #endif 87 #ifndef offsetofend 88 # define offsetofend(TYPE, FIELD) \ 89 (offsetof(TYPE, FIELD) + sizeof(((TYPE *)0)->FIELD)) 90 #endif 91 92 /* Symbol versioning is different between static and shared library. 93 * Properly versioned symbols are needed for shared library, but 94 * only the symbol of the new version is needed for static library. 95 * Starting with GNU C 10, use symver attribute instead of .symver assembler 96 * directive, which works better with GCC LTO builds. 97 */ 98 #if defined(SHARED) && defined(__GNUC__) && __GNUC__ >= 10 99 100 #define DEFAULT_VERSION(internal_name, api_name, version) \ 101 __attribute__((symver(#api_name "@@" #version))) 102 #define COMPAT_VERSION(internal_name, api_name, version) \ 103 __attribute__((symver(#api_name "@" #version))) 104 105 #elif defined(SHARED) 106 107 #define COMPAT_VERSION(internal_name, api_name, version) \ 108 asm(".symver " #internal_name "," #api_name "@" #version); 109 #define DEFAULT_VERSION(internal_name, api_name, version) \ 110 asm(".symver " #internal_name "," #api_name "@@" #version); 111 112 #else /* !SHARED */ 113 114 #define COMPAT_VERSION(internal_name, api_name, version) 115 #define DEFAULT_VERSION(internal_name, api_name, version) \ 116 extern typeof(internal_name) api_name \ 117 __attribute__((alias(#internal_name))); 118 119 #endif 120 121 extern void libbpf_print(enum libbpf_print_level level, 122 const char *format, ...) 123 __attribute__((format(printf, 2, 3))); 124 125 #define __pr(level, fmt, ...) \ 126 do { \ 127 libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \ 128 } while (0) 129 130 #define pr_warn(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__) 131 #define pr_info(fmt, ...) __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__) 132 #define pr_debug(fmt, ...) __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__) 133 134 #ifndef __has_builtin 135 #define __has_builtin(x) 0 136 #endif 137 /* 138 * Re-implement glibc's reallocarray() for libbpf internal-only use. 139 * reallocarray(), unfortunately, is not available in all versions of glibc, 140 * so requires extra feature detection and using reallocarray() stub from 141 * <tools/libc_compat.h> and COMPAT_NEED_REALLOCARRAY. All this complicates 142 * build of libbpf unnecessarily and is just a maintenance burden. Instead, 143 * it's trivial to implement libbpf-specific internal version and use it 144 * throughout libbpf. 145 */ 146 static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size) 147 { 148 size_t total; 149 150 #if __has_builtin(__builtin_mul_overflow) 151 if (unlikely(__builtin_mul_overflow(nmemb, size, &total))) 152 return NULL; 153 #else 154 if (size == 0 || nmemb > ULONG_MAX / size) 155 return NULL; 156 total = nmemb * size; 157 #endif 158 return realloc(ptr, total); 159 } 160 161 struct btf; 162 struct btf_type; 163 164 struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id); 165 const char *btf_kind_str(const struct btf_type *t); 166 const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); 167 168 static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) 169 { 170 return (enum btf_func_linkage)(int)btf_vlen(t); 171 } 172 173 static inline __u32 btf_type_info(int kind, int vlen, int kflag) 174 { 175 return (kflag << 31) | (kind << 24) | vlen; 176 } 177 178 enum map_def_parts { 179 MAP_DEF_MAP_TYPE = 0x001, 180 MAP_DEF_KEY_TYPE = 0x002, 181 MAP_DEF_KEY_SIZE = 0x004, 182 MAP_DEF_VALUE_TYPE = 0x008, 183 MAP_DEF_VALUE_SIZE = 0x010, 184 MAP_DEF_MAX_ENTRIES = 0x020, 185 MAP_DEF_MAP_FLAGS = 0x040, 186 MAP_DEF_NUMA_NODE = 0x080, 187 MAP_DEF_PINNING = 0x100, 188 MAP_DEF_INNER_MAP = 0x200, 189 190 MAP_DEF_ALL = 0x3ff, /* combination of all above */ 191 }; 192 193 struct btf_map_def { 194 enum map_def_parts parts; 195 __u32 map_type; 196 __u32 key_type_id; 197 __u32 key_size; 198 __u32 value_type_id; 199 __u32 value_size; 200 __u32 max_entries; 201 __u32 map_flags; 202 __u32 numa_node; 203 __u32 pinning; 204 }; 205 206 int parse_btf_map_def(const char *map_name, struct btf *btf, 207 const struct btf_type *def_t, bool strict, 208 struct btf_map_def *map_def, struct btf_map_def *inner_def); 209 210 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 211 size_t cur_cnt, size_t max_cnt, size_t add_cnt); 212 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt); 213 214 static inline bool libbpf_is_mem_zeroed(const char *p, ssize_t len) 215 { 216 while (len > 0) { 217 if (*p) 218 return false; 219 p++; 220 len--; 221 } 222 return true; 223 } 224 225 static inline bool libbpf_validate_opts(const char *opts, 226 size_t opts_sz, size_t user_sz, 227 const char *type_name) 228 { 229 if (user_sz < sizeof(size_t)) { 230 pr_warn("%s size (%zu) is too small\n", type_name, user_sz); 231 return false; 232 } 233 if (!libbpf_is_mem_zeroed(opts + opts_sz, (ssize_t)user_sz - opts_sz)) { 234 pr_warn("%s has non-zero extra bytes\n", type_name); 235 return false; 236 } 237 return true; 238 } 239 240 #define OPTS_VALID(opts, type) \ 241 (!(opts) || libbpf_validate_opts((const char *)opts, \ 242 offsetofend(struct type, \ 243 type##__last_field), \ 244 (opts)->sz, #type)) 245 #define OPTS_HAS(opts, field) \ 246 ((opts) && opts->sz >= offsetofend(typeof(*(opts)), field)) 247 #define OPTS_GET(opts, field, fallback_value) \ 248 (OPTS_HAS(opts, field) ? (opts)->field : fallback_value) 249 #define OPTS_SET(opts, field, value) \ 250 do { \ 251 if (OPTS_HAS(opts, field)) \ 252 (opts)->field = value; \ 253 } while (0) 254 255 #define OPTS_ZEROED(opts, last_nonzero_field) \ 256 ({ \ 257 ssize_t __off = offsetofend(typeof(*(opts)), last_nonzero_field); \ 258 !(opts) || libbpf_is_mem_zeroed((const void *)opts + __off, \ 259 (opts)->sz - __off); \ 260 }) 261 262 263 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); 264 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); 265 int libbpf__load_raw_btf(const char *raw_types, size_t types_len, 266 const char *str_sec, size_t str_len); 267 268 struct bpf_prog_load_params { 269 enum bpf_prog_type prog_type; 270 enum bpf_attach_type expected_attach_type; 271 const char *name; 272 const struct bpf_insn *insns; 273 size_t insn_cnt; 274 const char *license; 275 __u32 kern_version; 276 __u32 attach_prog_fd; 277 __u32 attach_btf_obj_fd; 278 __u32 attach_btf_id; 279 __u32 prog_ifindex; 280 __u32 prog_btf_fd; 281 __u32 prog_flags; 282 283 __u32 func_info_rec_size; 284 const void *func_info; 285 __u32 func_info_cnt; 286 287 __u32 line_info_rec_size; 288 const void *line_info; 289 __u32 line_info_cnt; 290 291 __u32 log_level; 292 char *log_buf; 293 size_t log_buf_sz; 294 }; 295 296 int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr); 297 298 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 299 __u32 *size); 300 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 301 __u32 *off); 302 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); 303 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 304 const char **prefix, int *kind); 305 306 struct btf_ext_info { 307 /* 308 * info points to the individual info section (e.g. func_info and 309 * line_info) from the .BTF.ext. It does not include the __u32 rec_size. 310 */ 311 void *info; 312 __u32 rec_size; 313 __u32 len; 314 }; 315 316 #define for_each_btf_ext_sec(seg, sec) \ 317 for (sec = (seg)->info; \ 318 (void *)sec < (seg)->info + (seg)->len; \ 319 sec = (void *)sec + sizeof(struct btf_ext_info_sec) + \ 320 (seg)->rec_size * sec->num_info) 321 322 #define for_each_btf_ext_rec(seg, sec, i, rec) \ 323 for (i = 0, rec = (void *)&(sec)->data; \ 324 i < (sec)->num_info; \ 325 i++, rec = (void *)rec + (seg)->rec_size) 326 327 /* 328 * The .BTF.ext ELF section layout defined as 329 * struct btf_ext_header 330 * func_info subsection 331 * 332 * The func_info subsection layout: 333 * record size for struct bpf_func_info in the func_info subsection 334 * struct btf_sec_func_info for section #1 335 * a list of bpf_func_info records for section #1 336 * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h 337 * but may not be identical 338 * struct btf_sec_func_info for section #2 339 * a list of bpf_func_info records for section #2 340 * ...... 341 * 342 * Note that the bpf_func_info record size in .BTF.ext may not 343 * be the same as the one defined in include/uapi/linux/bpf.h. 344 * The loader should ensure that record_size meets minimum 345 * requirement and pass the record as is to the kernel. The 346 * kernel will handle the func_info properly based on its contents. 347 */ 348 struct btf_ext_header { 349 __u16 magic; 350 __u8 version; 351 __u8 flags; 352 __u32 hdr_len; 353 354 /* All offsets are in bytes relative to the end of this header */ 355 __u32 func_info_off; 356 __u32 func_info_len; 357 __u32 line_info_off; 358 __u32 line_info_len; 359 360 /* optional part of .BTF.ext header */ 361 __u32 core_relo_off; 362 __u32 core_relo_len; 363 }; 364 365 struct btf_ext { 366 union { 367 struct btf_ext_header *hdr; 368 void *data; 369 }; 370 struct btf_ext_info func_info; 371 struct btf_ext_info line_info; 372 struct btf_ext_info core_relo_info; 373 __u32 data_size; 374 }; 375 376 struct btf_ext_info_sec { 377 __u32 sec_name_off; 378 __u32 num_info; 379 /* Followed by num_info * record_size number of bytes */ 380 __u8 data[]; 381 }; 382 383 /* The minimum bpf_func_info checked by the loader */ 384 struct bpf_func_info_min { 385 __u32 insn_off; 386 __u32 type_id; 387 }; 388 389 /* The minimum bpf_line_info checked by the loader */ 390 struct bpf_line_info_min { 391 __u32 insn_off; 392 __u32 file_name_off; 393 __u32 line_off; 394 __u32 line_col; 395 }; 396 397 398 typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); 399 typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); 400 int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx); 401 int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx); 402 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); 403 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); 404 405 extern enum libbpf_strict_mode libbpf_mode; 406 407 /* handle direct returned errors */ 408 static inline int libbpf_err(int ret) 409 { 410 if (ret < 0) 411 errno = -ret; 412 return ret; 413 } 414 415 /* handle errno-based (e.g., syscall or libc) errors according to libbpf's 416 * strict mode settings 417 */ 418 static inline int libbpf_err_errno(int ret) 419 { 420 if (libbpf_mode & LIBBPF_STRICT_DIRECT_ERRS) 421 /* errno is already assumed to be set on error */ 422 return ret < 0 ? -errno : ret; 423 424 /* legacy: on error return -1 directly and don't touch errno */ 425 return ret; 426 } 427 428 /* handle error for pointer-returning APIs, err is assumed to be < 0 always */ 429 static inline void *libbpf_err_ptr(int err) 430 { 431 /* set errno on error, this doesn't break anything */ 432 errno = -err; 433 434 if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS) 435 return NULL; 436 437 /* legacy: encode err as ptr */ 438 return ERR_PTR(err); 439 } 440 441 /* handle pointer-returning APIs' error handling */ 442 static inline void *libbpf_ptr(void *ret) 443 { 444 /* set errno on error, this doesn't break anything */ 445 if (IS_ERR(ret)) 446 errno = -PTR_ERR(ret); 447 448 if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS) 449 return IS_ERR(ret) ? NULL : ret; 450 451 /* legacy: pass-through original pointer */ 452 return ret; 453 } 454 455 static inline bool str_is_empty(const char *s) 456 { 457 return !s || !s[0]; 458 } 459 460 static inline bool is_ldimm64_insn(struct bpf_insn *insn) 461 { 462 return insn->code == (BPF_LD | BPF_IMM | BPF_DW); 463 } 464 465 #endif /* __LIBBPF_LIBBPF_INTERNAL_H */ 466