xref: /openbmc/linux/tools/lib/bpf/linker.c (revision c276044f)
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2 /*
3  * BPF static linker
4  *
5  * Copyright (c) 2021 Facebook
6  */
7 #include <stdbool.h>
8 #include <stddef.h>
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <unistd.h>
13 #include <errno.h>
14 #include <linux/err.h>
15 #include <linux/btf.h>
16 #include <elf.h>
17 #include <libelf.h>
18 #include <fcntl.h>
19 #include "libbpf.h"
20 #include "btf.h"
21 #include "libbpf_internal.h"
22 #include "strset.h"
23 
24 #define BTF_EXTERN_SEC ".extern"
25 
26 struct src_sec {
27 	const char *sec_name;
28 	/* positional (not necessarily ELF) index in an array of sections */
29 	int id;
30 	/* positional (not necessarily ELF) index of a matching section in a final object file */
31 	int dst_id;
32 	/* section data offset in a matching output section */
33 	int dst_off;
34 	/* whether section is omitted from the final ELF file */
35 	bool skipped;
36 	/* whether section is an ephemeral section, not mapped to an ELF section */
37 	bool ephemeral;
38 
39 	/* ELF info */
40 	size_t sec_idx;
41 	Elf_Scn *scn;
42 	Elf64_Shdr *shdr;
43 	Elf_Data *data;
44 
45 	/* corresponding BTF DATASEC type ID */
46 	int sec_type_id;
47 };
48 
49 struct src_obj {
50 	const char *filename;
51 	int fd;
52 	Elf *elf;
53 	/* Section header strings section index */
54 	size_t shstrs_sec_idx;
55 	/* SYMTAB section index */
56 	size_t symtab_sec_idx;
57 
58 	struct btf *btf;
59 	struct btf_ext *btf_ext;
60 
61 	/* List of sections (including ephemeral). Slot zero is unused. */
62 	struct src_sec *secs;
63 	int sec_cnt;
64 
65 	/* mapping of symbol indices from src to dst ELF */
66 	int *sym_map;
67 	/* mapping from the src BTF type IDs to dst ones */
68 	int *btf_type_map;
69 };
70 
71 /* single .BTF.ext data section */
72 struct btf_ext_sec_data {
73 	size_t rec_cnt;
74 	__u32 rec_sz;
75 	void *recs;
76 };
77 
78 struct glob_sym {
79 	/* ELF symbol index */
80 	int sym_idx;
81 	/* associated section id for .ksyms, .kconfig, etc, but not .extern */
82 	int sec_id;
83 	/* extern name offset in STRTAB */
84 	int name_off;
85 	/* optional associated BTF type ID */
86 	int btf_id;
87 	/* BTF type ID to which VAR/FUNC type is pointing to; used for
88 	 * rewriting types when extern VAR/FUNC is resolved to a concrete
89 	 * definition
90 	 */
91 	int underlying_btf_id;
92 	/* sec_var index in the corresponding dst_sec, if exists */
93 	int var_idx;
94 
95 	/* extern or resolved/global symbol */
96 	bool is_extern;
97 	/* weak or strong symbol, never goes back from strong to weak */
98 	bool is_weak;
99 };
100 
101 struct dst_sec {
102 	char *sec_name;
103 	/* positional (not necessarily ELF) index in an array of sections */
104 	int id;
105 
106 	bool ephemeral;
107 
108 	/* ELF info */
109 	size_t sec_idx;
110 	Elf_Scn *scn;
111 	Elf64_Shdr *shdr;
112 	Elf_Data *data;
113 
114 	/* final output section size */
115 	int sec_sz;
116 	/* final output contents of the section */
117 	void *raw_data;
118 
119 	/* corresponding STT_SECTION symbol index in SYMTAB */
120 	int sec_sym_idx;
121 
122 	/* section's DATASEC variable info, emitted on BTF finalization */
123 	bool has_btf;
124 	int sec_var_cnt;
125 	struct btf_var_secinfo *sec_vars;
126 
127 	/* section's .BTF.ext data */
128 	struct btf_ext_sec_data func_info;
129 	struct btf_ext_sec_data line_info;
130 	struct btf_ext_sec_data core_relo_info;
131 };
132 
133 struct bpf_linker {
134 	char *filename;
135 	int fd;
136 	Elf *elf;
137 	Elf64_Ehdr *elf_hdr;
138 
139 	/* Output sections metadata */
140 	struct dst_sec *secs;
141 	int sec_cnt;
142 
143 	struct strset *strtab_strs; /* STRTAB unique strings */
144 	size_t strtab_sec_idx; /* STRTAB section index */
145 	size_t symtab_sec_idx; /* SYMTAB section index */
146 
147 	struct btf *btf;
148 	struct btf_ext *btf_ext;
149 
150 	/* global (including extern) ELF symbols */
151 	int glob_sym_cnt;
152 	struct glob_sym *glob_syms;
153 };
154 
155 #define pr_warn_elf(fmt, ...)									\
156 	libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
157 
158 static int init_output_elf(struct bpf_linker *linker, const char *file);
159 
160 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
161 				const struct bpf_linker_file_opts *opts,
162 				struct src_obj *obj);
163 static int linker_sanity_check_elf(struct src_obj *obj);
164 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
165 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
166 static int linker_sanity_check_btf(struct src_obj *obj);
167 static int linker_sanity_check_btf_ext(struct src_obj *obj);
168 static int linker_fixup_btf(struct src_obj *obj);
169 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
170 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
171 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
172 				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
173 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
174 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
175 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
176 
177 static int finalize_btf(struct bpf_linker *linker);
178 static int finalize_btf_ext(struct bpf_linker *linker);
179 
bpf_linker__free(struct bpf_linker * linker)180 void bpf_linker__free(struct bpf_linker *linker)
181 {
182 	int i;
183 
184 	if (!linker)
185 		return;
186 
187 	free(linker->filename);
188 
189 	if (linker->elf)
190 		elf_end(linker->elf);
191 
192 	if (linker->fd >= 0)
193 		close(linker->fd);
194 
195 	strset__free(linker->strtab_strs);
196 
197 	btf__free(linker->btf);
198 	btf_ext__free(linker->btf_ext);
199 
200 	for (i = 1; i < linker->sec_cnt; i++) {
201 		struct dst_sec *sec = &linker->secs[i];
202 
203 		free(sec->sec_name);
204 		free(sec->raw_data);
205 		free(sec->sec_vars);
206 
207 		free(sec->func_info.recs);
208 		free(sec->line_info.recs);
209 		free(sec->core_relo_info.recs);
210 	}
211 	free(linker->secs);
212 
213 	free(linker->glob_syms);
214 	free(linker);
215 }
216 
bpf_linker__new(const char * filename,struct bpf_linker_opts * opts)217 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
218 {
219 	struct bpf_linker *linker;
220 	int err;
221 
222 	if (!OPTS_VALID(opts, bpf_linker_opts))
223 		return errno = EINVAL, NULL;
224 
225 	if (elf_version(EV_CURRENT) == EV_NONE) {
226 		pr_warn_elf("libelf initialization failed");
227 		return errno = EINVAL, NULL;
228 	}
229 
230 	linker = calloc(1, sizeof(*linker));
231 	if (!linker)
232 		return errno = ENOMEM, NULL;
233 
234 	linker->fd = -1;
235 
236 	err = init_output_elf(linker, filename);
237 	if (err)
238 		goto err_out;
239 
240 	return linker;
241 
242 err_out:
243 	bpf_linker__free(linker);
244 	return errno = -err, NULL;
245 }
246 
add_dst_sec(struct bpf_linker * linker,const char * sec_name)247 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
248 {
249 	struct dst_sec *secs = linker->secs, *sec;
250 	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
251 
252 	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
253 	if (!secs)
254 		return NULL;
255 
256 	/* zero out newly allocated memory */
257 	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
258 
259 	linker->secs = secs;
260 	linker->sec_cnt = new_cnt;
261 
262 	sec = &linker->secs[new_cnt - 1];
263 	sec->id = new_cnt - 1;
264 	sec->sec_name = strdup(sec_name);
265 	if (!sec->sec_name)
266 		return NULL;
267 
268 	return sec;
269 }
270 
add_new_sym(struct bpf_linker * linker,size_t * sym_idx)271 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
272 {
273 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
274 	Elf64_Sym *syms, *sym;
275 	size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
276 
277 	syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
278 	if (!syms)
279 		return NULL;
280 
281 	sym = &syms[sym_cnt];
282 	memset(sym, 0, sizeof(*sym));
283 
284 	symtab->raw_data = syms;
285 	symtab->sec_sz += sizeof(*sym);
286 	symtab->shdr->sh_size += sizeof(*sym);
287 	symtab->data->d_size += sizeof(*sym);
288 
289 	if (sym_idx)
290 		*sym_idx = sym_cnt;
291 
292 	return sym;
293 }
294 
init_output_elf(struct bpf_linker * linker,const char * file)295 static int init_output_elf(struct bpf_linker *linker, const char *file)
296 {
297 	int err, str_off;
298 	Elf64_Sym *init_sym;
299 	struct dst_sec *sec;
300 
301 	linker->filename = strdup(file);
302 	if (!linker->filename)
303 		return -ENOMEM;
304 
305 	linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
306 	if (linker->fd < 0) {
307 		err = -errno;
308 		pr_warn("failed to create '%s': %d\n", file, err);
309 		return err;
310 	}
311 
312 	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
313 	if (!linker->elf) {
314 		pr_warn_elf("failed to create ELF object");
315 		return -EINVAL;
316 	}
317 
318 	/* ELF header */
319 	linker->elf_hdr = elf64_newehdr(linker->elf);
320 	if (!linker->elf_hdr) {
321 		pr_warn_elf("failed to create ELF header");
322 		return -EINVAL;
323 	}
324 
325 	linker->elf_hdr->e_machine = EM_BPF;
326 	linker->elf_hdr->e_type = ET_REL;
327 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
328 	linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
330 	linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331 #else
332 #error "Unknown __BYTE_ORDER__"
333 #endif
334 
335 	/* STRTAB */
336 	/* initialize strset with an empty string to conform to ELF */
337 	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
338 	if (libbpf_get_error(linker->strtab_strs))
339 		return libbpf_get_error(linker->strtab_strs);
340 
341 	sec = add_dst_sec(linker, ".strtab");
342 	if (!sec)
343 		return -ENOMEM;
344 
345 	sec->scn = elf_newscn(linker->elf);
346 	if (!sec->scn) {
347 		pr_warn_elf("failed to create STRTAB section");
348 		return -EINVAL;
349 	}
350 
351 	sec->shdr = elf64_getshdr(sec->scn);
352 	if (!sec->shdr)
353 		return -EINVAL;
354 
355 	sec->data = elf_newdata(sec->scn);
356 	if (!sec->data) {
357 		pr_warn_elf("failed to create STRTAB data");
358 		return -EINVAL;
359 	}
360 
361 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
362 	if (str_off < 0)
363 		return str_off;
364 
365 	sec->sec_idx = elf_ndxscn(sec->scn);
366 	linker->elf_hdr->e_shstrndx = sec->sec_idx;
367 	linker->strtab_sec_idx = sec->sec_idx;
368 
369 	sec->shdr->sh_name = str_off;
370 	sec->shdr->sh_type = SHT_STRTAB;
371 	sec->shdr->sh_flags = SHF_STRINGS;
372 	sec->shdr->sh_offset = 0;
373 	sec->shdr->sh_link = 0;
374 	sec->shdr->sh_info = 0;
375 	sec->shdr->sh_addralign = 1;
376 	sec->shdr->sh_size = sec->sec_sz = 0;
377 	sec->shdr->sh_entsize = 0;
378 
379 	/* SYMTAB */
380 	sec = add_dst_sec(linker, ".symtab");
381 	if (!sec)
382 		return -ENOMEM;
383 
384 	sec->scn = elf_newscn(linker->elf);
385 	if (!sec->scn) {
386 		pr_warn_elf("failed to create SYMTAB section");
387 		return -EINVAL;
388 	}
389 
390 	sec->shdr = elf64_getshdr(sec->scn);
391 	if (!sec->shdr)
392 		return -EINVAL;
393 
394 	sec->data = elf_newdata(sec->scn);
395 	if (!sec->data) {
396 		pr_warn_elf("failed to create SYMTAB data");
397 		return -EINVAL;
398 	}
399 
400 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401 	if (str_off < 0)
402 		return str_off;
403 
404 	sec->sec_idx = elf_ndxscn(sec->scn);
405 	linker->symtab_sec_idx = sec->sec_idx;
406 
407 	sec->shdr->sh_name = str_off;
408 	sec->shdr->sh_type = SHT_SYMTAB;
409 	sec->shdr->sh_flags = 0;
410 	sec->shdr->sh_offset = 0;
411 	sec->shdr->sh_link = linker->strtab_sec_idx;
412 	/* sh_info should be one greater than the index of the last local
413 	 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
414 	 */
415 	sec->shdr->sh_info = 0;
416 	sec->shdr->sh_addralign = 8;
417 	sec->shdr->sh_entsize = sizeof(Elf64_Sym);
418 
419 	/* .BTF */
420 	linker->btf = btf__new_empty();
421 	err = libbpf_get_error(linker->btf);
422 	if (err)
423 		return err;
424 
425 	/* add the special all-zero symbol */
426 	init_sym = add_new_sym(linker, NULL);
427 	if (!init_sym)
428 		return -EINVAL;
429 
430 	init_sym->st_name = 0;
431 	init_sym->st_info = 0;
432 	init_sym->st_other = 0;
433 	init_sym->st_shndx = SHN_UNDEF;
434 	init_sym->st_value = 0;
435 	init_sym->st_size = 0;
436 
437 	return 0;
438 }
439 
bpf_linker__add_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts)440 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
441 			 const struct bpf_linker_file_opts *opts)
442 {
443 	struct src_obj obj = {};
444 	int err = 0;
445 
446 	if (!OPTS_VALID(opts, bpf_linker_file_opts))
447 		return libbpf_err(-EINVAL);
448 
449 	if (!linker->elf)
450 		return libbpf_err(-EINVAL);
451 
452 	err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
453 	err = err ?: linker_append_sec_data(linker, &obj);
454 	err = err ?: linker_append_elf_syms(linker, &obj);
455 	err = err ?: linker_append_elf_relos(linker, &obj);
456 	err = err ?: linker_append_btf(linker, &obj);
457 	err = err ?: linker_append_btf_ext(linker, &obj);
458 
459 	/* free up src_obj resources */
460 	free(obj.btf_type_map);
461 	btf__free(obj.btf);
462 	btf_ext__free(obj.btf_ext);
463 	free(obj.secs);
464 	free(obj.sym_map);
465 	if (obj.elf)
466 		elf_end(obj.elf);
467 	if (obj.fd >= 0)
468 		close(obj.fd);
469 
470 	return libbpf_err(err);
471 }
472 
is_dwarf_sec_name(const char * name)473 static bool is_dwarf_sec_name(const char *name)
474 {
475 	/* approximation, but the actual list is too long */
476 	return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
477 }
478 
is_ignored_sec(struct src_sec * sec)479 static bool is_ignored_sec(struct src_sec *sec)
480 {
481 	Elf64_Shdr *shdr = sec->shdr;
482 	const char *name = sec->sec_name;
483 
484 	/* no special handling of .strtab */
485 	if (shdr->sh_type == SHT_STRTAB)
486 		return true;
487 
488 	/* ignore .llvm_addrsig section as well */
489 	if (shdr->sh_type == SHT_LLVM_ADDRSIG)
490 		return true;
491 
492 	/* no subprograms will lead to an empty .text section, ignore it */
493 	if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
494 	    strcmp(sec->sec_name, ".text") == 0)
495 		return true;
496 
497 	/* DWARF sections */
498 	if (is_dwarf_sec_name(sec->sec_name))
499 		return true;
500 
501 	if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
502 		name += sizeof(".rel") - 1;
503 		/* DWARF section relocations */
504 		if (is_dwarf_sec_name(name))
505 			return true;
506 
507 		/* .BTF and .BTF.ext don't need relocations */
508 		if (strcmp(name, BTF_ELF_SEC) == 0 ||
509 		    strcmp(name, BTF_EXT_ELF_SEC) == 0)
510 			return true;
511 	}
512 
513 	return false;
514 }
515 
add_src_sec(struct src_obj * obj,const char * sec_name)516 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
517 {
518 	struct src_sec *secs = obj->secs, *sec;
519 	size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
520 
521 	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
522 	if (!secs)
523 		return NULL;
524 
525 	/* zero out newly allocated memory */
526 	memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
527 
528 	obj->secs = secs;
529 	obj->sec_cnt = new_cnt;
530 
531 	sec = &obj->secs[new_cnt - 1];
532 	sec->id = new_cnt - 1;
533 	sec->sec_name = sec_name;
534 
535 	return sec;
536 }
537 
linker_load_obj_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts,struct src_obj * obj)538 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
539 				const struct bpf_linker_file_opts *opts,
540 				struct src_obj *obj)
541 {
542 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
543 	const int host_endianness = ELFDATA2LSB;
544 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
545 	const int host_endianness = ELFDATA2MSB;
546 #else
547 #error "Unknown __BYTE_ORDER__"
548 #endif
549 	int err = 0;
550 	Elf_Scn *scn;
551 	Elf_Data *data;
552 	Elf64_Ehdr *ehdr;
553 	Elf64_Shdr *shdr;
554 	struct src_sec *sec;
555 
556 	pr_debug("linker: adding object file '%s'...\n", filename);
557 
558 	obj->filename = filename;
559 
560 	obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
561 	if (obj->fd < 0) {
562 		err = -errno;
563 		pr_warn("failed to open file '%s': %d\n", filename, err);
564 		return err;
565 	}
566 	obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
567 	if (!obj->elf) {
568 		err = -errno;
569 		pr_warn_elf("failed to parse ELF file '%s'", filename);
570 		return err;
571 	}
572 
573 	/* Sanity check ELF file high-level properties */
574 	ehdr = elf64_getehdr(obj->elf);
575 	if (!ehdr) {
576 		err = -errno;
577 		pr_warn_elf("failed to get ELF header for %s", filename);
578 		return err;
579 	}
580 	if (ehdr->e_ident[EI_DATA] != host_endianness) {
581 		err = -EOPNOTSUPP;
582 		pr_warn_elf("unsupported byte order of ELF file %s", filename);
583 		return err;
584 	}
585 	if (ehdr->e_type != ET_REL
586 	    || ehdr->e_machine != EM_BPF
587 	    || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
588 		err = -EOPNOTSUPP;
589 		pr_warn_elf("unsupported kind of ELF file %s", filename);
590 		return err;
591 	}
592 
593 	if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
594 		err = -errno;
595 		pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
596 		return err;
597 	}
598 
599 	scn = NULL;
600 	while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
601 		size_t sec_idx = elf_ndxscn(scn);
602 		const char *sec_name;
603 
604 		shdr = elf64_getshdr(scn);
605 		if (!shdr) {
606 			err = -errno;
607 			pr_warn_elf("failed to get section #%zu header for %s",
608 				    sec_idx, filename);
609 			return err;
610 		}
611 
612 		sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
613 		if (!sec_name) {
614 			err = -errno;
615 			pr_warn_elf("failed to get section #%zu name for %s",
616 				    sec_idx, filename);
617 			return err;
618 		}
619 
620 		data = elf_getdata(scn, 0);
621 		if (!data) {
622 			err = -errno;
623 			pr_warn_elf("failed to get section #%zu (%s) data from %s",
624 				    sec_idx, sec_name, filename);
625 			return err;
626 		}
627 
628 		sec = add_src_sec(obj, sec_name);
629 		if (!sec)
630 			return -ENOMEM;
631 
632 		sec->scn = scn;
633 		sec->shdr = shdr;
634 		sec->data = data;
635 		sec->sec_idx = elf_ndxscn(scn);
636 
637 		if (is_ignored_sec(sec)) {
638 			sec->skipped = true;
639 			continue;
640 		}
641 
642 		switch (shdr->sh_type) {
643 		case SHT_SYMTAB:
644 			if (obj->symtab_sec_idx) {
645 				err = -EOPNOTSUPP;
646 				pr_warn("multiple SYMTAB sections found, not supported\n");
647 				return err;
648 			}
649 			obj->symtab_sec_idx = sec_idx;
650 			break;
651 		case SHT_STRTAB:
652 			/* we'll construct our own string table */
653 			break;
654 		case SHT_PROGBITS:
655 			if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
656 				obj->btf = btf__new(data->d_buf, shdr->sh_size);
657 				err = libbpf_get_error(obj->btf);
658 				if (err) {
659 					pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
660 					return err;
661 				}
662 				sec->skipped = true;
663 				continue;
664 			}
665 			if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
666 				obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667 				err = libbpf_get_error(obj->btf_ext);
668 				if (err) {
669 					pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
670 					return err;
671 				}
672 				sec->skipped = true;
673 				continue;
674 			}
675 
676 			/* data & code */
677 			break;
678 		case SHT_NOBITS:
679 			/* BSS */
680 			break;
681 		case SHT_REL:
682 			/* relocations */
683 			break;
684 		default:
685 			pr_warn("unrecognized section #%zu (%s) in %s\n",
686 				sec_idx, sec_name, filename);
687 			err = -EINVAL;
688 			return err;
689 		}
690 	}
691 
692 	err = err ?: linker_sanity_check_elf(obj);
693 	err = err ?: linker_sanity_check_btf(obj);
694 	err = err ?: linker_sanity_check_btf_ext(obj);
695 	err = err ?: linker_fixup_btf(obj);
696 
697 	return err;
698 }
699 
linker_sanity_check_elf(struct src_obj * obj)700 static int linker_sanity_check_elf(struct src_obj *obj)
701 {
702 	struct src_sec *sec;
703 	int i, err;
704 
705 	if (!obj->symtab_sec_idx) {
706 		pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
707 		return -EINVAL;
708 	}
709 	if (!obj->shstrs_sec_idx) {
710 		pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
711 		return -EINVAL;
712 	}
713 
714 	for (i = 1; i < obj->sec_cnt; i++) {
715 		sec = &obj->secs[i];
716 
717 		if (sec->sec_name[0] == '\0') {
718 			pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
719 			return -EINVAL;
720 		}
721 
722 		if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
723 			return -EINVAL;
724 		if (sec->shdr->sh_addralign != sec->data->d_align)
725 			return -EINVAL;
726 
727 		if (sec->shdr->sh_size != sec->data->d_size)
728 			return -EINVAL;
729 
730 		switch (sec->shdr->sh_type) {
731 		case SHT_SYMTAB:
732 			err = linker_sanity_check_elf_symtab(obj, sec);
733 			if (err)
734 				return err;
735 			break;
736 		case SHT_STRTAB:
737 			break;
738 		case SHT_PROGBITS:
739 			if (sec->shdr->sh_flags & SHF_EXECINSTR) {
740 				if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
741 					return -EINVAL;
742 			}
743 			break;
744 		case SHT_NOBITS:
745 			break;
746 		case SHT_REL:
747 			err = linker_sanity_check_elf_relos(obj, sec);
748 			if (err)
749 				return err;
750 			break;
751 		case SHT_LLVM_ADDRSIG:
752 			break;
753 		default:
754 			pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
755 				sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
756 			return -EINVAL;
757 		}
758 	}
759 
760 	return 0;
761 }
762 
linker_sanity_check_elf_symtab(struct src_obj * obj,struct src_sec * sec)763 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
764 {
765 	struct src_sec *link_sec;
766 	Elf64_Sym *sym;
767 	int i, n;
768 
769 	if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
770 		return -EINVAL;
771 	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
772 		return -EINVAL;
773 
774 	if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
775 		pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
776 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
777 		return -EINVAL;
778 	}
779 	link_sec = &obj->secs[sec->shdr->sh_link];
780 	if (link_sec->shdr->sh_type != SHT_STRTAB) {
781 		pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
782 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
783 		return -EINVAL;
784 	}
785 
786 	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
787 	sym = sec->data->d_buf;
788 	for (i = 0; i < n; i++, sym++) {
789 		int sym_type = ELF64_ST_TYPE(sym->st_info);
790 		int sym_bind = ELF64_ST_BIND(sym->st_info);
791 		int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
792 
793 		if (i == 0) {
794 			if (sym->st_name != 0 || sym->st_info != 0
795 			    || sym->st_other != 0 || sym->st_shndx != 0
796 			    || sym->st_value != 0 || sym->st_size != 0) {
797 				pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
798 				return -EINVAL;
799 			}
800 			continue;
801 		}
802 		if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
803 			pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
804 				i, sec->sec_idx, sym_bind);
805 			return -EINVAL;
806 		}
807 		if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
808 			pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
809 				i, sec->sec_idx, sym_vis);
810 			return -EINVAL;
811 		}
812 		if (sym->st_shndx == 0) {
813 			if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
814 			    || sym->st_value != 0 || sym->st_size != 0) {
815 				pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
816 					i, obj->filename);
817 
818 				return -EINVAL;
819 			}
820 			continue;
821 		}
822 		if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
823 			pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
824 				i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
825 			return -EINVAL;
826 		}
827 		if (sym_type == STT_SECTION) {
828 			if (sym->st_value != 0)
829 				return -EINVAL;
830 			continue;
831 		}
832 	}
833 
834 	return 0;
835 }
836 
linker_sanity_check_elf_relos(struct src_obj * obj,struct src_sec * sec)837 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
838 {
839 	struct src_sec *link_sec, *sym_sec;
840 	Elf64_Rel *relo;
841 	int i, n;
842 
843 	if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
844 		return -EINVAL;
845 	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
846 		return -EINVAL;
847 
848 	/* SHT_REL's sh_link should point to SYMTAB */
849 	if (sec->shdr->sh_link != obj->symtab_sec_idx) {
850 		pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
851 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
852 		return -EINVAL;
853 	}
854 
855 	/* SHT_REL's sh_info points to relocated section */
856 	if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
857 		pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
858 			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
859 		return -EINVAL;
860 	}
861 	link_sec = &obj->secs[sec->shdr->sh_info];
862 
863 	/* .rel<secname> -> <secname> pattern is followed */
864 	if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
865 	    || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
866 		pr_warn("ELF relo section #%zu name has invalid name in %s\n",
867 			sec->sec_idx, obj->filename);
868 		return -EINVAL;
869 	}
870 
871 	/* don't further validate relocations for ignored sections */
872 	if (link_sec->skipped)
873 		return 0;
874 
875 	/* relocatable section is data or instructions */
876 	if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
877 		pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
878 			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
879 		return -EINVAL;
880 	}
881 
882 	/* check sanity of each relocation */
883 	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
884 	relo = sec->data->d_buf;
885 	sym_sec = &obj->secs[obj->symtab_sec_idx];
886 	for (i = 0; i < n; i++, relo++) {
887 		size_t sym_idx = ELF64_R_SYM(relo->r_info);
888 		size_t sym_type = ELF64_R_TYPE(relo->r_info);
889 
890 		if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
891 		    sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
892 			pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
893 				i, sec->sec_idx, sym_type, obj->filename);
894 			return -EINVAL;
895 		}
896 
897 		if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
898 			pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
899 				i, sec->sec_idx, sym_idx, obj->filename);
900 			return -EINVAL;
901 		}
902 
903 		if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
904 			if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
905 				pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
906 					i, sec->sec_idx, sym_idx, obj->filename);
907 				return -EINVAL;
908 			}
909 		}
910 	}
911 
912 	return 0;
913 }
914 
check_btf_type_id(__u32 * type_id,void * ctx)915 static int check_btf_type_id(__u32 *type_id, void *ctx)
916 {
917 	struct btf *btf = ctx;
918 
919 	if (*type_id >= btf__type_cnt(btf))
920 		return -EINVAL;
921 
922 	return 0;
923 }
924 
check_btf_str_off(__u32 * str_off,void * ctx)925 static int check_btf_str_off(__u32 *str_off, void *ctx)
926 {
927 	struct btf *btf = ctx;
928 	const char *s;
929 
930 	s = btf__str_by_offset(btf, *str_off);
931 
932 	if (!s)
933 		return -EINVAL;
934 
935 	return 0;
936 }
937 
linker_sanity_check_btf(struct src_obj * obj)938 static int linker_sanity_check_btf(struct src_obj *obj)
939 {
940 	struct btf_type *t;
941 	int i, n, err = 0;
942 
943 	if (!obj->btf)
944 		return 0;
945 
946 	n = btf__type_cnt(obj->btf);
947 	for (i = 1; i < n; i++) {
948 		t = btf_type_by_id(obj->btf, i);
949 
950 		err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
951 		err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
952 		if (err)
953 			return err;
954 	}
955 
956 	return 0;
957 }
958 
linker_sanity_check_btf_ext(struct src_obj * obj)959 static int linker_sanity_check_btf_ext(struct src_obj *obj)
960 {
961 	int err = 0;
962 
963 	if (!obj->btf_ext)
964 		return 0;
965 
966 	/* can't use .BTF.ext without .BTF */
967 	if (!obj->btf)
968 		return -EINVAL;
969 
970 	err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
971 	err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
972 	if (err)
973 		return err;
974 
975 	return 0;
976 }
977 
init_sec(struct bpf_linker * linker,struct dst_sec * dst_sec,struct src_sec * src_sec)978 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
979 {
980 	Elf_Scn *scn;
981 	Elf_Data *data;
982 	Elf64_Shdr *shdr;
983 	int name_off;
984 
985 	dst_sec->sec_sz = 0;
986 	dst_sec->sec_idx = 0;
987 	dst_sec->ephemeral = src_sec->ephemeral;
988 
989 	/* ephemeral sections are just thin section shells lacking most parts */
990 	if (src_sec->ephemeral)
991 		return 0;
992 
993 	scn = elf_newscn(linker->elf);
994 	if (!scn)
995 		return -ENOMEM;
996 	data = elf_newdata(scn);
997 	if (!data)
998 		return -ENOMEM;
999 	shdr = elf64_getshdr(scn);
1000 	if (!shdr)
1001 		return -ENOMEM;
1002 
1003 	dst_sec->scn = scn;
1004 	dst_sec->shdr = shdr;
1005 	dst_sec->data = data;
1006 	dst_sec->sec_idx = elf_ndxscn(scn);
1007 
1008 	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1009 	if (name_off < 0)
1010 		return name_off;
1011 
1012 	shdr->sh_name = name_off;
1013 	shdr->sh_type = src_sec->shdr->sh_type;
1014 	shdr->sh_flags = src_sec->shdr->sh_flags;
1015 	shdr->sh_size = 0;
1016 	/* sh_link and sh_info have different meaning for different types of
1017 	 * sections, so we leave it up to the caller code to fill them in, if
1018 	 * necessary
1019 	 */
1020 	shdr->sh_link = 0;
1021 	shdr->sh_info = 0;
1022 	shdr->sh_addralign = src_sec->shdr->sh_addralign;
1023 	shdr->sh_entsize = src_sec->shdr->sh_entsize;
1024 
1025 	data->d_type = src_sec->data->d_type;
1026 	data->d_size = 0;
1027 	data->d_buf = NULL;
1028 	data->d_align = src_sec->data->d_align;
1029 	data->d_off = 0;
1030 
1031 	return 0;
1032 }
1033 
find_dst_sec_by_name(struct bpf_linker * linker,const char * sec_name)1034 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1035 {
1036 	struct dst_sec *sec;
1037 	int i;
1038 
1039 	for (i = 1; i < linker->sec_cnt; i++) {
1040 		sec = &linker->secs[i];
1041 
1042 		if (strcmp(sec->sec_name, sec_name) == 0)
1043 			return sec;
1044 	}
1045 
1046 	return NULL;
1047 }
1048 
secs_match(struct dst_sec * dst,struct src_sec * src)1049 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1050 {
1051 	if (dst->ephemeral || src->ephemeral)
1052 		return true;
1053 
1054 	if (dst->shdr->sh_type != src->shdr->sh_type) {
1055 		pr_warn("sec %s types mismatch\n", dst->sec_name);
1056 		return false;
1057 	}
1058 	if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1059 		pr_warn("sec %s flags mismatch\n", dst->sec_name);
1060 		return false;
1061 	}
1062 	if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1063 		pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1064 		return false;
1065 	}
1066 
1067 	return true;
1068 }
1069 
sec_content_is_same(struct dst_sec * dst_sec,struct src_sec * src_sec)1070 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1071 {
1072 	if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1073 		return false;
1074 	if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1075 		return false;
1076 	return true;
1077 }
1078 
extend_sec(struct bpf_linker * linker,struct dst_sec * dst,struct src_sec * src)1079 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1080 {
1081 	void *tmp;
1082 	size_t dst_align, src_align;
1083 	size_t dst_align_sz, dst_final_sz;
1084 	int err;
1085 
1086 	/* Ephemeral source section doesn't contribute anything to ELF
1087 	 * section data.
1088 	 */
1089 	if (src->ephemeral)
1090 		return 0;
1091 
1092 	/* Some sections (like .maps) can contain both externs (and thus be
1093 	 * ephemeral) and non-externs (map definitions). So it's possible that
1094 	 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1095 	 * first non-ephemeral entity appears. In such case, we add ELF
1096 	 * section, data, etc.
1097 	 */
1098 	if (dst->ephemeral) {
1099 		err = init_sec(linker, dst, src);
1100 		if (err)
1101 			return err;
1102 	}
1103 
1104 	dst_align = dst->shdr->sh_addralign;
1105 	src_align = src->shdr->sh_addralign;
1106 	if (dst_align == 0)
1107 		dst_align = 1;
1108 	if (dst_align < src_align)
1109 		dst_align = src_align;
1110 
1111 	dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1112 
1113 	/* no need to re-align final size */
1114 	dst_final_sz = dst_align_sz + src->shdr->sh_size;
1115 
1116 	if (src->shdr->sh_type != SHT_NOBITS) {
1117 		tmp = realloc(dst->raw_data, dst_final_sz);
1118 		/* If dst_align_sz == 0, realloc() behaves in a special way:
1119 		 * 1. When dst->raw_data is NULL it returns:
1120 		 *    "either NULL or a pointer suitable to be passed to free()" [1].
1121 		 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1122 		 *    thus invalidating any "pointer suitable to be passed to free()" obtained
1123 		 *    at step (1).
1124 		 *
1125 		 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1126 		 * dst->raw_data would be freed again in bpf_linker__free().
1127 		 *
1128 		 * [1] man 3 realloc
1129 		 */
1130 		if (!tmp && dst_align_sz > 0)
1131 			return -ENOMEM;
1132 		dst->raw_data = tmp;
1133 
1134 		/* pad dst section, if it's alignment forced size increase */
1135 		memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1136 		/* now copy src data at a properly aligned offset */
1137 		memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1138 	}
1139 
1140 	dst->sec_sz = dst_final_sz;
1141 	dst->shdr->sh_size = dst_final_sz;
1142 	dst->data->d_size = dst_final_sz;
1143 
1144 	dst->shdr->sh_addralign = dst_align;
1145 	dst->data->d_align = dst_align;
1146 
1147 	src->dst_off = dst_align_sz;
1148 
1149 	return 0;
1150 }
1151 
is_data_sec(struct src_sec * sec)1152 static bool is_data_sec(struct src_sec *sec)
1153 {
1154 	if (!sec || sec->skipped)
1155 		return false;
1156 	/* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1157 	if (sec->ephemeral)
1158 		return true;
1159 	return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1160 }
1161 
is_relo_sec(struct src_sec * sec)1162 static bool is_relo_sec(struct src_sec *sec)
1163 {
1164 	if (!sec || sec->skipped || sec->ephemeral)
1165 		return false;
1166 	return sec->shdr->sh_type == SHT_REL;
1167 }
1168 
linker_append_sec_data(struct bpf_linker * linker,struct src_obj * obj)1169 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1170 {
1171 	int i, err;
1172 
1173 	for (i = 1; i < obj->sec_cnt; i++) {
1174 		struct src_sec *src_sec;
1175 		struct dst_sec *dst_sec;
1176 
1177 		src_sec = &obj->secs[i];
1178 		if (!is_data_sec(src_sec))
1179 			continue;
1180 
1181 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1182 		if (!dst_sec) {
1183 			dst_sec = add_dst_sec(linker, src_sec->sec_name);
1184 			if (!dst_sec)
1185 				return -ENOMEM;
1186 			err = init_sec(linker, dst_sec, src_sec);
1187 			if (err) {
1188 				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1189 				return err;
1190 			}
1191 		} else {
1192 			if (!secs_match(dst_sec, src_sec)) {
1193 				pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1194 				return -1;
1195 			}
1196 
1197 			/* "license" and "version" sections are deduped */
1198 			if (strcmp(src_sec->sec_name, "license") == 0
1199 			    || strcmp(src_sec->sec_name, "version") == 0) {
1200 				if (!sec_content_is_same(dst_sec, src_sec)) {
1201 					pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1202 					return -EINVAL;
1203 				}
1204 				src_sec->skipped = true;
1205 				src_sec->dst_id = dst_sec->id;
1206 				continue;
1207 			}
1208 		}
1209 
1210 		/* record mapped section index */
1211 		src_sec->dst_id = dst_sec->id;
1212 
1213 		err = extend_sec(linker, dst_sec, src_sec);
1214 		if (err)
1215 			return err;
1216 	}
1217 
1218 	return 0;
1219 }
1220 
linker_append_elf_syms(struct bpf_linker * linker,struct src_obj * obj)1221 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1222 {
1223 	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1224 	Elf64_Sym *sym = symtab->data->d_buf;
1225 	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1226 	int str_sec_idx = symtab->shdr->sh_link;
1227 	const char *sym_name;
1228 
1229 	obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1230 	if (!obj->sym_map)
1231 		return -ENOMEM;
1232 
1233 	for (i = 0; i < n; i++, sym++) {
1234 		/* We already validated all-zero symbol #0 and we already
1235 		 * appended it preventively to the final SYMTAB, so skip it.
1236 		 */
1237 		if (i == 0)
1238 			continue;
1239 
1240 		sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1241 		if (!sym_name) {
1242 			pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1243 			return -EINVAL;
1244 		}
1245 
1246 		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1247 		if (err)
1248 			return err;
1249 	}
1250 
1251 	return 0;
1252 }
1253 
get_sym_by_idx(struct bpf_linker * linker,size_t sym_idx)1254 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1255 {
1256 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1257 	Elf64_Sym *syms = symtab->raw_data;
1258 
1259 	return &syms[sym_idx];
1260 }
1261 
find_glob_sym(struct bpf_linker * linker,const char * sym_name)1262 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1263 {
1264 	struct glob_sym *glob_sym;
1265 	const char *name;
1266 	int i;
1267 
1268 	for (i = 0; i < linker->glob_sym_cnt; i++) {
1269 		glob_sym = &linker->glob_syms[i];
1270 		name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1271 
1272 		if (strcmp(name, sym_name) == 0)
1273 			return glob_sym;
1274 	}
1275 
1276 	return NULL;
1277 }
1278 
add_glob_sym(struct bpf_linker * linker)1279 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1280 {
1281 	struct glob_sym *syms, *sym;
1282 
1283 	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1284 				   sizeof(*linker->glob_syms));
1285 	if (!syms)
1286 		return NULL;
1287 
1288 	sym = &syms[linker->glob_sym_cnt];
1289 	memset(sym, 0, sizeof(*sym));
1290 	sym->var_idx = -1;
1291 
1292 	linker->glob_syms = syms;
1293 	linker->glob_sym_cnt++;
1294 
1295 	return sym;
1296 }
1297 
glob_sym_btf_matches(const char * sym_name,bool exact,const struct btf * btf1,__u32 id1,const struct btf * btf2,__u32 id2)1298 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1299 				 const struct btf *btf1, __u32 id1,
1300 				 const struct btf *btf2, __u32 id2)
1301 {
1302 	const struct btf_type *t1, *t2;
1303 	bool is_static1, is_static2;
1304 	const char *n1, *n2;
1305 	int i, n;
1306 
1307 recur:
1308 	n1 = n2 = NULL;
1309 	t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1310 	t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1311 
1312 	/* check if only one side is FWD, otherwise handle with common logic */
1313 	if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1314 		n1 = btf__str_by_offset(btf1, t1->name_off);
1315 		n2 = btf__str_by_offset(btf2, t2->name_off);
1316 		if (strcmp(n1, n2) != 0) {
1317 			pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1318 				sym_name, n1, n2);
1319 			return false;
1320 		}
1321 		/* validate if FWD kind matches concrete kind */
1322 		if (btf_is_fwd(t1)) {
1323 			if (btf_kflag(t1) && btf_is_union(t2))
1324 				return true;
1325 			if (!btf_kflag(t1) && btf_is_struct(t2))
1326 				return true;
1327 			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1328 				sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1329 		} else {
1330 			if (btf_kflag(t2) && btf_is_union(t1))
1331 				return true;
1332 			if (!btf_kflag(t2) && btf_is_struct(t1))
1333 				return true;
1334 			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1335 				sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1336 		}
1337 		return false;
1338 	}
1339 
1340 	if (btf_kind(t1) != btf_kind(t2)) {
1341 		pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1342 			sym_name, btf_kind_str(t1), btf_kind_str(t2));
1343 		return false;
1344 	}
1345 
1346 	switch (btf_kind(t1)) {
1347 	case BTF_KIND_STRUCT:
1348 	case BTF_KIND_UNION:
1349 	case BTF_KIND_ENUM:
1350 	case BTF_KIND_ENUM64:
1351 	case BTF_KIND_FWD:
1352 	case BTF_KIND_FUNC:
1353 	case BTF_KIND_VAR:
1354 		n1 = btf__str_by_offset(btf1, t1->name_off);
1355 		n2 = btf__str_by_offset(btf2, t2->name_off);
1356 		if (strcmp(n1, n2) != 0) {
1357 			pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1358 				sym_name, btf_kind_str(t1), n1, n2);
1359 			return false;
1360 		}
1361 		break;
1362 	default:
1363 		break;
1364 	}
1365 
1366 	switch (btf_kind(t1)) {
1367 	case BTF_KIND_UNKN: /* void */
1368 	case BTF_KIND_FWD:
1369 		return true;
1370 	case BTF_KIND_INT:
1371 	case BTF_KIND_FLOAT:
1372 	case BTF_KIND_ENUM:
1373 	case BTF_KIND_ENUM64:
1374 		/* ignore encoding for int and enum values for enum */
1375 		if (t1->size != t2->size) {
1376 			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1377 				sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1378 			return false;
1379 		}
1380 		return true;
1381 	case BTF_KIND_PTR:
1382 		/* just validate overall shape of the referenced type, so no
1383 		 * contents comparison for struct/union, and allowd fwd vs
1384 		 * struct/union
1385 		 */
1386 		exact = false;
1387 		id1 = t1->type;
1388 		id2 = t2->type;
1389 		goto recur;
1390 	case BTF_KIND_ARRAY:
1391 		/* ignore index type and array size */
1392 		id1 = btf_array(t1)->type;
1393 		id2 = btf_array(t2)->type;
1394 		goto recur;
1395 	case BTF_KIND_FUNC:
1396 		/* extern and global linkages are compatible */
1397 		is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1398 		is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1399 		if (is_static1 != is_static2) {
1400 			pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1401 			return false;
1402 		}
1403 
1404 		id1 = t1->type;
1405 		id2 = t2->type;
1406 		goto recur;
1407 	case BTF_KIND_VAR:
1408 		/* extern and global linkages are compatible */
1409 		is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1410 		is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1411 		if (is_static1 != is_static2) {
1412 			pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1413 			return false;
1414 		}
1415 
1416 		id1 = t1->type;
1417 		id2 = t2->type;
1418 		goto recur;
1419 	case BTF_KIND_STRUCT:
1420 	case BTF_KIND_UNION: {
1421 		const struct btf_member *m1, *m2;
1422 
1423 		if (!exact)
1424 			return true;
1425 
1426 		if (btf_vlen(t1) != btf_vlen(t2)) {
1427 			pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1428 				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1429 			return false;
1430 		}
1431 
1432 		n = btf_vlen(t1);
1433 		m1 = btf_members(t1);
1434 		m2 = btf_members(t2);
1435 		for (i = 0; i < n; i++, m1++, m2++) {
1436 			n1 = btf__str_by_offset(btf1, m1->name_off);
1437 			n2 = btf__str_by_offset(btf2, m2->name_off);
1438 			if (strcmp(n1, n2) != 0) {
1439 				pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1440 					sym_name, i, n1, n2);
1441 				return false;
1442 			}
1443 			if (m1->offset != m2->offset) {
1444 				pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1445 					sym_name, i, n1);
1446 				return false;
1447 			}
1448 			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1449 				return false;
1450 		}
1451 
1452 		return true;
1453 	}
1454 	case BTF_KIND_FUNC_PROTO: {
1455 		const struct btf_param *m1, *m2;
1456 
1457 		if (btf_vlen(t1) != btf_vlen(t2)) {
1458 			pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1459 				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1460 			return false;
1461 		}
1462 
1463 		n = btf_vlen(t1);
1464 		m1 = btf_params(t1);
1465 		m2 = btf_params(t2);
1466 		for (i = 0; i < n; i++, m1++, m2++) {
1467 			/* ignore func arg names */
1468 			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1469 				return false;
1470 		}
1471 
1472 		/* now check return type as well */
1473 		id1 = t1->type;
1474 		id2 = t2->type;
1475 		goto recur;
1476 	}
1477 
1478 	/* skip_mods_and_typedefs() make this impossible */
1479 	case BTF_KIND_TYPEDEF:
1480 	case BTF_KIND_VOLATILE:
1481 	case BTF_KIND_CONST:
1482 	case BTF_KIND_RESTRICT:
1483 	/* DATASECs are never compared with each other */
1484 	case BTF_KIND_DATASEC:
1485 	default:
1486 		pr_warn("global '%s': unsupported BTF kind %s\n",
1487 			sym_name, btf_kind_str(t1));
1488 		return false;
1489 	}
1490 }
1491 
map_defs_match(const char * sym_name,const struct btf * main_btf,const struct btf_map_def * main_def,const struct btf_map_def * main_inner_def,const struct btf * extra_btf,const struct btf_map_def * extra_def,const struct btf_map_def * extra_inner_def)1492 static bool map_defs_match(const char *sym_name,
1493 			   const struct btf *main_btf,
1494 			   const struct btf_map_def *main_def,
1495 			   const struct btf_map_def *main_inner_def,
1496 			   const struct btf *extra_btf,
1497 			   const struct btf_map_def *extra_def,
1498 			   const struct btf_map_def *extra_inner_def)
1499 {
1500 	const char *reason;
1501 
1502 	if (main_def->map_type != extra_def->map_type) {
1503 		reason = "type";
1504 		goto mismatch;
1505 	}
1506 
1507 	/* check key type/size match */
1508 	if (main_def->key_size != extra_def->key_size) {
1509 		reason = "key_size";
1510 		goto mismatch;
1511 	}
1512 	if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1513 		reason = "key type";
1514 		goto mismatch;
1515 	}
1516 	if ((main_def->parts & MAP_DEF_KEY_TYPE)
1517 	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1518 				      main_btf, main_def->key_type_id,
1519 				      extra_btf, extra_def->key_type_id)) {
1520 		reason = "key type";
1521 		goto mismatch;
1522 	}
1523 
1524 	/* validate value type/size match */
1525 	if (main_def->value_size != extra_def->value_size) {
1526 		reason = "value_size";
1527 		goto mismatch;
1528 	}
1529 	if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1530 		reason = "value type";
1531 		goto mismatch;
1532 	}
1533 	if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1534 	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1535 				      main_btf, main_def->value_type_id,
1536 				      extra_btf, extra_def->value_type_id)) {
1537 		reason = "key type";
1538 		goto mismatch;
1539 	}
1540 
1541 	if (main_def->max_entries != extra_def->max_entries) {
1542 		reason = "max_entries";
1543 		goto mismatch;
1544 	}
1545 	if (main_def->map_flags != extra_def->map_flags) {
1546 		reason = "map_flags";
1547 		goto mismatch;
1548 	}
1549 	if (main_def->numa_node != extra_def->numa_node) {
1550 		reason = "numa_node";
1551 		goto mismatch;
1552 	}
1553 	if (main_def->pinning != extra_def->pinning) {
1554 		reason = "pinning";
1555 		goto mismatch;
1556 	}
1557 
1558 	if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1559 		reason = "inner map";
1560 		goto mismatch;
1561 	}
1562 
1563 	if (main_def->parts & MAP_DEF_INNER_MAP) {
1564 		char inner_map_name[128];
1565 
1566 		snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1567 
1568 		return map_defs_match(inner_map_name,
1569 				      main_btf, main_inner_def, NULL,
1570 				      extra_btf, extra_inner_def, NULL);
1571 	}
1572 
1573 	return true;
1574 
1575 mismatch:
1576 	pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1577 	return false;
1578 }
1579 
glob_map_defs_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,int btf_id)1580 static bool glob_map_defs_match(const char *sym_name,
1581 				struct bpf_linker *linker, struct glob_sym *glob_sym,
1582 				struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1583 {
1584 	struct btf_map_def dst_def = {}, dst_inner_def = {};
1585 	struct btf_map_def src_def = {}, src_inner_def = {};
1586 	const struct btf_type *t;
1587 	int err;
1588 
1589 	t = btf__type_by_id(obj->btf, btf_id);
1590 	if (!btf_is_var(t)) {
1591 		pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1592 		return false;
1593 	}
1594 	t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1595 
1596 	err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1597 	if (err) {
1598 		pr_warn("global '%s': invalid map definition\n", sym_name);
1599 		return false;
1600 	}
1601 
1602 	/* re-parse existing map definition */
1603 	t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1604 	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1605 	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1606 	if (err) {
1607 		/* this should not happen, because we already validated it */
1608 		pr_warn("global '%s': invalid dst map definition\n", sym_name);
1609 		return false;
1610 	}
1611 
1612 	/* Currently extern map definition has to be complete and match
1613 	 * concrete map definition exactly. This restriction might be lifted
1614 	 * in the future.
1615 	 */
1616 	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1617 			      obj->btf, &src_def, &src_inner_def);
1618 }
1619 
glob_syms_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,size_t sym_idx,int btf_id)1620 static bool glob_syms_match(const char *sym_name,
1621 			    struct bpf_linker *linker, struct glob_sym *glob_sym,
1622 			    struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1623 {
1624 	const struct btf_type *src_t;
1625 
1626 	/* if we are dealing with externs, BTF types describing both global
1627 	 * and extern VARs/FUNCs should be completely present in all files
1628 	 */
1629 	if (!glob_sym->btf_id || !btf_id) {
1630 		pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1631 		return false;
1632 	}
1633 
1634 	src_t = btf__type_by_id(obj->btf, btf_id);
1635 	if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1636 		pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1637 			btf_kind_str(src_t), sym_name);
1638 		return false;
1639 	}
1640 
1641 	/* deal with .maps definitions specially */
1642 	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1643 		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1644 
1645 	if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1646 				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1647 		return false;
1648 
1649 	return true;
1650 }
1651 
btf_is_non_static(const struct btf_type * t)1652 static bool btf_is_non_static(const struct btf_type *t)
1653 {
1654 	return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1655 	       || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1656 }
1657 
find_glob_sym_btf(struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int * out_btf_sec_id,int * out_btf_id)1658 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1659 			     int *out_btf_sec_id, int *out_btf_id)
1660 {
1661 	int i, j, n, m, btf_id = 0;
1662 	const struct btf_type *t;
1663 	const struct btf_var_secinfo *vi;
1664 	const char *name;
1665 
1666 	if (!obj->btf) {
1667 		pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1668 		return -EINVAL;
1669 	}
1670 
1671 	n = btf__type_cnt(obj->btf);
1672 	for (i = 1; i < n; i++) {
1673 		t = btf__type_by_id(obj->btf, i);
1674 
1675 		/* some global and extern FUNCs and VARs might not be associated with any
1676 		 * DATASEC, so try to detect them in the same pass
1677 		 */
1678 		if (btf_is_non_static(t)) {
1679 			name = btf__str_by_offset(obj->btf, t->name_off);
1680 			if (strcmp(name, sym_name) != 0)
1681 				continue;
1682 
1683 			/* remember and still try to find DATASEC */
1684 			btf_id = i;
1685 			continue;
1686 		}
1687 
1688 		if (!btf_is_datasec(t))
1689 			continue;
1690 
1691 		vi = btf_var_secinfos(t);
1692 		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1693 			t = btf__type_by_id(obj->btf, vi->type);
1694 			name = btf__str_by_offset(obj->btf, t->name_off);
1695 
1696 			if (strcmp(name, sym_name) != 0)
1697 				continue;
1698 			if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1699 				continue;
1700 			if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1701 				continue;
1702 
1703 			if (btf_id && btf_id != vi->type) {
1704 				pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1705 					sym_name, btf_id, vi->type);
1706 				return -EINVAL;
1707 			}
1708 
1709 			*out_btf_sec_id = i;
1710 			*out_btf_id = vi->type;
1711 
1712 			return 0;
1713 		}
1714 	}
1715 
1716 	/* free-floating extern or global FUNC */
1717 	if (btf_id) {
1718 		*out_btf_sec_id = 0;
1719 		*out_btf_id = btf_id;
1720 		return 0;
1721 	}
1722 
1723 	pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1724 	return -ENOENT;
1725 }
1726 
find_src_sec_by_name(struct src_obj * obj,const char * sec_name)1727 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1728 {
1729 	struct src_sec *sec;
1730 	int i;
1731 
1732 	for (i = 1; i < obj->sec_cnt; i++) {
1733 		sec = &obj->secs[i];
1734 
1735 		if (strcmp(sec->sec_name, sec_name) == 0)
1736 			return sec;
1737 	}
1738 
1739 	return NULL;
1740 }
1741 
complete_extern_btf_info(struct btf * dst_btf,int dst_id,struct btf * src_btf,int src_id)1742 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1743 				    struct btf *src_btf, int src_id)
1744 {
1745 	struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1746 	struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1747 	struct btf_param *src_p, *dst_p;
1748 	const char *s;
1749 	int i, n, off;
1750 
1751 	/* We already made sure that source and destination types (FUNC or
1752 	 * VAR) match in terms of types and argument names.
1753 	 */
1754 	if (btf_is_var(dst_t)) {
1755 		btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1756 		return 0;
1757 	}
1758 
1759 	dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1760 
1761 	/* now onto FUNC_PROTO types */
1762 	src_t = btf_type_by_id(src_btf, src_t->type);
1763 	dst_t = btf_type_by_id(dst_btf, dst_t->type);
1764 
1765 	/* Fill in all the argument names, which for extern FUNCs are missing.
1766 	 * We'll end up with two copies of FUNCs/VARs for externs, but that
1767 	 * will be taken care of by BTF dedup at the very end.
1768 	 * It might be that BTF types for extern in one file has less/more BTF
1769 	 * information (e.g., FWD instead of full STRUCT/UNION information),
1770 	 * but that should be (in most cases, subject to BTF dedup rules)
1771 	 * handled and resolved by BTF dedup algorithm as well, so we won't
1772 	 * worry about it. Our only job is to make sure that argument names
1773 	 * are populated on both sides, otherwise BTF dedup will pedantically
1774 	 * consider them different.
1775 	 */
1776 	src_p = btf_params(src_t);
1777 	dst_p = btf_params(dst_t);
1778 	for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1779 		if (!src_p->name_off)
1780 			continue;
1781 
1782 		/* src_btf has more complete info, so add name to dst_btf */
1783 		s = btf__str_by_offset(src_btf, src_p->name_off);
1784 		off = btf__add_str(dst_btf, s);
1785 		if (off < 0)
1786 			return off;
1787 		dst_p->name_off = off;
1788 	}
1789 	return 0;
1790 }
1791 
sym_update_bind(Elf64_Sym * sym,int sym_bind)1792 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1793 {
1794 	sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1795 }
1796 
sym_update_type(Elf64_Sym * sym,int sym_type)1797 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1798 {
1799 	sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1800 }
1801 
sym_update_visibility(Elf64_Sym * sym,int sym_vis)1802 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1803 {
1804 	/* libelf doesn't provide setters for ST_VISIBILITY,
1805 	 * but it is stored in the lower 2 bits of st_other
1806 	 */
1807 	sym->st_other &= ~0x03;
1808 	sym->st_other |= sym_vis;
1809 }
1810 
linker_append_elf_sym(struct bpf_linker * linker,struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int src_sym_idx)1811 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1812 				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1813 {
1814 	struct src_sec *src_sec = NULL;
1815 	struct dst_sec *dst_sec = NULL;
1816 	struct glob_sym *glob_sym = NULL;
1817 	int name_off, sym_type, sym_bind, sym_vis, err;
1818 	int btf_sec_id = 0, btf_id = 0;
1819 	size_t dst_sym_idx;
1820 	Elf64_Sym *dst_sym;
1821 	bool sym_is_extern;
1822 
1823 	sym_type = ELF64_ST_TYPE(sym->st_info);
1824 	sym_bind = ELF64_ST_BIND(sym->st_info);
1825 	sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1826 	sym_is_extern = sym->st_shndx == SHN_UNDEF;
1827 
1828 	if (sym_is_extern) {
1829 		if (!obj->btf) {
1830 			pr_warn("externs without BTF info are not supported\n");
1831 			return -ENOTSUP;
1832 		}
1833 	} else if (sym->st_shndx < SHN_LORESERVE) {
1834 		src_sec = &obj->secs[sym->st_shndx];
1835 		if (src_sec->skipped)
1836 			return 0;
1837 		dst_sec = &linker->secs[src_sec->dst_id];
1838 
1839 		/* allow only one STT_SECTION symbol per section */
1840 		if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1841 			obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1842 			return 0;
1843 		}
1844 	}
1845 
1846 	if (sym_bind == STB_LOCAL)
1847 		goto add_sym;
1848 
1849 	/* find matching BTF info */
1850 	err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1851 	if (err)
1852 		return err;
1853 
1854 	if (sym_is_extern && btf_sec_id) {
1855 		const char *sec_name = NULL;
1856 		const struct btf_type *t;
1857 
1858 		t = btf__type_by_id(obj->btf, btf_sec_id);
1859 		sec_name = btf__str_by_offset(obj->btf, t->name_off);
1860 
1861 		/* Clang puts unannotated extern vars into
1862 		 * '.extern' BTF DATASEC. Treat them the same
1863 		 * as unannotated extern funcs (which are
1864 		 * currently not put into any DATASECs).
1865 		 * Those don't have associated src_sec/dst_sec.
1866 		 */
1867 		if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1868 			src_sec = find_src_sec_by_name(obj, sec_name);
1869 			if (!src_sec) {
1870 				pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1871 				return -ENOENT;
1872 			}
1873 			dst_sec = &linker->secs[src_sec->dst_id];
1874 		}
1875 	}
1876 
1877 	glob_sym = find_glob_sym(linker, sym_name);
1878 	if (glob_sym) {
1879 		/* Preventively resolve to existing symbol. This is
1880 		 * needed for further relocation symbol remapping in
1881 		 * the next step of linking.
1882 		 */
1883 		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1884 
1885 		/* If both symbols are non-externs, at least one of
1886 		 * them has to be STB_WEAK, otherwise they are in
1887 		 * a conflict with each other.
1888 		 */
1889 		if (!sym_is_extern && !glob_sym->is_extern
1890 		    && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1891 			pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1892 				src_sym_idx, sym_name, obj->filename);
1893 			return -EINVAL;
1894 		}
1895 
1896 		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1897 			return -EINVAL;
1898 
1899 		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1900 
1901 		/* If new symbol is strong, then force dst_sym to be strong as
1902 		 * well; this way a mix of weak and non-weak extern
1903 		 * definitions will end up being strong.
1904 		 */
1905 		if (sym_bind == STB_GLOBAL) {
1906 			/* We still need to preserve type (NOTYPE or
1907 			 * OBJECT/FUNC, depending on whether the symbol is
1908 			 * extern or not)
1909 			 */
1910 			sym_update_bind(dst_sym, STB_GLOBAL);
1911 			glob_sym->is_weak = false;
1912 		}
1913 
1914 		/* Non-default visibility is "contaminating", with stricter
1915 		 * visibility overwriting more permissive ones, even if more
1916 		 * permissive visibility comes from just an extern definition.
1917 		 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1918 		 * ensured by ELF symbol sanity checks above.
1919 		 */
1920 		if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1921 			sym_update_visibility(dst_sym, sym_vis);
1922 
1923 		/* If the new symbol is extern, then regardless if
1924 		 * existing symbol is extern or resolved global, just
1925 		 * keep the existing one untouched.
1926 		 */
1927 		if (sym_is_extern)
1928 			return 0;
1929 
1930 		/* If existing symbol is a strong resolved symbol, bail out,
1931 		 * because we lost resolution battle have nothing to
1932 		 * contribute. We already checked abover that there is no
1933 		 * strong-strong conflict. We also already tightened binding
1934 		 * and visibility, so nothing else to contribute at that point.
1935 		 */
1936 		if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1937 			return 0;
1938 
1939 		/* At this point, new symbol is strong non-extern,
1940 		 * so overwrite glob_sym with new symbol information.
1941 		 * Preserve binding and visibility.
1942 		 */
1943 		sym_update_type(dst_sym, sym_type);
1944 		dst_sym->st_shndx = dst_sec->sec_idx;
1945 		dst_sym->st_value = src_sec->dst_off + sym->st_value;
1946 		dst_sym->st_size = sym->st_size;
1947 
1948 		/* see comment below about dst_sec->id vs dst_sec->sec_idx */
1949 		glob_sym->sec_id = dst_sec->id;
1950 		glob_sym->is_extern = false;
1951 
1952 		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1953 					     obj->btf, btf_id))
1954 			return -EINVAL;
1955 
1956 		/* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1957 		glob_sym->underlying_btf_id = 0;
1958 
1959 		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1960 		return 0;
1961 	}
1962 
1963 add_sym:
1964 	name_off = strset__add_str(linker->strtab_strs, sym_name);
1965 	if (name_off < 0)
1966 		return name_off;
1967 
1968 	dst_sym = add_new_sym(linker, &dst_sym_idx);
1969 	if (!dst_sym)
1970 		return -ENOMEM;
1971 
1972 	dst_sym->st_name = name_off;
1973 	dst_sym->st_info = sym->st_info;
1974 	dst_sym->st_other = sym->st_other;
1975 	dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1976 	dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1977 	dst_sym->st_size = sym->st_size;
1978 
1979 	obj->sym_map[src_sym_idx] = dst_sym_idx;
1980 
1981 	if (sym_type == STT_SECTION && dst_sym) {
1982 		dst_sec->sec_sym_idx = dst_sym_idx;
1983 		dst_sym->st_value = 0;
1984 	}
1985 
1986 	if (sym_bind != STB_LOCAL) {
1987 		glob_sym = add_glob_sym(linker);
1988 		if (!glob_sym)
1989 			return -ENOMEM;
1990 
1991 		glob_sym->sym_idx = dst_sym_idx;
1992 		/* we use dst_sec->id (and not dst_sec->sec_idx), because
1993 		 * ephemeral sections (.kconfig, .ksyms, etc) don't have
1994 		 * sec_idx (as they don't have corresponding ELF section), but
1995 		 * still have id. .extern doesn't have even ephemeral section
1996 		 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
1997 		 */
1998 		glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
1999 		glob_sym->name_off = name_off;
2000 		/* we will fill btf_id in during BTF merging step */
2001 		glob_sym->btf_id = 0;
2002 		glob_sym->is_extern = sym_is_extern;
2003 		glob_sym->is_weak = sym_bind == STB_WEAK;
2004 	}
2005 
2006 	return 0;
2007 }
2008 
linker_append_elf_relos(struct bpf_linker * linker,struct src_obj * obj)2009 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2010 {
2011 	struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2012 	int i, err;
2013 
2014 	for (i = 1; i < obj->sec_cnt; i++) {
2015 		struct src_sec *src_sec, *src_linked_sec;
2016 		struct dst_sec *dst_sec, *dst_linked_sec;
2017 		Elf64_Rel *src_rel, *dst_rel;
2018 		int j, n;
2019 
2020 		src_sec = &obj->secs[i];
2021 		if (!is_relo_sec(src_sec))
2022 			continue;
2023 
2024 		/* shdr->sh_info points to relocatable section */
2025 		src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2026 		if (src_linked_sec->skipped)
2027 			continue;
2028 
2029 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2030 		if (!dst_sec) {
2031 			dst_sec = add_dst_sec(linker, src_sec->sec_name);
2032 			if (!dst_sec)
2033 				return -ENOMEM;
2034 			err = init_sec(linker, dst_sec, src_sec);
2035 			if (err) {
2036 				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2037 				return err;
2038 			}
2039 		} else if (!secs_match(dst_sec, src_sec)) {
2040 			pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2041 			return -1;
2042 		}
2043 
2044 		/* shdr->sh_link points to SYMTAB */
2045 		dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2046 
2047 		/* shdr->sh_info points to relocated section */
2048 		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2049 		dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2050 
2051 		src_sec->dst_id = dst_sec->id;
2052 		err = extend_sec(linker, dst_sec, src_sec);
2053 		if (err)
2054 			return err;
2055 
2056 		src_rel = src_sec->data->d_buf;
2057 		dst_rel = dst_sec->raw_data + src_sec->dst_off;
2058 		n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2059 		for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2060 			size_t src_sym_idx, dst_sym_idx, sym_type;
2061 			Elf64_Sym *src_sym;
2062 
2063 			src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2064 			src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2065 
2066 			dst_sym_idx = obj->sym_map[src_sym_idx];
2067 			dst_rel->r_offset += src_linked_sec->dst_off;
2068 			sym_type = ELF64_R_TYPE(src_rel->r_info);
2069 			dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2070 
2071 			if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2072 				struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2073 				struct bpf_insn *insn;
2074 
2075 				if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2076 					/* calls to the very first static function inside
2077 					 * .text section at offset 0 will
2078 					 * reference section symbol, not the
2079 					 * function symbol. Fix that up,
2080 					 * otherwise it won't be possible to
2081 					 * relocate calls to two different
2082 					 * static functions with the same name
2083 					 * (rom two different object files)
2084 					 */
2085 					insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2086 					if (insn->code == (BPF_JMP | BPF_CALL))
2087 						insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2088 					else
2089 						insn->imm += sec->dst_off;
2090 				} else {
2091 					pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2092 					return -EINVAL;
2093 				}
2094 			}
2095 
2096 		}
2097 	}
2098 
2099 	return 0;
2100 }
2101 
find_sym_by_name(struct src_obj * obj,size_t sec_idx,int sym_type,const char * sym_name)2102 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2103 				   int sym_type, const char *sym_name)
2104 {
2105 	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2106 	Elf64_Sym *sym = symtab->data->d_buf;
2107 	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2108 	int str_sec_idx = symtab->shdr->sh_link;
2109 	const char *name;
2110 
2111 	for (i = 0; i < n; i++, sym++) {
2112 		if (sym->st_shndx != sec_idx)
2113 			continue;
2114 		if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2115 			continue;
2116 
2117 		name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2118 		if (!name)
2119 			return NULL;
2120 
2121 		if (strcmp(sym_name, name) != 0)
2122 			continue;
2123 
2124 		return sym;
2125 	}
2126 
2127 	return NULL;
2128 }
2129 
linker_fixup_btf(struct src_obj * obj)2130 static int linker_fixup_btf(struct src_obj *obj)
2131 {
2132 	const char *sec_name;
2133 	struct src_sec *sec;
2134 	int i, j, n, m;
2135 
2136 	if (!obj->btf)
2137 		return 0;
2138 
2139 	n = btf__type_cnt(obj->btf);
2140 	for (i = 1; i < n; i++) {
2141 		struct btf_var_secinfo *vi;
2142 		struct btf_type *t;
2143 
2144 		t = btf_type_by_id(obj->btf, i);
2145 		if (btf_kind(t) != BTF_KIND_DATASEC)
2146 			continue;
2147 
2148 		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2149 		sec = find_src_sec_by_name(obj, sec_name);
2150 		if (sec) {
2151 			/* record actual section size, unless ephemeral */
2152 			if (sec->shdr)
2153 				t->size = sec->shdr->sh_size;
2154 		} else {
2155 			/* BTF can have some sections that are not represented
2156 			 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2157 			 * for special extern variables.
2158 			 *
2159 			 * For all but one such special (ephemeral)
2160 			 * sections, we pre-create "section shells" to be able
2161 			 * to keep track of extra per-section metadata later
2162 			 * (e.g., those BTF extern variables).
2163 			 *
2164 			 * .extern is even more special, though, because it
2165 			 * contains extern variables that need to be resolved
2166 			 * by static linker, not libbpf and kernel. When such
2167 			 * externs are resolved, we are going to remove them
2168 			 * from .extern BTF section and might end up not
2169 			 * needing it at all. Each resolved extern should have
2170 			 * matching non-extern VAR/FUNC in other sections.
2171 			 *
2172 			 * We do support leaving some of the externs
2173 			 * unresolved, though, to support cases of building
2174 			 * libraries, which will later be linked against final
2175 			 * BPF applications. So if at finalization we still
2176 			 * see unresolved externs, we'll create .extern
2177 			 * section on our own.
2178 			 */
2179 			if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2180 				continue;
2181 
2182 			sec = add_src_sec(obj, sec_name);
2183 			if (!sec)
2184 				return -ENOMEM;
2185 
2186 			sec->ephemeral = true;
2187 			sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2188 		}
2189 
2190 		/* remember ELF section and its BTF type ID match */
2191 		sec->sec_type_id = i;
2192 
2193 		/* fix up variable offsets */
2194 		vi = btf_var_secinfos(t);
2195 		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2196 			const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2197 			const char *var_name;
2198 			int var_linkage;
2199 			Elf64_Sym *sym;
2200 
2201 			/* could be a variable or function */
2202 			if (!btf_is_var(vt))
2203 				continue;
2204 
2205 			var_name = btf__str_by_offset(obj->btf, vt->name_off);
2206 			var_linkage = btf_var(vt)->linkage;
2207 
2208 			/* no need to patch up static or extern vars */
2209 			if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2210 				continue;
2211 
2212 			sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2213 			if (!sym) {
2214 				pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2215 				return -ENOENT;
2216 			}
2217 
2218 			vi->offset = sym->st_value;
2219 		}
2220 	}
2221 
2222 	return 0;
2223 }
2224 
remap_type_id(__u32 * type_id,void * ctx)2225 static int remap_type_id(__u32 *type_id, void *ctx)
2226 {
2227 	int *id_map = ctx;
2228 	int new_id = id_map[*type_id];
2229 
2230 	/* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2231 	if (new_id == 0 && *type_id != 0) {
2232 		pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2233 		return -EINVAL;
2234 	}
2235 
2236 	*type_id = id_map[*type_id];
2237 
2238 	return 0;
2239 }
2240 
linker_append_btf(struct bpf_linker * linker,struct src_obj * obj)2241 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2242 {
2243 	const struct btf_type *t;
2244 	int i, j, n, start_id, id;
2245 	const char *name;
2246 
2247 	if (!obj->btf)
2248 		return 0;
2249 
2250 	start_id = btf__type_cnt(linker->btf);
2251 	n = btf__type_cnt(obj->btf);
2252 
2253 	obj->btf_type_map = calloc(n + 1, sizeof(int));
2254 	if (!obj->btf_type_map)
2255 		return -ENOMEM;
2256 
2257 	for (i = 1; i < n; i++) {
2258 		struct glob_sym *glob_sym = NULL;
2259 
2260 		t = btf__type_by_id(obj->btf, i);
2261 
2262 		/* DATASECs are handled specially below */
2263 		if (btf_kind(t) == BTF_KIND_DATASEC)
2264 			continue;
2265 
2266 		if (btf_is_non_static(t)) {
2267 			/* there should be glob_sym already */
2268 			name = btf__str_by_offset(obj->btf, t->name_off);
2269 			glob_sym = find_glob_sym(linker, name);
2270 
2271 			/* VARs without corresponding glob_sym are those that
2272 			 * belong to skipped/deduplicated sections (i.e.,
2273 			 * license and version), so just skip them
2274 			 */
2275 			if (!glob_sym)
2276 				continue;
2277 
2278 			/* linker_append_elf_sym() might have requested
2279 			 * updating underlying type ID, if extern was resolved
2280 			 * to strong symbol or weak got upgraded to non-weak
2281 			 */
2282 			if (glob_sym->underlying_btf_id == 0)
2283 				glob_sym->underlying_btf_id = -t->type;
2284 
2285 			/* globals from previous object files that match our
2286 			 * VAR/FUNC already have a corresponding associated
2287 			 * BTF type, so just make sure to use it
2288 			 */
2289 			if (glob_sym->btf_id) {
2290 				/* reuse existing BTF type for global var/func */
2291 				obj->btf_type_map[i] = glob_sym->btf_id;
2292 				continue;
2293 			}
2294 		}
2295 
2296 		id = btf__add_type(linker->btf, obj->btf, t);
2297 		if (id < 0) {
2298 			pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2299 			return id;
2300 		}
2301 
2302 		obj->btf_type_map[i] = id;
2303 
2304 		/* record just appended BTF type for var/func */
2305 		if (glob_sym) {
2306 			glob_sym->btf_id = id;
2307 			glob_sym->underlying_btf_id = -t->type;
2308 		}
2309 	}
2310 
2311 	/* remap all the types except DATASECs */
2312 	n = btf__type_cnt(linker->btf);
2313 	for (i = start_id; i < n; i++) {
2314 		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2315 
2316 		if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2317 			return -EINVAL;
2318 	}
2319 
2320 	/* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2321 	 * actual type), if necessary
2322 	 */
2323 	for (i = 0; i < linker->glob_sym_cnt; i++) {
2324 		struct glob_sym *glob_sym = &linker->glob_syms[i];
2325 		struct btf_type *glob_t;
2326 
2327 		if (glob_sym->underlying_btf_id >= 0)
2328 			continue;
2329 
2330 		glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2331 
2332 		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2333 		glob_t->type = glob_sym->underlying_btf_id;
2334 	}
2335 
2336 	/* append DATASEC info */
2337 	for (i = 1; i < obj->sec_cnt; i++) {
2338 		struct src_sec *src_sec;
2339 		struct dst_sec *dst_sec;
2340 		const struct btf_var_secinfo *src_var;
2341 		struct btf_var_secinfo *dst_var;
2342 
2343 		src_sec = &obj->secs[i];
2344 		if (!src_sec->sec_type_id || src_sec->skipped)
2345 			continue;
2346 		dst_sec = &linker->secs[src_sec->dst_id];
2347 
2348 		/* Mark section as having BTF regardless of the presence of
2349 		 * variables. In some cases compiler might generate empty BTF
2350 		 * with no variables information. E.g., when promoting local
2351 		 * array/structure variable initial values and BPF object
2352 		 * file otherwise has no read-only static variables in
2353 		 * .rodata. We need to preserve such empty BTF and just set
2354 		 * correct section size.
2355 		 */
2356 		dst_sec->has_btf = true;
2357 
2358 		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2359 		src_var = btf_var_secinfos(t);
2360 		n = btf_vlen(t);
2361 		for (j = 0; j < n; j++, src_var++) {
2362 			void *sec_vars = dst_sec->sec_vars;
2363 			int new_id = obj->btf_type_map[src_var->type];
2364 			struct glob_sym *glob_sym = NULL;
2365 
2366 			t = btf_type_by_id(linker->btf, new_id);
2367 			if (btf_is_non_static(t)) {
2368 				name = btf__str_by_offset(linker->btf, t->name_off);
2369 				glob_sym = find_glob_sym(linker, name);
2370 				if (glob_sym->sec_id != dst_sec->id) {
2371 					pr_warn("global '%s': section mismatch %d vs %d\n",
2372 						name, glob_sym->sec_id, dst_sec->id);
2373 					return -EINVAL;
2374 				}
2375 			}
2376 
2377 			/* If there is already a member (VAR or FUNC) mapped
2378 			 * to the same type, don't add a duplicate entry.
2379 			 * This will happen when multiple object files define
2380 			 * the same extern VARs/FUNCs.
2381 			 */
2382 			if (glob_sym && glob_sym->var_idx >= 0) {
2383 				__s64 sz;
2384 
2385 				dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2386 				/* Because underlying BTF type might have
2387 				 * changed, so might its size have changed, so
2388 				 * re-calculate and update it in sec_var.
2389 				 */
2390 				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2391 				if (sz < 0) {
2392 					pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2393 						name, (int)sz);
2394 					return -EINVAL;
2395 				}
2396 				dst_var->size = sz;
2397 				continue;
2398 			}
2399 
2400 			sec_vars = libbpf_reallocarray(sec_vars,
2401 						       dst_sec->sec_var_cnt + 1,
2402 						       sizeof(*dst_sec->sec_vars));
2403 			if (!sec_vars)
2404 				return -ENOMEM;
2405 
2406 			dst_sec->sec_vars = sec_vars;
2407 			dst_sec->sec_var_cnt++;
2408 
2409 			dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2410 			dst_var->type = obj->btf_type_map[src_var->type];
2411 			dst_var->size = src_var->size;
2412 			dst_var->offset = src_sec->dst_off + src_var->offset;
2413 
2414 			if (glob_sym)
2415 				glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2416 		}
2417 	}
2418 
2419 	return 0;
2420 }
2421 
add_btf_ext_rec(struct btf_ext_sec_data * ext_data,const void * src_rec)2422 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2423 {
2424 	void *tmp;
2425 
2426 	tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2427 	if (!tmp)
2428 		return NULL;
2429 	ext_data->recs = tmp;
2430 
2431 	tmp += ext_data->rec_cnt * ext_data->rec_sz;
2432 	memcpy(tmp, src_rec, ext_data->rec_sz);
2433 
2434 	ext_data->rec_cnt++;
2435 
2436 	return tmp;
2437 }
2438 
linker_append_btf_ext(struct bpf_linker * linker,struct src_obj * obj)2439 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2440 {
2441 	const struct btf_ext_info_sec *ext_sec;
2442 	const char *sec_name, *s;
2443 	struct src_sec *src_sec;
2444 	struct dst_sec *dst_sec;
2445 	int rec_sz, str_off, i;
2446 
2447 	if (!obj->btf_ext)
2448 		return 0;
2449 
2450 	rec_sz = obj->btf_ext->func_info.rec_size;
2451 	for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2452 		struct bpf_func_info_min *src_rec, *dst_rec;
2453 
2454 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2455 		src_sec = find_src_sec_by_name(obj, sec_name);
2456 		if (!src_sec) {
2457 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2458 			return -EINVAL;
2459 		}
2460 		dst_sec = &linker->secs[src_sec->dst_id];
2461 
2462 		if (dst_sec->func_info.rec_sz == 0)
2463 			dst_sec->func_info.rec_sz = rec_sz;
2464 		if (dst_sec->func_info.rec_sz != rec_sz) {
2465 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2466 			return -EINVAL;
2467 		}
2468 
2469 		for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2470 			dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2471 			if (!dst_rec)
2472 				return -ENOMEM;
2473 
2474 			dst_rec->insn_off += src_sec->dst_off;
2475 			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2476 		}
2477 	}
2478 
2479 	rec_sz = obj->btf_ext->line_info.rec_size;
2480 	for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2481 		struct bpf_line_info_min *src_rec, *dst_rec;
2482 
2483 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2484 		src_sec = find_src_sec_by_name(obj, sec_name);
2485 		if (!src_sec) {
2486 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2487 			return -EINVAL;
2488 		}
2489 		dst_sec = &linker->secs[src_sec->dst_id];
2490 
2491 		if (dst_sec->line_info.rec_sz == 0)
2492 			dst_sec->line_info.rec_sz = rec_sz;
2493 		if (dst_sec->line_info.rec_sz != rec_sz) {
2494 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2495 			return -EINVAL;
2496 		}
2497 
2498 		for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2499 			dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2500 			if (!dst_rec)
2501 				return -ENOMEM;
2502 
2503 			dst_rec->insn_off += src_sec->dst_off;
2504 
2505 			s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2506 			str_off = btf__add_str(linker->btf, s);
2507 			if (str_off < 0)
2508 				return -ENOMEM;
2509 			dst_rec->file_name_off = str_off;
2510 
2511 			s = btf__str_by_offset(obj->btf, src_rec->line_off);
2512 			str_off = btf__add_str(linker->btf, s);
2513 			if (str_off < 0)
2514 				return -ENOMEM;
2515 			dst_rec->line_off = str_off;
2516 
2517 			/* dst_rec->line_col is fine */
2518 		}
2519 	}
2520 
2521 	rec_sz = obj->btf_ext->core_relo_info.rec_size;
2522 	for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2523 		struct bpf_core_relo *src_rec, *dst_rec;
2524 
2525 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2526 		src_sec = find_src_sec_by_name(obj, sec_name);
2527 		if (!src_sec) {
2528 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2529 			return -EINVAL;
2530 		}
2531 		dst_sec = &linker->secs[src_sec->dst_id];
2532 
2533 		if (dst_sec->core_relo_info.rec_sz == 0)
2534 			dst_sec->core_relo_info.rec_sz = rec_sz;
2535 		if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2536 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2537 			return -EINVAL;
2538 		}
2539 
2540 		for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2541 			dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2542 			if (!dst_rec)
2543 				return -ENOMEM;
2544 
2545 			dst_rec->insn_off += src_sec->dst_off;
2546 			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2547 
2548 			s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2549 			str_off = btf__add_str(linker->btf, s);
2550 			if (str_off < 0)
2551 				return -ENOMEM;
2552 			dst_rec->access_str_off = str_off;
2553 
2554 			/* dst_rec->kind is fine */
2555 		}
2556 	}
2557 
2558 	return 0;
2559 }
2560 
bpf_linker__finalize(struct bpf_linker * linker)2561 int bpf_linker__finalize(struct bpf_linker *linker)
2562 {
2563 	struct dst_sec *sec;
2564 	size_t strs_sz;
2565 	const void *strs;
2566 	int err, i;
2567 
2568 	if (!linker->elf)
2569 		return libbpf_err(-EINVAL);
2570 
2571 	err = finalize_btf(linker);
2572 	if (err)
2573 		return libbpf_err(err);
2574 
2575 	/* Finalize strings */
2576 	strs_sz = strset__data_size(linker->strtab_strs);
2577 	strs = strset__data(linker->strtab_strs);
2578 
2579 	sec = &linker->secs[linker->strtab_sec_idx];
2580 	sec->data->d_align = 1;
2581 	sec->data->d_off = 0LL;
2582 	sec->data->d_buf = (void *)strs;
2583 	sec->data->d_type = ELF_T_BYTE;
2584 	sec->data->d_size = strs_sz;
2585 	sec->shdr->sh_size = strs_sz;
2586 
2587 	for (i = 1; i < linker->sec_cnt; i++) {
2588 		sec = &linker->secs[i];
2589 
2590 		/* STRTAB is handled specially above */
2591 		if (sec->sec_idx == linker->strtab_sec_idx)
2592 			continue;
2593 
2594 		/* special ephemeral sections (.ksyms, .kconfig, etc) */
2595 		if (!sec->scn)
2596 			continue;
2597 
2598 		sec->data->d_buf = sec->raw_data;
2599 	}
2600 
2601 	/* Finalize ELF layout */
2602 	if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2603 		err = -errno;
2604 		pr_warn_elf("failed to finalize ELF layout");
2605 		return libbpf_err(err);
2606 	}
2607 
2608 	/* Write out final ELF contents */
2609 	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2610 		err = -errno;
2611 		pr_warn_elf("failed to write ELF contents");
2612 		return libbpf_err(err);
2613 	}
2614 
2615 	elf_end(linker->elf);
2616 	close(linker->fd);
2617 
2618 	linker->elf = NULL;
2619 	linker->fd = -1;
2620 
2621 	return 0;
2622 }
2623 
emit_elf_data_sec(struct bpf_linker * linker,const char * sec_name,size_t align,const void * raw_data,size_t raw_sz)2624 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2625 			     size_t align, const void *raw_data, size_t raw_sz)
2626 {
2627 	Elf_Scn *scn;
2628 	Elf_Data *data;
2629 	Elf64_Shdr *shdr;
2630 	int name_off;
2631 
2632 	name_off = strset__add_str(linker->strtab_strs, sec_name);
2633 	if (name_off < 0)
2634 		return name_off;
2635 
2636 	scn = elf_newscn(linker->elf);
2637 	if (!scn)
2638 		return -ENOMEM;
2639 	data = elf_newdata(scn);
2640 	if (!data)
2641 		return -ENOMEM;
2642 	shdr = elf64_getshdr(scn);
2643 	if (!shdr)
2644 		return -EINVAL;
2645 
2646 	shdr->sh_name = name_off;
2647 	shdr->sh_type = SHT_PROGBITS;
2648 	shdr->sh_flags = 0;
2649 	shdr->sh_size = raw_sz;
2650 	shdr->sh_link = 0;
2651 	shdr->sh_info = 0;
2652 	shdr->sh_addralign = align;
2653 	shdr->sh_entsize = 0;
2654 
2655 	data->d_type = ELF_T_BYTE;
2656 	data->d_size = raw_sz;
2657 	data->d_buf = (void *)raw_data;
2658 	data->d_align = align;
2659 	data->d_off = 0;
2660 
2661 	return 0;
2662 }
2663 
finalize_btf(struct bpf_linker * linker)2664 static int finalize_btf(struct bpf_linker *linker)
2665 {
2666 	LIBBPF_OPTS(btf_dedup_opts, opts);
2667 	struct btf *btf = linker->btf;
2668 	const void *raw_data;
2669 	int i, j, id, err;
2670 	__u32 raw_sz;
2671 
2672 	/* bail out if no BTF data was produced */
2673 	if (btf__type_cnt(linker->btf) == 1)
2674 		return 0;
2675 
2676 	for (i = 1; i < linker->sec_cnt; i++) {
2677 		struct dst_sec *sec = &linker->secs[i];
2678 
2679 		if (!sec->has_btf)
2680 			continue;
2681 
2682 		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2683 		if (id < 0) {
2684 			pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2685 				sec->sec_name, id);
2686 			return id;
2687 		}
2688 
2689 		for (j = 0; j < sec->sec_var_cnt; j++) {
2690 			struct btf_var_secinfo *vi = &sec->sec_vars[j];
2691 
2692 			if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2693 				return -EINVAL;
2694 		}
2695 	}
2696 
2697 	err = finalize_btf_ext(linker);
2698 	if (err) {
2699 		pr_warn(".BTF.ext generation failed: %d\n", err);
2700 		return err;
2701 	}
2702 
2703 	opts.btf_ext = linker->btf_ext;
2704 	err = btf__dedup(linker->btf, &opts);
2705 	if (err) {
2706 		pr_warn("BTF dedup failed: %d\n", err);
2707 		return err;
2708 	}
2709 
2710 	/* Emit .BTF section */
2711 	raw_data = btf__raw_data(linker->btf, &raw_sz);
2712 	if (!raw_data)
2713 		return -ENOMEM;
2714 
2715 	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2716 	if (err) {
2717 		pr_warn("failed to write out .BTF ELF section: %d\n", err);
2718 		return err;
2719 	}
2720 
2721 	/* Emit .BTF.ext section */
2722 	if (linker->btf_ext) {
2723 		raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2724 		if (!raw_data)
2725 			return -ENOMEM;
2726 
2727 		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2728 		if (err) {
2729 			pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2730 			return err;
2731 		}
2732 	}
2733 
2734 	return 0;
2735 }
2736 
emit_btf_ext_data(struct bpf_linker * linker,void * output,const char * sec_name,struct btf_ext_sec_data * sec_data)2737 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2738 			     const char *sec_name, struct btf_ext_sec_data *sec_data)
2739 {
2740 	struct btf_ext_info_sec *sec_info;
2741 	void *cur = output;
2742 	int str_off;
2743 	size_t sz;
2744 
2745 	if (!sec_data->rec_cnt)
2746 		return 0;
2747 
2748 	str_off = btf__add_str(linker->btf, sec_name);
2749 	if (str_off < 0)
2750 		return -ENOMEM;
2751 
2752 	sec_info = cur;
2753 	sec_info->sec_name_off = str_off;
2754 	sec_info->num_info = sec_data->rec_cnt;
2755 	cur += sizeof(struct btf_ext_info_sec);
2756 
2757 	sz = sec_data->rec_cnt * sec_data->rec_sz;
2758 	memcpy(cur, sec_data->recs, sz);
2759 	cur += sz;
2760 
2761 	return cur - output;
2762 }
2763 
finalize_btf_ext(struct bpf_linker * linker)2764 static int finalize_btf_ext(struct bpf_linker *linker)
2765 {
2766 	size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2767 	size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2768 	struct btf_ext_header *hdr;
2769 	void *data, *cur;
2770 	int i, err, sz;
2771 
2772 	/* validate that all sections have the same .BTF.ext record sizes
2773 	 * and calculate total data size for each type of data (func info,
2774 	 * line info, core relos)
2775 	 */
2776 	for (i = 1; i < linker->sec_cnt; i++) {
2777 		struct dst_sec *sec = &linker->secs[i];
2778 
2779 		if (sec->func_info.rec_cnt) {
2780 			if (func_rec_sz == 0)
2781 				func_rec_sz = sec->func_info.rec_sz;
2782 			if (func_rec_sz != sec->func_info.rec_sz) {
2783 				pr_warn("mismatch in func_info record size %zu != %u\n",
2784 					func_rec_sz, sec->func_info.rec_sz);
2785 				return -EINVAL;
2786 			}
2787 
2788 			funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2789 		}
2790 		if (sec->line_info.rec_cnt) {
2791 			if (line_rec_sz == 0)
2792 				line_rec_sz = sec->line_info.rec_sz;
2793 			if (line_rec_sz != sec->line_info.rec_sz) {
2794 				pr_warn("mismatch in line_info record size %zu != %u\n",
2795 					line_rec_sz, sec->line_info.rec_sz);
2796 				return -EINVAL;
2797 			}
2798 
2799 			lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2800 		}
2801 		if (sec->core_relo_info.rec_cnt) {
2802 			if (core_relo_rec_sz == 0)
2803 				core_relo_rec_sz = sec->core_relo_info.rec_sz;
2804 			if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2805 				pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2806 					core_relo_rec_sz, sec->core_relo_info.rec_sz);
2807 				return -EINVAL;
2808 			}
2809 
2810 			core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2811 		}
2812 	}
2813 
2814 	if (!funcs_sz && !lines_sz && !core_relos_sz)
2815 		return 0;
2816 
2817 	total_sz += sizeof(struct btf_ext_header);
2818 	if (funcs_sz) {
2819 		funcs_sz += sizeof(__u32); /* record size prefix */
2820 		total_sz += funcs_sz;
2821 	}
2822 	if (lines_sz) {
2823 		lines_sz += sizeof(__u32); /* record size prefix */
2824 		total_sz += lines_sz;
2825 	}
2826 	if (core_relos_sz) {
2827 		core_relos_sz += sizeof(__u32); /* record size prefix */
2828 		total_sz += core_relos_sz;
2829 	}
2830 
2831 	cur = data = calloc(1, total_sz);
2832 	if (!data)
2833 		return -ENOMEM;
2834 
2835 	hdr = cur;
2836 	hdr->magic = BTF_MAGIC;
2837 	hdr->version = BTF_VERSION;
2838 	hdr->flags = 0;
2839 	hdr->hdr_len = sizeof(struct btf_ext_header);
2840 	cur += sizeof(struct btf_ext_header);
2841 
2842 	/* All offsets are in bytes relative to the end of this header */
2843 	hdr->func_info_off = 0;
2844 	hdr->func_info_len = funcs_sz;
2845 	hdr->line_info_off = funcs_sz;
2846 	hdr->line_info_len = lines_sz;
2847 	hdr->core_relo_off = funcs_sz + lines_sz;
2848 	hdr->core_relo_len = core_relos_sz;
2849 
2850 	if (funcs_sz) {
2851 		*(__u32 *)cur = func_rec_sz;
2852 		cur += sizeof(__u32);
2853 
2854 		for (i = 1; i < linker->sec_cnt; i++) {
2855 			struct dst_sec *sec = &linker->secs[i];
2856 
2857 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2858 			if (sz < 0) {
2859 				err = sz;
2860 				goto out;
2861 			}
2862 
2863 			cur += sz;
2864 		}
2865 	}
2866 
2867 	if (lines_sz) {
2868 		*(__u32 *)cur = line_rec_sz;
2869 		cur += sizeof(__u32);
2870 
2871 		for (i = 1; i < linker->sec_cnt; i++) {
2872 			struct dst_sec *sec = &linker->secs[i];
2873 
2874 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2875 			if (sz < 0) {
2876 				err = sz;
2877 				goto out;
2878 			}
2879 
2880 			cur += sz;
2881 		}
2882 	}
2883 
2884 	if (core_relos_sz) {
2885 		*(__u32 *)cur = core_relo_rec_sz;
2886 		cur += sizeof(__u32);
2887 
2888 		for (i = 1; i < linker->sec_cnt; i++) {
2889 			struct dst_sec *sec = &linker->secs[i];
2890 
2891 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2892 			if (sz < 0) {
2893 				err = sz;
2894 				goto out;
2895 			}
2896 
2897 			cur += sz;
2898 		}
2899 	}
2900 
2901 	linker->btf_ext = btf_ext__new(data, total_sz);
2902 	err = libbpf_get_error(linker->btf_ext);
2903 	if (err) {
2904 		linker->btf_ext = NULL;
2905 		pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2906 		goto out;
2907 	}
2908 
2909 out:
2910 	free(data);
2911 	return err;
2912 }
2913