xref: /openbmc/linux/tools/lib/bpf/linker.c (revision b694e3c604e999343258c49e574abd7be012e726)
1  // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2  /*
3   * BPF static linker
4   *
5   * Copyright (c) 2021 Facebook
6   */
7  #include <stdbool.h>
8  #include <stddef.h>
9  #include <stdio.h>
10  #include <stdlib.h>
11  #include <string.h>
12  #include <unistd.h>
13  #include <errno.h>
14  #include <linux/err.h>
15  #include <linux/btf.h>
16  #include <elf.h>
17  #include <libelf.h>
18  #include <fcntl.h>
19  #include "libbpf.h"
20  #include "btf.h"
21  #include "libbpf_internal.h"
22  #include "strset.h"
23  
24  #define BTF_EXTERN_SEC ".extern"
25  
26  struct src_sec {
27  	const char *sec_name;
28  	/* positional (not necessarily ELF) index in an array of sections */
29  	int id;
30  	/* positional (not necessarily ELF) index of a matching section in a final object file */
31  	int dst_id;
32  	/* section data offset in a matching output section */
33  	int dst_off;
34  	/* whether section is omitted from the final ELF file */
35  	bool skipped;
36  	/* whether section is an ephemeral section, not mapped to an ELF section */
37  	bool ephemeral;
38  
39  	/* ELF info */
40  	size_t sec_idx;
41  	Elf_Scn *scn;
42  	Elf64_Shdr *shdr;
43  	Elf_Data *data;
44  
45  	/* corresponding BTF DATASEC type ID */
46  	int sec_type_id;
47  };
48  
49  struct src_obj {
50  	const char *filename;
51  	int fd;
52  	Elf *elf;
53  	/* Section header strings section index */
54  	size_t shstrs_sec_idx;
55  	/* SYMTAB section index */
56  	size_t symtab_sec_idx;
57  
58  	struct btf *btf;
59  	struct btf_ext *btf_ext;
60  
61  	/* List of sections (including ephemeral). Slot zero is unused. */
62  	struct src_sec *secs;
63  	int sec_cnt;
64  
65  	/* mapping of symbol indices from src to dst ELF */
66  	int *sym_map;
67  	/* mapping from the src BTF type IDs to dst ones */
68  	int *btf_type_map;
69  };
70  
71  /* single .BTF.ext data section */
72  struct btf_ext_sec_data {
73  	size_t rec_cnt;
74  	__u32 rec_sz;
75  	void *recs;
76  };
77  
78  struct glob_sym {
79  	/* ELF symbol index */
80  	int sym_idx;
81  	/* associated section id for .ksyms, .kconfig, etc, but not .extern */
82  	int sec_id;
83  	/* extern name offset in STRTAB */
84  	int name_off;
85  	/* optional associated BTF type ID */
86  	int btf_id;
87  	/* BTF type ID to which VAR/FUNC type is pointing to; used for
88  	 * rewriting types when extern VAR/FUNC is resolved to a concrete
89  	 * definition
90  	 */
91  	int underlying_btf_id;
92  	/* sec_var index in the corresponding dst_sec, if exists */
93  	int var_idx;
94  
95  	/* extern or resolved/global symbol */
96  	bool is_extern;
97  	/* weak or strong symbol, never goes back from strong to weak */
98  	bool is_weak;
99  };
100  
101  struct dst_sec {
102  	char *sec_name;
103  	/* positional (not necessarily ELF) index in an array of sections */
104  	int id;
105  
106  	bool ephemeral;
107  
108  	/* ELF info */
109  	size_t sec_idx;
110  	Elf_Scn *scn;
111  	Elf64_Shdr *shdr;
112  	Elf_Data *data;
113  
114  	/* final output section size */
115  	int sec_sz;
116  	/* final output contents of the section */
117  	void *raw_data;
118  
119  	/* corresponding STT_SECTION symbol index in SYMTAB */
120  	int sec_sym_idx;
121  
122  	/* section's DATASEC variable info, emitted on BTF finalization */
123  	bool has_btf;
124  	int sec_var_cnt;
125  	struct btf_var_secinfo *sec_vars;
126  
127  	/* section's .BTF.ext data */
128  	struct btf_ext_sec_data func_info;
129  	struct btf_ext_sec_data line_info;
130  	struct btf_ext_sec_data core_relo_info;
131  };
132  
133  struct bpf_linker {
134  	char *filename;
135  	int fd;
136  	Elf *elf;
137  	Elf64_Ehdr *elf_hdr;
138  
139  	/* Output sections metadata */
140  	struct dst_sec *secs;
141  	int sec_cnt;
142  
143  	struct strset *strtab_strs; /* STRTAB unique strings */
144  	size_t strtab_sec_idx; /* STRTAB section index */
145  	size_t symtab_sec_idx; /* SYMTAB section index */
146  
147  	struct btf *btf;
148  	struct btf_ext *btf_ext;
149  
150  	/* global (including extern) ELF symbols */
151  	int glob_sym_cnt;
152  	struct glob_sym *glob_syms;
153  };
154  
155  #define pr_warn_elf(fmt, ...)									\
156  	libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
157  
158  static int init_output_elf(struct bpf_linker *linker, const char *file);
159  
160  static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
161  				const struct bpf_linker_file_opts *opts,
162  				struct src_obj *obj);
163  static int linker_sanity_check_elf(struct src_obj *obj);
164  static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
165  static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
166  static int linker_sanity_check_btf(struct src_obj *obj);
167  static int linker_sanity_check_btf_ext(struct src_obj *obj);
168  static int linker_fixup_btf(struct src_obj *obj);
169  static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
170  static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
171  static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
172  				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
173  static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
174  static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
175  static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
176  
177  static int finalize_btf(struct bpf_linker *linker);
178  static int finalize_btf_ext(struct bpf_linker *linker);
179  
bpf_linker__free(struct bpf_linker * linker)180  void bpf_linker__free(struct bpf_linker *linker)
181  {
182  	int i;
183  
184  	if (!linker)
185  		return;
186  
187  	free(linker->filename);
188  
189  	if (linker->elf)
190  		elf_end(linker->elf);
191  
192  	if (linker->fd >= 0)
193  		close(linker->fd);
194  
195  	strset__free(linker->strtab_strs);
196  
197  	btf__free(linker->btf);
198  	btf_ext__free(linker->btf_ext);
199  
200  	for (i = 1; i < linker->sec_cnt; i++) {
201  		struct dst_sec *sec = &linker->secs[i];
202  
203  		free(sec->sec_name);
204  		free(sec->raw_data);
205  		free(sec->sec_vars);
206  
207  		free(sec->func_info.recs);
208  		free(sec->line_info.recs);
209  		free(sec->core_relo_info.recs);
210  	}
211  	free(linker->secs);
212  
213  	free(linker->glob_syms);
214  	free(linker);
215  }
216  
bpf_linker__new(const char * filename,struct bpf_linker_opts * opts)217  struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
218  {
219  	struct bpf_linker *linker;
220  	int err;
221  
222  	if (!OPTS_VALID(opts, bpf_linker_opts))
223  		return errno = EINVAL, NULL;
224  
225  	if (elf_version(EV_CURRENT) == EV_NONE) {
226  		pr_warn_elf("libelf initialization failed");
227  		return errno = EINVAL, NULL;
228  	}
229  
230  	linker = calloc(1, sizeof(*linker));
231  	if (!linker)
232  		return errno = ENOMEM, NULL;
233  
234  	linker->fd = -1;
235  
236  	err = init_output_elf(linker, filename);
237  	if (err)
238  		goto err_out;
239  
240  	return linker;
241  
242  err_out:
243  	bpf_linker__free(linker);
244  	return errno = -err, NULL;
245  }
246  
add_dst_sec(struct bpf_linker * linker,const char * sec_name)247  static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
248  {
249  	struct dst_sec *secs = linker->secs, *sec;
250  	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
251  
252  	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
253  	if (!secs)
254  		return NULL;
255  
256  	/* zero out newly allocated memory */
257  	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
258  
259  	linker->secs = secs;
260  	linker->sec_cnt = new_cnt;
261  
262  	sec = &linker->secs[new_cnt - 1];
263  	sec->id = new_cnt - 1;
264  	sec->sec_name = strdup(sec_name);
265  	if (!sec->sec_name)
266  		return NULL;
267  
268  	return sec;
269  }
270  
add_new_sym(struct bpf_linker * linker,size_t * sym_idx)271  static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
272  {
273  	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
274  	Elf64_Sym *syms, *sym;
275  	size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
276  
277  	syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
278  	if (!syms)
279  		return NULL;
280  
281  	sym = &syms[sym_cnt];
282  	memset(sym, 0, sizeof(*sym));
283  
284  	symtab->raw_data = syms;
285  	symtab->sec_sz += sizeof(*sym);
286  	symtab->shdr->sh_size += sizeof(*sym);
287  	symtab->data->d_size += sizeof(*sym);
288  
289  	if (sym_idx)
290  		*sym_idx = sym_cnt;
291  
292  	return sym;
293  }
294  
init_output_elf(struct bpf_linker * linker,const char * file)295  static int init_output_elf(struct bpf_linker *linker, const char *file)
296  {
297  	int err, str_off;
298  	Elf64_Sym *init_sym;
299  	struct dst_sec *sec;
300  
301  	linker->filename = strdup(file);
302  	if (!linker->filename)
303  		return -ENOMEM;
304  
305  	linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
306  	if (linker->fd < 0) {
307  		err = -errno;
308  		pr_warn("failed to create '%s': %d\n", file, err);
309  		return err;
310  	}
311  
312  	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
313  	if (!linker->elf) {
314  		pr_warn_elf("failed to create ELF object");
315  		return -EINVAL;
316  	}
317  
318  	/* ELF header */
319  	linker->elf_hdr = elf64_newehdr(linker->elf);
320  	if (!linker->elf_hdr) {
321  		pr_warn_elf("failed to create ELF header");
322  		return -EINVAL;
323  	}
324  
325  	linker->elf_hdr->e_machine = EM_BPF;
326  	linker->elf_hdr->e_type = ET_REL;
327  #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
328  	linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329  #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
330  	linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331  #else
332  #error "Unknown __BYTE_ORDER__"
333  #endif
334  
335  	/* STRTAB */
336  	/* initialize strset with an empty string to conform to ELF */
337  	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
338  	if (libbpf_get_error(linker->strtab_strs))
339  		return libbpf_get_error(linker->strtab_strs);
340  
341  	sec = add_dst_sec(linker, ".strtab");
342  	if (!sec)
343  		return -ENOMEM;
344  
345  	sec->scn = elf_newscn(linker->elf);
346  	if (!sec->scn) {
347  		pr_warn_elf("failed to create STRTAB section");
348  		return -EINVAL;
349  	}
350  
351  	sec->shdr = elf64_getshdr(sec->scn);
352  	if (!sec->shdr)
353  		return -EINVAL;
354  
355  	sec->data = elf_newdata(sec->scn);
356  	if (!sec->data) {
357  		pr_warn_elf("failed to create STRTAB data");
358  		return -EINVAL;
359  	}
360  
361  	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
362  	if (str_off < 0)
363  		return str_off;
364  
365  	sec->sec_idx = elf_ndxscn(sec->scn);
366  	linker->elf_hdr->e_shstrndx = sec->sec_idx;
367  	linker->strtab_sec_idx = sec->sec_idx;
368  
369  	sec->shdr->sh_name = str_off;
370  	sec->shdr->sh_type = SHT_STRTAB;
371  	sec->shdr->sh_flags = SHF_STRINGS;
372  	sec->shdr->sh_offset = 0;
373  	sec->shdr->sh_link = 0;
374  	sec->shdr->sh_info = 0;
375  	sec->shdr->sh_addralign = 1;
376  	sec->shdr->sh_size = sec->sec_sz = 0;
377  	sec->shdr->sh_entsize = 0;
378  
379  	/* SYMTAB */
380  	sec = add_dst_sec(linker, ".symtab");
381  	if (!sec)
382  		return -ENOMEM;
383  
384  	sec->scn = elf_newscn(linker->elf);
385  	if (!sec->scn) {
386  		pr_warn_elf("failed to create SYMTAB section");
387  		return -EINVAL;
388  	}
389  
390  	sec->shdr = elf64_getshdr(sec->scn);
391  	if (!sec->shdr)
392  		return -EINVAL;
393  
394  	sec->data = elf_newdata(sec->scn);
395  	if (!sec->data) {
396  		pr_warn_elf("failed to create SYMTAB data");
397  		return -EINVAL;
398  	}
399  	/* Ensure libelf translates byte-order of symbol records */
400  	sec->data->d_type = ELF_T_SYM;
401  
402  	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
403  	if (str_off < 0)
404  		return str_off;
405  
406  	sec->sec_idx = elf_ndxscn(sec->scn);
407  	linker->symtab_sec_idx = sec->sec_idx;
408  
409  	sec->shdr->sh_name = str_off;
410  	sec->shdr->sh_type = SHT_SYMTAB;
411  	sec->shdr->sh_flags = 0;
412  	sec->shdr->sh_offset = 0;
413  	sec->shdr->sh_link = linker->strtab_sec_idx;
414  	/* sh_info should be one greater than the index of the last local
415  	 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
416  	 */
417  	sec->shdr->sh_info = 0;
418  	sec->shdr->sh_addralign = 8;
419  	sec->shdr->sh_entsize = sizeof(Elf64_Sym);
420  
421  	/* .BTF */
422  	linker->btf = btf__new_empty();
423  	err = libbpf_get_error(linker->btf);
424  	if (err)
425  		return err;
426  
427  	/* add the special all-zero symbol */
428  	init_sym = add_new_sym(linker, NULL);
429  	if (!init_sym)
430  		return -EINVAL;
431  
432  	init_sym->st_name = 0;
433  	init_sym->st_info = 0;
434  	init_sym->st_other = 0;
435  	init_sym->st_shndx = SHN_UNDEF;
436  	init_sym->st_value = 0;
437  	init_sym->st_size = 0;
438  
439  	return 0;
440  }
441  
bpf_linker__add_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts)442  int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
443  			 const struct bpf_linker_file_opts *opts)
444  {
445  	struct src_obj obj = {};
446  	int err = 0;
447  
448  	if (!OPTS_VALID(opts, bpf_linker_file_opts))
449  		return libbpf_err(-EINVAL);
450  
451  	if (!linker->elf)
452  		return libbpf_err(-EINVAL);
453  
454  	err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
455  	err = err ?: linker_append_sec_data(linker, &obj);
456  	err = err ?: linker_append_elf_syms(linker, &obj);
457  	err = err ?: linker_append_elf_relos(linker, &obj);
458  	err = err ?: linker_append_btf(linker, &obj);
459  	err = err ?: linker_append_btf_ext(linker, &obj);
460  
461  	/* free up src_obj resources */
462  	free(obj.btf_type_map);
463  	btf__free(obj.btf);
464  	btf_ext__free(obj.btf_ext);
465  	free(obj.secs);
466  	free(obj.sym_map);
467  	if (obj.elf)
468  		elf_end(obj.elf);
469  	if (obj.fd >= 0)
470  		close(obj.fd);
471  
472  	return libbpf_err(err);
473  }
474  
is_dwarf_sec_name(const char * name)475  static bool is_dwarf_sec_name(const char *name)
476  {
477  	/* approximation, but the actual list is too long */
478  	return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
479  }
480  
is_ignored_sec(struct src_sec * sec)481  static bool is_ignored_sec(struct src_sec *sec)
482  {
483  	Elf64_Shdr *shdr = sec->shdr;
484  	const char *name = sec->sec_name;
485  
486  	/* no special handling of .strtab */
487  	if (shdr->sh_type == SHT_STRTAB)
488  		return true;
489  
490  	/* ignore .llvm_addrsig section as well */
491  	if (shdr->sh_type == SHT_LLVM_ADDRSIG)
492  		return true;
493  
494  	/* no subprograms will lead to an empty .text section, ignore it */
495  	if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
496  	    strcmp(sec->sec_name, ".text") == 0)
497  		return true;
498  
499  	/* DWARF sections */
500  	if (is_dwarf_sec_name(sec->sec_name))
501  		return true;
502  
503  	if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
504  		name += sizeof(".rel") - 1;
505  		/* DWARF section relocations */
506  		if (is_dwarf_sec_name(name))
507  			return true;
508  
509  		/* .BTF and .BTF.ext don't need relocations */
510  		if (strcmp(name, BTF_ELF_SEC) == 0 ||
511  		    strcmp(name, BTF_EXT_ELF_SEC) == 0)
512  			return true;
513  	}
514  
515  	return false;
516  }
517  
add_src_sec(struct src_obj * obj,const char * sec_name)518  static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
519  {
520  	struct src_sec *secs = obj->secs, *sec;
521  	size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
522  
523  	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
524  	if (!secs)
525  		return NULL;
526  
527  	/* zero out newly allocated memory */
528  	memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
529  
530  	obj->secs = secs;
531  	obj->sec_cnt = new_cnt;
532  
533  	sec = &obj->secs[new_cnt - 1];
534  	sec->id = new_cnt - 1;
535  	sec->sec_name = sec_name;
536  
537  	return sec;
538  }
539  
linker_load_obj_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts,struct src_obj * obj)540  static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
541  				const struct bpf_linker_file_opts *opts,
542  				struct src_obj *obj)
543  {
544  #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
545  	const int host_endianness = ELFDATA2LSB;
546  #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
547  	const int host_endianness = ELFDATA2MSB;
548  #else
549  #error "Unknown __BYTE_ORDER__"
550  #endif
551  	int err = 0;
552  	Elf_Scn *scn;
553  	Elf_Data *data;
554  	Elf64_Ehdr *ehdr;
555  	Elf64_Shdr *shdr;
556  	struct src_sec *sec;
557  
558  	pr_debug("linker: adding object file '%s'...\n", filename);
559  
560  	obj->filename = filename;
561  
562  	obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
563  	if (obj->fd < 0) {
564  		err = -errno;
565  		pr_warn("failed to open file '%s': %d\n", filename, err);
566  		return err;
567  	}
568  	obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
569  	if (!obj->elf) {
570  		pr_warn_elf("failed to parse ELF file '%s'", filename);
571  		return -EINVAL;
572  	}
573  
574  	/* Sanity check ELF file high-level properties */
575  	ehdr = elf64_getehdr(obj->elf);
576  	if (!ehdr) {
577  		pr_warn_elf("failed to get ELF header for %s", filename);
578  		return -EINVAL;
579  	}
580  	if (ehdr->e_ident[EI_DATA] != host_endianness) {
581  		err = -EOPNOTSUPP;
582  		pr_warn_elf("unsupported byte order of ELF file %s", filename);
583  		return err;
584  	}
585  	if (ehdr->e_type != ET_REL
586  	    || ehdr->e_machine != EM_BPF
587  	    || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
588  		err = -EOPNOTSUPP;
589  		pr_warn_elf("unsupported kind of ELF file %s", filename);
590  		return err;
591  	}
592  
593  	if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
594  		pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
595  		return -EINVAL;
596  	}
597  
598  	scn = NULL;
599  	while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
600  		size_t sec_idx = elf_ndxscn(scn);
601  		const char *sec_name;
602  
603  		shdr = elf64_getshdr(scn);
604  		if (!shdr) {
605  			pr_warn_elf("failed to get section #%zu header for %s",
606  				    sec_idx, filename);
607  			return -EINVAL;
608  		}
609  
610  		sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
611  		if (!sec_name) {
612  			pr_warn_elf("failed to get section #%zu name for %s",
613  				    sec_idx, filename);
614  			return -EINVAL;
615  		}
616  
617  		data = elf_getdata(scn, 0);
618  		if (!data) {
619  			pr_warn_elf("failed to get section #%zu (%s) data from %s",
620  				    sec_idx, sec_name, filename);
621  			return -EINVAL;
622  		}
623  
624  		sec = add_src_sec(obj, sec_name);
625  		if (!sec)
626  			return -ENOMEM;
627  
628  		sec->scn = scn;
629  		sec->shdr = shdr;
630  		sec->data = data;
631  		sec->sec_idx = elf_ndxscn(scn);
632  
633  		if (is_ignored_sec(sec)) {
634  			sec->skipped = true;
635  			continue;
636  		}
637  
638  		switch (shdr->sh_type) {
639  		case SHT_SYMTAB:
640  			if (obj->symtab_sec_idx) {
641  				err = -EOPNOTSUPP;
642  				pr_warn("multiple SYMTAB sections found, not supported\n");
643  				return err;
644  			}
645  			obj->symtab_sec_idx = sec_idx;
646  			break;
647  		case SHT_STRTAB:
648  			/* we'll construct our own string table */
649  			break;
650  		case SHT_PROGBITS:
651  			if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
652  				obj->btf = btf__new(data->d_buf, shdr->sh_size);
653  				err = libbpf_get_error(obj->btf);
654  				if (err) {
655  					pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
656  					return err;
657  				}
658  				sec->skipped = true;
659  				continue;
660  			}
661  			if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
662  				obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
663  				err = libbpf_get_error(obj->btf_ext);
664  				if (err) {
665  					pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
666  					return err;
667  				}
668  				sec->skipped = true;
669  				continue;
670  			}
671  
672  			/* data & code */
673  			break;
674  		case SHT_NOBITS:
675  			/* BSS */
676  			break;
677  		case SHT_REL:
678  			/* relocations */
679  			break;
680  		default:
681  			pr_warn("unrecognized section #%zu (%s) in %s\n",
682  				sec_idx, sec_name, filename);
683  			err = -EINVAL;
684  			return err;
685  		}
686  	}
687  
688  	err = err ?: linker_sanity_check_elf(obj);
689  	err = err ?: linker_sanity_check_btf(obj);
690  	err = err ?: linker_sanity_check_btf_ext(obj);
691  	err = err ?: linker_fixup_btf(obj);
692  
693  	return err;
694  }
695  
linker_sanity_check_elf(struct src_obj * obj)696  static int linker_sanity_check_elf(struct src_obj *obj)
697  {
698  	struct src_sec *sec;
699  	int i, err;
700  
701  	if (!obj->symtab_sec_idx) {
702  		pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
703  		return -EINVAL;
704  	}
705  	if (!obj->shstrs_sec_idx) {
706  		pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
707  		return -EINVAL;
708  	}
709  
710  	for (i = 1; i < obj->sec_cnt; i++) {
711  		sec = &obj->secs[i];
712  
713  		if (sec->sec_name[0] == '\0') {
714  			pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
715  			return -EINVAL;
716  		}
717  
718  		if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
719  			return -EINVAL;
720  		if (sec->shdr->sh_addralign != sec->data->d_align)
721  			return -EINVAL;
722  
723  		if (sec->shdr->sh_size != sec->data->d_size)
724  			return -EINVAL;
725  
726  		switch (sec->shdr->sh_type) {
727  		case SHT_SYMTAB:
728  			err = linker_sanity_check_elf_symtab(obj, sec);
729  			if (err)
730  				return err;
731  			break;
732  		case SHT_STRTAB:
733  			break;
734  		case SHT_PROGBITS:
735  			if (sec->shdr->sh_flags & SHF_EXECINSTR) {
736  				if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
737  					return -EINVAL;
738  			}
739  			break;
740  		case SHT_NOBITS:
741  			break;
742  		case SHT_REL:
743  			err = linker_sanity_check_elf_relos(obj, sec);
744  			if (err)
745  				return err;
746  			break;
747  		case SHT_LLVM_ADDRSIG:
748  			break;
749  		default:
750  			pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
751  				sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
752  			return -EINVAL;
753  		}
754  	}
755  
756  	return 0;
757  }
758  
linker_sanity_check_elf_symtab(struct src_obj * obj,struct src_sec * sec)759  static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
760  {
761  	struct src_sec *link_sec;
762  	Elf64_Sym *sym;
763  	int i, n;
764  
765  	if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
766  		return -EINVAL;
767  	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
768  		return -EINVAL;
769  
770  	if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
771  		pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
772  			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
773  		return -EINVAL;
774  	}
775  	link_sec = &obj->secs[sec->shdr->sh_link];
776  	if (link_sec->shdr->sh_type != SHT_STRTAB) {
777  		pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
778  			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
779  		return -EINVAL;
780  	}
781  
782  	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
783  	sym = sec->data->d_buf;
784  	for (i = 0; i < n; i++, sym++) {
785  		int sym_type = ELF64_ST_TYPE(sym->st_info);
786  		int sym_bind = ELF64_ST_BIND(sym->st_info);
787  		int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
788  
789  		if (i == 0) {
790  			if (sym->st_name != 0 || sym->st_info != 0
791  			    || sym->st_other != 0 || sym->st_shndx != 0
792  			    || sym->st_value != 0 || sym->st_size != 0) {
793  				pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
794  				return -EINVAL;
795  			}
796  			continue;
797  		}
798  		if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
799  			pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
800  				i, sec->sec_idx, sym_bind);
801  			return -EINVAL;
802  		}
803  		if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
804  			pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
805  				i, sec->sec_idx, sym_vis);
806  			return -EINVAL;
807  		}
808  		if (sym->st_shndx == 0) {
809  			if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
810  			    || sym->st_value != 0 || sym->st_size != 0) {
811  				pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
812  					i, obj->filename);
813  
814  				return -EINVAL;
815  			}
816  			continue;
817  		}
818  		if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
819  			pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
820  				i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
821  			return -EINVAL;
822  		}
823  		if (sym_type == STT_SECTION) {
824  			if (sym->st_value != 0)
825  				return -EINVAL;
826  			continue;
827  		}
828  	}
829  
830  	return 0;
831  }
832  
linker_sanity_check_elf_relos(struct src_obj * obj,struct src_sec * sec)833  static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
834  {
835  	struct src_sec *link_sec, *sym_sec;
836  	Elf64_Rel *relo;
837  	int i, n;
838  
839  	if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
840  		return -EINVAL;
841  	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
842  		return -EINVAL;
843  
844  	/* SHT_REL's sh_link should point to SYMTAB */
845  	if (sec->shdr->sh_link != obj->symtab_sec_idx) {
846  		pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
847  			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
848  		return -EINVAL;
849  	}
850  
851  	/* SHT_REL's sh_info points to relocated section */
852  	if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
853  		pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
854  			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
855  		return -EINVAL;
856  	}
857  	link_sec = &obj->secs[sec->shdr->sh_info];
858  
859  	/* .rel<secname> -> <secname> pattern is followed */
860  	if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
861  	    || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
862  		pr_warn("ELF relo section #%zu name has invalid name in %s\n",
863  			sec->sec_idx, obj->filename);
864  		return -EINVAL;
865  	}
866  
867  	/* don't further validate relocations for ignored sections */
868  	if (link_sec->skipped)
869  		return 0;
870  
871  	/* relocatable section is data or instructions */
872  	if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
873  		pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
874  			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
875  		return -EINVAL;
876  	}
877  
878  	/* check sanity of each relocation */
879  	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
880  	relo = sec->data->d_buf;
881  	sym_sec = &obj->secs[obj->symtab_sec_idx];
882  	for (i = 0; i < n; i++, relo++) {
883  		size_t sym_idx = ELF64_R_SYM(relo->r_info);
884  		size_t sym_type = ELF64_R_TYPE(relo->r_info);
885  
886  		if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
887  		    sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
888  			pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
889  				i, sec->sec_idx, sym_type, obj->filename);
890  			return -EINVAL;
891  		}
892  
893  		if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
894  			pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
895  				i, sec->sec_idx, sym_idx, obj->filename);
896  			return -EINVAL;
897  		}
898  
899  		if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
900  			if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
901  				pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
902  					i, sec->sec_idx, sym_idx, obj->filename);
903  				return -EINVAL;
904  			}
905  		}
906  	}
907  
908  	return 0;
909  }
910  
check_btf_type_id(__u32 * type_id,void * ctx)911  static int check_btf_type_id(__u32 *type_id, void *ctx)
912  {
913  	struct btf *btf = ctx;
914  
915  	if (*type_id >= btf__type_cnt(btf))
916  		return -EINVAL;
917  
918  	return 0;
919  }
920  
check_btf_str_off(__u32 * str_off,void * ctx)921  static int check_btf_str_off(__u32 *str_off, void *ctx)
922  {
923  	struct btf *btf = ctx;
924  	const char *s;
925  
926  	s = btf__str_by_offset(btf, *str_off);
927  
928  	if (!s)
929  		return -EINVAL;
930  
931  	return 0;
932  }
933  
linker_sanity_check_btf(struct src_obj * obj)934  static int linker_sanity_check_btf(struct src_obj *obj)
935  {
936  	struct btf_type *t;
937  	int i, n, err = 0;
938  
939  	if (!obj->btf)
940  		return 0;
941  
942  	n = btf__type_cnt(obj->btf);
943  	for (i = 1; i < n; i++) {
944  		t = btf_type_by_id(obj->btf, i);
945  
946  		err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
947  		err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
948  		if (err)
949  			return err;
950  	}
951  
952  	return 0;
953  }
954  
linker_sanity_check_btf_ext(struct src_obj * obj)955  static int linker_sanity_check_btf_ext(struct src_obj *obj)
956  {
957  	int err = 0;
958  
959  	if (!obj->btf_ext)
960  		return 0;
961  
962  	/* can't use .BTF.ext without .BTF */
963  	if (!obj->btf)
964  		return -EINVAL;
965  
966  	err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
967  	err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
968  	if (err)
969  		return err;
970  
971  	return 0;
972  }
973  
init_sec(struct bpf_linker * linker,struct dst_sec * dst_sec,struct src_sec * src_sec)974  static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
975  {
976  	Elf_Scn *scn;
977  	Elf_Data *data;
978  	Elf64_Shdr *shdr;
979  	int name_off;
980  
981  	dst_sec->sec_sz = 0;
982  	dst_sec->sec_idx = 0;
983  	dst_sec->ephemeral = src_sec->ephemeral;
984  
985  	/* ephemeral sections are just thin section shells lacking most parts */
986  	if (src_sec->ephemeral)
987  		return 0;
988  
989  	scn = elf_newscn(linker->elf);
990  	if (!scn)
991  		return -ENOMEM;
992  	data = elf_newdata(scn);
993  	if (!data)
994  		return -ENOMEM;
995  	shdr = elf64_getshdr(scn);
996  	if (!shdr)
997  		return -ENOMEM;
998  
999  	dst_sec->scn = scn;
1000  	dst_sec->shdr = shdr;
1001  	dst_sec->data = data;
1002  	dst_sec->sec_idx = elf_ndxscn(scn);
1003  
1004  	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1005  	if (name_off < 0)
1006  		return name_off;
1007  
1008  	shdr->sh_name = name_off;
1009  	shdr->sh_type = src_sec->shdr->sh_type;
1010  	shdr->sh_flags = src_sec->shdr->sh_flags;
1011  	shdr->sh_size = 0;
1012  	/* sh_link and sh_info have different meaning for different types of
1013  	 * sections, so we leave it up to the caller code to fill them in, if
1014  	 * necessary
1015  	 */
1016  	shdr->sh_link = 0;
1017  	shdr->sh_info = 0;
1018  	shdr->sh_addralign = src_sec->shdr->sh_addralign;
1019  	shdr->sh_entsize = src_sec->shdr->sh_entsize;
1020  
1021  	data->d_type = src_sec->data->d_type;
1022  	data->d_size = 0;
1023  	data->d_buf = NULL;
1024  	data->d_align = src_sec->data->d_align;
1025  	data->d_off = 0;
1026  
1027  	return 0;
1028  }
1029  
find_dst_sec_by_name(struct bpf_linker * linker,const char * sec_name)1030  static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1031  {
1032  	struct dst_sec *sec;
1033  	int i;
1034  
1035  	for (i = 1; i < linker->sec_cnt; i++) {
1036  		sec = &linker->secs[i];
1037  
1038  		if (strcmp(sec->sec_name, sec_name) == 0)
1039  			return sec;
1040  	}
1041  
1042  	return NULL;
1043  }
1044  
secs_match(struct dst_sec * dst,struct src_sec * src)1045  static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1046  {
1047  	if (dst->ephemeral || src->ephemeral)
1048  		return true;
1049  
1050  	if (dst->shdr->sh_type != src->shdr->sh_type) {
1051  		pr_warn("sec %s types mismatch\n", dst->sec_name);
1052  		return false;
1053  	}
1054  	if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1055  		pr_warn("sec %s flags mismatch\n", dst->sec_name);
1056  		return false;
1057  	}
1058  	if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1059  		pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1060  		return false;
1061  	}
1062  
1063  	return true;
1064  }
1065  
sec_content_is_same(struct dst_sec * dst_sec,struct src_sec * src_sec)1066  static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1067  {
1068  	if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1069  		return false;
1070  	if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1071  		return false;
1072  	return true;
1073  }
1074  
extend_sec(struct bpf_linker * linker,struct dst_sec * dst,struct src_sec * src)1075  static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1076  {
1077  	void *tmp;
1078  	size_t dst_align, src_align;
1079  	size_t dst_align_sz, dst_final_sz;
1080  	int err;
1081  
1082  	/* Ephemeral source section doesn't contribute anything to ELF
1083  	 * section data.
1084  	 */
1085  	if (src->ephemeral)
1086  		return 0;
1087  
1088  	/* Some sections (like .maps) can contain both externs (and thus be
1089  	 * ephemeral) and non-externs (map definitions). So it's possible that
1090  	 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1091  	 * first non-ephemeral entity appears. In such case, we add ELF
1092  	 * section, data, etc.
1093  	 */
1094  	if (dst->ephemeral) {
1095  		err = init_sec(linker, dst, src);
1096  		if (err)
1097  			return err;
1098  	}
1099  
1100  	dst_align = dst->shdr->sh_addralign;
1101  	src_align = src->shdr->sh_addralign;
1102  	if (dst_align == 0)
1103  		dst_align = 1;
1104  	if (dst_align < src_align)
1105  		dst_align = src_align;
1106  
1107  	dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1108  
1109  	/* no need to re-align final size */
1110  	dst_final_sz = dst_align_sz + src->shdr->sh_size;
1111  
1112  	if (src->shdr->sh_type != SHT_NOBITS) {
1113  		tmp = realloc(dst->raw_data, dst_final_sz);
1114  		/* If dst_align_sz == 0, realloc() behaves in a special way:
1115  		 * 1. When dst->raw_data is NULL it returns:
1116  		 *    "either NULL or a pointer suitable to be passed to free()" [1].
1117  		 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1118  		 *    thus invalidating any "pointer suitable to be passed to free()" obtained
1119  		 *    at step (1).
1120  		 *
1121  		 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1122  		 * dst->raw_data would be freed again in bpf_linker__free().
1123  		 *
1124  		 * [1] man 3 realloc
1125  		 */
1126  		if (!tmp && dst_align_sz > 0)
1127  			return -ENOMEM;
1128  		dst->raw_data = tmp;
1129  
1130  		/* pad dst section, if it's alignment forced size increase */
1131  		memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1132  		/* now copy src data at a properly aligned offset */
1133  		memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1134  	}
1135  
1136  	dst->sec_sz = dst_final_sz;
1137  	dst->shdr->sh_size = dst_final_sz;
1138  	dst->data->d_size = dst_final_sz;
1139  
1140  	dst->shdr->sh_addralign = dst_align;
1141  	dst->data->d_align = dst_align;
1142  
1143  	src->dst_off = dst_align_sz;
1144  
1145  	return 0;
1146  }
1147  
is_data_sec(struct src_sec * sec)1148  static bool is_data_sec(struct src_sec *sec)
1149  {
1150  	if (!sec || sec->skipped)
1151  		return false;
1152  	/* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1153  	if (sec->ephemeral)
1154  		return true;
1155  	return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1156  }
1157  
is_relo_sec(struct src_sec * sec)1158  static bool is_relo_sec(struct src_sec *sec)
1159  {
1160  	if (!sec || sec->skipped || sec->ephemeral)
1161  		return false;
1162  	return sec->shdr->sh_type == SHT_REL;
1163  }
1164  
linker_append_sec_data(struct bpf_linker * linker,struct src_obj * obj)1165  static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1166  {
1167  	int i, err;
1168  
1169  	for (i = 1; i < obj->sec_cnt; i++) {
1170  		struct src_sec *src_sec;
1171  		struct dst_sec *dst_sec;
1172  
1173  		src_sec = &obj->secs[i];
1174  		if (!is_data_sec(src_sec))
1175  			continue;
1176  
1177  		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1178  		if (!dst_sec) {
1179  			dst_sec = add_dst_sec(linker, src_sec->sec_name);
1180  			if (!dst_sec)
1181  				return -ENOMEM;
1182  			err = init_sec(linker, dst_sec, src_sec);
1183  			if (err) {
1184  				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1185  				return err;
1186  			}
1187  		} else {
1188  			if (!secs_match(dst_sec, src_sec)) {
1189  				pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1190  				return -1;
1191  			}
1192  
1193  			/* "license" and "version" sections are deduped */
1194  			if (strcmp(src_sec->sec_name, "license") == 0
1195  			    || strcmp(src_sec->sec_name, "version") == 0) {
1196  				if (!sec_content_is_same(dst_sec, src_sec)) {
1197  					pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1198  					return -EINVAL;
1199  				}
1200  				src_sec->skipped = true;
1201  				src_sec->dst_id = dst_sec->id;
1202  				continue;
1203  			}
1204  		}
1205  
1206  		/* record mapped section index */
1207  		src_sec->dst_id = dst_sec->id;
1208  
1209  		err = extend_sec(linker, dst_sec, src_sec);
1210  		if (err)
1211  			return err;
1212  	}
1213  
1214  	return 0;
1215  }
1216  
linker_append_elf_syms(struct bpf_linker * linker,struct src_obj * obj)1217  static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1218  {
1219  	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1220  	Elf64_Sym *sym = symtab->data->d_buf;
1221  	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1222  	int str_sec_idx = symtab->shdr->sh_link;
1223  	const char *sym_name;
1224  
1225  	obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1226  	if (!obj->sym_map)
1227  		return -ENOMEM;
1228  
1229  	for (i = 0; i < n; i++, sym++) {
1230  		/* We already validated all-zero symbol #0 and we already
1231  		 * appended it preventively to the final SYMTAB, so skip it.
1232  		 */
1233  		if (i == 0)
1234  			continue;
1235  
1236  		sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1237  		if (!sym_name) {
1238  			pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1239  			return -EINVAL;
1240  		}
1241  
1242  		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1243  		if (err)
1244  			return err;
1245  	}
1246  
1247  	return 0;
1248  }
1249  
get_sym_by_idx(struct bpf_linker * linker,size_t sym_idx)1250  static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1251  {
1252  	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1253  	Elf64_Sym *syms = symtab->raw_data;
1254  
1255  	return &syms[sym_idx];
1256  }
1257  
find_glob_sym(struct bpf_linker * linker,const char * sym_name)1258  static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1259  {
1260  	struct glob_sym *glob_sym;
1261  	const char *name;
1262  	int i;
1263  
1264  	for (i = 0; i < linker->glob_sym_cnt; i++) {
1265  		glob_sym = &linker->glob_syms[i];
1266  		name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1267  
1268  		if (strcmp(name, sym_name) == 0)
1269  			return glob_sym;
1270  	}
1271  
1272  	return NULL;
1273  }
1274  
add_glob_sym(struct bpf_linker * linker)1275  static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1276  {
1277  	struct glob_sym *syms, *sym;
1278  
1279  	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1280  				   sizeof(*linker->glob_syms));
1281  	if (!syms)
1282  		return NULL;
1283  
1284  	sym = &syms[linker->glob_sym_cnt];
1285  	memset(sym, 0, sizeof(*sym));
1286  	sym->var_idx = -1;
1287  
1288  	linker->glob_syms = syms;
1289  	linker->glob_sym_cnt++;
1290  
1291  	return sym;
1292  }
1293  
glob_sym_btf_matches(const char * sym_name,bool exact,const struct btf * btf1,__u32 id1,const struct btf * btf2,__u32 id2)1294  static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1295  				 const struct btf *btf1, __u32 id1,
1296  				 const struct btf *btf2, __u32 id2)
1297  {
1298  	const struct btf_type *t1, *t2;
1299  	bool is_static1, is_static2;
1300  	const char *n1, *n2;
1301  	int i, n;
1302  
1303  recur:
1304  	n1 = n2 = NULL;
1305  	t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1306  	t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1307  
1308  	/* check if only one side is FWD, otherwise handle with common logic */
1309  	if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1310  		n1 = btf__str_by_offset(btf1, t1->name_off);
1311  		n2 = btf__str_by_offset(btf2, t2->name_off);
1312  		if (strcmp(n1, n2) != 0) {
1313  			pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1314  				sym_name, n1, n2);
1315  			return false;
1316  		}
1317  		/* validate if FWD kind matches concrete kind */
1318  		if (btf_is_fwd(t1)) {
1319  			if (btf_kflag(t1) && btf_is_union(t2))
1320  				return true;
1321  			if (!btf_kflag(t1) && btf_is_struct(t2))
1322  				return true;
1323  			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1324  				sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1325  		} else {
1326  			if (btf_kflag(t2) && btf_is_union(t1))
1327  				return true;
1328  			if (!btf_kflag(t2) && btf_is_struct(t1))
1329  				return true;
1330  			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1331  				sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1332  		}
1333  		return false;
1334  	}
1335  
1336  	if (btf_kind(t1) != btf_kind(t2)) {
1337  		pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1338  			sym_name, btf_kind_str(t1), btf_kind_str(t2));
1339  		return false;
1340  	}
1341  
1342  	switch (btf_kind(t1)) {
1343  	case BTF_KIND_STRUCT:
1344  	case BTF_KIND_UNION:
1345  	case BTF_KIND_ENUM:
1346  	case BTF_KIND_ENUM64:
1347  	case BTF_KIND_FWD:
1348  	case BTF_KIND_FUNC:
1349  	case BTF_KIND_VAR:
1350  		n1 = btf__str_by_offset(btf1, t1->name_off);
1351  		n2 = btf__str_by_offset(btf2, t2->name_off);
1352  		if (strcmp(n1, n2) != 0) {
1353  			pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1354  				sym_name, btf_kind_str(t1), n1, n2);
1355  			return false;
1356  		}
1357  		break;
1358  	default:
1359  		break;
1360  	}
1361  
1362  	switch (btf_kind(t1)) {
1363  	case BTF_KIND_UNKN: /* void */
1364  	case BTF_KIND_FWD:
1365  		return true;
1366  	case BTF_KIND_INT:
1367  	case BTF_KIND_FLOAT:
1368  	case BTF_KIND_ENUM:
1369  	case BTF_KIND_ENUM64:
1370  		/* ignore encoding for int and enum values for enum */
1371  		if (t1->size != t2->size) {
1372  			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1373  				sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1374  			return false;
1375  		}
1376  		return true;
1377  	case BTF_KIND_PTR:
1378  		/* just validate overall shape of the referenced type, so no
1379  		 * contents comparison for struct/union, and allowd fwd vs
1380  		 * struct/union
1381  		 */
1382  		exact = false;
1383  		id1 = t1->type;
1384  		id2 = t2->type;
1385  		goto recur;
1386  	case BTF_KIND_ARRAY:
1387  		/* ignore index type and array size */
1388  		id1 = btf_array(t1)->type;
1389  		id2 = btf_array(t2)->type;
1390  		goto recur;
1391  	case BTF_KIND_FUNC:
1392  		/* extern and global linkages are compatible */
1393  		is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1394  		is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1395  		if (is_static1 != is_static2) {
1396  			pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1397  			return false;
1398  		}
1399  
1400  		id1 = t1->type;
1401  		id2 = t2->type;
1402  		goto recur;
1403  	case BTF_KIND_VAR:
1404  		/* extern and global linkages are compatible */
1405  		is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1406  		is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1407  		if (is_static1 != is_static2) {
1408  			pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1409  			return false;
1410  		}
1411  
1412  		id1 = t1->type;
1413  		id2 = t2->type;
1414  		goto recur;
1415  	case BTF_KIND_STRUCT:
1416  	case BTF_KIND_UNION: {
1417  		const struct btf_member *m1, *m2;
1418  
1419  		if (!exact)
1420  			return true;
1421  
1422  		if (btf_vlen(t1) != btf_vlen(t2)) {
1423  			pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1424  				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1425  			return false;
1426  		}
1427  
1428  		n = btf_vlen(t1);
1429  		m1 = btf_members(t1);
1430  		m2 = btf_members(t2);
1431  		for (i = 0; i < n; i++, m1++, m2++) {
1432  			n1 = btf__str_by_offset(btf1, m1->name_off);
1433  			n2 = btf__str_by_offset(btf2, m2->name_off);
1434  			if (strcmp(n1, n2) != 0) {
1435  				pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1436  					sym_name, i, n1, n2);
1437  				return false;
1438  			}
1439  			if (m1->offset != m2->offset) {
1440  				pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1441  					sym_name, i, n1);
1442  				return false;
1443  			}
1444  			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1445  				return false;
1446  		}
1447  
1448  		return true;
1449  	}
1450  	case BTF_KIND_FUNC_PROTO: {
1451  		const struct btf_param *m1, *m2;
1452  
1453  		if (btf_vlen(t1) != btf_vlen(t2)) {
1454  			pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1455  				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1456  			return false;
1457  		}
1458  
1459  		n = btf_vlen(t1);
1460  		m1 = btf_params(t1);
1461  		m2 = btf_params(t2);
1462  		for (i = 0; i < n; i++, m1++, m2++) {
1463  			/* ignore func arg names */
1464  			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1465  				return false;
1466  		}
1467  
1468  		/* now check return type as well */
1469  		id1 = t1->type;
1470  		id2 = t2->type;
1471  		goto recur;
1472  	}
1473  
1474  	/* skip_mods_and_typedefs() make this impossible */
1475  	case BTF_KIND_TYPEDEF:
1476  	case BTF_KIND_VOLATILE:
1477  	case BTF_KIND_CONST:
1478  	case BTF_KIND_RESTRICT:
1479  	/* DATASECs are never compared with each other */
1480  	case BTF_KIND_DATASEC:
1481  	default:
1482  		pr_warn("global '%s': unsupported BTF kind %s\n",
1483  			sym_name, btf_kind_str(t1));
1484  		return false;
1485  	}
1486  }
1487  
map_defs_match(const char * sym_name,const struct btf * main_btf,const struct btf_map_def * main_def,const struct btf_map_def * main_inner_def,const struct btf * extra_btf,const struct btf_map_def * extra_def,const struct btf_map_def * extra_inner_def)1488  static bool map_defs_match(const char *sym_name,
1489  			   const struct btf *main_btf,
1490  			   const struct btf_map_def *main_def,
1491  			   const struct btf_map_def *main_inner_def,
1492  			   const struct btf *extra_btf,
1493  			   const struct btf_map_def *extra_def,
1494  			   const struct btf_map_def *extra_inner_def)
1495  {
1496  	const char *reason;
1497  
1498  	if (main_def->map_type != extra_def->map_type) {
1499  		reason = "type";
1500  		goto mismatch;
1501  	}
1502  
1503  	/* check key type/size match */
1504  	if (main_def->key_size != extra_def->key_size) {
1505  		reason = "key_size";
1506  		goto mismatch;
1507  	}
1508  	if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1509  		reason = "key type";
1510  		goto mismatch;
1511  	}
1512  	if ((main_def->parts & MAP_DEF_KEY_TYPE)
1513  	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1514  				      main_btf, main_def->key_type_id,
1515  				      extra_btf, extra_def->key_type_id)) {
1516  		reason = "key type";
1517  		goto mismatch;
1518  	}
1519  
1520  	/* validate value type/size match */
1521  	if (main_def->value_size != extra_def->value_size) {
1522  		reason = "value_size";
1523  		goto mismatch;
1524  	}
1525  	if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1526  		reason = "value type";
1527  		goto mismatch;
1528  	}
1529  	if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1530  	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1531  				      main_btf, main_def->value_type_id,
1532  				      extra_btf, extra_def->value_type_id)) {
1533  		reason = "key type";
1534  		goto mismatch;
1535  	}
1536  
1537  	if (main_def->max_entries != extra_def->max_entries) {
1538  		reason = "max_entries";
1539  		goto mismatch;
1540  	}
1541  	if (main_def->map_flags != extra_def->map_flags) {
1542  		reason = "map_flags";
1543  		goto mismatch;
1544  	}
1545  	if (main_def->numa_node != extra_def->numa_node) {
1546  		reason = "numa_node";
1547  		goto mismatch;
1548  	}
1549  	if (main_def->pinning != extra_def->pinning) {
1550  		reason = "pinning";
1551  		goto mismatch;
1552  	}
1553  
1554  	if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1555  		reason = "inner map";
1556  		goto mismatch;
1557  	}
1558  
1559  	if (main_def->parts & MAP_DEF_INNER_MAP) {
1560  		char inner_map_name[128];
1561  
1562  		snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1563  
1564  		return map_defs_match(inner_map_name,
1565  				      main_btf, main_inner_def, NULL,
1566  				      extra_btf, extra_inner_def, NULL);
1567  	}
1568  
1569  	return true;
1570  
1571  mismatch:
1572  	pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1573  	return false;
1574  }
1575  
glob_map_defs_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,int btf_id)1576  static bool glob_map_defs_match(const char *sym_name,
1577  				struct bpf_linker *linker, struct glob_sym *glob_sym,
1578  				struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1579  {
1580  	struct btf_map_def dst_def = {}, dst_inner_def = {};
1581  	struct btf_map_def src_def = {}, src_inner_def = {};
1582  	const struct btf_type *t;
1583  	int err;
1584  
1585  	t = btf__type_by_id(obj->btf, btf_id);
1586  	if (!btf_is_var(t)) {
1587  		pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1588  		return false;
1589  	}
1590  	t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1591  
1592  	err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1593  	if (err) {
1594  		pr_warn("global '%s': invalid map definition\n", sym_name);
1595  		return false;
1596  	}
1597  
1598  	/* re-parse existing map definition */
1599  	t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1600  	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1601  	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1602  	if (err) {
1603  		/* this should not happen, because we already validated it */
1604  		pr_warn("global '%s': invalid dst map definition\n", sym_name);
1605  		return false;
1606  	}
1607  
1608  	/* Currently extern map definition has to be complete and match
1609  	 * concrete map definition exactly. This restriction might be lifted
1610  	 * in the future.
1611  	 */
1612  	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1613  			      obj->btf, &src_def, &src_inner_def);
1614  }
1615  
glob_syms_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,size_t sym_idx,int btf_id)1616  static bool glob_syms_match(const char *sym_name,
1617  			    struct bpf_linker *linker, struct glob_sym *glob_sym,
1618  			    struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1619  {
1620  	const struct btf_type *src_t;
1621  
1622  	/* if we are dealing with externs, BTF types describing both global
1623  	 * and extern VARs/FUNCs should be completely present in all files
1624  	 */
1625  	if (!glob_sym->btf_id || !btf_id) {
1626  		pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1627  		return false;
1628  	}
1629  
1630  	src_t = btf__type_by_id(obj->btf, btf_id);
1631  	if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1632  		pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1633  			btf_kind_str(src_t), sym_name);
1634  		return false;
1635  	}
1636  
1637  	/* deal with .maps definitions specially */
1638  	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1639  		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1640  
1641  	if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1642  				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1643  		return false;
1644  
1645  	return true;
1646  }
1647  
btf_is_non_static(const struct btf_type * t)1648  static bool btf_is_non_static(const struct btf_type *t)
1649  {
1650  	return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1651  	       || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1652  }
1653  
find_glob_sym_btf(struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int * out_btf_sec_id,int * out_btf_id)1654  static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1655  			     int *out_btf_sec_id, int *out_btf_id)
1656  {
1657  	int i, j, n, m, btf_id = 0;
1658  	const struct btf_type *t;
1659  	const struct btf_var_secinfo *vi;
1660  	const char *name;
1661  
1662  	if (!obj->btf) {
1663  		pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1664  		return -EINVAL;
1665  	}
1666  
1667  	n = btf__type_cnt(obj->btf);
1668  	for (i = 1; i < n; i++) {
1669  		t = btf__type_by_id(obj->btf, i);
1670  
1671  		/* some global and extern FUNCs and VARs might not be associated with any
1672  		 * DATASEC, so try to detect them in the same pass
1673  		 */
1674  		if (btf_is_non_static(t)) {
1675  			name = btf__str_by_offset(obj->btf, t->name_off);
1676  			if (strcmp(name, sym_name) != 0)
1677  				continue;
1678  
1679  			/* remember and still try to find DATASEC */
1680  			btf_id = i;
1681  			continue;
1682  		}
1683  
1684  		if (!btf_is_datasec(t))
1685  			continue;
1686  
1687  		vi = btf_var_secinfos(t);
1688  		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1689  			t = btf__type_by_id(obj->btf, vi->type);
1690  			name = btf__str_by_offset(obj->btf, t->name_off);
1691  
1692  			if (strcmp(name, sym_name) != 0)
1693  				continue;
1694  			if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1695  				continue;
1696  			if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1697  				continue;
1698  
1699  			if (btf_id && btf_id != vi->type) {
1700  				pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1701  					sym_name, btf_id, vi->type);
1702  				return -EINVAL;
1703  			}
1704  
1705  			*out_btf_sec_id = i;
1706  			*out_btf_id = vi->type;
1707  
1708  			return 0;
1709  		}
1710  	}
1711  
1712  	/* free-floating extern or global FUNC */
1713  	if (btf_id) {
1714  		*out_btf_sec_id = 0;
1715  		*out_btf_id = btf_id;
1716  		return 0;
1717  	}
1718  
1719  	pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1720  	return -ENOENT;
1721  }
1722  
find_src_sec_by_name(struct src_obj * obj,const char * sec_name)1723  static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1724  {
1725  	struct src_sec *sec;
1726  	int i;
1727  
1728  	for (i = 1; i < obj->sec_cnt; i++) {
1729  		sec = &obj->secs[i];
1730  
1731  		if (strcmp(sec->sec_name, sec_name) == 0)
1732  			return sec;
1733  	}
1734  
1735  	return NULL;
1736  }
1737  
complete_extern_btf_info(struct btf * dst_btf,int dst_id,struct btf * src_btf,int src_id)1738  static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1739  				    struct btf *src_btf, int src_id)
1740  {
1741  	struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1742  	struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1743  	struct btf_param *src_p, *dst_p;
1744  	const char *s;
1745  	int i, n, off;
1746  
1747  	/* We already made sure that source and destination types (FUNC or
1748  	 * VAR) match in terms of types and argument names.
1749  	 */
1750  	if (btf_is_var(dst_t)) {
1751  		btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1752  		return 0;
1753  	}
1754  
1755  	dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1756  
1757  	/* now onto FUNC_PROTO types */
1758  	src_t = btf_type_by_id(src_btf, src_t->type);
1759  	dst_t = btf_type_by_id(dst_btf, dst_t->type);
1760  
1761  	/* Fill in all the argument names, which for extern FUNCs are missing.
1762  	 * We'll end up with two copies of FUNCs/VARs for externs, but that
1763  	 * will be taken care of by BTF dedup at the very end.
1764  	 * It might be that BTF types for extern in one file has less/more BTF
1765  	 * information (e.g., FWD instead of full STRUCT/UNION information),
1766  	 * but that should be (in most cases, subject to BTF dedup rules)
1767  	 * handled and resolved by BTF dedup algorithm as well, so we won't
1768  	 * worry about it. Our only job is to make sure that argument names
1769  	 * are populated on both sides, otherwise BTF dedup will pedantically
1770  	 * consider them different.
1771  	 */
1772  	src_p = btf_params(src_t);
1773  	dst_p = btf_params(dst_t);
1774  	for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1775  		if (!src_p->name_off)
1776  			continue;
1777  
1778  		/* src_btf has more complete info, so add name to dst_btf */
1779  		s = btf__str_by_offset(src_btf, src_p->name_off);
1780  		off = btf__add_str(dst_btf, s);
1781  		if (off < 0)
1782  			return off;
1783  		dst_p->name_off = off;
1784  	}
1785  	return 0;
1786  }
1787  
sym_update_bind(Elf64_Sym * sym,int sym_bind)1788  static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1789  {
1790  	sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1791  }
1792  
sym_update_type(Elf64_Sym * sym,int sym_type)1793  static void sym_update_type(Elf64_Sym *sym, int sym_type)
1794  {
1795  	sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1796  }
1797  
sym_update_visibility(Elf64_Sym * sym,int sym_vis)1798  static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1799  {
1800  	/* libelf doesn't provide setters for ST_VISIBILITY,
1801  	 * but it is stored in the lower 2 bits of st_other
1802  	 */
1803  	sym->st_other &= ~0x03;
1804  	sym->st_other |= sym_vis;
1805  }
1806  
linker_append_elf_sym(struct bpf_linker * linker,struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int src_sym_idx)1807  static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1808  				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1809  {
1810  	struct src_sec *src_sec = NULL;
1811  	struct dst_sec *dst_sec = NULL;
1812  	struct glob_sym *glob_sym = NULL;
1813  	int name_off, sym_type, sym_bind, sym_vis, err;
1814  	int btf_sec_id = 0, btf_id = 0;
1815  	size_t dst_sym_idx;
1816  	Elf64_Sym *dst_sym;
1817  	bool sym_is_extern;
1818  
1819  	sym_type = ELF64_ST_TYPE(sym->st_info);
1820  	sym_bind = ELF64_ST_BIND(sym->st_info);
1821  	sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1822  	sym_is_extern = sym->st_shndx == SHN_UNDEF;
1823  
1824  	if (sym_is_extern) {
1825  		if (!obj->btf) {
1826  			pr_warn("externs without BTF info are not supported\n");
1827  			return -ENOTSUP;
1828  		}
1829  	} else if (sym->st_shndx < SHN_LORESERVE) {
1830  		src_sec = &obj->secs[sym->st_shndx];
1831  		if (src_sec->skipped)
1832  			return 0;
1833  		dst_sec = &linker->secs[src_sec->dst_id];
1834  
1835  		/* allow only one STT_SECTION symbol per section */
1836  		if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1837  			obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1838  			return 0;
1839  		}
1840  	}
1841  
1842  	if (sym_bind == STB_LOCAL)
1843  		goto add_sym;
1844  
1845  	/* find matching BTF info */
1846  	err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1847  	if (err)
1848  		return err;
1849  
1850  	if (sym_is_extern && btf_sec_id) {
1851  		const char *sec_name = NULL;
1852  		const struct btf_type *t;
1853  
1854  		t = btf__type_by_id(obj->btf, btf_sec_id);
1855  		sec_name = btf__str_by_offset(obj->btf, t->name_off);
1856  
1857  		/* Clang puts unannotated extern vars into
1858  		 * '.extern' BTF DATASEC. Treat them the same
1859  		 * as unannotated extern funcs (which are
1860  		 * currently not put into any DATASECs).
1861  		 * Those don't have associated src_sec/dst_sec.
1862  		 */
1863  		if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1864  			src_sec = find_src_sec_by_name(obj, sec_name);
1865  			if (!src_sec) {
1866  				pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1867  				return -ENOENT;
1868  			}
1869  			dst_sec = &linker->secs[src_sec->dst_id];
1870  		}
1871  	}
1872  
1873  	glob_sym = find_glob_sym(linker, sym_name);
1874  	if (glob_sym) {
1875  		/* Preventively resolve to existing symbol. This is
1876  		 * needed for further relocation symbol remapping in
1877  		 * the next step of linking.
1878  		 */
1879  		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1880  
1881  		/* If both symbols are non-externs, at least one of
1882  		 * them has to be STB_WEAK, otherwise they are in
1883  		 * a conflict with each other.
1884  		 */
1885  		if (!sym_is_extern && !glob_sym->is_extern
1886  		    && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1887  			pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1888  				src_sym_idx, sym_name, obj->filename);
1889  			return -EINVAL;
1890  		}
1891  
1892  		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1893  			return -EINVAL;
1894  
1895  		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1896  
1897  		/* If new symbol is strong, then force dst_sym to be strong as
1898  		 * well; this way a mix of weak and non-weak extern
1899  		 * definitions will end up being strong.
1900  		 */
1901  		if (sym_bind == STB_GLOBAL) {
1902  			/* We still need to preserve type (NOTYPE or
1903  			 * OBJECT/FUNC, depending on whether the symbol is
1904  			 * extern or not)
1905  			 */
1906  			sym_update_bind(dst_sym, STB_GLOBAL);
1907  			glob_sym->is_weak = false;
1908  		}
1909  
1910  		/* Non-default visibility is "contaminating", with stricter
1911  		 * visibility overwriting more permissive ones, even if more
1912  		 * permissive visibility comes from just an extern definition.
1913  		 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1914  		 * ensured by ELF symbol sanity checks above.
1915  		 */
1916  		if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1917  			sym_update_visibility(dst_sym, sym_vis);
1918  
1919  		/* If the new symbol is extern, then regardless if
1920  		 * existing symbol is extern or resolved global, just
1921  		 * keep the existing one untouched.
1922  		 */
1923  		if (sym_is_extern)
1924  			return 0;
1925  
1926  		/* If existing symbol is a strong resolved symbol, bail out,
1927  		 * because we lost resolution battle have nothing to
1928  		 * contribute. We already checked abover that there is no
1929  		 * strong-strong conflict. We also already tightened binding
1930  		 * and visibility, so nothing else to contribute at that point.
1931  		 */
1932  		if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1933  			return 0;
1934  
1935  		/* At this point, new symbol is strong non-extern,
1936  		 * so overwrite glob_sym with new symbol information.
1937  		 * Preserve binding and visibility.
1938  		 */
1939  		sym_update_type(dst_sym, sym_type);
1940  		dst_sym->st_shndx = dst_sec->sec_idx;
1941  		dst_sym->st_value = src_sec->dst_off + sym->st_value;
1942  		dst_sym->st_size = sym->st_size;
1943  
1944  		/* see comment below about dst_sec->id vs dst_sec->sec_idx */
1945  		glob_sym->sec_id = dst_sec->id;
1946  		glob_sym->is_extern = false;
1947  
1948  		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1949  					     obj->btf, btf_id))
1950  			return -EINVAL;
1951  
1952  		/* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1953  		glob_sym->underlying_btf_id = 0;
1954  
1955  		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1956  		return 0;
1957  	}
1958  
1959  add_sym:
1960  	name_off = strset__add_str(linker->strtab_strs, sym_name);
1961  	if (name_off < 0)
1962  		return name_off;
1963  
1964  	dst_sym = add_new_sym(linker, &dst_sym_idx);
1965  	if (!dst_sym)
1966  		return -ENOMEM;
1967  
1968  	dst_sym->st_name = name_off;
1969  	dst_sym->st_info = sym->st_info;
1970  	dst_sym->st_other = sym->st_other;
1971  	dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1972  	dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1973  	dst_sym->st_size = sym->st_size;
1974  
1975  	obj->sym_map[src_sym_idx] = dst_sym_idx;
1976  
1977  	if (sym_type == STT_SECTION && dst_sec) {
1978  		dst_sec->sec_sym_idx = dst_sym_idx;
1979  		dst_sym->st_value = 0;
1980  	}
1981  
1982  	if (sym_bind != STB_LOCAL) {
1983  		glob_sym = add_glob_sym(linker);
1984  		if (!glob_sym)
1985  			return -ENOMEM;
1986  
1987  		glob_sym->sym_idx = dst_sym_idx;
1988  		/* we use dst_sec->id (and not dst_sec->sec_idx), because
1989  		 * ephemeral sections (.kconfig, .ksyms, etc) don't have
1990  		 * sec_idx (as they don't have corresponding ELF section), but
1991  		 * still have id. .extern doesn't have even ephemeral section
1992  		 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
1993  		 */
1994  		glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
1995  		glob_sym->name_off = name_off;
1996  		/* we will fill btf_id in during BTF merging step */
1997  		glob_sym->btf_id = 0;
1998  		glob_sym->is_extern = sym_is_extern;
1999  		glob_sym->is_weak = sym_bind == STB_WEAK;
2000  	}
2001  
2002  	return 0;
2003  }
2004  
linker_append_elf_relos(struct bpf_linker * linker,struct src_obj * obj)2005  static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2006  {
2007  	struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2008  	int i, err;
2009  
2010  	for (i = 1; i < obj->sec_cnt; i++) {
2011  		struct src_sec *src_sec, *src_linked_sec;
2012  		struct dst_sec *dst_sec, *dst_linked_sec;
2013  		Elf64_Rel *src_rel, *dst_rel;
2014  		int j, n;
2015  
2016  		src_sec = &obj->secs[i];
2017  		if (!is_relo_sec(src_sec))
2018  			continue;
2019  
2020  		/* shdr->sh_info points to relocatable section */
2021  		src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2022  		if (src_linked_sec->skipped)
2023  			continue;
2024  
2025  		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2026  		if (!dst_sec) {
2027  			dst_sec = add_dst_sec(linker, src_sec->sec_name);
2028  			if (!dst_sec)
2029  				return -ENOMEM;
2030  			err = init_sec(linker, dst_sec, src_sec);
2031  			if (err) {
2032  				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2033  				return err;
2034  			}
2035  		} else if (!secs_match(dst_sec, src_sec)) {
2036  			pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2037  			return -1;
2038  		}
2039  
2040  		/* shdr->sh_link points to SYMTAB */
2041  		dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2042  
2043  		/* shdr->sh_info points to relocated section */
2044  		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2045  		dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2046  
2047  		src_sec->dst_id = dst_sec->id;
2048  		err = extend_sec(linker, dst_sec, src_sec);
2049  		if (err)
2050  			return err;
2051  
2052  		src_rel = src_sec->data->d_buf;
2053  		dst_rel = dst_sec->raw_data + src_sec->dst_off;
2054  		n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2055  		for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2056  			size_t src_sym_idx, dst_sym_idx, sym_type;
2057  			Elf64_Sym *src_sym;
2058  
2059  			src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2060  			src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2061  
2062  			dst_sym_idx = obj->sym_map[src_sym_idx];
2063  			dst_rel->r_offset += src_linked_sec->dst_off;
2064  			sym_type = ELF64_R_TYPE(src_rel->r_info);
2065  			dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2066  
2067  			if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2068  				struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2069  				struct bpf_insn *insn;
2070  
2071  				if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2072  					/* calls to the very first static function inside
2073  					 * .text section at offset 0 will
2074  					 * reference section symbol, not the
2075  					 * function symbol. Fix that up,
2076  					 * otherwise it won't be possible to
2077  					 * relocate calls to two different
2078  					 * static functions with the same name
2079  					 * (rom two different object files)
2080  					 */
2081  					insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2082  					if (insn->code == (BPF_JMP | BPF_CALL))
2083  						insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2084  					else
2085  						insn->imm += sec->dst_off;
2086  				} else {
2087  					pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2088  					return -EINVAL;
2089  				}
2090  			}
2091  
2092  		}
2093  	}
2094  
2095  	return 0;
2096  }
2097  
find_sym_by_name(struct src_obj * obj,size_t sec_idx,int sym_type,const char * sym_name)2098  static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2099  				   int sym_type, const char *sym_name)
2100  {
2101  	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2102  	Elf64_Sym *sym = symtab->data->d_buf;
2103  	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2104  	int str_sec_idx = symtab->shdr->sh_link;
2105  	const char *name;
2106  
2107  	for (i = 0; i < n; i++, sym++) {
2108  		if (sym->st_shndx != sec_idx)
2109  			continue;
2110  		if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2111  			continue;
2112  
2113  		name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2114  		if (!name)
2115  			return NULL;
2116  
2117  		if (strcmp(sym_name, name) != 0)
2118  			continue;
2119  
2120  		return sym;
2121  	}
2122  
2123  	return NULL;
2124  }
2125  
linker_fixup_btf(struct src_obj * obj)2126  static int linker_fixup_btf(struct src_obj *obj)
2127  {
2128  	const char *sec_name;
2129  	struct src_sec *sec;
2130  	int i, j, n, m;
2131  
2132  	if (!obj->btf)
2133  		return 0;
2134  
2135  	n = btf__type_cnt(obj->btf);
2136  	for (i = 1; i < n; i++) {
2137  		struct btf_var_secinfo *vi;
2138  		struct btf_type *t;
2139  
2140  		t = btf_type_by_id(obj->btf, i);
2141  		if (btf_kind(t) != BTF_KIND_DATASEC)
2142  			continue;
2143  
2144  		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2145  		sec = find_src_sec_by_name(obj, sec_name);
2146  		if (sec) {
2147  			/* record actual section size, unless ephemeral */
2148  			if (sec->shdr)
2149  				t->size = sec->shdr->sh_size;
2150  		} else {
2151  			/* BTF can have some sections that are not represented
2152  			 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2153  			 * for special extern variables.
2154  			 *
2155  			 * For all but one such special (ephemeral)
2156  			 * sections, we pre-create "section shells" to be able
2157  			 * to keep track of extra per-section metadata later
2158  			 * (e.g., those BTF extern variables).
2159  			 *
2160  			 * .extern is even more special, though, because it
2161  			 * contains extern variables that need to be resolved
2162  			 * by static linker, not libbpf and kernel. When such
2163  			 * externs are resolved, we are going to remove them
2164  			 * from .extern BTF section and might end up not
2165  			 * needing it at all. Each resolved extern should have
2166  			 * matching non-extern VAR/FUNC in other sections.
2167  			 *
2168  			 * We do support leaving some of the externs
2169  			 * unresolved, though, to support cases of building
2170  			 * libraries, which will later be linked against final
2171  			 * BPF applications. So if at finalization we still
2172  			 * see unresolved externs, we'll create .extern
2173  			 * section on our own.
2174  			 */
2175  			if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2176  				continue;
2177  
2178  			sec = add_src_sec(obj, sec_name);
2179  			if (!sec)
2180  				return -ENOMEM;
2181  
2182  			sec->ephemeral = true;
2183  			sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2184  		}
2185  
2186  		/* remember ELF section and its BTF type ID match */
2187  		sec->sec_type_id = i;
2188  
2189  		/* fix up variable offsets */
2190  		vi = btf_var_secinfos(t);
2191  		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2192  			const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2193  			const char *var_name;
2194  			int var_linkage;
2195  			Elf64_Sym *sym;
2196  
2197  			/* could be a variable or function */
2198  			if (!btf_is_var(vt))
2199  				continue;
2200  
2201  			var_name = btf__str_by_offset(obj->btf, vt->name_off);
2202  			var_linkage = btf_var(vt)->linkage;
2203  
2204  			/* no need to patch up static or extern vars */
2205  			if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2206  				continue;
2207  
2208  			sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2209  			if (!sym) {
2210  				pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2211  				return -ENOENT;
2212  			}
2213  
2214  			vi->offset = sym->st_value;
2215  		}
2216  	}
2217  
2218  	return 0;
2219  }
2220  
remap_type_id(__u32 * type_id,void * ctx)2221  static int remap_type_id(__u32 *type_id, void *ctx)
2222  {
2223  	int *id_map = ctx;
2224  	int new_id = id_map[*type_id];
2225  
2226  	/* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2227  	if (new_id == 0 && *type_id != 0) {
2228  		pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2229  		return -EINVAL;
2230  	}
2231  
2232  	*type_id = id_map[*type_id];
2233  
2234  	return 0;
2235  }
2236  
linker_append_btf(struct bpf_linker * linker,struct src_obj * obj)2237  static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2238  {
2239  	const struct btf_type *t;
2240  	int i, j, n, start_id, id;
2241  	const char *name;
2242  
2243  	if (!obj->btf)
2244  		return 0;
2245  
2246  	start_id = btf__type_cnt(linker->btf);
2247  	n = btf__type_cnt(obj->btf);
2248  
2249  	obj->btf_type_map = calloc(n + 1, sizeof(int));
2250  	if (!obj->btf_type_map)
2251  		return -ENOMEM;
2252  
2253  	for (i = 1; i < n; i++) {
2254  		struct glob_sym *glob_sym = NULL;
2255  
2256  		t = btf__type_by_id(obj->btf, i);
2257  
2258  		/* DATASECs are handled specially below */
2259  		if (btf_kind(t) == BTF_KIND_DATASEC)
2260  			continue;
2261  
2262  		if (btf_is_non_static(t)) {
2263  			/* there should be glob_sym already */
2264  			name = btf__str_by_offset(obj->btf, t->name_off);
2265  			glob_sym = find_glob_sym(linker, name);
2266  
2267  			/* VARs without corresponding glob_sym are those that
2268  			 * belong to skipped/deduplicated sections (i.e.,
2269  			 * license and version), so just skip them
2270  			 */
2271  			if (!glob_sym)
2272  				continue;
2273  
2274  			/* linker_append_elf_sym() might have requested
2275  			 * updating underlying type ID, if extern was resolved
2276  			 * to strong symbol or weak got upgraded to non-weak
2277  			 */
2278  			if (glob_sym->underlying_btf_id == 0)
2279  				glob_sym->underlying_btf_id = -t->type;
2280  
2281  			/* globals from previous object files that match our
2282  			 * VAR/FUNC already have a corresponding associated
2283  			 * BTF type, so just make sure to use it
2284  			 */
2285  			if (glob_sym->btf_id) {
2286  				/* reuse existing BTF type for global var/func */
2287  				obj->btf_type_map[i] = glob_sym->btf_id;
2288  				continue;
2289  			}
2290  		}
2291  
2292  		id = btf__add_type(linker->btf, obj->btf, t);
2293  		if (id < 0) {
2294  			pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2295  			return id;
2296  		}
2297  
2298  		obj->btf_type_map[i] = id;
2299  
2300  		/* record just appended BTF type for var/func */
2301  		if (glob_sym) {
2302  			glob_sym->btf_id = id;
2303  			glob_sym->underlying_btf_id = -t->type;
2304  		}
2305  	}
2306  
2307  	/* remap all the types except DATASECs */
2308  	n = btf__type_cnt(linker->btf);
2309  	for (i = start_id; i < n; i++) {
2310  		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2311  
2312  		if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2313  			return -EINVAL;
2314  	}
2315  
2316  	/* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2317  	 * actual type), if necessary
2318  	 */
2319  	for (i = 0; i < linker->glob_sym_cnt; i++) {
2320  		struct glob_sym *glob_sym = &linker->glob_syms[i];
2321  		struct btf_type *glob_t;
2322  
2323  		if (glob_sym->underlying_btf_id >= 0)
2324  			continue;
2325  
2326  		glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2327  
2328  		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2329  		glob_t->type = glob_sym->underlying_btf_id;
2330  	}
2331  
2332  	/* append DATASEC info */
2333  	for (i = 1; i < obj->sec_cnt; i++) {
2334  		struct src_sec *src_sec;
2335  		struct dst_sec *dst_sec;
2336  		const struct btf_var_secinfo *src_var;
2337  		struct btf_var_secinfo *dst_var;
2338  
2339  		src_sec = &obj->secs[i];
2340  		if (!src_sec->sec_type_id || src_sec->skipped)
2341  			continue;
2342  		dst_sec = &linker->secs[src_sec->dst_id];
2343  
2344  		/* Mark section as having BTF regardless of the presence of
2345  		 * variables. In some cases compiler might generate empty BTF
2346  		 * with no variables information. E.g., when promoting local
2347  		 * array/structure variable initial values and BPF object
2348  		 * file otherwise has no read-only static variables in
2349  		 * .rodata. We need to preserve such empty BTF and just set
2350  		 * correct section size.
2351  		 */
2352  		dst_sec->has_btf = true;
2353  
2354  		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2355  		src_var = btf_var_secinfos(t);
2356  		n = btf_vlen(t);
2357  		for (j = 0; j < n; j++, src_var++) {
2358  			void *sec_vars = dst_sec->sec_vars;
2359  			int new_id = obj->btf_type_map[src_var->type];
2360  			struct glob_sym *glob_sym = NULL;
2361  
2362  			t = btf_type_by_id(linker->btf, new_id);
2363  			if (btf_is_non_static(t)) {
2364  				name = btf__str_by_offset(linker->btf, t->name_off);
2365  				glob_sym = find_glob_sym(linker, name);
2366  				if (glob_sym->sec_id != dst_sec->id) {
2367  					pr_warn("global '%s': section mismatch %d vs %d\n",
2368  						name, glob_sym->sec_id, dst_sec->id);
2369  					return -EINVAL;
2370  				}
2371  			}
2372  
2373  			/* If there is already a member (VAR or FUNC) mapped
2374  			 * to the same type, don't add a duplicate entry.
2375  			 * This will happen when multiple object files define
2376  			 * the same extern VARs/FUNCs.
2377  			 */
2378  			if (glob_sym && glob_sym->var_idx >= 0) {
2379  				__s64 sz;
2380  
2381  				dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2382  				/* Because underlying BTF type might have
2383  				 * changed, so might its size have changed, so
2384  				 * re-calculate and update it in sec_var.
2385  				 */
2386  				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2387  				if (sz < 0) {
2388  					pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2389  						name, (int)sz);
2390  					return -EINVAL;
2391  				}
2392  				dst_var->size = sz;
2393  				continue;
2394  			}
2395  
2396  			sec_vars = libbpf_reallocarray(sec_vars,
2397  						       dst_sec->sec_var_cnt + 1,
2398  						       sizeof(*dst_sec->sec_vars));
2399  			if (!sec_vars)
2400  				return -ENOMEM;
2401  
2402  			dst_sec->sec_vars = sec_vars;
2403  			dst_sec->sec_var_cnt++;
2404  
2405  			dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2406  			dst_var->type = obj->btf_type_map[src_var->type];
2407  			dst_var->size = src_var->size;
2408  			dst_var->offset = src_sec->dst_off + src_var->offset;
2409  
2410  			if (glob_sym)
2411  				glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2412  		}
2413  	}
2414  
2415  	return 0;
2416  }
2417  
add_btf_ext_rec(struct btf_ext_sec_data * ext_data,const void * src_rec)2418  static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2419  {
2420  	void *tmp;
2421  
2422  	tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2423  	if (!tmp)
2424  		return NULL;
2425  	ext_data->recs = tmp;
2426  
2427  	tmp += ext_data->rec_cnt * ext_data->rec_sz;
2428  	memcpy(tmp, src_rec, ext_data->rec_sz);
2429  
2430  	ext_data->rec_cnt++;
2431  
2432  	return tmp;
2433  }
2434  
linker_append_btf_ext(struct bpf_linker * linker,struct src_obj * obj)2435  static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2436  {
2437  	const struct btf_ext_info_sec *ext_sec;
2438  	const char *sec_name, *s;
2439  	struct src_sec *src_sec;
2440  	struct dst_sec *dst_sec;
2441  	int rec_sz, str_off, i;
2442  
2443  	if (!obj->btf_ext)
2444  		return 0;
2445  
2446  	rec_sz = obj->btf_ext->func_info.rec_size;
2447  	for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2448  		struct bpf_func_info_min *src_rec, *dst_rec;
2449  
2450  		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2451  		src_sec = find_src_sec_by_name(obj, sec_name);
2452  		if (!src_sec) {
2453  			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2454  			return -EINVAL;
2455  		}
2456  		dst_sec = &linker->secs[src_sec->dst_id];
2457  
2458  		if (dst_sec->func_info.rec_sz == 0)
2459  			dst_sec->func_info.rec_sz = rec_sz;
2460  		if (dst_sec->func_info.rec_sz != rec_sz) {
2461  			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2462  			return -EINVAL;
2463  		}
2464  
2465  		for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2466  			dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2467  			if (!dst_rec)
2468  				return -ENOMEM;
2469  
2470  			dst_rec->insn_off += src_sec->dst_off;
2471  			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2472  		}
2473  	}
2474  
2475  	rec_sz = obj->btf_ext->line_info.rec_size;
2476  	for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2477  		struct bpf_line_info_min *src_rec, *dst_rec;
2478  
2479  		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2480  		src_sec = find_src_sec_by_name(obj, sec_name);
2481  		if (!src_sec) {
2482  			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2483  			return -EINVAL;
2484  		}
2485  		dst_sec = &linker->secs[src_sec->dst_id];
2486  
2487  		if (dst_sec->line_info.rec_sz == 0)
2488  			dst_sec->line_info.rec_sz = rec_sz;
2489  		if (dst_sec->line_info.rec_sz != rec_sz) {
2490  			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2491  			return -EINVAL;
2492  		}
2493  
2494  		for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2495  			dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2496  			if (!dst_rec)
2497  				return -ENOMEM;
2498  
2499  			dst_rec->insn_off += src_sec->dst_off;
2500  
2501  			s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2502  			str_off = btf__add_str(linker->btf, s);
2503  			if (str_off < 0)
2504  				return -ENOMEM;
2505  			dst_rec->file_name_off = str_off;
2506  
2507  			s = btf__str_by_offset(obj->btf, src_rec->line_off);
2508  			str_off = btf__add_str(linker->btf, s);
2509  			if (str_off < 0)
2510  				return -ENOMEM;
2511  			dst_rec->line_off = str_off;
2512  
2513  			/* dst_rec->line_col is fine */
2514  		}
2515  	}
2516  
2517  	rec_sz = obj->btf_ext->core_relo_info.rec_size;
2518  	for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2519  		struct bpf_core_relo *src_rec, *dst_rec;
2520  
2521  		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2522  		src_sec = find_src_sec_by_name(obj, sec_name);
2523  		if (!src_sec) {
2524  			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2525  			return -EINVAL;
2526  		}
2527  		dst_sec = &linker->secs[src_sec->dst_id];
2528  
2529  		if (dst_sec->core_relo_info.rec_sz == 0)
2530  			dst_sec->core_relo_info.rec_sz = rec_sz;
2531  		if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2532  			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2533  			return -EINVAL;
2534  		}
2535  
2536  		for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2537  			dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2538  			if (!dst_rec)
2539  				return -ENOMEM;
2540  
2541  			dst_rec->insn_off += src_sec->dst_off;
2542  			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2543  
2544  			s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2545  			str_off = btf__add_str(linker->btf, s);
2546  			if (str_off < 0)
2547  				return -ENOMEM;
2548  			dst_rec->access_str_off = str_off;
2549  
2550  			/* dst_rec->kind is fine */
2551  		}
2552  	}
2553  
2554  	return 0;
2555  }
2556  
bpf_linker__finalize(struct bpf_linker * linker)2557  int bpf_linker__finalize(struct bpf_linker *linker)
2558  {
2559  	struct dst_sec *sec;
2560  	size_t strs_sz;
2561  	const void *strs;
2562  	int err, i;
2563  
2564  	if (!linker->elf)
2565  		return libbpf_err(-EINVAL);
2566  
2567  	err = finalize_btf(linker);
2568  	if (err)
2569  		return libbpf_err(err);
2570  
2571  	/* Finalize strings */
2572  	strs_sz = strset__data_size(linker->strtab_strs);
2573  	strs = strset__data(linker->strtab_strs);
2574  
2575  	sec = &linker->secs[linker->strtab_sec_idx];
2576  	sec->data->d_align = 1;
2577  	sec->data->d_off = 0LL;
2578  	sec->data->d_buf = (void *)strs;
2579  	sec->data->d_type = ELF_T_BYTE;
2580  	sec->data->d_size = strs_sz;
2581  	sec->shdr->sh_size = strs_sz;
2582  
2583  	for (i = 1; i < linker->sec_cnt; i++) {
2584  		sec = &linker->secs[i];
2585  
2586  		/* STRTAB is handled specially above */
2587  		if (sec->sec_idx == linker->strtab_sec_idx)
2588  			continue;
2589  
2590  		/* special ephemeral sections (.ksyms, .kconfig, etc) */
2591  		if (!sec->scn)
2592  			continue;
2593  
2594  		sec->data->d_buf = sec->raw_data;
2595  	}
2596  
2597  	/* Finalize ELF layout */
2598  	if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2599  		err = -EINVAL;
2600  		pr_warn_elf("failed to finalize ELF layout");
2601  		return libbpf_err(err);
2602  	}
2603  
2604  	/* Write out final ELF contents */
2605  	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2606  		err = -EINVAL;
2607  		pr_warn_elf("failed to write ELF contents");
2608  		return libbpf_err(err);
2609  	}
2610  
2611  	elf_end(linker->elf);
2612  	close(linker->fd);
2613  
2614  	linker->elf = NULL;
2615  	linker->fd = -1;
2616  
2617  	return 0;
2618  }
2619  
emit_elf_data_sec(struct bpf_linker * linker,const char * sec_name,size_t align,const void * raw_data,size_t raw_sz)2620  static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2621  			     size_t align, const void *raw_data, size_t raw_sz)
2622  {
2623  	Elf_Scn *scn;
2624  	Elf_Data *data;
2625  	Elf64_Shdr *shdr;
2626  	int name_off;
2627  
2628  	name_off = strset__add_str(linker->strtab_strs, sec_name);
2629  	if (name_off < 0)
2630  		return name_off;
2631  
2632  	scn = elf_newscn(linker->elf);
2633  	if (!scn)
2634  		return -ENOMEM;
2635  	data = elf_newdata(scn);
2636  	if (!data)
2637  		return -ENOMEM;
2638  	shdr = elf64_getshdr(scn);
2639  	if (!shdr)
2640  		return -EINVAL;
2641  
2642  	shdr->sh_name = name_off;
2643  	shdr->sh_type = SHT_PROGBITS;
2644  	shdr->sh_flags = 0;
2645  	shdr->sh_size = raw_sz;
2646  	shdr->sh_link = 0;
2647  	shdr->sh_info = 0;
2648  	shdr->sh_addralign = align;
2649  	shdr->sh_entsize = 0;
2650  
2651  	data->d_type = ELF_T_BYTE;
2652  	data->d_size = raw_sz;
2653  	data->d_buf = (void *)raw_data;
2654  	data->d_align = align;
2655  	data->d_off = 0;
2656  
2657  	return 0;
2658  }
2659  
finalize_btf(struct bpf_linker * linker)2660  static int finalize_btf(struct bpf_linker *linker)
2661  {
2662  	LIBBPF_OPTS(btf_dedup_opts, opts);
2663  	struct btf *btf = linker->btf;
2664  	const void *raw_data;
2665  	int i, j, id, err;
2666  	__u32 raw_sz;
2667  
2668  	/* bail out if no BTF data was produced */
2669  	if (btf__type_cnt(linker->btf) == 1)
2670  		return 0;
2671  
2672  	for (i = 1; i < linker->sec_cnt; i++) {
2673  		struct dst_sec *sec = &linker->secs[i];
2674  
2675  		if (!sec->has_btf)
2676  			continue;
2677  
2678  		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2679  		if (id < 0) {
2680  			pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2681  				sec->sec_name, id);
2682  			return id;
2683  		}
2684  
2685  		for (j = 0; j < sec->sec_var_cnt; j++) {
2686  			struct btf_var_secinfo *vi = &sec->sec_vars[j];
2687  
2688  			if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2689  				return -EINVAL;
2690  		}
2691  	}
2692  
2693  	err = finalize_btf_ext(linker);
2694  	if (err) {
2695  		pr_warn(".BTF.ext generation failed: %d\n", err);
2696  		return err;
2697  	}
2698  
2699  	opts.btf_ext = linker->btf_ext;
2700  	err = btf__dedup(linker->btf, &opts);
2701  	if (err) {
2702  		pr_warn("BTF dedup failed: %d\n", err);
2703  		return err;
2704  	}
2705  
2706  	/* Emit .BTF section */
2707  	raw_data = btf__raw_data(linker->btf, &raw_sz);
2708  	if (!raw_data)
2709  		return -ENOMEM;
2710  
2711  	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2712  	if (err) {
2713  		pr_warn("failed to write out .BTF ELF section: %d\n", err);
2714  		return err;
2715  	}
2716  
2717  	/* Emit .BTF.ext section */
2718  	if (linker->btf_ext) {
2719  		raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2720  		if (!raw_data)
2721  			return -ENOMEM;
2722  
2723  		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2724  		if (err) {
2725  			pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2726  			return err;
2727  		}
2728  	}
2729  
2730  	return 0;
2731  }
2732  
emit_btf_ext_data(struct bpf_linker * linker,void * output,const char * sec_name,struct btf_ext_sec_data * sec_data)2733  static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2734  			     const char *sec_name, struct btf_ext_sec_data *sec_data)
2735  {
2736  	struct btf_ext_info_sec *sec_info;
2737  	void *cur = output;
2738  	int str_off;
2739  	size_t sz;
2740  
2741  	if (!sec_data->rec_cnt)
2742  		return 0;
2743  
2744  	str_off = btf__add_str(linker->btf, sec_name);
2745  	if (str_off < 0)
2746  		return -ENOMEM;
2747  
2748  	sec_info = cur;
2749  	sec_info->sec_name_off = str_off;
2750  	sec_info->num_info = sec_data->rec_cnt;
2751  	cur += sizeof(struct btf_ext_info_sec);
2752  
2753  	sz = sec_data->rec_cnt * sec_data->rec_sz;
2754  	memcpy(cur, sec_data->recs, sz);
2755  	cur += sz;
2756  
2757  	return cur - output;
2758  }
2759  
finalize_btf_ext(struct bpf_linker * linker)2760  static int finalize_btf_ext(struct bpf_linker *linker)
2761  {
2762  	size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2763  	size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2764  	struct btf_ext_header *hdr;
2765  	void *data, *cur;
2766  	int i, err, sz;
2767  
2768  	/* validate that all sections have the same .BTF.ext record sizes
2769  	 * and calculate total data size for each type of data (func info,
2770  	 * line info, core relos)
2771  	 */
2772  	for (i = 1; i < linker->sec_cnt; i++) {
2773  		struct dst_sec *sec = &linker->secs[i];
2774  
2775  		if (sec->func_info.rec_cnt) {
2776  			if (func_rec_sz == 0)
2777  				func_rec_sz = sec->func_info.rec_sz;
2778  			if (func_rec_sz != sec->func_info.rec_sz) {
2779  				pr_warn("mismatch in func_info record size %zu != %u\n",
2780  					func_rec_sz, sec->func_info.rec_sz);
2781  				return -EINVAL;
2782  			}
2783  
2784  			funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2785  		}
2786  		if (sec->line_info.rec_cnt) {
2787  			if (line_rec_sz == 0)
2788  				line_rec_sz = sec->line_info.rec_sz;
2789  			if (line_rec_sz != sec->line_info.rec_sz) {
2790  				pr_warn("mismatch in line_info record size %zu != %u\n",
2791  					line_rec_sz, sec->line_info.rec_sz);
2792  				return -EINVAL;
2793  			}
2794  
2795  			lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2796  		}
2797  		if (sec->core_relo_info.rec_cnt) {
2798  			if (core_relo_rec_sz == 0)
2799  				core_relo_rec_sz = sec->core_relo_info.rec_sz;
2800  			if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2801  				pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2802  					core_relo_rec_sz, sec->core_relo_info.rec_sz);
2803  				return -EINVAL;
2804  			}
2805  
2806  			core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2807  		}
2808  	}
2809  
2810  	if (!funcs_sz && !lines_sz && !core_relos_sz)
2811  		return 0;
2812  
2813  	total_sz += sizeof(struct btf_ext_header);
2814  	if (funcs_sz) {
2815  		funcs_sz += sizeof(__u32); /* record size prefix */
2816  		total_sz += funcs_sz;
2817  	}
2818  	if (lines_sz) {
2819  		lines_sz += sizeof(__u32); /* record size prefix */
2820  		total_sz += lines_sz;
2821  	}
2822  	if (core_relos_sz) {
2823  		core_relos_sz += sizeof(__u32); /* record size prefix */
2824  		total_sz += core_relos_sz;
2825  	}
2826  
2827  	cur = data = calloc(1, total_sz);
2828  	if (!data)
2829  		return -ENOMEM;
2830  
2831  	hdr = cur;
2832  	hdr->magic = BTF_MAGIC;
2833  	hdr->version = BTF_VERSION;
2834  	hdr->flags = 0;
2835  	hdr->hdr_len = sizeof(struct btf_ext_header);
2836  	cur += sizeof(struct btf_ext_header);
2837  
2838  	/* All offsets are in bytes relative to the end of this header */
2839  	hdr->func_info_off = 0;
2840  	hdr->func_info_len = funcs_sz;
2841  	hdr->line_info_off = funcs_sz;
2842  	hdr->line_info_len = lines_sz;
2843  	hdr->core_relo_off = funcs_sz + lines_sz;
2844  	hdr->core_relo_len = core_relos_sz;
2845  
2846  	if (funcs_sz) {
2847  		*(__u32 *)cur = func_rec_sz;
2848  		cur += sizeof(__u32);
2849  
2850  		for (i = 1; i < linker->sec_cnt; i++) {
2851  			struct dst_sec *sec = &linker->secs[i];
2852  
2853  			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2854  			if (sz < 0) {
2855  				err = sz;
2856  				goto out;
2857  			}
2858  
2859  			cur += sz;
2860  		}
2861  	}
2862  
2863  	if (lines_sz) {
2864  		*(__u32 *)cur = line_rec_sz;
2865  		cur += sizeof(__u32);
2866  
2867  		for (i = 1; i < linker->sec_cnt; i++) {
2868  			struct dst_sec *sec = &linker->secs[i];
2869  
2870  			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2871  			if (sz < 0) {
2872  				err = sz;
2873  				goto out;
2874  			}
2875  
2876  			cur += sz;
2877  		}
2878  	}
2879  
2880  	if (core_relos_sz) {
2881  		*(__u32 *)cur = core_relo_rec_sz;
2882  		cur += sizeof(__u32);
2883  
2884  		for (i = 1; i < linker->sec_cnt; i++) {
2885  			struct dst_sec *sec = &linker->secs[i];
2886  
2887  			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2888  			if (sz < 0) {
2889  				err = sz;
2890  				goto out;
2891  			}
2892  
2893  			cur += sz;
2894  		}
2895  	}
2896  
2897  	linker->btf_ext = btf_ext__new(data, total_sz);
2898  	err = libbpf_get_error(linker->btf_ext);
2899  	if (err) {
2900  		linker->btf_ext = NULL;
2901  		pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2902  		goto out;
2903  	}
2904  
2905  out:
2906  	free(data);
2907  	return err;
2908  }
2909