1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
27d485f64SArd Biesheuvel /*
3b7ede5a1SArd Biesheuvel * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
47d485f64SArd Biesheuvel */
57d485f64SArd Biesheuvel
67d485f64SArd Biesheuvel #include <linux/elf.h>
779f32b22SAlex Sverdlin #include <linux/ftrace.h>
87d485f64SArd Biesheuvel #include <linux/kernel.h>
97d485f64SArd Biesheuvel #include <linux/module.h>
101031a7e6SArd Biesheuvel #include <linux/sort.h>
11a959f887SBen Dooks #include <linux/moduleloader.h>
127d485f64SArd Biesheuvel
137d485f64SArd Biesheuvel #include <asm/cache.h>
147d485f64SArd Biesheuvel #include <asm/opcodes.h>
157d485f64SArd Biesheuvel
167d485f64SArd Biesheuvel #ifdef CONFIG_THUMB2_KERNEL
177d485f64SArd Biesheuvel #define PLT_ENT_LDR __opcode_to_mem_thumb32(0xf8dff000 | \
187d485f64SArd Biesheuvel (PLT_ENT_STRIDE - 4))
197d485f64SArd Biesheuvel #else
207d485f64SArd Biesheuvel #define PLT_ENT_LDR __opcode_to_mem_arm(0xe59ff000 | \
217d485f64SArd Biesheuvel (PLT_ENT_STRIDE - 8))
227d485f64SArd Biesheuvel #endif
237d485f64SArd Biesheuvel
2479f32b22SAlex Sverdlin static const u32 fixed_plts[] = {
256fa630bfSAlex Sverdlin #ifdef CONFIG_DYNAMIC_FTRACE
2679f32b22SAlex Sverdlin FTRACE_ADDR,
2779f32b22SAlex Sverdlin MCOUNT_ADDR,
2879f32b22SAlex Sverdlin #endif
2979f32b22SAlex Sverdlin };
3079f32b22SAlex Sverdlin
prealloc_fixed(struct mod_plt_sec * pltsec,struct plt_entries * plt)3179f32b22SAlex Sverdlin static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt)
3279f32b22SAlex Sverdlin {
3379f32b22SAlex Sverdlin int i;
3479f32b22SAlex Sverdlin
3579f32b22SAlex Sverdlin if (!ARRAY_SIZE(fixed_plts) || pltsec->plt_count)
3679f32b22SAlex Sverdlin return;
3779f32b22SAlex Sverdlin pltsec->plt_count = ARRAY_SIZE(fixed_plts);
3879f32b22SAlex Sverdlin
3979f32b22SAlex Sverdlin for (i = 0; i < ARRAY_SIZE(plt->ldr); ++i)
4079f32b22SAlex Sverdlin plt->ldr[i] = PLT_ENT_LDR;
4179f32b22SAlex Sverdlin
4279f32b22SAlex Sverdlin BUILD_BUG_ON(sizeof(fixed_plts) > sizeof(plt->lit));
4379f32b22SAlex Sverdlin memcpy(plt->lit, fixed_plts, sizeof(fixed_plts));
4479f32b22SAlex Sverdlin }
4579f32b22SAlex Sverdlin
get_module_plt(struct module * mod,unsigned long loc,Elf32_Addr val)467d485f64SArd Biesheuvel u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val)
477d485f64SArd Biesheuvel {
48ac3b4328SSong Liu struct mod_plt_sec *pltsec = !within_module_init(loc, mod) ?
49ac3b4328SSong Liu &mod->arch.core : &mod->arch.init;
5079f32b22SAlex Sverdlin struct plt_entries *plt;
5179f32b22SAlex Sverdlin int idx;
52b7ede5a1SArd Biesheuvel
5379f32b22SAlex Sverdlin /* cache the address, ELF header is available only during module load */
5479f32b22SAlex Sverdlin if (!pltsec->plt_ent)
5579f32b22SAlex Sverdlin pltsec->plt_ent = (struct plt_entries *)pltsec->plt->sh_addr;
5679f32b22SAlex Sverdlin plt = pltsec->plt_ent;
577d485f64SArd Biesheuvel
5879f32b22SAlex Sverdlin prealloc_fixed(pltsec, plt);
5979f32b22SAlex Sverdlin
6079f32b22SAlex Sverdlin for (idx = 0; idx < ARRAY_SIZE(fixed_plts); ++idx)
6179f32b22SAlex Sverdlin if (plt->lit[idx] == val)
6279f32b22SAlex Sverdlin return (u32)&plt->ldr[idx];
6379f32b22SAlex Sverdlin
6479f32b22SAlex Sverdlin idx = 0;
6566e94ba3SArd Biesheuvel /*
6666e94ba3SArd Biesheuvel * Look for an existing entry pointing to 'val'. Given that the
6766e94ba3SArd Biesheuvel * relocations are sorted, this will be the last entry we allocated.
6866e94ba3SArd Biesheuvel * (if one exists).
6966e94ba3SArd Biesheuvel */
70b7ede5a1SArd Biesheuvel if (pltsec->plt_count > 0) {
71b7ede5a1SArd Biesheuvel plt += (pltsec->plt_count - 1) / PLT_ENT_COUNT;
72b7ede5a1SArd Biesheuvel idx = (pltsec->plt_count - 1) % PLT_ENT_COUNT;
737d485f64SArd Biesheuvel
7466e94ba3SArd Biesheuvel if (plt->lit[idx] == val)
7566e94ba3SArd Biesheuvel return (u32)&plt->ldr[idx];
767d485f64SArd Biesheuvel
7766e94ba3SArd Biesheuvel idx = (idx + 1) % PLT_ENT_COUNT;
7866e94ba3SArd Biesheuvel if (!idx)
7966e94ba3SArd Biesheuvel plt++;
8066e94ba3SArd Biesheuvel }
8166e94ba3SArd Biesheuvel
82b7ede5a1SArd Biesheuvel pltsec->plt_count++;
83b7ede5a1SArd Biesheuvel BUG_ON(pltsec->plt_count * PLT_ENT_SIZE > pltsec->plt->sh_size);
8466e94ba3SArd Biesheuvel
8566e94ba3SArd Biesheuvel if (!idx)
867d485f64SArd Biesheuvel /* Populate a new set of entries */
877d485f64SArd Biesheuvel *plt = (struct plt_entries){
887d485f64SArd Biesheuvel { [0 ... PLT_ENT_COUNT - 1] = PLT_ENT_LDR, },
897d485f64SArd Biesheuvel { val, }
907d485f64SArd Biesheuvel };
9166e94ba3SArd Biesheuvel else
9266e94ba3SArd Biesheuvel plt->lit[idx] = val;
9366e94ba3SArd Biesheuvel
9466e94ba3SArd Biesheuvel return (u32)&plt->ldr[idx];
957d485f64SArd Biesheuvel }
967d485f64SArd Biesheuvel
971031a7e6SArd Biesheuvel #define cmp_3way(a,b) ((a) < (b) ? -1 : (a) > (b))
981031a7e6SArd Biesheuvel
cmp_rel(const void * a,const void * b)991031a7e6SArd Biesheuvel static int cmp_rel(const void *a, const void *b)
1007d485f64SArd Biesheuvel {
1011031a7e6SArd Biesheuvel const Elf32_Rel *x = a, *y = b;
1027d485f64SArd Biesheuvel int i;
1037d485f64SArd Biesheuvel
1041031a7e6SArd Biesheuvel /* sort by type and symbol index */
1051031a7e6SArd Biesheuvel i = cmp_3way(ELF32_R_TYPE(x->r_info), ELF32_R_TYPE(y->r_info));
1061031a7e6SArd Biesheuvel if (i == 0)
1071031a7e6SArd Biesheuvel i = cmp_3way(ELF32_R_SYM(x->r_info), ELF32_R_SYM(y->r_info));
1081031a7e6SArd Biesheuvel return i;
1091031a7e6SArd Biesheuvel }
1101031a7e6SArd Biesheuvel
is_zero_addend_relocation(Elf32_Addr base,const Elf32_Rel * rel)1111031a7e6SArd Biesheuvel static bool is_zero_addend_relocation(Elf32_Addr base, const Elf32_Rel *rel)
1121031a7e6SArd Biesheuvel {
1131031a7e6SArd Biesheuvel u32 *tval = (u32 *)(base + rel->r_offset);
1147d485f64SArd Biesheuvel
1157d485f64SArd Biesheuvel /*
1161031a7e6SArd Biesheuvel * Do a bitwise compare on the raw addend rather than fully decoding
1171031a7e6SArd Biesheuvel * the offset and doing an arithmetic comparison.
1181031a7e6SArd Biesheuvel * Note that a zero-addend jump/call relocation is encoded taking the
1191031a7e6SArd Biesheuvel * PC bias into account, i.e., -8 for ARM and -4 for Thumb2.
1207d485f64SArd Biesheuvel */
1211031a7e6SArd Biesheuvel switch (ELF32_R_TYPE(rel->r_info)) {
1221031a7e6SArd Biesheuvel u16 upper, lower;
1231031a7e6SArd Biesheuvel
1241031a7e6SArd Biesheuvel case R_ARM_THM_CALL:
1251031a7e6SArd Biesheuvel case R_ARM_THM_JUMP24:
1261031a7e6SArd Biesheuvel upper = __mem_to_opcode_thumb16(((u16 *)tval)[0]);
1271031a7e6SArd Biesheuvel lower = __mem_to_opcode_thumb16(((u16 *)tval)[1]);
1281031a7e6SArd Biesheuvel
1291031a7e6SArd Biesheuvel return (upper & 0x7ff) == 0x7ff && (lower & 0x2fff) == 0x2ffe;
1301031a7e6SArd Biesheuvel
1311031a7e6SArd Biesheuvel case R_ARM_CALL:
1321031a7e6SArd Biesheuvel case R_ARM_PC24:
1331031a7e6SArd Biesheuvel case R_ARM_JUMP24:
1341031a7e6SArd Biesheuvel return (__mem_to_opcode_arm(*tval) & 0xffffff) == 0xfffffe;
1357d485f64SArd Biesheuvel }
1361031a7e6SArd Biesheuvel BUG();
1371031a7e6SArd Biesheuvel }
1381031a7e6SArd Biesheuvel
duplicate_rel(Elf32_Addr base,const Elf32_Rel * rel,int num)1391031a7e6SArd Biesheuvel static bool duplicate_rel(Elf32_Addr base, const Elf32_Rel *rel, int num)
1401031a7e6SArd Biesheuvel {
1411031a7e6SArd Biesheuvel const Elf32_Rel *prev;
1421031a7e6SArd Biesheuvel
1431031a7e6SArd Biesheuvel /*
1441031a7e6SArd Biesheuvel * Entries are sorted by type and symbol index. That means that,
1451031a7e6SArd Biesheuvel * if a duplicate entry exists, it must be in the preceding
1461031a7e6SArd Biesheuvel * slot.
1471031a7e6SArd Biesheuvel */
1481031a7e6SArd Biesheuvel if (!num)
1491031a7e6SArd Biesheuvel return false;
1501031a7e6SArd Biesheuvel
1511031a7e6SArd Biesheuvel prev = rel + num - 1;
1521031a7e6SArd Biesheuvel return cmp_rel(rel + num, prev) == 0 &&
1531031a7e6SArd Biesheuvel is_zero_addend_relocation(base, prev);
1547d485f64SArd Biesheuvel }
1557d485f64SArd Biesheuvel
1567d485f64SArd Biesheuvel /* Count how many PLT entries we may need */
count_plts(const Elf32_Sym * syms,Elf32_Addr base,const Elf32_Rel * rel,int num,Elf32_Word dstidx)15705123fefSArd Biesheuvel static unsigned int count_plts(const Elf32_Sym *syms, Elf32_Addr base,
158b7ede5a1SArd Biesheuvel const Elf32_Rel *rel, int num, Elf32_Word dstidx)
1597d485f64SArd Biesheuvel {
1607d485f64SArd Biesheuvel unsigned int ret = 0;
16105123fefSArd Biesheuvel const Elf32_Sym *s;
1627d485f64SArd Biesheuvel int i;
1637d485f64SArd Biesheuvel
16405123fefSArd Biesheuvel for (i = 0; i < num; i++) {
1657d485f64SArd Biesheuvel switch (ELF32_R_TYPE(rel[i].r_info)) {
1667d485f64SArd Biesheuvel case R_ARM_CALL:
1677d485f64SArd Biesheuvel case R_ARM_PC24:
1687d485f64SArd Biesheuvel case R_ARM_JUMP24:
1697d485f64SArd Biesheuvel case R_ARM_THM_CALL:
1707d485f64SArd Biesheuvel case R_ARM_THM_JUMP24:
17105123fefSArd Biesheuvel /*
17205123fefSArd Biesheuvel * We only have to consider branch targets that resolve
173b7ede5a1SArd Biesheuvel * to symbols that are defined in a different section.
174b7ede5a1SArd Biesheuvel * This is not simply a heuristic, it is a fundamental
175b7ede5a1SArd Biesheuvel * limitation, since there is no guaranteed way to emit
176b7ede5a1SArd Biesheuvel * PLT entries sufficiently close to the branch if the
177b7ede5a1SArd Biesheuvel * section size exceeds the range of a branch
178b7ede5a1SArd Biesheuvel * instruction. So ignore relocations against defined
179b7ede5a1SArd Biesheuvel * symbols if they live in the same section as the
180b7ede5a1SArd Biesheuvel * relocation target.
18105123fefSArd Biesheuvel */
18205123fefSArd Biesheuvel s = syms + ELF32_R_SYM(rel[i].r_info);
183b7ede5a1SArd Biesheuvel if (s->st_shndx == dstidx)
18405123fefSArd Biesheuvel break;
18505123fefSArd Biesheuvel
1861031a7e6SArd Biesheuvel /*
1871031a7e6SArd Biesheuvel * Jump relocations with non-zero addends against
1881031a7e6SArd Biesheuvel * undefined symbols are supported by the ELF spec, but
1891031a7e6SArd Biesheuvel * do not occur in practice (e.g., 'jump n bytes past
1901031a7e6SArd Biesheuvel * the entry point of undefined function symbol f').
1911031a7e6SArd Biesheuvel * So we need to support them, but there is no need to
1921031a7e6SArd Biesheuvel * take them into consideration when trying to optimize
1931031a7e6SArd Biesheuvel * this code. So let's only check for duplicates when
194b7ede5a1SArd Biesheuvel * the addend is zero. (Note that calls into the core
195b7ede5a1SArd Biesheuvel * module via init PLT entries could involve section
196b7ede5a1SArd Biesheuvel * relative symbol references with non-zero addends, for
197b7ede5a1SArd Biesheuvel * which we may end up emitting duplicates, but the init
198b7ede5a1SArd Biesheuvel * PLT is released along with the rest of the .init
199b7ede5a1SArd Biesheuvel * region as soon as module loading completes.)
2001031a7e6SArd Biesheuvel */
2011031a7e6SArd Biesheuvel if (!is_zero_addend_relocation(base, rel + i) ||
2021031a7e6SArd Biesheuvel !duplicate_rel(base, rel, i))
2037d485f64SArd Biesheuvel ret++;
20405123fefSArd Biesheuvel }
2057d485f64SArd Biesheuvel }
2067d485f64SArd Biesheuvel return ret;
2077d485f64SArd Biesheuvel }
2087d485f64SArd Biesheuvel
module_frob_arch_sections(Elf_Ehdr * ehdr,Elf_Shdr * sechdrs,char * secstrings,struct module * mod)2097d485f64SArd Biesheuvel int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
2107d485f64SArd Biesheuvel char *secstrings, struct module *mod)
2117d485f64SArd Biesheuvel {
21279f32b22SAlex Sverdlin unsigned long core_plts = ARRAY_SIZE(fixed_plts);
21379f32b22SAlex Sverdlin unsigned long init_plts = ARRAY_SIZE(fixed_plts);
2147d485f64SArd Biesheuvel Elf32_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum;
21505123fefSArd Biesheuvel Elf32_Sym *syms = NULL;
2167d485f64SArd Biesheuvel
2177d485f64SArd Biesheuvel /*
2187d485f64SArd Biesheuvel * To store the PLTs, we expand the .text section for core module code
21935fa91eeSArd Biesheuvel * and for initialization code.
2207d485f64SArd Biesheuvel */
22105123fefSArd Biesheuvel for (s = sechdrs; s < sechdrs_end; ++s) {
22235fa91eeSArd Biesheuvel if (strcmp(".plt", secstrings + s->sh_name) == 0)
223b7ede5a1SArd Biesheuvel mod->arch.core.plt = s;
224b7ede5a1SArd Biesheuvel else if (strcmp(".init.plt", secstrings + s->sh_name) == 0)
225b7ede5a1SArd Biesheuvel mod->arch.init.plt = s;
22605123fefSArd Biesheuvel else if (s->sh_type == SHT_SYMTAB)
22705123fefSArd Biesheuvel syms = (Elf32_Sym *)s->sh_addr;
22805123fefSArd Biesheuvel }
2297d485f64SArd Biesheuvel
230b7ede5a1SArd Biesheuvel if (!mod->arch.core.plt || !mod->arch.init.plt) {
231b7ede5a1SArd Biesheuvel pr_err("%s: module PLT section(s) missing\n", mod->name);
2327d485f64SArd Biesheuvel return -ENOEXEC;
2337d485f64SArd Biesheuvel }
23405123fefSArd Biesheuvel if (!syms) {
23505123fefSArd Biesheuvel pr_err("%s: module symtab section missing\n", mod->name);
23605123fefSArd Biesheuvel return -ENOEXEC;
23705123fefSArd Biesheuvel }
2387d485f64SArd Biesheuvel
2397d485f64SArd Biesheuvel for (s = sechdrs + 1; s < sechdrs_end; ++s) {
2401031a7e6SArd Biesheuvel Elf32_Rel *rels = (void *)ehdr + s->sh_offset;
2417d485f64SArd Biesheuvel int numrels = s->sh_size / sizeof(Elf32_Rel);
2427d485f64SArd Biesheuvel Elf32_Shdr *dstsec = sechdrs + s->sh_info;
2437d485f64SArd Biesheuvel
2447d485f64SArd Biesheuvel if (s->sh_type != SHT_REL)
2457d485f64SArd Biesheuvel continue;
2467d485f64SArd Biesheuvel
24705123fefSArd Biesheuvel /* ignore relocations that operate on non-exec sections */
24805123fefSArd Biesheuvel if (!(dstsec->sh_flags & SHF_EXECINSTR))
24905123fefSArd Biesheuvel continue;
25005123fefSArd Biesheuvel
2511031a7e6SArd Biesheuvel /* sort by type and symbol index */
2521031a7e6SArd Biesheuvel sort(rels, numrels, sizeof(Elf32_Rel), cmp_rel, NULL);
2531031a7e6SArd Biesheuvel
254*a6846234SJames Morse if (!module_init_layout_section(secstrings + dstsec->sh_name))
255b7ede5a1SArd Biesheuvel core_plts += count_plts(syms, dstsec->sh_addr, rels,
256b7ede5a1SArd Biesheuvel numrels, s->sh_info);
257b7ede5a1SArd Biesheuvel else
258b7ede5a1SArd Biesheuvel init_plts += count_plts(syms, dstsec->sh_addr, rels,
259b7ede5a1SArd Biesheuvel numrels, s->sh_info);
2607d485f64SArd Biesheuvel }
2617d485f64SArd Biesheuvel
262b7ede5a1SArd Biesheuvel mod->arch.core.plt->sh_type = SHT_NOBITS;
263b7ede5a1SArd Biesheuvel mod->arch.core.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
264b7ede5a1SArd Biesheuvel mod->arch.core.plt->sh_addralign = L1_CACHE_BYTES;
265b7ede5a1SArd Biesheuvel mod->arch.core.plt->sh_size = round_up(core_plts * PLT_ENT_SIZE,
2667d485f64SArd Biesheuvel sizeof(struct plt_entries));
267b7ede5a1SArd Biesheuvel mod->arch.core.plt_count = 0;
26879f32b22SAlex Sverdlin mod->arch.core.plt_ent = NULL;
2697d485f64SArd Biesheuvel
270b7ede5a1SArd Biesheuvel mod->arch.init.plt->sh_type = SHT_NOBITS;
271b7ede5a1SArd Biesheuvel mod->arch.init.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
272b7ede5a1SArd Biesheuvel mod->arch.init.plt->sh_addralign = L1_CACHE_BYTES;
273b7ede5a1SArd Biesheuvel mod->arch.init.plt->sh_size = round_up(init_plts * PLT_ENT_SIZE,
274b7ede5a1SArd Biesheuvel sizeof(struct plt_entries));
275b7ede5a1SArd Biesheuvel mod->arch.init.plt_count = 0;
27679f32b22SAlex Sverdlin mod->arch.init.plt_ent = NULL;
277b7ede5a1SArd Biesheuvel
278b7ede5a1SArd Biesheuvel pr_debug("%s: plt=%x, init.plt=%x\n", __func__,
279b7ede5a1SArd Biesheuvel mod->arch.core.plt->sh_size, mod->arch.init.plt->sh_size);
2807d485f64SArd Biesheuvel return 0;
2817d485f64SArd Biesheuvel }
2824ab07fd3SAlex Sverdlin
in_module_plt(unsigned long loc)2834ab07fd3SAlex Sverdlin bool in_module_plt(unsigned long loc)
2844ab07fd3SAlex Sverdlin {
2854ab07fd3SAlex Sverdlin struct module *mod;
2864ab07fd3SAlex Sverdlin bool ret;
2874ab07fd3SAlex Sverdlin
2884ab07fd3SAlex Sverdlin preempt_disable();
2894ab07fd3SAlex Sverdlin mod = __module_text_address(loc);
2904ab07fd3SAlex Sverdlin ret = mod && (loc - (u32)mod->arch.core.plt_ent < mod->arch.core.plt_count * PLT_ENT_SIZE ||
2914ab07fd3SAlex Sverdlin loc - (u32)mod->arch.init.plt_ent < mod->arch.init.plt_count * PLT_ENT_SIZE);
2924ab07fd3SAlex Sverdlin preempt_enable();
2934ab07fd3SAlex Sverdlin
2944ab07fd3SAlex Sverdlin return ret;
2954ab07fd3SAlex Sverdlin }
296