11a59d1b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2ed981856SPaul Mackerras /* Kernel module help for PPC. 3ed981856SPaul Mackerras Copyright (C) 2001 Rusty Russell. 4ed981856SPaul Mackerras 5ed981856SPaul Mackerras */ 6c7d1f6afSAnton Blanchard 7c7d1f6afSAnton Blanchard #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8c7d1f6afSAnton Blanchard 9ed981856SPaul Mackerras #include <linux/module.h> 10ed981856SPaul Mackerras #include <linux/moduleloader.h> 11ed981856SPaul Mackerras #include <linux/elf.h> 12ed981856SPaul Mackerras #include <linux/vmalloc.h> 13ed981856SPaul Mackerras #include <linux/fs.h> 14ed981856SPaul Mackerras #include <linux/string.h> 15ed981856SPaul Mackerras #include <linux/kernel.h> 167cc45e64SSteven Rostedt #include <linux/ftrace.h> 17ed981856SPaul Mackerras #include <linux/cache.h> 1873c9ceabSJeremy Fitzhardinge #include <linux/bug.h> 19eda09fbdSEmil Medve #include <linux/sort.h> 20b88c4767SRobert Jennings #include <asm/setup.h> 210c850965SChristophe Leroy #include <asm/code-patching.h> 2221c4ff80SBenjamin Herrenschmidt 23ed981856SPaul Mackerras /* Count how many different relocations (different symbol, different 24ed981856SPaul Mackerras addend) */ 25ed981856SPaul Mackerras static unsigned int count_relocs(const Elf32_Rela *rela, unsigned int num) 26ed981856SPaul Mackerras { 27eda09fbdSEmil Medve unsigned int i, r_info, r_addend, _count_relocs; 28ed981856SPaul Mackerras 29eda09fbdSEmil Medve _count_relocs = 0; 30eda09fbdSEmil Medve r_info = 0; 31eda09fbdSEmil Medve r_addend = 0; 32eda09fbdSEmil Medve for (i = 0; i < num; i++) 33eda09fbdSEmil Medve /* Only count 24-bit relocs, others don't need stubs */ 34eda09fbdSEmil Medve if (ELF32_R_TYPE(rela[i].r_info) == R_PPC_REL24 && 35eda09fbdSEmil Medve (r_info != ELF32_R_SYM(rela[i].r_info) || 36eda09fbdSEmil Medve r_addend != rela[i].r_addend)) { 37eda09fbdSEmil Medve _count_relocs++; 38eda09fbdSEmil Medve r_info = ELF32_R_SYM(rela[i].r_info); 39eda09fbdSEmil Medve r_addend = rela[i].r_addend; 40ed981856SPaul Mackerras } 41eda09fbdSEmil Medve 427cc45e64SSteven Rostedt #ifdef CONFIG_DYNAMIC_FTRACE 437cc45e64SSteven Rostedt _count_relocs++; /* add one for ftrace_caller */ 447cc45e64SSteven Rostedt #endif 45eda09fbdSEmil Medve return _count_relocs; 46ed981856SPaul Mackerras } 47eda09fbdSEmil Medve 48eda09fbdSEmil Medve static int relacmp(const void *_x, const void *_y) 49eda09fbdSEmil Medve { 50eda09fbdSEmil Medve const Elf32_Rela *x, *y; 51eda09fbdSEmil Medve 52eda09fbdSEmil Medve y = (Elf32_Rela *)_x; 53eda09fbdSEmil Medve x = (Elf32_Rela *)_y; 54eda09fbdSEmil Medve 55eda09fbdSEmil Medve /* Compare the entire r_info (as opposed to ELF32_R_SYM(r_info) only) to 56eda09fbdSEmil Medve * make the comparison cheaper/faster. It won't affect the sorting or 57eda09fbdSEmil Medve * the counting algorithms' performance 58eda09fbdSEmil Medve */ 59eda09fbdSEmil Medve if (x->r_info < y->r_info) 60eda09fbdSEmil Medve return -1; 61eda09fbdSEmil Medve else if (x->r_info > y->r_info) 62eda09fbdSEmil Medve return 1; 63eda09fbdSEmil Medve else if (x->r_addend < y->r_addend) 64eda09fbdSEmil Medve return -1; 65eda09fbdSEmil Medve else if (x->r_addend > y->r_addend) 66eda09fbdSEmil Medve return 1; 67eda09fbdSEmil Medve else 68eda09fbdSEmil Medve return 0; 69eda09fbdSEmil Medve } 70eda09fbdSEmil Medve 71ed981856SPaul Mackerras /* Get the potential trampolines size required of the init and 72ed981856SPaul Mackerras non-init sections */ 73ed981856SPaul Mackerras static unsigned long get_plt_size(const Elf32_Ehdr *hdr, 74ed981856SPaul Mackerras const Elf32_Shdr *sechdrs, 75ed981856SPaul Mackerras const char *secstrings, 76ed981856SPaul Mackerras int is_init) 77ed981856SPaul Mackerras { 78ed981856SPaul Mackerras unsigned long ret = 0; 79ed981856SPaul Mackerras unsigned i; 80ed981856SPaul Mackerras 81ed981856SPaul Mackerras /* Everything marked ALLOC (this includes the exported 82ed981856SPaul Mackerras symbols) */ 83ed981856SPaul Mackerras for (i = 1; i < hdr->e_shnum; i++) { 84ed981856SPaul Mackerras /* If it's called *.init*, and we're not init, we're 85ed981856SPaul Mackerras not interested */ 86d8731527SMathieu Malaterre if ((strstr(secstrings + sechdrs[i].sh_name, ".init") != NULL) 87ed981856SPaul Mackerras != is_init) 88ed981856SPaul Mackerras continue; 89ed981856SPaul Mackerras 90ed981856SPaul Mackerras /* We don't want to look at debug sections. */ 91d8731527SMathieu Malaterre if (strstr(secstrings + sechdrs[i].sh_name, ".debug")) 92ed981856SPaul Mackerras continue; 93ed981856SPaul Mackerras 94ed981856SPaul Mackerras if (sechdrs[i].sh_type == SHT_RELA) { 95c7d1f6afSAnton Blanchard pr_debug("Found relocations in section %u\n", i); 96c7d1f6afSAnton Blanchard pr_debug("Ptr: %p. Number: %u\n", 97ed981856SPaul Mackerras (void *)hdr + sechdrs[i].sh_offset, 98ed981856SPaul Mackerras sechdrs[i].sh_size / sizeof(Elf32_Rela)); 99eda09fbdSEmil Medve 100eda09fbdSEmil Medve /* Sort the relocation information based on a symbol and 101eda09fbdSEmil Medve * addend key. This is a stable O(n*log n) complexity 1021fd02f66SJulia Lawall * algorithm but it will reduce the complexity of 103eda09fbdSEmil Medve * count_relocs() to linear complexity O(n) 104eda09fbdSEmil Medve */ 105eda09fbdSEmil Medve sort((void *)hdr + sechdrs[i].sh_offset, 106eda09fbdSEmil Medve sechdrs[i].sh_size / sizeof(Elf32_Rela), 107bac7ca7bSAndrey Abramov sizeof(Elf32_Rela), relacmp, NULL); 108eda09fbdSEmil Medve 109ed981856SPaul Mackerras ret += count_relocs((void *)hdr 110ed981856SPaul Mackerras + sechdrs[i].sh_offset, 111ed981856SPaul Mackerras sechdrs[i].sh_size 112ed981856SPaul Mackerras / sizeof(Elf32_Rela)) 113ed981856SPaul Mackerras * sizeof(struct ppc_plt_entry); 114ed981856SPaul Mackerras } 115ed981856SPaul Mackerras } 116ed981856SPaul Mackerras 117ed981856SPaul Mackerras return ret; 118ed981856SPaul Mackerras } 119ed981856SPaul Mackerras 120ed981856SPaul Mackerras int module_frob_arch_sections(Elf32_Ehdr *hdr, 121ed981856SPaul Mackerras Elf32_Shdr *sechdrs, 122ed981856SPaul Mackerras char *secstrings, 123ed981856SPaul Mackerras struct module *me) 124ed981856SPaul Mackerras { 125ed981856SPaul Mackerras unsigned int i; 126ed981856SPaul Mackerras 127ed981856SPaul Mackerras /* Find .plt and .init.plt sections */ 128ed981856SPaul Mackerras for (i = 0; i < hdr->e_shnum; i++) { 129ed981856SPaul Mackerras if (strcmp(secstrings + sechdrs[i].sh_name, ".init.plt") == 0) 130ed981856SPaul Mackerras me->arch.init_plt_section = i; 131ed981856SPaul Mackerras else if (strcmp(secstrings + sechdrs[i].sh_name, ".plt") == 0) 132ed981856SPaul Mackerras me->arch.core_plt_section = i; 133ed981856SPaul Mackerras } 134ed981856SPaul Mackerras if (!me->arch.core_plt_section || !me->arch.init_plt_section) { 135c7d1f6afSAnton Blanchard pr_err("Module doesn't contain .plt or .init.plt sections.\n"); 136ed981856SPaul Mackerras return -ENOEXEC; 137ed981856SPaul Mackerras } 138ed981856SPaul Mackerras 139ed981856SPaul Mackerras /* Override their sizes */ 140ed981856SPaul Mackerras sechdrs[me->arch.core_plt_section].sh_size 141ed981856SPaul Mackerras = get_plt_size(hdr, sechdrs, secstrings, 0); 142ed981856SPaul Mackerras sechdrs[me->arch.init_plt_section].sh_size 143ed981856SPaul Mackerras = get_plt_size(hdr, sechdrs, secstrings, 1); 144ed981856SPaul Mackerras return 0; 145ed981856SPaul Mackerras } 146ed981856SPaul Mackerras 147ed981856SPaul Mackerras static inline int entry_matches(struct ppc_plt_entry *entry, Elf32_Addr val) 148ed981856SPaul Mackerras { 14947b04699SChristophe Leroy if (entry->jump[0] != PPC_RAW_LIS(_R12, PPC_HA(val))) 150ed981856SPaul Mackerras return 0; 15147b04699SChristophe Leroy if (entry->jump[1] != PPC_RAW_ADDI(_R12, _R12, PPC_LO(val))) 1524eb4516eSChristophe Leroy return 0; 1534eb4516eSChristophe Leroy return 1; 154ed981856SPaul Mackerras } 155ed981856SPaul Mackerras 156ed981856SPaul Mackerras /* Set up a trampoline in the PLT to bounce us to the distant function */ 157ed981856SPaul Mackerras static uint32_t do_plt_call(void *location, 158ed981856SPaul Mackerras Elf32_Addr val, 159136cd345SMichael Ellerman const Elf32_Shdr *sechdrs, 160ed981856SPaul Mackerras struct module *mod) 161ed981856SPaul Mackerras { 162ed981856SPaul Mackerras struct ppc_plt_entry *entry; 163ed981856SPaul Mackerras 164c7d1f6afSAnton Blanchard pr_debug("Doing plt for call to 0x%x at 0x%x\n", val, (unsigned int)location); 165ed981856SPaul Mackerras /* Init, or core PLT? */ 1667523e4dcSRusty Russell if (location >= mod->core_layout.base 1677523e4dcSRusty Russell && location < mod->core_layout.base + mod->core_layout.size) 168ed981856SPaul Mackerras entry = (void *)sechdrs[mod->arch.core_plt_section].sh_addr; 169ed981856SPaul Mackerras else 170ed981856SPaul Mackerras entry = (void *)sechdrs[mod->arch.init_plt_section].sh_addr; 171ed981856SPaul Mackerras 172ed981856SPaul Mackerras /* Find this entry, or if that fails, the next avail. entry */ 173ed981856SPaul Mackerras while (entry->jump[0]) { 174ed981856SPaul Mackerras if (entry_matches(entry, val)) return (uint32_t)entry; 175ed981856SPaul Mackerras entry++; 176ed981856SPaul Mackerras } 177ed981856SPaul Mackerras 1780c850965SChristophe Leroy if (patch_instruction(&entry->jump[0], ppc_inst(PPC_RAW_LIS(_R12, PPC_HA(val))))) 1790c850965SChristophe Leroy return 0; 1800c850965SChristophe Leroy if (patch_instruction(&entry->jump[1], ppc_inst(PPC_RAW_ADDI(_R12, _R12, PPC_LO(val))))) 1810c850965SChristophe Leroy return 0; 1820c850965SChristophe Leroy if (patch_instruction(&entry->jump[2], ppc_inst(PPC_RAW_MTCTR(_R12)))) 1830c850965SChristophe Leroy return 0; 1840c850965SChristophe Leroy if (patch_instruction(&entry->jump[3], ppc_inst(PPC_RAW_BCTR()))) 1850c850965SChristophe Leroy return 0; 186ed981856SPaul Mackerras 187c7d1f6afSAnton Blanchard pr_debug("Initialized plt for 0x%x at %p\n", val, entry); 188ed981856SPaul Mackerras return (uint32_t)entry; 189ed981856SPaul Mackerras } 190ed981856SPaul Mackerras 1910c850965SChristophe Leroy static int patch_location_16(uint32_t *loc, u16 value) 1920c850965SChristophe Leroy { 1930c850965SChristophe Leroy loc = PTR_ALIGN_DOWN(loc, sizeof(u32)); 1940c850965SChristophe Leroy return patch_instruction(loc, ppc_inst((*loc & 0xffff0000) | value)); 1950c850965SChristophe Leroy } 1960c850965SChristophe Leroy 197ed981856SPaul Mackerras int apply_relocate_add(Elf32_Shdr *sechdrs, 198ed981856SPaul Mackerras const char *strtab, 199ed981856SPaul Mackerras unsigned int symindex, 200ed981856SPaul Mackerras unsigned int relsec, 201ed981856SPaul Mackerras struct module *module) 202ed981856SPaul Mackerras { 203ed981856SPaul Mackerras unsigned int i; 204ed981856SPaul Mackerras Elf32_Rela *rela = (void *)sechdrs[relsec].sh_addr; 205ed981856SPaul Mackerras Elf32_Sym *sym; 206ed981856SPaul Mackerras uint32_t *location; 207ed981856SPaul Mackerras uint32_t value; 208ed981856SPaul Mackerras 209c7d1f6afSAnton Blanchard pr_debug("Applying ADD relocate section %u to %u\n", relsec, 210ed981856SPaul Mackerras sechdrs[relsec].sh_info); 211ed981856SPaul Mackerras for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) { 212ed981856SPaul Mackerras /* This is where to make the change */ 213ed981856SPaul Mackerras location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 214ed981856SPaul Mackerras + rela[i].r_offset; 215ed981856SPaul Mackerras /* This is the symbol it is referring to. Note that all 216ed981856SPaul Mackerras undefined symbols have been resolved. */ 217ed981856SPaul Mackerras sym = (Elf32_Sym *)sechdrs[symindex].sh_addr 218ed981856SPaul Mackerras + ELF32_R_SYM(rela[i].r_info); 219ed981856SPaul Mackerras /* `Everything is relative'. */ 220ed981856SPaul Mackerras value = sym->st_value + rela[i].r_addend; 221ed981856SPaul Mackerras 222ed981856SPaul Mackerras switch (ELF32_R_TYPE(rela[i].r_info)) { 223ed981856SPaul Mackerras case R_PPC_ADDR32: 224ed981856SPaul Mackerras /* Simply set it */ 225ed981856SPaul Mackerras *(uint32_t *)location = value; 226ed981856SPaul Mackerras break; 227ed981856SPaul Mackerras 228ed981856SPaul Mackerras case R_PPC_ADDR16_LO: 229ed981856SPaul Mackerras /* Low half of the symbol */ 2300c850965SChristophe Leroy if (patch_location_16(location, PPC_LO(value))) 2310c850965SChristophe Leroy return -EFAULT; 232ed981856SPaul Mackerras break; 233ed981856SPaul Mackerras 2349a3d6458SSimon Vallet case R_PPC_ADDR16_HI: 2359a3d6458SSimon Vallet /* Higher half of the symbol */ 2360c850965SChristophe Leroy if (patch_location_16(location, PPC_HI(value))) 2370c850965SChristophe Leroy return -EFAULT; 2389a3d6458SSimon Vallet break; 2399a3d6458SSimon Vallet 240ed981856SPaul Mackerras case R_PPC_ADDR16_HA: 2410c850965SChristophe Leroy if (patch_location_16(location, PPC_HA(value))) 2420c850965SChristophe Leroy return -EFAULT; 243ed981856SPaul Mackerras break; 244ed981856SPaul Mackerras 245ed981856SPaul Mackerras case R_PPC_REL24: 246ed981856SPaul Mackerras if ((int)(value - (uint32_t)location) < -0x02000000 2470c850965SChristophe Leroy || (int)(value - (uint32_t)location) >= 0x02000000) { 248ed981856SPaul Mackerras value = do_plt_call(location, value, 249ed981856SPaul Mackerras sechdrs, module); 2500c850965SChristophe Leroy if (!value) 2510c850965SChristophe Leroy return -EFAULT; 2520c850965SChristophe Leroy } 253ed981856SPaul Mackerras 254ed981856SPaul Mackerras /* Only replace bits 2 through 26 */ 255c7d1f6afSAnton Blanchard pr_debug("REL24 value = %08X. location = %08X\n", 256ed981856SPaul Mackerras value, (uint32_t)location); 257c7d1f6afSAnton Blanchard pr_debug("Location before: %08X.\n", 258ed981856SPaul Mackerras *(uint32_t *)location); 259*e0c2ef43SChristophe Leroy value = (*(uint32_t *)location & ~PPC_LI_MASK) | 260*e0c2ef43SChristophe Leroy PPC_LI(value - (uint32_t)location); 2610c850965SChristophe Leroy 2620c850965SChristophe Leroy if (patch_instruction(location, ppc_inst(value))) 2630c850965SChristophe Leroy return -EFAULT; 2640c850965SChristophe Leroy 265c7d1f6afSAnton Blanchard pr_debug("Location after: %08X.\n", 266ed981856SPaul Mackerras *(uint32_t *)location); 267c7d1f6afSAnton Blanchard pr_debug("ie. jump to %08X+%08X = %08X\n", 268*e0c2ef43SChristophe Leroy *(uint32_t *)PPC_LI((uint32_t)location), (uint32_t)location, 269*e0c2ef43SChristophe Leroy (*(uint32_t *)PPC_LI((uint32_t)location)) + (uint32_t)location); 270ed981856SPaul Mackerras break; 271ed981856SPaul Mackerras 272ed981856SPaul Mackerras case R_PPC_REL32: 273ed981856SPaul Mackerras /* 32-bit relative jump. */ 274ed981856SPaul Mackerras *(uint32_t *)location = value - (uint32_t)location; 275ed981856SPaul Mackerras break; 276ed981856SPaul Mackerras 277ed981856SPaul Mackerras default: 278c7d1f6afSAnton Blanchard pr_err("%s: unknown ADD relocation: %u\n", 279ed981856SPaul Mackerras module->name, 280ed981856SPaul Mackerras ELF32_R_TYPE(rela[i].r_info)); 281ed981856SPaul Mackerras return -ENOEXEC; 282ed981856SPaul Mackerras } 283ed981856SPaul Mackerras } 284136cd345SMichael Ellerman 285ed981856SPaul Mackerras return 0; 286ed981856SPaul Mackerras } 287136cd345SMichael Ellerman 288136cd345SMichael Ellerman #ifdef CONFIG_DYNAMIC_FTRACE 2898052d043SChristophe Leroy notrace int module_trampoline_target(struct module *mod, unsigned long addr, 290c93d4f6eSChristophe Leroy unsigned long *target) 291c93d4f6eSChristophe Leroy { 2928052d043SChristophe Leroy ppc_inst_t jmp[4]; 293c93d4f6eSChristophe Leroy 294c93d4f6eSChristophe Leroy /* Find where the trampoline jumps to */ 2958052d043SChristophe Leroy if (copy_inst_from_kernel_nofault(jmp, (void *)addr)) 2968052d043SChristophe Leroy return -EFAULT; 2978052d043SChristophe Leroy if (__copy_inst_from_kernel_nofault(jmp + 1, (void *)addr + 4)) 2988052d043SChristophe Leroy return -EFAULT; 2998052d043SChristophe Leroy if (__copy_inst_from_kernel_nofault(jmp + 2, (void *)addr + 8)) 3008052d043SChristophe Leroy return -EFAULT; 3018052d043SChristophe Leroy if (__copy_inst_from_kernel_nofault(jmp + 3, (void *)addr + 12)) 302c93d4f6eSChristophe Leroy return -EFAULT; 303c93d4f6eSChristophe Leroy 304c93d4f6eSChristophe Leroy /* verify that this is what we expect it to be */ 3058052d043SChristophe Leroy if ((ppc_inst_val(jmp[0]) & 0xffff0000) != PPC_RAW_LIS(_R12, 0)) 3068052d043SChristophe Leroy return -EINVAL; 3078052d043SChristophe Leroy if ((ppc_inst_val(jmp[1]) & 0xffff0000) != PPC_RAW_ADDI(_R12, _R12, 0)) 3088052d043SChristophe Leroy return -EINVAL; 3098052d043SChristophe Leroy if (ppc_inst_val(jmp[2]) != PPC_RAW_MTCTR(_R12)) 3108052d043SChristophe Leroy return -EINVAL; 3118052d043SChristophe Leroy if (ppc_inst_val(jmp[3]) != PPC_RAW_BCTR()) 312c93d4f6eSChristophe Leroy return -EINVAL; 313c93d4f6eSChristophe Leroy 3148052d043SChristophe Leroy addr = (ppc_inst_val(jmp[1]) & 0xffff) | ((ppc_inst_val(jmp[0]) & 0xffff) << 16); 315c93d4f6eSChristophe Leroy if (addr & 0x8000) 316c93d4f6eSChristophe Leroy addr -= 0x10000; 317c93d4f6eSChristophe Leroy 318c93d4f6eSChristophe Leroy *target = addr; 319c93d4f6eSChristophe Leroy 320c93d4f6eSChristophe Leroy return 0; 321c93d4f6eSChristophe Leroy } 322c93d4f6eSChristophe Leroy 323136cd345SMichael Ellerman int module_finalize_ftrace(struct module *module, const Elf_Shdr *sechdrs) 324136cd345SMichael Ellerman { 325136cd345SMichael Ellerman module->arch.tramp = do_plt_call(module->core_layout.base, 326136cd345SMichael Ellerman (unsigned long)ftrace_caller, 327136cd345SMichael Ellerman sechdrs, module); 328136cd345SMichael Ellerman if (!module->arch.tramp) 329136cd345SMichael Ellerman return -ENOENT; 330136cd345SMichael Ellerman 3317dfbfb87SChristophe Leroy #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 3327dfbfb87SChristophe Leroy module->arch.tramp_regs = do_plt_call(module->core_layout.base, 3337dfbfb87SChristophe Leroy (unsigned long)ftrace_regs_caller, 3347dfbfb87SChristophe Leroy sechdrs, module); 3357dfbfb87SChristophe Leroy if (!module->arch.tramp_regs) 3367dfbfb87SChristophe Leroy return -ENOENT; 3377dfbfb87SChristophe Leroy #endif 3387dfbfb87SChristophe Leroy 339136cd345SMichael Ellerman return 0; 340136cd345SMichael Ellerman } 341136cd345SMichael Ellerman #endif 342