1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AArch64 loadable module support. 4 * 5 * Copyright (C) 2012 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/elf.h> 12 #include <linux/ftrace.h> 13 #include <linux/gfp.h> 14 #include <linux/kasan.h> 15 #include <linux/kernel.h> 16 #include <linux/mm.h> 17 #include <linux/moduleloader.h> 18 #include <linux/scs.h> 19 #include <linux/vmalloc.h> 20 #include <asm/alternative.h> 21 #include <asm/insn.h> 22 #include <asm/scs.h> 23 #include <asm/sections.h> 24 25 void *module_alloc(unsigned long size) 26 { 27 u64 module_alloc_end = module_alloc_base + MODULES_VSIZE; 28 gfp_t gfp_mask = GFP_KERNEL; 29 void *p; 30 31 /* Silence the initial allocation */ 32 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) 33 gfp_mask |= __GFP_NOWARN; 34 35 p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, 36 module_alloc_end, gfp_mask, PAGE_KERNEL, 0, 37 NUMA_NO_NODE, __builtin_return_address(0)); 38 39 if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) { 40 p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, 41 module_alloc_base + SZ_2G, GFP_KERNEL, 42 PAGE_KERNEL, 0, NUMA_NO_NODE, 43 __builtin_return_address(0)); 44 } 45 46 if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) { 47 vfree(p); 48 return NULL; 49 } 50 51 /* Memory is intended to be executable, reset the pointer tag. */ 52 return kasan_reset_tag(p); 53 } 54 55 enum aarch64_reloc_op { 56 RELOC_OP_NONE, 57 RELOC_OP_ABS, 58 RELOC_OP_PREL, 59 RELOC_OP_PAGE, 60 }; 61 62 static u64 do_reloc(enum aarch64_reloc_op reloc_op, __le32 *place, u64 val) 63 { 64 switch (reloc_op) { 65 case RELOC_OP_ABS: 66 return val; 67 case RELOC_OP_PREL: 68 return val - (u64)place; 69 case RELOC_OP_PAGE: 70 return (val & ~0xfff) - ((u64)place & ~0xfff); 71 case RELOC_OP_NONE: 72 return 0; 73 } 74 75 pr_err("do_reloc: unknown relocation operation %d\n", reloc_op); 76 return 0; 77 } 78 79 static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len) 80 { 81 s64 sval = do_reloc(op, place, val); 82 83 /* 84 * The ELF psABI for AArch64 documents the 16-bit and 32-bit place 85 * relative and absolute relocations as having a range of [-2^15, 2^16) 86 * or [-2^31, 2^32), respectively. However, in order to be able to 87 * detect overflows reliably, we have to choose whether we interpret 88 * such quantities as signed or as unsigned, and stick with it. 89 * The way we organize our address space requires a signed 90 * interpretation of 32-bit relative references, so let's use that 91 * for all R_AARCH64_PRELxx relocations. This means our upper 92 * bound for overflow detection should be Sxx_MAX rather than Uxx_MAX. 93 */ 94 95 switch (len) { 96 case 16: 97 *(s16 *)place = sval; 98 switch (op) { 99 case RELOC_OP_ABS: 100 if (sval < 0 || sval > U16_MAX) 101 return -ERANGE; 102 break; 103 case RELOC_OP_PREL: 104 if (sval < S16_MIN || sval > S16_MAX) 105 return -ERANGE; 106 break; 107 default: 108 pr_err("Invalid 16-bit data relocation (%d)\n", op); 109 return 0; 110 } 111 break; 112 case 32: 113 *(s32 *)place = sval; 114 switch (op) { 115 case RELOC_OP_ABS: 116 if (sval < 0 || sval > U32_MAX) 117 return -ERANGE; 118 break; 119 case RELOC_OP_PREL: 120 if (sval < S32_MIN || sval > S32_MAX) 121 return -ERANGE; 122 break; 123 default: 124 pr_err("Invalid 32-bit data relocation (%d)\n", op); 125 return 0; 126 } 127 break; 128 case 64: 129 *(s64 *)place = sval; 130 break; 131 default: 132 pr_err("Invalid length (%d) for data relocation\n", len); 133 return 0; 134 } 135 return 0; 136 } 137 138 enum aarch64_insn_movw_imm_type { 139 AARCH64_INSN_IMM_MOVNZ, 140 AARCH64_INSN_IMM_MOVKZ, 141 }; 142 143 static int reloc_insn_movw(enum aarch64_reloc_op op, __le32 *place, u64 val, 144 int lsb, enum aarch64_insn_movw_imm_type imm_type) 145 { 146 u64 imm; 147 s64 sval; 148 u32 insn = le32_to_cpu(*place); 149 150 sval = do_reloc(op, place, val); 151 imm = sval >> lsb; 152 153 if (imm_type == AARCH64_INSN_IMM_MOVNZ) { 154 /* 155 * For signed MOVW relocations, we have to manipulate the 156 * instruction encoding depending on whether or not the 157 * immediate is less than zero. 158 */ 159 insn &= ~(3 << 29); 160 if (sval >= 0) { 161 /* >=0: Set the instruction to MOVZ (opcode 10b). */ 162 insn |= 2 << 29; 163 } else { 164 /* 165 * <0: Set the instruction to MOVN (opcode 00b). 166 * Since we've masked the opcode already, we 167 * don't need to do anything other than 168 * inverting the new immediate field. 169 */ 170 imm = ~imm; 171 } 172 } 173 174 /* Update the instruction with the new encoding. */ 175 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); 176 *place = cpu_to_le32(insn); 177 178 if (imm > U16_MAX) 179 return -ERANGE; 180 181 return 0; 182 } 183 184 static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val, 185 int lsb, int len, enum aarch64_insn_imm_type imm_type) 186 { 187 u64 imm, imm_mask; 188 s64 sval; 189 u32 insn = le32_to_cpu(*place); 190 191 /* Calculate the relocation value. */ 192 sval = do_reloc(op, place, val); 193 sval >>= lsb; 194 195 /* Extract the value bits and shift them to bit 0. */ 196 imm_mask = (BIT(lsb + len) - 1) >> lsb; 197 imm = sval & imm_mask; 198 199 /* Update the instruction's immediate field. */ 200 insn = aarch64_insn_encode_immediate(imm_type, insn, imm); 201 *place = cpu_to_le32(insn); 202 203 /* 204 * Extract the upper value bits (including the sign bit) and 205 * shift them to bit 0. 206 */ 207 sval = (s64)(sval & ~(imm_mask >> 1)) >> (len - 1); 208 209 /* 210 * Overflow has occurred if the upper bits are not all equal to 211 * the sign bit of the value. 212 */ 213 if ((u64)(sval + 1) >= 2) 214 return -ERANGE; 215 216 return 0; 217 } 218 219 static int reloc_insn_adrp(struct module *mod, Elf64_Shdr *sechdrs, 220 __le32 *place, u64 val) 221 { 222 u32 insn; 223 224 if (!is_forbidden_offset_for_adrp(place)) 225 return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21, 226 AARCH64_INSN_IMM_ADR); 227 228 /* patch ADRP to ADR if it is in range */ 229 if (!reloc_insn_imm(RELOC_OP_PREL, place, val & ~0xfff, 0, 21, 230 AARCH64_INSN_IMM_ADR)) { 231 insn = le32_to_cpu(*place); 232 insn &= ~BIT(31); 233 } else { 234 /* out of range for ADR -> emit a veneer */ 235 val = module_emit_veneer_for_adrp(mod, sechdrs, place, val & ~0xfff); 236 if (!val) 237 return -ENOEXEC; 238 insn = aarch64_insn_gen_branch_imm((u64)place, val, 239 AARCH64_INSN_BRANCH_NOLINK); 240 } 241 242 *place = cpu_to_le32(insn); 243 return 0; 244 } 245 246 int apply_relocate_add(Elf64_Shdr *sechdrs, 247 const char *strtab, 248 unsigned int symindex, 249 unsigned int relsec, 250 struct module *me) 251 { 252 unsigned int i; 253 int ovf; 254 bool overflow_check; 255 Elf64_Sym *sym; 256 void *loc; 257 u64 val; 258 Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr; 259 260 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 261 /* loc corresponds to P in the AArch64 ELF document. */ 262 loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 263 + rel[i].r_offset; 264 265 /* sym is the ELF symbol we're referring to. */ 266 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr 267 + ELF64_R_SYM(rel[i].r_info); 268 269 /* val corresponds to (S + A) in the AArch64 ELF document. */ 270 val = sym->st_value + rel[i].r_addend; 271 272 /* Check for overflow by default. */ 273 overflow_check = true; 274 275 /* Perform the static relocation. */ 276 switch (ELF64_R_TYPE(rel[i].r_info)) { 277 /* Null relocations. */ 278 case R_ARM_NONE: 279 case R_AARCH64_NONE: 280 ovf = 0; 281 break; 282 283 /* Data relocations. */ 284 case R_AARCH64_ABS64: 285 overflow_check = false; 286 ovf = reloc_data(RELOC_OP_ABS, loc, val, 64); 287 break; 288 case R_AARCH64_ABS32: 289 ovf = reloc_data(RELOC_OP_ABS, loc, val, 32); 290 break; 291 case R_AARCH64_ABS16: 292 ovf = reloc_data(RELOC_OP_ABS, loc, val, 16); 293 break; 294 case R_AARCH64_PREL64: 295 overflow_check = false; 296 ovf = reloc_data(RELOC_OP_PREL, loc, val, 64); 297 break; 298 case R_AARCH64_PREL32: 299 ovf = reloc_data(RELOC_OP_PREL, loc, val, 32); 300 break; 301 case R_AARCH64_PREL16: 302 ovf = reloc_data(RELOC_OP_PREL, loc, val, 16); 303 break; 304 305 /* MOVW instruction relocations. */ 306 case R_AARCH64_MOVW_UABS_G0_NC: 307 overflow_check = false; 308 fallthrough; 309 case R_AARCH64_MOVW_UABS_G0: 310 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 311 AARCH64_INSN_IMM_MOVKZ); 312 break; 313 case R_AARCH64_MOVW_UABS_G1_NC: 314 overflow_check = false; 315 fallthrough; 316 case R_AARCH64_MOVW_UABS_G1: 317 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 318 AARCH64_INSN_IMM_MOVKZ); 319 break; 320 case R_AARCH64_MOVW_UABS_G2_NC: 321 overflow_check = false; 322 fallthrough; 323 case R_AARCH64_MOVW_UABS_G2: 324 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 325 AARCH64_INSN_IMM_MOVKZ); 326 break; 327 case R_AARCH64_MOVW_UABS_G3: 328 /* We're using the top bits so we can't overflow. */ 329 overflow_check = false; 330 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48, 331 AARCH64_INSN_IMM_MOVKZ); 332 break; 333 case R_AARCH64_MOVW_SABS_G0: 334 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 335 AARCH64_INSN_IMM_MOVNZ); 336 break; 337 case R_AARCH64_MOVW_SABS_G1: 338 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 339 AARCH64_INSN_IMM_MOVNZ); 340 break; 341 case R_AARCH64_MOVW_SABS_G2: 342 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 343 AARCH64_INSN_IMM_MOVNZ); 344 break; 345 case R_AARCH64_MOVW_PREL_G0_NC: 346 overflow_check = false; 347 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 348 AARCH64_INSN_IMM_MOVKZ); 349 break; 350 case R_AARCH64_MOVW_PREL_G0: 351 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 352 AARCH64_INSN_IMM_MOVNZ); 353 break; 354 case R_AARCH64_MOVW_PREL_G1_NC: 355 overflow_check = false; 356 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 357 AARCH64_INSN_IMM_MOVKZ); 358 break; 359 case R_AARCH64_MOVW_PREL_G1: 360 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 361 AARCH64_INSN_IMM_MOVNZ); 362 break; 363 case R_AARCH64_MOVW_PREL_G2_NC: 364 overflow_check = false; 365 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 366 AARCH64_INSN_IMM_MOVKZ); 367 break; 368 case R_AARCH64_MOVW_PREL_G2: 369 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 370 AARCH64_INSN_IMM_MOVNZ); 371 break; 372 case R_AARCH64_MOVW_PREL_G3: 373 /* We're using the top bits so we can't overflow. */ 374 overflow_check = false; 375 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48, 376 AARCH64_INSN_IMM_MOVNZ); 377 break; 378 379 /* Immediate instruction relocations. */ 380 case R_AARCH64_LD_PREL_LO19: 381 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 382 AARCH64_INSN_IMM_19); 383 break; 384 case R_AARCH64_ADR_PREL_LO21: 385 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21, 386 AARCH64_INSN_IMM_ADR); 387 break; 388 case R_AARCH64_ADR_PREL_PG_HI21_NC: 389 overflow_check = false; 390 fallthrough; 391 case R_AARCH64_ADR_PREL_PG_HI21: 392 ovf = reloc_insn_adrp(me, sechdrs, loc, val); 393 if (ovf && ovf != -ERANGE) 394 return ovf; 395 break; 396 case R_AARCH64_ADD_ABS_LO12_NC: 397 case R_AARCH64_LDST8_ABS_LO12_NC: 398 overflow_check = false; 399 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12, 400 AARCH64_INSN_IMM_12); 401 break; 402 case R_AARCH64_LDST16_ABS_LO12_NC: 403 overflow_check = false; 404 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11, 405 AARCH64_INSN_IMM_12); 406 break; 407 case R_AARCH64_LDST32_ABS_LO12_NC: 408 overflow_check = false; 409 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10, 410 AARCH64_INSN_IMM_12); 411 break; 412 case R_AARCH64_LDST64_ABS_LO12_NC: 413 overflow_check = false; 414 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, 415 AARCH64_INSN_IMM_12); 416 break; 417 case R_AARCH64_LDST128_ABS_LO12_NC: 418 overflow_check = false; 419 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8, 420 AARCH64_INSN_IMM_12); 421 break; 422 case R_AARCH64_TSTBR14: 423 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14, 424 AARCH64_INSN_IMM_14); 425 break; 426 case R_AARCH64_CONDBR19: 427 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 428 AARCH64_INSN_IMM_19); 429 break; 430 case R_AARCH64_JUMP26: 431 case R_AARCH64_CALL26: 432 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, 433 AARCH64_INSN_IMM_26); 434 435 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && 436 ovf == -ERANGE) { 437 val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym); 438 if (!val) 439 return -ENOEXEC; 440 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 441 26, AARCH64_INSN_IMM_26); 442 } 443 break; 444 445 default: 446 pr_err("module %s: unsupported RELA relocation: %llu\n", 447 me->name, ELF64_R_TYPE(rel[i].r_info)); 448 return -ENOEXEC; 449 } 450 451 if (overflow_check && ovf == -ERANGE) 452 goto overflow; 453 454 } 455 456 return 0; 457 458 overflow: 459 pr_err("module %s: overflow in relocation type %d val %Lx\n", 460 me->name, (int)ELF64_R_TYPE(rel[i].r_info), val); 461 return -ENOEXEC; 462 } 463 464 static inline void __init_plt(struct plt_entry *plt, unsigned long addr) 465 { 466 *plt = get_plt_entry(addr, plt); 467 } 468 469 static int module_init_ftrace_plt(const Elf_Ehdr *hdr, 470 const Elf_Shdr *sechdrs, 471 struct module *mod) 472 { 473 #if defined(CONFIG_ARM64_MODULE_PLTS) && defined(CONFIG_DYNAMIC_FTRACE) 474 const Elf_Shdr *s; 475 struct plt_entry *plts; 476 477 s = find_section(hdr, sechdrs, ".text.ftrace_trampoline"); 478 if (!s) 479 return -ENOEXEC; 480 481 plts = (void *)s->sh_addr; 482 483 __init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR); 484 485 mod->arch.ftrace_trampolines = plts; 486 #endif 487 return 0; 488 } 489 490 int module_finalize(const Elf_Ehdr *hdr, 491 const Elf_Shdr *sechdrs, 492 struct module *me) 493 { 494 const Elf_Shdr *s; 495 s = find_section(hdr, sechdrs, ".altinstructions"); 496 if (s) 497 apply_alternatives_module((void *)s->sh_addr, s->sh_size); 498 499 if (scs_is_dynamic()) { 500 s = find_section(hdr, sechdrs, ".init.eh_frame"); 501 if (s) 502 scs_patch((void *)s->sh_addr, s->sh_size); 503 } 504 505 return module_init_ftrace_plt(hdr, sechdrs, me); 506 } 507