1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AArch64 loadable module support. 4 * 5 * Copyright (C) 2012 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/elf.h> 12 #include <linux/ftrace.h> 13 #include <linux/gfp.h> 14 #include <linux/kasan.h> 15 #include <linux/kernel.h> 16 #include <linux/mm.h> 17 #include <linux/moduleloader.h> 18 #include <linux/vmalloc.h> 19 #include <asm/alternative.h> 20 #include <asm/insn.h> 21 #include <asm/sections.h> 22 23 void *module_alloc(unsigned long size) 24 { 25 u64 module_alloc_end = module_alloc_base + MODULES_VSIZE; 26 gfp_t gfp_mask = GFP_KERNEL; 27 void *p; 28 29 /* Silence the initial allocation */ 30 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) 31 gfp_mask |= __GFP_NOWARN; 32 33 if (IS_ENABLED(CONFIG_KASAN_GENERIC) || 34 IS_ENABLED(CONFIG_KASAN_SW_TAGS)) 35 /* don't exceed the static module region - see below */ 36 module_alloc_end = MODULES_END; 37 38 p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, 39 module_alloc_end, gfp_mask, PAGE_KERNEL, VM_DEFER_KMEMLEAK, 40 NUMA_NO_NODE, __builtin_return_address(0)); 41 42 if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && 43 (IS_ENABLED(CONFIG_KASAN_VMALLOC) || 44 (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 45 !IS_ENABLED(CONFIG_KASAN_SW_TAGS)))) 46 /* 47 * KASAN without KASAN_VMALLOC can only deal with module 48 * allocations being served from the reserved module region, 49 * since the remainder of the vmalloc region is already 50 * backed by zero shadow pages, and punching holes into it 51 * is non-trivial. Since the module region is not randomized 52 * when KASAN is enabled without KASAN_VMALLOC, it is even 53 * less likely that the module region gets exhausted, so we 54 * can simply omit this fallback in that case. 55 */ 56 p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, 57 module_alloc_base + SZ_2G, GFP_KERNEL, 58 PAGE_KERNEL, 0, NUMA_NO_NODE, 59 __builtin_return_address(0)); 60 61 if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) { 62 vfree(p); 63 return NULL; 64 } 65 66 /* Memory is intended to be executable, reset the pointer tag. */ 67 return kasan_reset_tag(p); 68 } 69 70 enum aarch64_reloc_op { 71 RELOC_OP_NONE, 72 RELOC_OP_ABS, 73 RELOC_OP_PREL, 74 RELOC_OP_PAGE, 75 }; 76 77 static u64 do_reloc(enum aarch64_reloc_op reloc_op, __le32 *place, u64 val) 78 { 79 switch (reloc_op) { 80 case RELOC_OP_ABS: 81 return val; 82 case RELOC_OP_PREL: 83 return val - (u64)place; 84 case RELOC_OP_PAGE: 85 return (val & ~0xfff) - ((u64)place & ~0xfff); 86 case RELOC_OP_NONE: 87 return 0; 88 } 89 90 pr_err("do_reloc: unknown relocation operation %d\n", reloc_op); 91 return 0; 92 } 93 94 static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len) 95 { 96 s64 sval = do_reloc(op, place, val); 97 98 /* 99 * The ELF psABI for AArch64 documents the 16-bit and 32-bit place 100 * relative and absolute relocations as having a range of [-2^15, 2^16) 101 * or [-2^31, 2^32), respectively. However, in order to be able to 102 * detect overflows reliably, we have to choose whether we interpret 103 * such quantities as signed or as unsigned, and stick with it. 104 * The way we organize our address space requires a signed 105 * interpretation of 32-bit relative references, so let's use that 106 * for all R_AARCH64_PRELxx relocations. This means our upper 107 * bound for overflow detection should be Sxx_MAX rather than Uxx_MAX. 108 */ 109 110 switch (len) { 111 case 16: 112 *(s16 *)place = sval; 113 switch (op) { 114 case RELOC_OP_ABS: 115 if (sval < 0 || sval > U16_MAX) 116 return -ERANGE; 117 break; 118 case RELOC_OP_PREL: 119 if (sval < S16_MIN || sval > S16_MAX) 120 return -ERANGE; 121 break; 122 default: 123 pr_err("Invalid 16-bit data relocation (%d)\n", op); 124 return 0; 125 } 126 break; 127 case 32: 128 *(s32 *)place = sval; 129 switch (op) { 130 case RELOC_OP_ABS: 131 if (sval < 0 || sval > U32_MAX) 132 return -ERANGE; 133 break; 134 case RELOC_OP_PREL: 135 if (sval < S32_MIN || sval > S32_MAX) 136 return -ERANGE; 137 break; 138 default: 139 pr_err("Invalid 32-bit data relocation (%d)\n", op); 140 return 0; 141 } 142 break; 143 case 64: 144 *(s64 *)place = sval; 145 break; 146 default: 147 pr_err("Invalid length (%d) for data relocation\n", len); 148 return 0; 149 } 150 return 0; 151 } 152 153 enum aarch64_insn_movw_imm_type { 154 AARCH64_INSN_IMM_MOVNZ, 155 AARCH64_INSN_IMM_MOVKZ, 156 }; 157 158 static int reloc_insn_movw(enum aarch64_reloc_op op, __le32 *place, u64 val, 159 int lsb, enum aarch64_insn_movw_imm_type imm_type) 160 { 161 u64 imm; 162 s64 sval; 163 u32 insn = le32_to_cpu(*place); 164 165 sval = do_reloc(op, place, val); 166 imm = sval >> lsb; 167 168 if (imm_type == AARCH64_INSN_IMM_MOVNZ) { 169 /* 170 * For signed MOVW relocations, we have to manipulate the 171 * instruction encoding depending on whether or not the 172 * immediate is less than zero. 173 */ 174 insn &= ~(3 << 29); 175 if (sval >= 0) { 176 /* >=0: Set the instruction to MOVZ (opcode 10b). */ 177 insn |= 2 << 29; 178 } else { 179 /* 180 * <0: Set the instruction to MOVN (opcode 00b). 181 * Since we've masked the opcode already, we 182 * don't need to do anything other than 183 * inverting the new immediate field. 184 */ 185 imm = ~imm; 186 } 187 } 188 189 /* Update the instruction with the new encoding. */ 190 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); 191 *place = cpu_to_le32(insn); 192 193 if (imm > U16_MAX) 194 return -ERANGE; 195 196 return 0; 197 } 198 199 static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val, 200 int lsb, int len, enum aarch64_insn_imm_type imm_type) 201 { 202 u64 imm, imm_mask; 203 s64 sval; 204 u32 insn = le32_to_cpu(*place); 205 206 /* Calculate the relocation value. */ 207 sval = do_reloc(op, place, val); 208 sval >>= lsb; 209 210 /* Extract the value bits and shift them to bit 0. */ 211 imm_mask = (BIT(lsb + len) - 1) >> lsb; 212 imm = sval & imm_mask; 213 214 /* Update the instruction's immediate field. */ 215 insn = aarch64_insn_encode_immediate(imm_type, insn, imm); 216 *place = cpu_to_le32(insn); 217 218 /* 219 * Extract the upper value bits (including the sign bit) and 220 * shift them to bit 0. 221 */ 222 sval = (s64)(sval & ~(imm_mask >> 1)) >> (len - 1); 223 224 /* 225 * Overflow has occurred if the upper bits are not all equal to 226 * the sign bit of the value. 227 */ 228 if ((u64)(sval + 1) >= 2) 229 return -ERANGE; 230 231 return 0; 232 } 233 234 static int reloc_insn_adrp(struct module *mod, Elf64_Shdr *sechdrs, 235 __le32 *place, u64 val) 236 { 237 u32 insn; 238 239 if (!is_forbidden_offset_for_adrp(place)) 240 return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21, 241 AARCH64_INSN_IMM_ADR); 242 243 /* patch ADRP to ADR if it is in range */ 244 if (!reloc_insn_imm(RELOC_OP_PREL, place, val & ~0xfff, 0, 21, 245 AARCH64_INSN_IMM_ADR)) { 246 insn = le32_to_cpu(*place); 247 insn &= ~BIT(31); 248 } else { 249 /* out of range for ADR -> emit a veneer */ 250 val = module_emit_veneer_for_adrp(mod, sechdrs, place, val & ~0xfff); 251 if (!val) 252 return -ENOEXEC; 253 insn = aarch64_insn_gen_branch_imm((u64)place, val, 254 AARCH64_INSN_BRANCH_NOLINK); 255 } 256 257 *place = cpu_to_le32(insn); 258 return 0; 259 } 260 261 int apply_relocate_add(Elf64_Shdr *sechdrs, 262 const char *strtab, 263 unsigned int symindex, 264 unsigned int relsec, 265 struct module *me) 266 { 267 unsigned int i; 268 int ovf; 269 bool overflow_check; 270 Elf64_Sym *sym; 271 void *loc; 272 u64 val; 273 Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr; 274 275 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 276 /* loc corresponds to P in the AArch64 ELF document. */ 277 loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 278 + rel[i].r_offset; 279 280 /* sym is the ELF symbol we're referring to. */ 281 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr 282 + ELF64_R_SYM(rel[i].r_info); 283 284 /* val corresponds to (S + A) in the AArch64 ELF document. */ 285 val = sym->st_value + rel[i].r_addend; 286 287 /* Check for overflow by default. */ 288 overflow_check = true; 289 290 /* Perform the static relocation. */ 291 switch (ELF64_R_TYPE(rel[i].r_info)) { 292 /* Null relocations. */ 293 case R_ARM_NONE: 294 case R_AARCH64_NONE: 295 ovf = 0; 296 break; 297 298 /* Data relocations. */ 299 case R_AARCH64_ABS64: 300 overflow_check = false; 301 ovf = reloc_data(RELOC_OP_ABS, loc, val, 64); 302 break; 303 case R_AARCH64_ABS32: 304 ovf = reloc_data(RELOC_OP_ABS, loc, val, 32); 305 break; 306 case R_AARCH64_ABS16: 307 ovf = reloc_data(RELOC_OP_ABS, loc, val, 16); 308 break; 309 case R_AARCH64_PREL64: 310 overflow_check = false; 311 ovf = reloc_data(RELOC_OP_PREL, loc, val, 64); 312 break; 313 case R_AARCH64_PREL32: 314 ovf = reloc_data(RELOC_OP_PREL, loc, val, 32); 315 break; 316 case R_AARCH64_PREL16: 317 ovf = reloc_data(RELOC_OP_PREL, loc, val, 16); 318 break; 319 320 /* MOVW instruction relocations. */ 321 case R_AARCH64_MOVW_UABS_G0_NC: 322 overflow_check = false; 323 fallthrough; 324 case R_AARCH64_MOVW_UABS_G0: 325 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 326 AARCH64_INSN_IMM_MOVKZ); 327 break; 328 case R_AARCH64_MOVW_UABS_G1_NC: 329 overflow_check = false; 330 fallthrough; 331 case R_AARCH64_MOVW_UABS_G1: 332 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 333 AARCH64_INSN_IMM_MOVKZ); 334 break; 335 case R_AARCH64_MOVW_UABS_G2_NC: 336 overflow_check = false; 337 fallthrough; 338 case R_AARCH64_MOVW_UABS_G2: 339 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 340 AARCH64_INSN_IMM_MOVKZ); 341 break; 342 case R_AARCH64_MOVW_UABS_G3: 343 /* We're using the top bits so we can't overflow. */ 344 overflow_check = false; 345 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48, 346 AARCH64_INSN_IMM_MOVKZ); 347 break; 348 case R_AARCH64_MOVW_SABS_G0: 349 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 350 AARCH64_INSN_IMM_MOVNZ); 351 break; 352 case R_AARCH64_MOVW_SABS_G1: 353 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 354 AARCH64_INSN_IMM_MOVNZ); 355 break; 356 case R_AARCH64_MOVW_SABS_G2: 357 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 358 AARCH64_INSN_IMM_MOVNZ); 359 break; 360 case R_AARCH64_MOVW_PREL_G0_NC: 361 overflow_check = false; 362 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 363 AARCH64_INSN_IMM_MOVKZ); 364 break; 365 case R_AARCH64_MOVW_PREL_G0: 366 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 367 AARCH64_INSN_IMM_MOVNZ); 368 break; 369 case R_AARCH64_MOVW_PREL_G1_NC: 370 overflow_check = false; 371 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 372 AARCH64_INSN_IMM_MOVKZ); 373 break; 374 case R_AARCH64_MOVW_PREL_G1: 375 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 376 AARCH64_INSN_IMM_MOVNZ); 377 break; 378 case R_AARCH64_MOVW_PREL_G2_NC: 379 overflow_check = false; 380 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 381 AARCH64_INSN_IMM_MOVKZ); 382 break; 383 case R_AARCH64_MOVW_PREL_G2: 384 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 385 AARCH64_INSN_IMM_MOVNZ); 386 break; 387 case R_AARCH64_MOVW_PREL_G3: 388 /* We're using the top bits so we can't overflow. */ 389 overflow_check = false; 390 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48, 391 AARCH64_INSN_IMM_MOVNZ); 392 break; 393 394 /* Immediate instruction relocations. */ 395 case R_AARCH64_LD_PREL_LO19: 396 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 397 AARCH64_INSN_IMM_19); 398 break; 399 case R_AARCH64_ADR_PREL_LO21: 400 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21, 401 AARCH64_INSN_IMM_ADR); 402 break; 403 case R_AARCH64_ADR_PREL_PG_HI21_NC: 404 overflow_check = false; 405 fallthrough; 406 case R_AARCH64_ADR_PREL_PG_HI21: 407 ovf = reloc_insn_adrp(me, sechdrs, loc, val); 408 if (ovf && ovf != -ERANGE) 409 return ovf; 410 break; 411 case R_AARCH64_ADD_ABS_LO12_NC: 412 case R_AARCH64_LDST8_ABS_LO12_NC: 413 overflow_check = false; 414 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12, 415 AARCH64_INSN_IMM_12); 416 break; 417 case R_AARCH64_LDST16_ABS_LO12_NC: 418 overflow_check = false; 419 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11, 420 AARCH64_INSN_IMM_12); 421 break; 422 case R_AARCH64_LDST32_ABS_LO12_NC: 423 overflow_check = false; 424 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10, 425 AARCH64_INSN_IMM_12); 426 break; 427 case R_AARCH64_LDST64_ABS_LO12_NC: 428 overflow_check = false; 429 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, 430 AARCH64_INSN_IMM_12); 431 break; 432 case R_AARCH64_LDST128_ABS_LO12_NC: 433 overflow_check = false; 434 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8, 435 AARCH64_INSN_IMM_12); 436 break; 437 case R_AARCH64_TSTBR14: 438 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14, 439 AARCH64_INSN_IMM_14); 440 break; 441 case R_AARCH64_CONDBR19: 442 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 443 AARCH64_INSN_IMM_19); 444 break; 445 case R_AARCH64_JUMP26: 446 case R_AARCH64_CALL26: 447 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, 448 AARCH64_INSN_IMM_26); 449 450 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && 451 ovf == -ERANGE) { 452 val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym); 453 if (!val) 454 return -ENOEXEC; 455 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 456 26, AARCH64_INSN_IMM_26); 457 } 458 break; 459 460 default: 461 pr_err("module %s: unsupported RELA relocation: %llu\n", 462 me->name, ELF64_R_TYPE(rel[i].r_info)); 463 return -ENOEXEC; 464 } 465 466 if (overflow_check && ovf == -ERANGE) 467 goto overflow; 468 469 } 470 471 return 0; 472 473 overflow: 474 pr_err("module %s: overflow in relocation type %d val %Lx\n", 475 me->name, (int)ELF64_R_TYPE(rel[i].r_info), val); 476 return -ENOEXEC; 477 } 478 479 static inline void __init_plt(struct plt_entry *plt, unsigned long addr) 480 { 481 *plt = get_plt_entry(addr, plt); 482 } 483 484 static int module_init_ftrace_plt(const Elf_Ehdr *hdr, 485 const Elf_Shdr *sechdrs, 486 struct module *mod) 487 { 488 #if defined(CONFIG_ARM64_MODULE_PLTS) && defined(CONFIG_DYNAMIC_FTRACE) 489 const Elf_Shdr *s; 490 struct plt_entry *plts; 491 492 s = find_section(hdr, sechdrs, ".text.ftrace_trampoline"); 493 if (!s) 494 return -ENOEXEC; 495 496 plts = (void *)s->sh_addr; 497 498 __init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR); 499 500 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) 501 __init_plt(&plts[FTRACE_REGS_PLT_IDX], FTRACE_REGS_ADDR); 502 503 mod->arch.ftrace_trampolines = plts; 504 #endif 505 return 0; 506 } 507 508 int module_finalize(const Elf_Ehdr *hdr, 509 const Elf_Shdr *sechdrs, 510 struct module *me) 511 { 512 const Elf_Shdr *s; 513 s = find_section(hdr, sechdrs, ".altinstructions"); 514 if (s) 515 apply_alternatives_module((void *)s->sh_addr, s->sh_size); 516 517 return module_init_ftrace_plt(hdr, sechdrs, me); 518 } 519