1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Author: Hanlu Li <lihanlu@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 9 #define pr_fmt(fmt) "kmod: " fmt 10 11 #include <linux/moduleloader.h> 12 #include <linux/elf.h> 13 #include <linux/mm.h> 14 #include <linux/numa.h> 15 #include <linux/vmalloc.h> 16 #include <linux/slab.h> 17 #include <linux/fs.h> 18 #include <linux/string.h> 19 #include <linux/kernel.h> 20 #include <asm/alternative.h> 21 22 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 23 { 24 if (*rela_stack_top >= RELA_STACK_DEPTH) 25 return -ENOEXEC; 26 27 rela_stack[(*rela_stack_top)++] = stack_value; 28 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); 29 30 return 0; 31 } 32 33 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) 34 { 35 if (*rela_stack_top == 0) 36 return -ENOEXEC; 37 38 *stack_value = rela_stack[--(*rela_stack_top)]; 39 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); 40 41 return 0; 42 } 43 44 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, 45 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 46 { 47 return 0; 48 } 49 50 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, 51 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 52 { 53 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); 54 return -EINVAL; 55 } 56 57 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, 58 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 59 { 60 *location = v; 61 return 0; 62 } 63 64 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, 65 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 66 { 67 *(Elf_Addr *)location = v; 68 return 0; 69 } 70 71 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, 72 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 73 { 74 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); 75 } 76 77 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, 78 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 79 { 80 return rela_stack_push(v, rela_stack, rela_stack_top); 81 } 82 83 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, 84 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 85 { 86 int err = 0; 87 s64 opr1; 88 89 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 90 if (err) 91 return err; 92 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 93 if (err) 94 return err; 95 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 96 if (err) 97 return err; 98 99 return 0; 100 } 101 102 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, 103 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 104 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 105 { 106 ptrdiff_t offset = (void *)v - (void *)location; 107 108 if (offset >= SZ_128M) 109 v = module_emit_plt_entry(mod, sechdrs, v); 110 111 if (offset < -SZ_128M) 112 v = module_emit_plt_entry(mod, sechdrs, v); 113 114 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); 115 } 116 117 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, 118 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 119 { 120 int err = 0; 121 s64 opr1, opr2, opr3; 122 123 if (type == R_LARCH_SOP_IF_ELSE) { 124 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); 125 if (err) 126 return err; 127 } 128 129 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); 130 if (err) 131 return err; 132 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 133 if (err) 134 return err; 135 136 switch (type) { 137 case R_LARCH_SOP_AND: 138 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); 139 break; 140 case R_LARCH_SOP_ADD: 141 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); 142 break; 143 case R_LARCH_SOP_SUB: 144 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); 145 break; 146 case R_LARCH_SOP_SL: 147 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); 148 break; 149 case R_LARCH_SOP_SR: 150 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); 151 break; 152 case R_LARCH_SOP_IF_ELSE: 153 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); 154 break; 155 default: 156 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 157 return -EINVAL; 158 } 159 160 return err; 161 } 162 163 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, 164 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 165 { 166 int err = 0; 167 s64 opr1; 168 union loongarch_instruction *insn = (union loongarch_instruction *)location; 169 170 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 171 if (err) 172 return err; 173 174 switch (type) { 175 case R_LARCH_SOP_POP_32_U_10_12: 176 if (!unsigned_imm_check(opr1, 12)) 177 goto overflow; 178 179 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ 180 insn->reg2i12_format.immediate = opr1 & 0xfff; 181 return 0; 182 case R_LARCH_SOP_POP_32_S_10_12: 183 if (!signed_imm_check(opr1, 12)) 184 goto overflow; 185 186 insn->reg2i12_format.immediate = opr1 & 0xfff; 187 return 0; 188 case R_LARCH_SOP_POP_32_S_10_16: 189 if (!signed_imm_check(opr1, 16)) 190 goto overflow; 191 192 insn->reg2i16_format.immediate = opr1 & 0xffff; 193 return 0; 194 case R_LARCH_SOP_POP_32_S_10_16_S2: 195 if (opr1 % 4) 196 goto unaligned; 197 198 if (!signed_imm_check(opr1, 18)) 199 goto overflow; 200 201 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff; 202 return 0; 203 case R_LARCH_SOP_POP_32_S_5_20: 204 if (!signed_imm_check(opr1, 20)) 205 goto overflow; 206 207 insn->reg1i20_format.immediate = (opr1) & 0xfffff; 208 return 0; 209 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: 210 if (opr1 % 4) 211 goto unaligned; 212 213 if (!signed_imm_check(opr1, 23)) 214 goto overflow; 215 216 opr1 >>= 2; 217 insn->reg1i21_format.immediate_l = opr1 & 0xffff; 218 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f; 219 return 0; 220 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: 221 if (opr1 % 4) 222 goto unaligned; 223 224 if (!signed_imm_check(opr1, 28)) 225 goto overflow; 226 227 opr1 >>= 2; 228 insn->reg0i26_format.immediate_l = opr1 & 0xffff; 229 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff; 230 return 0; 231 case R_LARCH_SOP_POP_32_U: 232 if (!unsigned_imm_check(opr1, 32)) 233 goto overflow; 234 235 /* (*(uint32_t *) PC) = opr */ 236 *location = (u32)opr1; 237 return 0; 238 default: 239 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 240 return -EINVAL; 241 } 242 243 overflow: 244 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", 245 mod->name, opr1, __func__, type); 246 return -ENOEXEC; 247 248 unaligned: 249 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", 250 mod->name, opr1, __func__, type); 251 return -ENOEXEC; 252 } 253 254 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, 255 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 256 { 257 switch (type) { 258 case R_LARCH_ADD32: 259 *(s32 *)location += v; 260 return 0; 261 case R_LARCH_ADD64: 262 *(s64 *)location += v; 263 return 0; 264 case R_LARCH_SUB32: 265 *(s32 *)location -= v; 266 return 0; 267 case R_LARCH_SUB64: 268 *(s64 *)location -= v; 269 return 0; 270 default: 271 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 272 return -EINVAL; 273 } 274 } 275 276 static int apply_r_larch_b26(struct module *mod, 277 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 278 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 279 { 280 ptrdiff_t offset = (void *)v - (void *)location; 281 union loongarch_instruction *insn = (union loongarch_instruction *)location; 282 283 if (offset >= SZ_128M) 284 v = module_emit_plt_entry(mod, sechdrs, v); 285 286 if (offset < -SZ_128M) 287 v = module_emit_plt_entry(mod, sechdrs, v); 288 289 offset = (void *)v - (void *)location; 290 291 if (offset & 3) { 292 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", 293 mod->name, (long long)offset, type); 294 return -ENOEXEC; 295 } 296 297 if (!signed_imm_check(offset, 28)) { 298 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", 299 mod->name, (long long)offset, type); 300 return -ENOEXEC; 301 } 302 303 offset >>= 2; 304 insn->reg0i26_format.immediate_l = offset & 0xffff; 305 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff; 306 307 return 0; 308 } 309 310 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, 311 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 312 { 313 union loongarch_instruction *insn = (union loongarch_instruction *)location; 314 /* Use s32 for a sign-extension deliberately. */ 315 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - 316 (void *)((Elf_Addr)location & ~0xfff); 317 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; 318 ptrdiff_t offset_rem = (void *)v - (void *)anchor; 319 320 switch (type) { 321 case R_LARCH_PCALA_LO12: 322 insn->reg2i12_format.immediate = v & 0xfff; 323 break; 324 case R_LARCH_PCALA_HI20: 325 v = offset_hi20 >> 12; 326 insn->reg1i20_format.immediate = v & 0xfffff; 327 break; 328 case R_LARCH_PCALA64_LO20: 329 v = offset_rem >> 32; 330 insn->reg1i20_format.immediate = v & 0xfffff; 331 break; 332 case R_LARCH_PCALA64_HI12: 333 v = offset_rem >> 52; 334 insn->reg2i12_format.immediate = v & 0xfff; 335 break; 336 default: 337 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 338 return -EINVAL; 339 } 340 341 return 0; 342 } 343 344 static int apply_r_larch_got_pc(struct module *mod, 345 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 346 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 347 { 348 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v); 349 350 if (!got) 351 return -EINVAL; 352 353 switch (type) { 354 case R_LARCH_GOT_PC_LO12: 355 type = R_LARCH_PCALA_LO12; 356 break; 357 case R_LARCH_GOT_PC_HI20: 358 type = R_LARCH_PCALA_HI20; 359 break; 360 default: 361 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 362 return -EINVAL; 363 } 364 365 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); 366 } 367 368 /* 369 * reloc_handlers_rela() - Apply a particular relocation to a module 370 * @mod: the module to apply the reloc to 371 * @location: the address at which the reloc is to be applied 372 * @v: the value of the reloc, with addend for RELA-style 373 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module 374 * @rela_stac_top: where the stack operation(pop/push) applies to 375 * 376 * Return: 0 upon success, else -ERRNO 377 */ 378 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, 379 s64 *rela_stack, size_t *rela_stack_top, unsigned int type); 380 381 /* The handlers for known reloc types */ 382 static reloc_rela_handler reloc_rela_handlers[] = { 383 [R_LARCH_NONE ... R_LARCH_RELAX] = apply_r_larch_error, 384 385 [R_LARCH_NONE] = apply_r_larch_none, 386 [R_LARCH_32] = apply_r_larch_32, 387 [R_LARCH_64] = apply_r_larch_64, 388 [R_LARCH_MARK_LA] = apply_r_larch_none, 389 [R_LARCH_MARK_PCREL] = apply_r_larch_none, 390 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, 391 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, 392 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, 393 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, 394 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, 395 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, 396 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, 397 }; 398 399 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 400 unsigned int symindex, unsigned int relsec, 401 struct module *mod) 402 { 403 int i, err; 404 unsigned int type; 405 s64 rela_stack[RELA_STACK_DEPTH]; 406 size_t rela_stack_top = 0; 407 reloc_rela_handler handler; 408 void *location; 409 Elf_Addr v; 410 Elf_Sym *sym; 411 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; 412 413 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, 414 sechdrs[relsec].sh_info); 415 416 rela_stack_top = 0; 417 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 418 /* This is where to make the change */ 419 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; 420 /* This is the symbol it is referring to */ 421 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); 422 if (IS_ERR_VALUE(sym->st_value)) { 423 /* Ignore unresolved weak symbol */ 424 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 425 continue; 426 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); 427 return -ENOENT; 428 } 429 430 type = ELF_R_TYPE(rel[i].r_info); 431 432 if (type < ARRAY_SIZE(reloc_rela_handlers)) 433 handler = reloc_rela_handlers[type]; 434 else 435 handler = NULL; 436 437 if (!handler) { 438 pr_err("%s: Unknown relocation type %u\n", mod->name, type); 439 return -EINVAL; 440 } 441 442 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", 443 (int)ELF_R_TYPE(rel[i].r_info), 444 sym->st_value, rel[i].r_addend, (u64)location); 445 446 v = sym->st_value + rel[i].r_addend; 447 switch (type) { 448 case R_LARCH_B26: 449 err = apply_r_larch_b26(mod, sechdrs, location, 450 v, rela_stack, &rela_stack_top, type); 451 break; 452 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12: 453 err = apply_r_larch_got_pc(mod, sechdrs, location, 454 v, rela_stack, &rela_stack_top, type); 455 break; 456 case R_LARCH_SOP_PUSH_PLT_PCREL: 457 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location, 458 v, rela_stack, &rela_stack_top, type); 459 break; 460 default: 461 err = handler(mod, location, v, rela_stack, &rela_stack_top, type); 462 } 463 if (err) 464 return err; 465 } 466 467 return 0; 468 } 469 470 void *module_alloc(unsigned long size) 471 { 472 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, 473 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); 474 } 475 476 int module_finalize(const Elf_Ehdr *hdr, 477 const Elf_Shdr *sechdrs, struct module *mod) 478 { 479 const Elf_Shdr *s, *se; 480 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 481 482 for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { 483 if (!strcmp(".altinstructions", secstrs + s->sh_name)) 484 apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size); 485 } 486 487 return 0; 488 } 489