1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Author: Hanlu Li <lihanlu@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 9 #define pr_fmt(fmt) "kmod: " fmt 10 11 #include <linux/moduleloader.h> 12 #include <linux/elf.h> 13 #include <linux/mm.h> 14 #include <linux/numa.h> 15 #include <linux/vmalloc.h> 16 #include <linux/slab.h> 17 #include <linux/fs.h> 18 #include <linux/string.h> 19 #include <linux/kernel.h> 20 #include <asm/alternative.h> 21 22 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 23 { 24 if (*rela_stack_top >= RELA_STACK_DEPTH) 25 return -ENOEXEC; 26 27 rela_stack[(*rela_stack_top)++] = stack_value; 28 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); 29 30 return 0; 31 } 32 33 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) 34 { 35 if (*rela_stack_top == 0) 36 return -ENOEXEC; 37 38 *stack_value = rela_stack[--(*rela_stack_top)]; 39 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); 40 41 return 0; 42 } 43 44 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, 45 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 46 { 47 return 0; 48 } 49 50 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, 51 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 52 { 53 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); 54 return -EINVAL; 55 } 56 57 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, 58 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 59 { 60 *location = v; 61 return 0; 62 } 63 64 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, 65 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 66 { 67 *(Elf_Addr *)location = v; 68 return 0; 69 } 70 71 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, 72 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 73 { 74 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); 75 } 76 77 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, 78 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 79 { 80 return rela_stack_push(v, rela_stack, rela_stack_top); 81 } 82 83 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, 84 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 85 { 86 int err = 0; 87 s64 opr1; 88 89 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 90 if (err) 91 return err; 92 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 93 if (err) 94 return err; 95 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 96 if (err) 97 return err; 98 99 return 0; 100 } 101 102 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, u32 *location, Elf_Addr v, 103 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 104 { 105 ptrdiff_t offset = (void *)v - (void *)location; 106 107 if (offset >= SZ_128M) 108 v = module_emit_plt_entry(mod, v); 109 110 if (offset < -SZ_128M) 111 v = module_emit_plt_entry(mod, v); 112 113 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); 114 } 115 116 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, 117 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 118 { 119 int err = 0; 120 s64 opr1, opr2, opr3; 121 122 if (type == R_LARCH_SOP_IF_ELSE) { 123 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); 124 if (err) 125 return err; 126 } 127 128 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); 129 if (err) 130 return err; 131 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 132 if (err) 133 return err; 134 135 switch (type) { 136 case R_LARCH_SOP_AND: 137 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); 138 break; 139 case R_LARCH_SOP_ADD: 140 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); 141 break; 142 case R_LARCH_SOP_SUB: 143 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); 144 break; 145 case R_LARCH_SOP_SL: 146 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); 147 break; 148 case R_LARCH_SOP_SR: 149 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); 150 break; 151 case R_LARCH_SOP_IF_ELSE: 152 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); 153 break; 154 default: 155 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 156 return -EINVAL; 157 } 158 159 return err; 160 } 161 162 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, 163 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 164 { 165 int err = 0; 166 s64 opr1; 167 union loongarch_instruction *insn = (union loongarch_instruction *)location; 168 169 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 170 if (err) 171 return err; 172 173 switch (type) { 174 case R_LARCH_SOP_POP_32_U_10_12: 175 if (!unsigned_imm_check(opr1, 12)) 176 goto overflow; 177 178 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ 179 insn->reg2i12_format.immediate = opr1 & 0xfff; 180 return 0; 181 case R_LARCH_SOP_POP_32_S_10_12: 182 if (!signed_imm_check(opr1, 12)) 183 goto overflow; 184 185 insn->reg2i12_format.immediate = opr1 & 0xfff; 186 return 0; 187 case R_LARCH_SOP_POP_32_S_10_16: 188 if (!signed_imm_check(opr1, 16)) 189 goto overflow; 190 191 insn->reg2i16_format.immediate = opr1 & 0xffff; 192 return 0; 193 case R_LARCH_SOP_POP_32_S_10_16_S2: 194 if (opr1 % 4) 195 goto unaligned; 196 197 if (!signed_imm_check(opr1, 18)) 198 goto overflow; 199 200 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff; 201 return 0; 202 case R_LARCH_SOP_POP_32_S_5_20: 203 if (!signed_imm_check(opr1, 20)) 204 goto overflow; 205 206 insn->reg1i20_format.immediate = (opr1) & 0xfffff; 207 return 0; 208 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: 209 if (opr1 % 4) 210 goto unaligned; 211 212 if (!signed_imm_check(opr1, 23)) 213 goto overflow; 214 215 opr1 >>= 2; 216 insn->reg1i21_format.immediate_l = opr1 & 0xffff; 217 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f; 218 return 0; 219 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: 220 if (opr1 % 4) 221 goto unaligned; 222 223 if (!signed_imm_check(opr1, 28)) 224 goto overflow; 225 226 opr1 >>= 2; 227 insn->reg0i26_format.immediate_l = opr1 & 0xffff; 228 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff; 229 return 0; 230 case R_LARCH_SOP_POP_32_U: 231 if (!unsigned_imm_check(opr1, 32)) 232 goto overflow; 233 234 /* (*(uint32_t *) PC) = opr */ 235 *location = (u32)opr1; 236 return 0; 237 default: 238 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 239 return -EINVAL; 240 } 241 242 overflow: 243 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", 244 mod->name, opr1, __func__, type); 245 return -ENOEXEC; 246 247 unaligned: 248 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", 249 mod->name, opr1, __func__, type); 250 return -ENOEXEC; 251 } 252 253 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, 254 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 255 { 256 switch (type) { 257 case R_LARCH_ADD32: 258 *(s32 *)location += v; 259 return 0; 260 case R_LARCH_ADD64: 261 *(s64 *)location += v; 262 return 0; 263 case R_LARCH_SUB32: 264 *(s32 *)location -= v; 265 return 0; 266 case R_LARCH_SUB64: 267 *(s64 *)location -= v; 268 return 0; 269 default: 270 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 271 return -EINVAL; 272 } 273 } 274 275 static int apply_r_larch_b26(struct module *mod, u32 *location, Elf_Addr v, 276 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 277 { 278 ptrdiff_t offset = (void *)v - (void *)location; 279 union loongarch_instruction *insn = (union loongarch_instruction *)location; 280 281 if (offset >= SZ_128M) 282 v = module_emit_plt_entry(mod, v); 283 284 if (offset < -SZ_128M) 285 v = module_emit_plt_entry(mod, v); 286 287 offset = (void *)v - (void *)location; 288 289 if (offset & 3) { 290 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", 291 mod->name, (long long)offset, type); 292 return -ENOEXEC; 293 } 294 295 if (!signed_imm_check(offset, 28)) { 296 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", 297 mod->name, (long long)offset, type); 298 return -ENOEXEC; 299 } 300 301 offset >>= 2; 302 insn->reg0i26_format.immediate_l = offset & 0xffff; 303 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff; 304 305 return 0; 306 } 307 308 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, 309 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 310 { 311 union loongarch_instruction *insn = (union loongarch_instruction *)location; 312 /* Use s32 for a sign-extension deliberately. */ 313 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - 314 (void *)((Elf_Addr)location & ~0xfff); 315 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; 316 ptrdiff_t offset_rem = (void *)v - (void *)anchor; 317 318 switch (type) { 319 case R_LARCH_PCALA_LO12: 320 insn->reg2i12_format.immediate = v & 0xfff; 321 break; 322 case R_LARCH_PCALA_HI20: 323 v = offset_hi20 >> 12; 324 insn->reg1i20_format.immediate = v & 0xfffff; 325 break; 326 case R_LARCH_PCALA64_LO20: 327 v = offset_rem >> 32; 328 insn->reg1i20_format.immediate = v & 0xfffff; 329 break; 330 case R_LARCH_PCALA64_HI12: 331 v = offset_rem >> 52; 332 insn->reg2i12_format.immediate = v & 0xfff; 333 break; 334 default: 335 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 336 return -EINVAL; 337 } 338 339 return 0; 340 } 341 342 static int apply_r_larch_got_pc(struct module *mod, u32 *location, Elf_Addr v, 343 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 344 { 345 Elf_Addr got = module_emit_got_entry(mod, v); 346 347 if (!got) 348 return -EINVAL; 349 350 switch (type) { 351 case R_LARCH_GOT_PC_LO12: 352 type = R_LARCH_PCALA_LO12; 353 break; 354 case R_LARCH_GOT_PC_HI20: 355 type = R_LARCH_PCALA_HI20; 356 break; 357 default: 358 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 359 return -EINVAL; 360 } 361 362 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); 363 } 364 365 /* 366 * reloc_handlers_rela() - Apply a particular relocation to a module 367 * @mod: the module to apply the reloc to 368 * @location: the address at which the reloc is to be applied 369 * @v: the value of the reloc, with addend for RELA-style 370 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module 371 * @rela_stac_top: where the stack operation(pop/push) applies to 372 * 373 * Return: 0 upon success, else -ERRNO 374 */ 375 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, 376 s64 *rela_stack, size_t *rela_stack_top, unsigned int type); 377 378 /* The handlers for known reloc types */ 379 static reloc_rela_handler reloc_rela_handlers[] = { 380 [R_LARCH_NONE ... R_LARCH_RELAX] = apply_r_larch_error, 381 382 [R_LARCH_NONE] = apply_r_larch_none, 383 [R_LARCH_32] = apply_r_larch_32, 384 [R_LARCH_64] = apply_r_larch_64, 385 [R_LARCH_MARK_LA] = apply_r_larch_none, 386 [R_LARCH_MARK_PCREL] = apply_r_larch_none, 387 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, 388 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, 389 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, 390 [R_LARCH_SOP_PUSH_PLT_PCREL] = apply_r_larch_sop_push_plt_pcrel, 391 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, 392 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, 393 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, 394 [R_LARCH_B26] = apply_r_larch_b26, 395 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, 396 [R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12] = apply_r_larch_got_pc, 397 }; 398 399 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 400 unsigned int symindex, unsigned int relsec, 401 struct module *mod) 402 { 403 int i, err; 404 unsigned int type; 405 s64 rela_stack[RELA_STACK_DEPTH]; 406 size_t rela_stack_top = 0; 407 reloc_rela_handler handler; 408 void *location; 409 Elf_Addr v; 410 Elf_Sym *sym; 411 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; 412 413 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, 414 sechdrs[relsec].sh_info); 415 416 rela_stack_top = 0; 417 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 418 /* This is where to make the change */ 419 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; 420 /* This is the symbol it is referring to */ 421 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); 422 if (IS_ERR_VALUE(sym->st_value)) { 423 /* Ignore unresolved weak symbol */ 424 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 425 continue; 426 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); 427 return -ENOENT; 428 } 429 430 type = ELF_R_TYPE(rel[i].r_info); 431 432 if (type < ARRAY_SIZE(reloc_rela_handlers)) 433 handler = reloc_rela_handlers[type]; 434 else 435 handler = NULL; 436 437 if (!handler) { 438 pr_err("%s: Unknown relocation type %u\n", mod->name, type); 439 return -EINVAL; 440 } 441 442 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", 443 (int)ELF_R_TYPE(rel[i].r_info), 444 sym->st_value, rel[i].r_addend, (u64)location); 445 446 v = sym->st_value + rel[i].r_addend; 447 err = handler(mod, location, v, rela_stack, &rela_stack_top, type); 448 if (err) 449 return err; 450 } 451 452 return 0; 453 } 454 455 void *module_alloc(unsigned long size) 456 { 457 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, 458 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); 459 } 460 461 int module_finalize(const Elf_Ehdr *hdr, 462 const Elf_Shdr *sechdrs, struct module *mod) 463 { 464 const Elf_Shdr *s, *se; 465 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 466 467 for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { 468 if (!strcmp(".altinstructions", secstrs + s->sh_name)) 469 apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size); 470 } 471 472 return 0; 473 } 474