1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Author: Hanlu Li <lihanlu@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 9 #define pr_fmt(fmt) "kmod: " fmt 10 11 #include <linux/moduleloader.h> 12 #include <linux/elf.h> 13 #include <linux/mm.h> 14 #include <linux/numa.h> 15 #include <linux/vmalloc.h> 16 #include <linux/slab.h> 17 #include <linux/fs.h> 18 #include <linux/ftrace.h> 19 #include <linux/string.h> 20 #include <linux/kernel.h> 21 #include <asm/alternative.h> 22 #include <asm/inst.h> 23 rela_stack_push(s64 stack_value,s64 * rela_stack,size_t * rela_stack_top)24 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 25 { 26 if (*rela_stack_top >= RELA_STACK_DEPTH) 27 return -ENOEXEC; 28 29 rela_stack[(*rela_stack_top)++] = stack_value; 30 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); 31 32 return 0; 33 } 34 rela_stack_pop(s64 * stack_value,s64 * rela_stack,size_t * rela_stack_top)35 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) 36 { 37 if (*rela_stack_top == 0) 38 return -ENOEXEC; 39 40 *stack_value = rela_stack[--(*rela_stack_top)]; 41 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); 42 43 return 0; 44 } 45 apply_r_larch_none(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)46 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, 47 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 48 { 49 return 0; 50 } 51 apply_r_larch_error(struct module * me,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)52 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, 53 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 54 { 55 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); 56 return -EINVAL; 57 } 58 apply_r_larch_32(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)59 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, 60 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 61 { 62 *location = v; 63 return 0; 64 } 65 apply_r_larch_64(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)66 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, 67 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 68 { 69 *(Elf_Addr *)location = v; 70 return 0; 71 } 72 apply_r_larch_sop_push_pcrel(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)73 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, 74 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 75 { 76 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); 77 } 78 apply_r_larch_sop_push_absolute(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)79 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, 80 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 81 { 82 return rela_stack_push(v, rela_stack, rela_stack_top); 83 } 84 apply_r_larch_sop_push_dup(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)85 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, 86 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 87 { 88 int err = 0; 89 s64 opr1; 90 91 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 92 if (err) 93 return err; 94 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 95 if (err) 96 return err; 97 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 98 if (err) 99 return err; 100 101 return 0; 102 } 103 apply_r_larch_sop_push_plt_pcrel(struct module * mod,Elf_Shdr * sechdrs,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)104 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, 105 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 106 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 107 { 108 ptrdiff_t offset = (void *)v - (void *)location; 109 110 if (offset >= SZ_128M) 111 v = module_emit_plt_entry(mod, sechdrs, v); 112 113 if (offset < -SZ_128M) 114 v = module_emit_plt_entry(mod, sechdrs, v); 115 116 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); 117 } 118 apply_r_larch_sop(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)119 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, 120 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 121 { 122 int err = 0; 123 s64 opr1, opr2, opr3; 124 125 if (type == R_LARCH_SOP_IF_ELSE) { 126 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); 127 if (err) 128 return err; 129 } 130 131 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); 132 if (err) 133 return err; 134 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 135 if (err) 136 return err; 137 138 switch (type) { 139 case R_LARCH_SOP_AND: 140 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); 141 break; 142 case R_LARCH_SOP_ADD: 143 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); 144 break; 145 case R_LARCH_SOP_SUB: 146 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); 147 break; 148 case R_LARCH_SOP_SL: 149 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); 150 break; 151 case R_LARCH_SOP_SR: 152 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); 153 break; 154 case R_LARCH_SOP_IF_ELSE: 155 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); 156 break; 157 default: 158 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 159 return -EINVAL; 160 } 161 162 return err; 163 } 164 apply_r_larch_sop_imm_field(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)165 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, 166 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 167 { 168 int err = 0; 169 s64 opr1; 170 union loongarch_instruction *insn = (union loongarch_instruction *)location; 171 172 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 173 if (err) 174 return err; 175 176 switch (type) { 177 case R_LARCH_SOP_POP_32_U_10_12: 178 if (!unsigned_imm_check(opr1, 12)) 179 goto overflow; 180 181 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ 182 insn->reg2i12_format.immediate = opr1 & 0xfff; 183 return 0; 184 case R_LARCH_SOP_POP_32_S_10_12: 185 if (!signed_imm_check(opr1, 12)) 186 goto overflow; 187 188 insn->reg2i12_format.immediate = opr1 & 0xfff; 189 return 0; 190 case R_LARCH_SOP_POP_32_S_10_16: 191 if (!signed_imm_check(opr1, 16)) 192 goto overflow; 193 194 insn->reg2i16_format.immediate = opr1 & 0xffff; 195 return 0; 196 case R_LARCH_SOP_POP_32_S_10_16_S2: 197 if (opr1 % 4) 198 goto unaligned; 199 200 if (!signed_imm_check(opr1, 18)) 201 goto overflow; 202 203 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff; 204 return 0; 205 case R_LARCH_SOP_POP_32_S_5_20: 206 if (!signed_imm_check(opr1, 20)) 207 goto overflow; 208 209 insn->reg1i20_format.immediate = (opr1) & 0xfffff; 210 return 0; 211 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: 212 if (opr1 % 4) 213 goto unaligned; 214 215 if (!signed_imm_check(opr1, 23)) 216 goto overflow; 217 218 opr1 >>= 2; 219 insn->reg1i21_format.immediate_l = opr1 & 0xffff; 220 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f; 221 return 0; 222 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: 223 if (opr1 % 4) 224 goto unaligned; 225 226 if (!signed_imm_check(opr1, 28)) 227 goto overflow; 228 229 opr1 >>= 2; 230 insn->reg0i26_format.immediate_l = opr1 & 0xffff; 231 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff; 232 return 0; 233 case R_LARCH_SOP_POP_32_U: 234 if (!unsigned_imm_check(opr1, 32)) 235 goto overflow; 236 237 /* (*(uint32_t *) PC) = opr */ 238 *location = (u32)opr1; 239 return 0; 240 default: 241 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 242 return -EINVAL; 243 } 244 245 overflow: 246 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", 247 mod->name, opr1, __func__, type); 248 return -ENOEXEC; 249 250 unaligned: 251 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", 252 mod->name, opr1, __func__, type); 253 return -ENOEXEC; 254 } 255 apply_r_larch_add_sub(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)256 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, 257 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 258 { 259 switch (type) { 260 case R_LARCH_ADD32: 261 *(s32 *)location += v; 262 return 0; 263 case R_LARCH_ADD64: 264 *(s64 *)location += v; 265 return 0; 266 case R_LARCH_SUB32: 267 *(s32 *)location -= v; 268 return 0; 269 case R_LARCH_SUB64: 270 *(s64 *)location -= v; 271 return 0; 272 default: 273 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 274 return -EINVAL; 275 } 276 } 277 apply_r_larch_b26(struct module * mod,Elf_Shdr * sechdrs,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)278 static int apply_r_larch_b26(struct module *mod, 279 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 280 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 281 { 282 ptrdiff_t offset = (void *)v - (void *)location; 283 union loongarch_instruction *insn = (union loongarch_instruction *)location; 284 285 if (offset >= SZ_128M) 286 v = module_emit_plt_entry(mod, sechdrs, v); 287 288 if (offset < -SZ_128M) 289 v = module_emit_plt_entry(mod, sechdrs, v); 290 291 offset = (void *)v - (void *)location; 292 293 if (offset & 3) { 294 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", 295 mod->name, (long long)offset, type); 296 return -ENOEXEC; 297 } 298 299 if (!signed_imm_check(offset, 28)) { 300 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", 301 mod->name, (long long)offset, type); 302 return -ENOEXEC; 303 } 304 305 offset >>= 2; 306 insn->reg0i26_format.immediate_l = offset & 0xffff; 307 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff; 308 309 return 0; 310 } 311 apply_r_larch_pcala(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)312 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, 313 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 314 { 315 union loongarch_instruction *insn = (union loongarch_instruction *)location; 316 /* Use s32 for a sign-extension deliberately. */ 317 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - 318 (void *)((Elf_Addr)location & ~0xfff); 319 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; 320 ptrdiff_t offset_rem = (void *)v - (void *)anchor; 321 322 switch (type) { 323 case R_LARCH_PCALA_LO12: 324 insn->reg2i12_format.immediate = v & 0xfff; 325 break; 326 case R_LARCH_PCALA_HI20: 327 v = offset_hi20 >> 12; 328 insn->reg1i20_format.immediate = v & 0xfffff; 329 break; 330 case R_LARCH_PCALA64_LO20: 331 v = offset_rem >> 32; 332 insn->reg1i20_format.immediate = v & 0xfffff; 333 break; 334 case R_LARCH_PCALA64_HI12: 335 v = offset_rem >> 52; 336 insn->reg2i12_format.immediate = v & 0xfff; 337 break; 338 default: 339 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 340 return -EINVAL; 341 } 342 343 return 0; 344 } 345 apply_r_larch_got_pc(struct module * mod,Elf_Shdr * sechdrs,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)346 static int apply_r_larch_got_pc(struct module *mod, 347 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 348 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 349 { 350 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v); 351 352 if (!got) 353 return -EINVAL; 354 355 switch (type) { 356 case R_LARCH_GOT_PC_LO12: 357 type = R_LARCH_PCALA_LO12; 358 break; 359 case R_LARCH_GOT_PC_HI20: 360 type = R_LARCH_PCALA_HI20; 361 break; 362 default: 363 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 364 return -EINVAL; 365 } 366 367 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); 368 } 369 apply_r_larch_32_pcrel(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)370 static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v, 371 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 372 { 373 ptrdiff_t offset = (void *)v - (void *)location; 374 375 *(u32 *)location = offset; 376 return 0; 377 } 378 apply_r_larch_64_pcrel(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)379 static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v, 380 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 381 { 382 ptrdiff_t offset = (void *)v - (void *)location; 383 384 *(u64 *)location = offset; 385 return 0; 386 } 387 388 /* 389 * reloc_handlers_rela() - Apply a particular relocation to a module 390 * @mod: the module to apply the reloc to 391 * @location: the address at which the reloc is to be applied 392 * @v: the value of the reloc, with addend for RELA-style 393 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module 394 * @rela_stac_top: where the stack operation(pop/push) applies to 395 * 396 * Return: 0 upon success, else -ERRNO 397 */ 398 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, 399 s64 *rela_stack, size_t *rela_stack_top, unsigned int type); 400 401 /* The handlers for known reloc types */ 402 static reloc_rela_handler reloc_rela_handlers[] = { 403 [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error, 404 405 [R_LARCH_NONE] = apply_r_larch_none, 406 [R_LARCH_32] = apply_r_larch_32, 407 [R_LARCH_64] = apply_r_larch_64, 408 [R_LARCH_MARK_LA] = apply_r_larch_none, 409 [R_LARCH_MARK_PCREL] = apply_r_larch_none, 410 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, 411 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, 412 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, 413 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, 414 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, 415 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, 416 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, 417 [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel, 418 [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel, 419 }; 420 apply_relocate_add(Elf_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * mod)421 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 422 unsigned int symindex, unsigned int relsec, 423 struct module *mod) 424 { 425 int i, err; 426 unsigned int type; 427 s64 rela_stack[RELA_STACK_DEPTH]; 428 size_t rela_stack_top = 0; 429 reloc_rela_handler handler; 430 void *location; 431 Elf_Addr v; 432 Elf_Sym *sym; 433 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; 434 435 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, 436 sechdrs[relsec].sh_info); 437 438 rela_stack_top = 0; 439 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 440 /* This is where to make the change */ 441 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; 442 /* This is the symbol it is referring to */ 443 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); 444 if (IS_ERR_VALUE(sym->st_value)) { 445 /* Ignore unresolved weak symbol */ 446 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 447 continue; 448 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); 449 return -ENOENT; 450 } 451 452 type = ELF_R_TYPE(rel[i].r_info); 453 454 if (type < ARRAY_SIZE(reloc_rela_handlers)) 455 handler = reloc_rela_handlers[type]; 456 else 457 handler = NULL; 458 459 if (!handler) { 460 pr_err("%s: Unknown relocation type %u\n", mod->name, type); 461 return -EINVAL; 462 } 463 464 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", 465 (int)ELF_R_TYPE(rel[i].r_info), 466 sym->st_value, rel[i].r_addend, (u64)location); 467 468 v = sym->st_value + rel[i].r_addend; 469 switch (type) { 470 case R_LARCH_B26: 471 err = apply_r_larch_b26(mod, sechdrs, location, 472 v, rela_stack, &rela_stack_top, type); 473 break; 474 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12: 475 err = apply_r_larch_got_pc(mod, sechdrs, location, 476 v, rela_stack, &rela_stack_top, type); 477 break; 478 case R_LARCH_SOP_PUSH_PLT_PCREL: 479 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location, 480 v, rela_stack, &rela_stack_top, type); 481 break; 482 default: 483 err = handler(mod, location, v, rela_stack, &rela_stack_top, type); 484 } 485 if (err) 486 return err; 487 } 488 489 return 0; 490 } 491 module_alloc(unsigned long size)492 void *module_alloc(unsigned long size) 493 { 494 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, 495 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); 496 } 497 module_init_ftrace_plt(const Elf_Ehdr * hdr,const Elf_Shdr * sechdrs,struct module * mod)498 static void module_init_ftrace_plt(const Elf_Ehdr *hdr, 499 const Elf_Shdr *sechdrs, struct module *mod) 500 { 501 #ifdef CONFIG_DYNAMIC_FTRACE 502 struct plt_entry *ftrace_plts; 503 504 ftrace_plts = (void *)sechdrs->sh_addr; 505 506 ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR); 507 508 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) 509 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR); 510 511 mod->arch.ftrace_trampolines = ftrace_plts; 512 #endif 513 } 514 module_finalize(const Elf_Ehdr * hdr,const Elf_Shdr * sechdrs,struct module * mod)515 int module_finalize(const Elf_Ehdr *hdr, 516 const Elf_Shdr *sechdrs, struct module *mod) 517 { 518 const Elf_Shdr *s, *se; 519 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 520 521 for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { 522 if (!strcmp(".altinstructions", secstrs + s->sh_name)) 523 apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size); 524 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name)) 525 module_init_ftrace_plt(hdr, s, mod); 526 } 527 528 return 0; 529 } 530