1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Author: Hanlu Li <lihanlu@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 9 #define pr_fmt(fmt) "kmod: " fmt 10 11 #include <linux/moduleloader.h> 12 #include <linux/elf.h> 13 #include <linux/mm.h> 14 #include <linux/numa.h> 15 #include <linux/vmalloc.h> 16 #include <linux/slab.h> 17 #include <linux/fs.h> 18 #include <linux/string.h> 19 #include <linux/kernel.h> 20 21 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 22 { 23 if (*rela_stack_top >= RELA_STACK_DEPTH) 24 return -ENOEXEC; 25 26 rela_stack[(*rela_stack_top)++] = stack_value; 27 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); 28 29 return 0; 30 } 31 32 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) 33 { 34 if (*rela_stack_top == 0) 35 return -ENOEXEC; 36 37 *stack_value = rela_stack[--(*rela_stack_top)]; 38 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); 39 40 return 0; 41 } 42 43 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, 44 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 45 { 46 return 0; 47 } 48 49 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, 50 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 51 { 52 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); 53 return -EINVAL; 54 } 55 56 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, 57 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 58 { 59 *location = v; 60 return 0; 61 } 62 63 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, 64 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 65 { 66 *(Elf_Addr *)location = v; 67 return 0; 68 } 69 70 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, 71 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 72 { 73 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); 74 } 75 76 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, 77 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 78 { 79 return rela_stack_push(v, rela_stack, rela_stack_top); 80 } 81 82 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, 83 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 84 { 85 int err = 0; 86 s64 opr1; 87 88 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 89 if (err) 90 return err; 91 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 92 if (err) 93 return err; 94 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 95 if (err) 96 return err; 97 98 return 0; 99 } 100 101 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, u32 *location, Elf_Addr v, 102 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 103 { 104 ptrdiff_t offset = (void *)v - (void *)location; 105 106 if (offset >= SZ_128M) 107 v = module_emit_plt_entry(mod, v); 108 109 if (offset < -SZ_128M) 110 v = module_emit_plt_entry(mod, v); 111 112 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); 113 } 114 115 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, 116 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 117 { 118 int err = 0; 119 s64 opr1, opr2, opr3; 120 121 if (type == R_LARCH_SOP_IF_ELSE) { 122 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); 123 if (err) 124 return err; 125 } 126 127 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); 128 if (err) 129 return err; 130 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 131 if (err) 132 return err; 133 134 switch (type) { 135 case R_LARCH_SOP_AND: 136 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); 137 break; 138 case R_LARCH_SOP_ADD: 139 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); 140 break; 141 case R_LARCH_SOP_SUB: 142 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); 143 break; 144 case R_LARCH_SOP_SL: 145 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); 146 break; 147 case R_LARCH_SOP_SR: 148 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); 149 break; 150 case R_LARCH_SOP_IF_ELSE: 151 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); 152 break; 153 default: 154 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 155 return -EINVAL; 156 } 157 158 return err; 159 } 160 161 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, 162 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 163 { 164 int err = 0; 165 s64 opr1; 166 union loongarch_instruction *insn = (union loongarch_instruction *)location; 167 168 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 169 if (err) 170 return err; 171 172 switch (type) { 173 case R_LARCH_SOP_POP_32_U_10_12: 174 if (!unsigned_imm_check(opr1, 12)) 175 goto overflow; 176 177 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ 178 insn->reg2i12_format.immediate = opr1 & 0xfff; 179 return 0; 180 case R_LARCH_SOP_POP_32_S_10_12: 181 if (!signed_imm_check(opr1, 12)) 182 goto overflow; 183 184 insn->reg2i12_format.immediate = opr1 & 0xfff; 185 return 0; 186 case R_LARCH_SOP_POP_32_S_10_16: 187 if (!signed_imm_check(opr1, 16)) 188 goto overflow; 189 190 insn->reg2i16_format.immediate = opr1 & 0xffff; 191 return 0; 192 case R_LARCH_SOP_POP_32_S_10_16_S2: 193 if (opr1 % 4) 194 goto unaligned; 195 196 if (!signed_imm_check(opr1, 18)) 197 goto overflow; 198 199 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff; 200 return 0; 201 case R_LARCH_SOP_POP_32_S_5_20: 202 if (!signed_imm_check(opr1, 20)) 203 goto overflow; 204 205 insn->reg1i20_format.immediate = (opr1) & 0xfffff; 206 return 0; 207 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: 208 if (opr1 % 4) 209 goto unaligned; 210 211 if (!signed_imm_check(opr1, 23)) 212 goto overflow; 213 214 opr1 >>= 2; 215 insn->reg1i21_format.immediate_l = opr1 & 0xffff; 216 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f; 217 return 0; 218 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: 219 if (opr1 % 4) 220 goto unaligned; 221 222 if (!signed_imm_check(opr1, 28)) 223 goto overflow; 224 225 opr1 >>= 2; 226 insn->reg0i26_format.immediate_l = opr1 & 0xffff; 227 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff; 228 return 0; 229 case R_LARCH_SOP_POP_32_U: 230 if (!unsigned_imm_check(opr1, 32)) 231 goto overflow; 232 233 /* (*(uint32_t *) PC) = opr */ 234 *location = (u32)opr1; 235 return 0; 236 default: 237 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 238 return -EINVAL; 239 } 240 241 overflow: 242 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", 243 mod->name, opr1, __func__, type); 244 return -ENOEXEC; 245 246 unaligned: 247 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", 248 mod->name, opr1, __func__, type); 249 return -ENOEXEC; 250 } 251 252 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, 253 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 254 { 255 switch (type) { 256 case R_LARCH_ADD32: 257 *(s32 *)location += v; 258 return 0; 259 case R_LARCH_ADD64: 260 *(s64 *)location += v; 261 return 0; 262 case R_LARCH_SUB32: 263 *(s32 *)location -= v; 264 return 0; 265 case R_LARCH_SUB64: 266 *(s64 *)location -= v; 267 return 0; 268 default: 269 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 270 return -EINVAL; 271 } 272 } 273 274 static int apply_r_larch_b26(struct module *mod, u32 *location, Elf_Addr v, 275 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 276 { 277 ptrdiff_t offset = (void *)v - (void *)location; 278 union loongarch_instruction *insn = (union loongarch_instruction *)location; 279 280 if (offset >= SZ_128M) 281 v = module_emit_plt_entry(mod, v); 282 283 if (offset < -SZ_128M) 284 v = module_emit_plt_entry(mod, v); 285 286 offset = (void *)v - (void *)location; 287 288 if (offset & 3) { 289 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", 290 mod->name, (long long)offset, type); 291 return -ENOEXEC; 292 } 293 294 if (!signed_imm_check(offset, 28)) { 295 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", 296 mod->name, (long long)offset, type); 297 return -ENOEXEC; 298 } 299 300 offset >>= 2; 301 insn->reg0i26_format.immediate_l = offset & 0xffff; 302 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff; 303 304 return 0; 305 } 306 307 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, 308 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 309 { 310 union loongarch_instruction *insn = (union loongarch_instruction *)location; 311 /* Use s32 for a sign-extension deliberately. */ 312 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - 313 (void *)((Elf_Addr)location & ~0xfff); 314 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; 315 ptrdiff_t offset_rem = (void *)v - (void *)anchor; 316 317 switch (type) { 318 case R_LARCH_PCALA_LO12: 319 insn->reg2i12_format.immediate = v & 0xfff; 320 break; 321 case R_LARCH_PCALA_HI20: 322 v = offset_hi20 >> 12; 323 insn->reg1i20_format.immediate = v & 0xfffff; 324 break; 325 case R_LARCH_PCALA64_LO20: 326 v = offset_rem >> 32; 327 insn->reg1i20_format.immediate = v & 0xfffff; 328 break; 329 case R_LARCH_PCALA64_HI12: 330 v = offset_rem >> 52; 331 insn->reg2i12_format.immediate = v & 0xfff; 332 break; 333 default: 334 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 335 return -EINVAL; 336 } 337 338 return 0; 339 } 340 341 static int apply_r_larch_got_pc(struct module *mod, u32 *location, Elf_Addr v, 342 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 343 { 344 Elf_Addr got = module_emit_got_entry(mod, v); 345 346 if (!got) 347 return -EINVAL; 348 349 switch (type) { 350 case R_LARCH_GOT_PC_LO12: 351 type = R_LARCH_PCALA_LO12; 352 break; 353 case R_LARCH_GOT_PC_HI20: 354 type = R_LARCH_PCALA_HI20; 355 break; 356 default: 357 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 358 return -EINVAL; 359 } 360 361 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); 362 } 363 364 /* 365 * reloc_handlers_rela() - Apply a particular relocation to a module 366 * @mod: the module to apply the reloc to 367 * @location: the address at which the reloc is to be applied 368 * @v: the value of the reloc, with addend for RELA-style 369 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module 370 * @rela_stac_top: where the stack operation(pop/push) applies to 371 * 372 * Return: 0 upon success, else -ERRNO 373 */ 374 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, 375 s64 *rela_stack, size_t *rela_stack_top, unsigned int type); 376 377 /* The handlers for known reloc types */ 378 static reloc_rela_handler reloc_rela_handlers[] = { 379 [R_LARCH_NONE ... R_LARCH_RELAX] = apply_r_larch_error, 380 381 [R_LARCH_NONE] = apply_r_larch_none, 382 [R_LARCH_32] = apply_r_larch_32, 383 [R_LARCH_64] = apply_r_larch_64, 384 [R_LARCH_MARK_LA] = apply_r_larch_none, 385 [R_LARCH_MARK_PCREL] = apply_r_larch_none, 386 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, 387 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, 388 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, 389 [R_LARCH_SOP_PUSH_PLT_PCREL] = apply_r_larch_sop_push_plt_pcrel, 390 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, 391 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, 392 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, 393 [R_LARCH_B26] = apply_r_larch_b26, 394 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, 395 [R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12] = apply_r_larch_got_pc, 396 }; 397 398 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 399 unsigned int symindex, unsigned int relsec, 400 struct module *mod) 401 { 402 int i, err; 403 unsigned int type; 404 s64 rela_stack[RELA_STACK_DEPTH]; 405 size_t rela_stack_top = 0; 406 reloc_rela_handler handler; 407 void *location; 408 Elf_Addr v; 409 Elf_Sym *sym; 410 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; 411 412 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, 413 sechdrs[relsec].sh_info); 414 415 rela_stack_top = 0; 416 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 417 /* This is where to make the change */ 418 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; 419 /* This is the symbol it is referring to */ 420 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); 421 if (IS_ERR_VALUE(sym->st_value)) { 422 /* Ignore unresolved weak symbol */ 423 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 424 continue; 425 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); 426 return -ENOENT; 427 } 428 429 type = ELF_R_TYPE(rel[i].r_info); 430 431 if (type < ARRAY_SIZE(reloc_rela_handlers)) 432 handler = reloc_rela_handlers[type]; 433 else 434 handler = NULL; 435 436 if (!handler) { 437 pr_err("%s: Unknown relocation type %u\n", mod->name, type); 438 return -EINVAL; 439 } 440 441 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", 442 (int)ELF_R_TYPE(rel[i].r_info), 443 sym->st_value, rel[i].r_addend, (u64)location); 444 445 v = sym->st_value + rel[i].r_addend; 446 err = handler(mod, location, v, rela_stack, &rela_stack_top, type); 447 if (err) 448 return err; 449 } 450 451 return 0; 452 } 453 454 void *module_alloc(unsigned long size) 455 { 456 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, 457 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); 458 } 459