1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * IA-64-specific support for kernel module loader. 4 * 5 * Copyright (C) 2003 Hewlett-Packard Co 6 * David Mosberger-Tang <davidm@hpl.hp.com> 7 * 8 * Loosely based on patch by Rusty Russell. 9 */ 10 11 /* relocs tested so far: 12 13 DIR64LSB 14 FPTR64LSB 15 GPREL22 16 LDXMOV 17 LDXMOV 18 LTOFF22 19 LTOFF22X 20 LTOFF22X 21 LTOFF_FPTR22 22 PCREL21B (for br.call only; br.cond is not supported out of modules!) 23 PCREL60B (for brl.cond only; brl.call is not supported for modules!) 24 PCREL64LSB 25 SECREL32LSB 26 SEGREL64LSB 27 */ 28 29 30 #include <linux/kernel.h> 31 #include <linux/sched.h> 32 #include <linux/elf.h> 33 #include <linux/moduleloader.h> 34 #include <linux/string.h> 35 #include <linux/vmalloc.h> 36 37 #include <asm/patch.h> 38 #include <asm/unaligned.h> 39 40 #define ARCH_MODULE_DEBUG 0 41 42 #if ARCH_MODULE_DEBUG 43 # define DEBUGP printk 44 # define inline 45 #else 46 # define DEBUGP(fmt , a...) 47 #endif 48 49 #ifdef CONFIG_ITANIUM 50 # define USE_BRL 0 51 #else 52 # define USE_BRL 1 53 #endif 54 55 #define MAX_LTOFF ((uint64_t) (1 << 22)) /* max. allowable linkage-table offset */ 56 57 /* Define some relocation helper macros/types: */ 58 59 #define FORMAT_SHIFT 0 60 #define FORMAT_BITS 3 61 #define FORMAT_MASK ((1 << FORMAT_BITS) - 1) 62 #define VALUE_SHIFT 3 63 #define VALUE_BITS 5 64 #define VALUE_MASK ((1 << VALUE_BITS) - 1) 65 66 enum reloc_target_format { 67 /* direct encoded formats: */ 68 RF_NONE = 0, 69 RF_INSN14 = 1, 70 RF_INSN22 = 2, 71 RF_INSN64 = 3, 72 RF_32MSB = 4, 73 RF_32LSB = 5, 74 RF_64MSB = 6, 75 RF_64LSB = 7, 76 77 /* formats that cannot be directly decoded: */ 78 RF_INSN60, 79 RF_INSN21B, /* imm21 form 1 */ 80 RF_INSN21M, /* imm21 form 2 */ 81 RF_INSN21F /* imm21 form 3 */ 82 }; 83 84 enum reloc_value_formula { 85 RV_DIRECT = 4, /* S + A */ 86 RV_GPREL = 5, /* @gprel(S + A) */ 87 RV_LTREL = 6, /* @ltoff(S + A) */ 88 RV_PLTREL = 7, /* @pltoff(S + A) */ 89 RV_FPTR = 8, /* @fptr(S + A) */ 90 RV_PCREL = 9, /* S + A - P */ 91 RV_LTREL_FPTR = 10, /* @ltoff(@fptr(S + A)) */ 92 RV_SEGREL = 11, /* @segrel(S + A) */ 93 RV_SECREL = 12, /* @secrel(S + A) */ 94 RV_BDREL = 13, /* BD + A */ 95 RV_LTV = 14, /* S + A (like RV_DIRECT, except frozen at static link-time) */ 96 RV_PCREL2 = 15, /* S + A - P */ 97 RV_SPECIAL = 16, /* various (see below) */ 98 RV_RSVD17 = 17, 99 RV_TPREL = 18, /* @tprel(S + A) */ 100 RV_LTREL_TPREL = 19, /* @ltoff(@tprel(S + A)) */ 101 RV_DTPMOD = 20, /* @dtpmod(S + A) */ 102 RV_LTREL_DTPMOD = 21, /* @ltoff(@dtpmod(S + A)) */ 103 RV_DTPREL = 22, /* @dtprel(S + A) */ 104 RV_LTREL_DTPREL = 23, /* @ltoff(@dtprel(S + A)) */ 105 RV_RSVD24 = 24, 106 RV_RSVD25 = 25, 107 RV_RSVD26 = 26, 108 RV_RSVD27 = 27 109 /* 28-31 reserved for implementation-specific purposes. */ 110 }; 111 112 #define N(reloc) [R_IA64_##reloc] = #reloc 113 114 static const char *reloc_name[256] = { 115 N(NONE), N(IMM14), N(IMM22), N(IMM64), 116 N(DIR32MSB), N(DIR32LSB), N(DIR64MSB), N(DIR64LSB), 117 N(GPREL22), N(GPREL64I), N(GPREL32MSB), N(GPREL32LSB), 118 N(GPREL64MSB), N(GPREL64LSB), N(LTOFF22), N(LTOFF64I), 119 N(PLTOFF22), N(PLTOFF64I), N(PLTOFF64MSB), N(PLTOFF64LSB), 120 N(FPTR64I), N(FPTR32MSB), N(FPTR32LSB), N(FPTR64MSB), 121 N(FPTR64LSB), N(PCREL60B), N(PCREL21B), N(PCREL21M), 122 N(PCREL21F), N(PCREL32MSB), N(PCREL32LSB), N(PCREL64MSB), 123 N(PCREL64LSB), N(LTOFF_FPTR22), N(LTOFF_FPTR64I), N(LTOFF_FPTR32MSB), 124 N(LTOFF_FPTR32LSB), N(LTOFF_FPTR64MSB), N(LTOFF_FPTR64LSB), N(SEGREL32MSB), 125 N(SEGREL32LSB), N(SEGREL64MSB), N(SEGREL64LSB), N(SECREL32MSB), 126 N(SECREL32LSB), N(SECREL64MSB), N(SECREL64LSB), N(REL32MSB), 127 N(REL32LSB), N(REL64MSB), N(REL64LSB), N(LTV32MSB), 128 N(LTV32LSB), N(LTV64MSB), N(LTV64LSB), N(PCREL21BI), 129 N(PCREL22), N(PCREL64I), N(IPLTMSB), N(IPLTLSB), 130 N(COPY), N(LTOFF22X), N(LDXMOV), N(TPREL14), 131 N(TPREL22), N(TPREL64I), N(TPREL64MSB), N(TPREL64LSB), 132 N(LTOFF_TPREL22), N(DTPMOD64MSB), N(DTPMOD64LSB), N(LTOFF_DTPMOD22), 133 N(DTPREL14), N(DTPREL22), N(DTPREL64I), N(DTPREL32MSB), 134 N(DTPREL32LSB), N(DTPREL64MSB), N(DTPREL64LSB), N(LTOFF_DTPREL22) 135 }; 136 137 #undef N 138 139 /* Opaque struct for insns, to protect against derefs. */ 140 struct insn; 141 142 static inline uint64_t 143 bundle (const struct insn *insn) 144 { 145 return (uint64_t) insn & ~0xfUL; 146 } 147 148 static inline int 149 slot (const struct insn *insn) 150 { 151 return (uint64_t) insn & 0x3; 152 } 153 154 static int 155 apply_imm64 (struct module *mod, struct insn *insn, uint64_t val) 156 { 157 if (slot(insn) != 1 && slot(insn) != 2) { 158 printk(KERN_ERR "%s: invalid slot number %d for IMM64\n", 159 mod->name, slot(insn)); 160 return 0; 161 } 162 ia64_patch_imm64((u64) insn, val); 163 return 1; 164 } 165 166 static int 167 apply_imm60 (struct module *mod, struct insn *insn, uint64_t val) 168 { 169 if (slot(insn) != 1 && slot(insn) != 2) { 170 printk(KERN_ERR "%s: invalid slot number %d for IMM60\n", 171 mod->name, slot(insn)); 172 return 0; 173 } 174 if (val + ((uint64_t) 1 << 59) >= (1UL << 60)) { 175 printk(KERN_ERR "%s: value %ld out of IMM60 range\n", 176 mod->name, (long) val); 177 return 0; 178 } 179 ia64_patch_imm60((u64) insn, val); 180 return 1; 181 } 182 183 static int 184 apply_imm22 (struct module *mod, struct insn *insn, uint64_t val) 185 { 186 if (val + (1 << 21) >= (1 << 22)) { 187 printk(KERN_ERR "%s: value %li out of IMM22 range\n", 188 mod->name, (long)val); 189 return 0; 190 } 191 ia64_patch((u64) insn, 0x01fffcfe000UL, ( ((val & 0x200000UL) << 15) /* bit 21 -> 36 */ 192 | ((val & 0x1f0000UL) << 6) /* bit 16 -> 22 */ 193 | ((val & 0x00ff80UL) << 20) /* bit 7 -> 27 */ 194 | ((val & 0x00007fUL) << 13) /* bit 0 -> 13 */)); 195 return 1; 196 } 197 198 static int 199 apply_imm21b (struct module *mod, struct insn *insn, uint64_t val) 200 { 201 if (val + (1 << 20) >= (1 << 21)) { 202 printk(KERN_ERR "%s: value %li out of IMM21b range\n", 203 mod->name, (long)val); 204 return 0; 205 } 206 ia64_patch((u64) insn, 0x11ffffe000UL, ( ((val & 0x100000UL) << 16) /* bit 20 -> 36 */ 207 | ((val & 0x0fffffUL) << 13) /* bit 0 -> 13 */)); 208 return 1; 209 } 210 211 #if USE_BRL 212 213 struct plt_entry { 214 /* Three instruction bundles in PLT. */ 215 unsigned char bundle[2][16]; 216 }; 217 218 static const struct plt_entry ia64_plt_template = { 219 { 220 { 221 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 222 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */ 223 0x00, 0x00, 0x00, 0x60 224 }, 225 { 226 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 227 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* brl.many gp=TARGET_GP */ 228 0x08, 0x00, 0x00, 0xc0 229 } 230 } 231 }; 232 233 static int 234 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp) 235 { 236 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_gp) 237 && apply_imm60(mod, (struct insn *) (plt->bundle[1] + 2), 238 (target_ip - (int64_t) plt->bundle[1]) / 16)) 239 return 1; 240 return 0; 241 } 242 243 unsigned long 244 plt_target (struct plt_entry *plt) 245 { 246 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[1]; 247 long off; 248 249 b0 = b[0]; b1 = b[1]; 250 off = ( ((b1 & 0x00fffff000000000UL) >> 36) /* imm20b -> bit 0 */ 251 | ((b0 >> 48) << 20) | ((b1 & 0x7fffffUL) << 36) /* imm39 -> bit 20 */ 252 | ((b1 & 0x0800000000000000UL) << 0)); /* i -> bit 59 */ 253 return (long) plt->bundle[1] + 16*off; 254 } 255 256 #else /* !USE_BRL */ 257 258 struct plt_entry { 259 /* Three instruction bundles in PLT. */ 260 unsigned char bundle[3][16]; 261 }; 262 263 static const struct plt_entry ia64_plt_template = { 264 { 265 { 266 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 267 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* movl r16=TARGET_IP */ 268 0x02, 0x00, 0x00, 0x60 269 }, 270 { 271 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 272 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */ 273 0x00, 0x00, 0x00, 0x60 274 }, 275 { 276 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MIB] nop.m 0 */ 277 0x60, 0x80, 0x04, 0x80, 0x03, 0x00, /* mov b6=r16 */ 278 0x60, 0x00, 0x80, 0x00 /* br.few b6 */ 279 } 280 } 281 }; 282 283 static int 284 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp) 285 { 286 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_ip) 287 && apply_imm64(mod, (struct insn *) (plt->bundle[1] + 2), target_gp)) 288 return 1; 289 return 0; 290 } 291 292 unsigned long 293 plt_target (struct plt_entry *plt) 294 { 295 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[0]; 296 297 b0 = b[0]; b1 = b[1]; 298 return ( ((b1 & 0x000007f000000000) >> 36) /* imm7b -> bit 0 */ 299 | ((b1 & 0x07fc000000000000) >> 43) /* imm9d -> bit 7 */ 300 | ((b1 & 0x0003e00000000000) >> 29) /* imm5c -> bit 16 */ 301 | ((b1 & 0x0000100000000000) >> 23) /* ic -> bit 21 */ 302 | ((b0 >> 46) << 22) | ((b1 & 0x7fffff) << 40) /* imm41 -> bit 22 */ 303 | ((b1 & 0x0800000000000000) << 4)); /* i -> bit 63 */ 304 } 305 306 #endif /* !USE_BRL */ 307 308 void 309 module_arch_freeing_init (struct module *mod) 310 { 311 if (mod->arch.init_unw_table) { 312 unw_remove_unwind_table(mod->arch.init_unw_table); 313 mod->arch.init_unw_table = NULL; 314 } 315 } 316 317 /* Have we already seen one of these relocations? */ 318 /* FIXME: we could look in other sections, too --RR */ 319 static int 320 duplicate_reloc (const Elf64_Rela *rela, unsigned int num) 321 { 322 unsigned int i; 323 324 for (i = 0; i < num; i++) { 325 if (rela[i].r_info == rela[num].r_info && rela[i].r_addend == rela[num].r_addend) 326 return 1; 327 } 328 return 0; 329 } 330 331 /* Count how many GOT entries we may need */ 332 static unsigned int 333 count_gots (const Elf64_Rela *rela, unsigned int num) 334 { 335 unsigned int i, ret = 0; 336 337 /* Sure, this is order(n^2), but it's usually short, and not 338 time critical */ 339 for (i = 0; i < num; i++) { 340 switch (ELF64_R_TYPE(rela[i].r_info)) { 341 case R_IA64_LTOFF22: 342 case R_IA64_LTOFF22X: 343 case R_IA64_LTOFF64I: 344 case R_IA64_LTOFF_FPTR22: 345 case R_IA64_LTOFF_FPTR64I: 346 case R_IA64_LTOFF_FPTR32MSB: 347 case R_IA64_LTOFF_FPTR32LSB: 348 case R_IA64_LTOFF_FPTR64MSB: 349 case R_IA64_LTOFF_FPTR64LSB: 350 if (!duplicate_reloc(rela, i)) 351 ret++; 352 break; 353 } 354 } 355 return ret; 356 } 357 358 /* Count how many PLT entries we may need */ 359 static unsigned int 360 count_plts (const Elf64_Rela *rela, unsigned int num) 361 { 362 unsigned int i, ret = 0; 363 364 /* Sure, this is order(n^2), but it's usually short, and not 365 time critical */ 366 for (i = 0; i < num; i++) { 367 switch (ELF64_R_TYPE(rela[i].r_info)) { 368 case R_IA64_PCREL21B: 369 case R_IA64_PLTOFF22: 370 case R_IA64_PLTOFF64I: 371 case R_IA64_PLTOFF64MSB: 372 case R_IA64_PLTOFF64LSB: 373 case R_IA64_IPLTMSB: 374 case R_IA64_IPLTLSB: 375 if (!duplicate_reloc(rela, i)) 376 ret++; 377 break; 378 } 379 } 380 return ret; 381 } 382 383 /* We need to create an function-descriptors for any internal function 384 which is referenced. */ 385 static unsigned int 386 count_fdescs (const Elf64_Rela *rela, unsigned int num) 387 { 388 unsigned int i, ret = 0; 389 390 /* Sure, this is order(n^2), but it's usually short, and not time critical. */ 391 for (i = 0; i < num; i++) { 392 switch (ELF64_R_TYPE(rela[i].r_info)) { 393 case R_IA64_FPTR64I: 394 case R_IA64_FPTR32LSB: 395 case R_IA64_FPTR32MSB: 396 case R_IA64_FPTR64LSB: 397 case R_IA64_FPTR64MSB: 398 case R_IA64_LTOFF_FPTR22: 399 case R_IA64_LTOFF_FPTR32LSB: 400 case R_IA64_LTOFF_FPTR32MSB: 401 case R_IA64_LTOFF_FPTR64I: 402 case R_IA64_LTOFF_FPTR64LSB: 403 case R_IA64_LTOFF_FPTR64MSB: 404 case R_IA64_IPLTMSB: 405 case R_IA64_IPLTLSB: 406 /* 407 * Jumps to static functions sometimes go straight to their 408 * offset. Of course, that may not be possible if the jump is 409 * from init -> core or vice. versa, so we need to generate an 410 * FDESC (and PLT etc) for that. 411 */ 412 case R_IA64_PCREL21B: 413 if (!duplicate_reloc(rela, i)) 414 ret++; 415 break; 416 } 417 } 418 return ret; 419 } 420 421 int 422 module_frob_arch_sections (Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings, 423 struct module *mod) 424 { 425 unsigned long core_plts = 0, init_plts = 0, gots = 0, fdescs = 0; 426 Elf64_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum; 427 428 /* 429 * To store the PLTs and function-descriptors, we expand the .text section for 430 * core module-code and the .init.text section for initialization code. 431 */ 432 for (s = sechdrs; s < sechdrs_end; ++s) 433 if (strcmp(".core.plt", secstrings + s->sh_name) == 0) 434 mod->arch.core_plt = s; 435 else if (strcmp(".init.plt", secstrings + s->sh_name) == 0) 436 mod->arch.init_plt = s; 437 else if (strcmp(".got", secstrings + s->sh_name) == 0) 438 mod->arch.got = s; 439 else if (strcmp(".opd", secstrings + s->sh_name) == 0) 440 mod->arch.opd = s; 441 else if (strcmp(".IA_64.unwind", secstrings + s->sh_name) == 0) 442 mod->arch.unwind = s; 443 444 if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) { 445 printk(KERN_ERR "%s: sections missing\n", mod->name); 446 return -ENOEXEC; 447 } 448 449 /* GOT and PLTs can occur in any relocated section... */ 450 for (s = sechdrs + 1; s < sechdrs_end; ++s) { 451 const Elf64_Rela *rels = (void *)ehdr + s->sh_offset; 452 unsigned long numrels = s->sh_size/sizeof(Elf64_Rela); 453 454 if (s->sh_type != SHT_RELA) 455 continue; 456 457 gots += count_gots(rels, numrels); 458 fdescs += count_fdescs(rels, numrels); 459 if (strstr(secstrings + s->sh_name, ".init")) 460 init_plts += count_plts(rels, numrels); 461 else 462 core_plts += count_plts(rels, numrels); 463 } 464 465 mod->arch.core_plt->sh_type = SHT_NOBITS; 466 mod->arch.core_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC; 467 mod->arch.core_plt->sh_addralign = 16; 468 mod->arch.core_plt->sh_size = core_plts * sizeof(struct plt_entry); 469 mod->arch.init_plt->sh_type = SHT_NOBITS; 470 mod->arch.init_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC; 471 mod->arch.init_plt->sh_addralign = 16; 472 mod->arch.init_plt->sh_size = init_plts * sizeof(struct plt_entry); 473 mod->arch.got->sh_type = SHT_NOBITS; 474 mod->arch.got->sh_flags = ARCH_SHF_SMALL | SHF_ALLOC; 475 mod->arch.got->sh_addralign = 8; 476 mod->arch.got->sh_size = gots * sizeof(struct got_entry); 477 mod->arch.opd->sh_type = SHT_NOBITS; 478 mod->arch.opd->sh_flags = SHF_ALLOC; 479 mod->arch.opd->sh_addralign = 8; 480 mod->arch.opd->sh_size = fdescs * sizeof(struct fdesc); 481 DEBUGP("%s: core.plt=%lx, init.plt=%lx, got=%lx, fdesc=%lx\n", 482 __func__, mod->arch.core_plt->sh_size, mod->arch.init_plt->sh_size, 483 mod->arch.got->sh_size, mod->arch.opd->sh_size); 484 return 0; 485 } 486 487 static inline int 488 in_init (const struct module *mod, uint64_t addr) 489 { 490 return addr - (uint64_t) mod->init_layout.base < mod->init_layout.size; 491 } 492 493 static inline int 494 in_core (const struct module *mod, uint64_t addr) 495 { 496 return addr - (uint64_t) mod->core_layout.base < mod->core_layout.size; 497 } 498 499 static inline int 500 is_internal (const struct module *mod, uint64_t value) 501 { 502 return in_init(mod, value) || in_core(mod, value); 503 } 504 505 /* 506 * Get gp-relative offset for the linkage-table entry of VALUE. 507 */ 508 static uint64_t 509 get_ltoff (struct module *mod, uint64_t value, int *okp) 510 { 511 struct got_entry *got, *e; 512 513 if (!*okp) 514 return 0; 515 516 got = (void *) mod->arch.got->sh_addr; 517 for (e = got; e < got + mod->arch.next_got_entry; ++e) 518 if (e->val == value) 519 goto found; 520 521 /* Not enough GOT entries? */ 522 BUG_ON(e >= (struct got_entry *) (mod->arch.got->sh_addr + mod->arch.got->sh_size)); 523 524 e->val = value; 525 ++mod->arch.next_got_entry; 526 found: 527 return (uint64_t) e - mod->arch.gp; 528 } 529 530 static inline int 531 gp_addressable (struct module *mod, uint64_t value) 532 { 533 return value - mod->arch.gp + MAX_LTOFF/2 < MAX_LTOFF; 534 } 535 536 /* Get PC-relative PLT entry for this value. Returns 0 on failure. */ 537 static uint64_t 538 get_plt (struct module *mod, const struct insn *insn, uint64_t value, int *okp) 539 { 540 struct plt_entry *plt, *plt_end; 541 uint64_t target_ip, target_gp; 542 543 if (!*okp) 544 return 0; 545 546 if (in_init(mod, (uint64_t) insn)) { 547 plt = (void *) mod->arch.init_plt->sh_addr; 548 plt_end = (void *) plt + mod->arch.init_plt->sh_size; 549 } else { 550 plt = (void *) mod->arch.core_plt->sh_addr; 551 plt_end = (void *) plt + mod->arch.core_plt->sh_size; 552 } 553 554 /* "value" is a pointer to a function-descriptor; fetch the target ip/gp from it: */ 555 target_ip = ((uint64_t *) value)[0]; 556 target_gp = ((uint64_t *) value)[1]; 557 558 /* Look for existing PLT entry. */ 559 while (plt->bundle[0][0]) { 560 if (plt_target(plt) == target_ip) 561 goto found; 562 if (++plt >= plt_end) 563 BUG(); 564 } 565 *plt = ia64_plt_template; 566 if (!patch_plt(mod, plt, target_ip, target_gp)) { 567 *okp = 0; 568 return 0; 569 } 570 #if ARCH_MODULE_DEBUG 571 if (plt_target(plt) != target_ip) { 572 printk("%s: mistargeted PLT: wanted %lx, got %lx\n", 573 __func__, target_ip, plt_target(plt)); 574 *okp = 0; 575 return 0; 576 } 577 #endif 578 found: 579 return (uint64_t) plt; 580 } 581 582 /* Get function descriptor for VALUE. */ 583 static uint64_t 584 get_fdesc (struct module *mod, uint64_t value, int *okp) 585 { 586 struct fdesc *fdesc = (void *) mod->arch.opd->sh_addr; 587 588 if (!*okp) 589 return 0; 590 591 if (!value) { 592 printk(KERN_ERR "%s: fdesc for zero requested!\n", mod->name); 593 return 0; 594 } 595 596 if (!is_internal(mod, value)) 597 /* 598 * If it's not a module-local entry-point, "value" already points to a 599 * function-descriptor. 600 */ 601 return value; 602 603 /* Look for existing function descriptor. */ 604 while (fdesc->ip) { 605 if (fdesc->ip == value) 606 return (uint64_t)fdesc; 607 if ((uint64_t) ++fdesc >= mod->arch.opd->sh_addr + mod->arch.opd->sh_size) 608 BUG(); 609 } 610 611 /* Create new one */ 612 fdesc->ip = value; 613 fdesc->gp = mod->arch.gp; 614 return (uint64_t) fdesc; 615 } 616 617 static inline int 618 do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend, 619 Elf64_Shdr *sec, void *location) 620 { 621 enum reloc_target_format format = (r_type >> FORMAT_SHIFT) & FORMAT_MASK; 622 enum reloc_value_formula formula = (r_type >> VALUE_SHIFT) & VALUE_MASK; 623 uint64_t val; 624 int ok = 1; 625 626 val = sym->st_value + addend; 627 628 switch (formula) { 629 case RV_SEGREL: /* segment base is arbitrarily chosen to be 0 for kernel modules */ 630 case RV_DIRECT: 631 break; 632 633 case RV_GPREL: val -= mod->arch.gp; break; 634 case RV_LTREL: val = get_ltoff(mod, val, &ok); break; 635 case RV_PLTREL: val = get_plt(mod, location, val, &ok); break; 636 case RV_FPTR: val = get_fdesc(mod, val, &ok); break; 637 case RV_SECREL: val -= sec->sh_addr; break; 638 case RV_LTREL_FPTR: val = get_ltoff(mod, get_fdesc(mod, val, &ok), &ok); break; 639 640 case RV_PCREL: 641 switch (r_type) { 642 case R_IA64_PCREL21B: 643 if ((in_init(mod, val) && in_core(mod, (uint64_t)location)) || 644 (in_core(mod, val) && in_init(mod, (uint64_t)location))) { 645 /* 646 * Init section may have been allocated far away from core, 647 * if the branch won't reach, then allocate a plt for it. 648 */ 649 uint64_t delta = ((int64_t)val - (int64_t)location) / 16; 650 if (delta + (1 << 20) >= (1 << 21)) { 651 val = get_fdesc(mod, val, &ok); 652 val = get_plt(mod, location, val, &ok); 653 } 654 } else if (!is_internal(mod, val)) 655 val = get_plt(mod, location, val, &ok); 656 /* FALL THROUGH */ 657 default: 658 val -= bundle(location); 659 break; 660 661 case R_IA64_PCREL32MSB: 662 case R_IA64_PCREL32LSB: 663 case R_IA64_PCREL64MSB: 664 case R_IA64_PCREL64LSB: 665 val -= (uint64_t) location; 666 break; 667 668 } 669 switch (r_type) { 670 case R_IA64_PCREL60B: format = RF_INSN60; break; 671 case R_IA64_PCREL21B: format = RF_INSN21B; break; 672 case R_IA64_PCREL21M: format = RF_INSN21M; break; 673 case R_IA64_PCREL21F: format = RF_INSN21F; break; 674 default: break; 675 } 676 break; 677 678 case RV_BDREL: 679 val -= (uint64_t) (in_init(mod, val) ? mod->init_layout.base : mod->core_layout.base); 680 break; 681 682 case RV_LTV: 683 /* can link-time value relocs happen here? */ 684 BUG(); 685 break; 686 687 case RV_PCREL2: 688 if (r_type == R_IA64_PCREL21BI) { 689 if (!is_internal(mod, val)) { 690 printk(KERN_ERR "%s: %s reloc against " 691 "non-local symbol (%lx)\n", __func__, 692 reloc_name[r_type], (unsigned long)val); 693 return -ENOEXEC; 694 } 695 format = RF_INSN21B; 696 } 697 val -= bundle(location); 698 break; 699 700 case RV_SPECIAL: 701 switch (r_type) { 702 case R_IA64_IPLTMSB: 703 case R_IA64_IPLTLSB: 704 val = get_fdesc(mod, get_plt(mod, location, val, &ok), &ok); 705 format = RF_64LSB; 706 if (r_type == R_IA64_IPLTMSB) 707 format = RF_64MSB; 708 break; 709 710 case R_IA64_SUB: 711 val = addend - sym->st_value; 712 format = RF_INSN64; 713 break; 714 715 case R_IA64_LTOFF22X: 716 if (gp_addressable(mod, val)) 717 val -= mod->arch.gp; 718 else 719 val = get_ltoff(mod, val, &ok); 720 format = RF_INSN22; 721 break; 722 723 case R_IA64_LDXMOV: 724 if (gp_addressable(mod, val)) { 725 /* turn "ld8" into "mov": */ 726 DEBUGP("%s: patching ld8 at %p to mov\n", __func__, location); 727 ia64_patch((u64) location, 0x1fff80fe000UL, 0x10000000000UL); 728 } 729 return 0; 730 731 default: 732 if (reloc_name[r_type]) 733 printk(KERN_ERR "%s: special reloc %s not supported", 734 mod->name, reloc_name[r_type]); 735 else 736 printk(KERN_ERR "%s: unknown special reloc %x\n", 737 mod->name, r_type); 738 return -ENOEXEC; 739 } 740 break; 741 742 case RV_TPREL: 743 case RV_LTREL_TPREL: 744 case RV_DTPMOD: 745 case RV_LTREL_DTPMOD: 746 case RV_DTPREL: 747 case RV_LTREL_DTPREL: 748 printk(KERN_ERR "%s: %s reloc not supported\n", 749 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?"); 750 return -ENOEXEC; 751 752 default: 753 printk(KERN_ERR "%s: unknown reloc %x\n", mod->name, r_type); 754 return -ENOEXEC; 755 } 756 757 if (!ok) 758 return -ENOEXEC; 759 760 DEBUGP("%s: [%p]<-%016lx = %s(%lx)\n", __func__, location, val, 761 reloc_name[r_type] ? reloc_name[r_type] : "?", sym->st_value + addend); 762 763 switch (format) { 764 case RF_INSN21B: ok = apply_imm21b(mod, location, (int64_t) val / 16); break; 765 case RF_INSN22: ok = apply_imm22(mod, location, val); break; 766 case RF_INSN64: ok = apply_imm64(mod, location, val); break; 767 case RF_INSN60: ok = apply_imm60(mod, location, (int64_t) val / 16); break; 768 case RF_32LSB: put_unaligned(val, (uint32_t *) location); break; 769 case RF_64LSB: put_unaligned(val, (uint64_t *) location); break; 770 case RF_32MSB: /* ia64 Linux is little-endian... */ 771 case RF_64MSB: /* ia64 Linux is little-endian... */ 772 case RF_INSN14: /* must be within-module, i.e., resolved by "ld -r" */ 773 case RF_INSN21M: /* must be within-module, i.e., resolved by "ld -r" */ 774 case RF_INSN21F: /* must be within-module, i.e., resolved by "ld -r" */ 775 printk(KERN_ERR "%s: format %u needed by %s reloc is not supported\n", 776 mod->name, format, reloc_name[r_type] ? reloc_name[r_type] : "?"); 777 return -ENOEXEC; 778 779 default: 780 printk(KERN_ERR "%s: relocation %s resulted in unknown format %u\n", 781 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?", format); 782 return -ENOEXEC; 783 } 784 return ok ? 0 : -ENOEXEC; 785 } 786 787 int 788 apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, 789 unsigned int relsec, struct module *mod) 790 { 791 unsigned int i, n = sechdrs[relsec].sh_size / sizeof(Elf64_Rela); 792 Elf64_Rela *rela = (void *) sechdrs[relsec].sh_addr; 793 Elf64_Shdr *target_sec; 794 int ret; 795 796 DEBUGP("%s: applying section %u (%u relocs) to %u\n", __func__, 797 relsec, n, sechdrs[relsec].sh_info); 798 799 target_sec = sechdrs + sechdrs[relsec].sh_info; 800 801 if (target_sec->sh_entsize == ~0UL) 802 /* 803 * If target section wasn't allocated, we don't need to relocate it. 804 * Happens, e.g., for debug sections. 805 */ 806 return 0; 807 808 if (!mod->arch.gp) { 809 /* 810 * XXX Should have an arch-hook for running this after final section 811 * addresses have been selected... 812 */ 813 uint64_t gp; 814 if (mod->core_layout.size > MAX_LTOFF) 815 /* 816 * This takes advantage of fact that SHF_ARCH_SMALL gets allocated 817 * at the end of the module. 818 */ 819 gp = mod->core_layout.size - MAX_LTOFF / 2; 820 else 821 gp = mod->core_layout.size / 2; 822 gp = (uint64_t) mod->core_layout.base + ((gp + 7) & -8); 823 mod->arch.gp = gp; 824 DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp); 825 } 826 827 for (i = 0; i < n; i++) { 828 ret = do_reloc(mod, ELF64_R_TYPE(rela[i].r_info), 829 ((Elf64_Sym *) sechdrs[symindex].sh_addr 830 + ELF64_R_SYM(rela[i].r_info)), 831 rela[i].r_addend, target_sec, 832 (void *) target_sec->sh_addr + rela[i].r_offset); 833 if (ret < 0) 834 return ret; 835 } 836 return 0; 837 } 838 839 /* 840 * Modules contain a single unwind table which covers both the core and the init text 841 * sections but since the two are not contiguous, we need to split this table up such that 842 * we can register (and unregister) each "segment" separately. Fortunately, this sounds 843 * more complicated than it really is. 844 */ 845 static void 846 register_unwind_table (struct module *mod) 847 { 848 struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr; 849 struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start); 850 struct unw_table_entry tmp, *e1, *e2, *core, *init; 851 unsigned long num_init = 0, num_core = 0; 852 853 /* First, count how many init and core unwind-table entries there are. */ 854 for (e1 = start; e1 < end; ++e1) 855 if (in_init(mod, e1->start_offset)) 856 ++num_init; 857 else 858 ++num_core; 859 /* 860 * Second, sort the table such that all unwind-table entries for the init and core 861 * text sections are nicely separated. We do this with a stupid bubble sort 862 * (unwind tables don't get ridiculously huge). 863 */ 864 for (e1 = start; e1 < end; ++e1) { 865 for (e2 = e1 + 1; e2 < end; ++e2) { 866 if (e2->start_offset < e1->start_offset) { 867 tmp = *e1; 868 *e1 = *e2; 869 *e2 = tmp; 870 } 871 } 872 } 873 /* 874 * Third, locate the init and core segments in the unwind table: 875 */ 876 if (in_init(mod, start->start_offset)) { 877 init = start; 878 core = start + num_init; 879 } else { 880 core = start; 881 init = start + num_core; 882 } 883 884 DEBUGP("%s: name=%s, gp=%lx, num_init=%lu, num_core=%lu\n", __func__, 885 mod->name, mod->arch.gp, num_init, num_core); 886 887 /* 888 * Fourth, register both tables (if not empty). 889 */ 890 if (num_core > 0) { 891 mod->arch.core_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp, 892 core, core + num_core); 893 DEBUGP("%s: core: handle=%p [%p-%p)\n", __func__, 894 mod->arch.core_unw_table, core, core + num_core); 895 } 896 if (num_init > 0) { 897 mod->arch.init_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp, 898 init, init + num_init); 899 DEBUGP("%s: init: handle=%p [%p-%p)\n", __func__, 900 mod->arch.init_unw_table, init, init + num_init); 901 } 902 } 903 904 int 905 module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) 906 { 907 DEBUGP("%s: init: entry=%p\n", __func__, mod->init); 908 if (mod->arch.unwind) 909 register_unwind_table(mod); 910 return 0; 911 } 912 913 void 914 module_arch_cleanup (struct module *mod) 915 { 916 if (mod->arch.init_unw_table) 917 unw_remove_unwind_table(mod->arch.init_unw_table); 918 if (mod->arch.core_unw_table) 919 unw_remove_unwind_table(mod->arch.core_unw_table); 920 } 921