1 /* 2 * IA-64-specific support for kernel module loader. 3 * 4 * Copyright (C) 2003 Hewlett-Packard Co 5 * David Mosberger-Tang <davidm@hpl.hp.com> 6 * 7 * Loosely based on patch by Rusty Russell. 8 */ 9 10 /* relocs tested so far: 11 12 DIR64LSB 13 FPTR64LSB 14 GPREL22 15 LDXMOV 16 LDXMOV 17 LTOFF22 18 LTOFF22X 19 LTOFF22X 20 LTOFF_FPTR22 21 PCREL21B (for br.call only; br.cond is not supported out of modules!) 22 PCREL60B (for brl.cond only; brl.call is not supported for modules!) 23 PCREL64LSB 24 SECREL32LSB 25 SEGREL64LSB 26 */ 27 28 29 #include <linux/kernel.h> 30 #include <linux/sched.h> 31 #include <linux/elf.h> 32 #include <linux/moduleloader.h> 33 #include <linux/string.h> 34 #include <linux/vmalloc.h> 35 36 #include <asm/patch.h> 37 #include <asm/unaligned.h> 38 39 #define ARCH_MODULE_DEBUG 0 40 41 #if ARCH_MODULE_DEBUG 42 # define DEBUGP printk 43 # define inline 44 #else 45 # define DEBUGP(fmt , a...) 46 #endif 47 48 #ifdef CONFIG_ITANIUM 49 # define USE_BRL 0 50 #else 51 # define USE_BRL 1 52 #endif 53 54 #define MAX_LTOFF ((uint64_t) (1 << 22)) /* max. allowable linkage-table offset */ 55 56 /* Define some relocation helper macros/types: */ 57 58 #define FORMAT_SHIFT 0 59 #define FORMAT_BITS 3 60 #define FORMAT_MASK ((1 << FORMAT_BITS) - 1) 61 #define VALUE_SHIFT 3 62 #define VALUE_BITS 5 63 #define VALUE_MASK ((1 << VALUE_BITS) - 1) 64 65 enum reloc_target_format { 66 /* direct encoded formats: */ 67 RF_NONE = 0, 68 RF_INSN14 = 1, 69 RF_INSN22 = 2, 70 RF_INSN64 = 3, 71 RF_32MSB = 4, 72 RF_32LSB = 5, 73 RF_64MSB = 6, 74 RF_64LSB = 7, 75 76 /* formats that cannot be directly decoded: */ 77 RF_INSN60, 78 RF_INSN21B, /* imm21 form 1 */ 79 RF_INSN21M, /* imm21 form 2 */ 80 RF_INSN21F /* imm21 form 3 */ 81 }; 82 83 enum reloc_value_formula { 84 RV_DIRECT = 4, /* S + A */ 85 RV_GPREL = 5, /* @gprel(S + A) */ 86 RV_LTREL = 6, /* @ltoff(S + A) */ 87 RV_PLTREL = 7, /* @pltoff(S + A) */ 88 RV_FPTR = 8, /* @fptr(S + A) */ 89 RV_PCREL = 9, /* S + A - P */ 90 RV_LTREL_FPTR = 10, /* @ltoff(@fptr(S + A)) */ 91 RV_SEGREL = 11, /* @segrel(S + A) */ 92 RV_SECREL = 12, /* @secrel(S + A) */ 93 RV_BDREL = 13, /* BD + A */ 94 RV_LTV = 14, /* S + A (like RV_DIRECT, except frozen at static link-time) */ 95 RV_PCREL2 = 15, /* S + A - P */ 96 RV_SPECIAL = 16, /* various (see below) */ 97 RV_RSVD17 = 17, 98 RV_TPREL = 18, /* @tprel(S + A) */ 99 RV_LTREL_TPREL = 19, /* @ltoff(@tprel(S + A)) */ 100 RV_DTPMOD = 20, /* @dtpmod(S + A) */ 101 RV_LTREL_DTPMOD = 21, /* @ltoff(@dtpmod(S + A)) */ 102 RV_DTPREL = 22, /* @dtprel(S + A) */ 103 RV_LTREL_DTPREL = 23, /* @ltoff(@dtprel(S + A)) */ 104 RV_RSVD24 = 24, 105 RV_RSVD25 = 25, 106 RV_RSVD26 = 26, 107 RV_RSVD27 = 27 108 /* 28-31 reserved for implementation-specific purposes. */ 109 }; 110 111 #define N(reloc) [R_IA64_##reloc] = #reloc 112 113 static const char *reloc_name[256] = { 114 N(NONE), N(IMM14), N(IMM22), N(IMM64), 115 N(DIR32MSB), N(DIR32LSB), N(DIR64MSB), N(DIR64LSB), 116 N(GPREL22), N(GPREL64I), N(GPREL32MSB), N(GPREL32LSB), 117 N(GPREL64MSB), N(GPREL64LSB), N(LTOFF22), N(LTOFF64I), 118 N(PLTOFF22), N(PLTOFF64I), N(PLTOFF64MSB), N(PLTOFF64LSB), 119 N(FPTR64I), N(FPTR32MSB), N(FPTR32LSB), N(FPTR64MSB), 120 N(FPTR64LSB), N(PCREL60B), N(PCREL21B), N(PCREL21M), 121 N(PCREL21F), N(PCREL32MSB), N(PCREL32LSB), N(PCREL64MSB), 122 N(PCREL64LSB), N(LTOFF_FPTR22), N(LTOFF_FPTR64I), N(LTOFF_FPTR32MSB), 123 N(LTOFF_FPTR32LSB), N(LTOFF_FPTR64MSB), N(LTOFF_FPTR64LSB), N(SEGREL32MSB), 124 N(SEGREL32LSB), N(SEGREL64MSB), N(SEGREL64LSB), N(SECREL32MSB), 125 N(SECREL32LSB), N(SECREL64MSB), N(SECREL64LSB), N(REL32MSB), 126 N(REL32LSB), N(REL64MSB), N(REL64LSB), N(LTV32MSB), 127 N(LTV32LSB), N(LTV64MSB), N(LTV64LSB), N(PCREL21BI), 128 N(PCREL22), N(PCREL64I), N(IPLTMSB), N(IPLTLSB), 129 N(COPY), N(LTOFF22X), N(LDXMOV), N(TPREL14), 130 N(TPREL22), N(TPREL64I), N(TPREL64MSB), N(TPREL64LSB), 131 N(LTOFF_TPREL22), N(DTPMOD64MSB), N(DTPMOD64LSB), N(LTOFF_DTPMOD22), 132 N(DTPREL14), N(DTPREL22), N(DTPREL64I), N(DTPREL32MSB), 133 N(DTPREL32LSB), N(DTPREL64MSB), N(DTPREL64LSB), N(LTOFF_DTPREL22) 134 }; 135 136 #undef N 137 138 /* Opaque struct for insns, to protect against derefs. */ 139 struct insn; 140 141 static inline uint64_t 142 bundle (const struct insn *insn) 143 { 144 return (uint64_t) insn & ~0xfUL; 145 } 146 147 static inline int 148 slot (const struct insn *insn) 149 { 150 return (uint64_t) insn & 0x3; 151 } 152 153 static int 154 apply_imm64 (struct module *mod, struct insn *insn, uint64_t val) 155 { 156 if (slot(insn) != 2) { 157 printk(KERN_ERR "%s: invalid slot number %d for IMM64\n", 158 mod->name, slot(insn)); 159 return 0; 160 } 161 ia64_patch_imm64((u64) insn, val); 162 return 1; 163 } 164 165 static int 166 apply_imm60 (struct module *mod, struct insn *insn, uint64_t val) 167 { 168 if (slot(insn) != 2) { 169 printk(KERN_ERR "%s: invalid slot number %d for IMM60\n", 170 mod->name, slot(insn)); 171 return 0; 172 } 173 if (val + ((uint64_t) 1 << 59) >= (1UL << 60)) { 174 printk(KERN_ERR "%s: value %ld out of IMM60 range\n", 175 mod->name, (long) val); 176 return 0; 177 } 178 ia64_patch_imm60((u64) insn, val); 179 return 1; 180 } 181 182 static int 183 apply_imm22 (struct module *mod, struct insn *insn, uint64_t val) 184 { 185 if (val + (1 << 21) >= (1 << 22)) { 186 printk(KERN_ERR "%s: value %li out of IMM22 range\n", 187 mod->name, (long)val); 188 return 0; 189 } 190 ia64_patch((u64) insn, 0x01fffcfe000UL, ( ((val & 0x200000UL) << 15) /* bit 21 -> 36 */ 191 | ((val & 0x1f0000UL) << 6) /* bit 16 -> 22 */ 192 | ((val & 0x00ff80UL) << 20) /* bit 7 -> 27 */ 193 | ((val & 0x00007fUL) << 13) /* bit 0 -> 13 */)); 194 return 1; 195 } 196 197 static int 198 apply_imm21b (struct module *mod, struct insn *insn, uint64_t val) 199 { 200 if (val + (1 << 20) >= (1 << 21)) { 201 printk(KERN_ERR "%s: value %li out of IMM21b range\n", 202 mod->name, (long)val); 203 return 0; 204 } 205 ia64_patch((u64) insn, 0x11ffffe000UL, ( ((val & 0x100000UL) << 16) /* bit 20 -> 36 */ 206 | ((val & 0x0fffffUL) << 13) /* bit 0 -> 13 */)); 207 return 1; 208 } 209 210 #if USE_BRL 211 212 struct plt_entry { 213 /* Three instruction bundles in PLT. */ 214 unsigned char bundle[2][16]; 215 }; 216 217 static const struct plt_entry ia64_plt_template = { 218 { 219 { 220 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 221 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */ 222 0x00, 0x00, 0x00, 0x60 223 }, 224 { 225 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 226 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* brl.many gp=TARGET_GP */ 227 0x08, 0x00, 0x00, 0xc0 228 } 229 } 230 }; 231 232 static int 233 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp) 234 { 235 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_gp) 236 && apply_imm60(mod, (struct insn *) (plt->bundle[1] + 2), 237 (target_ip - (int64_t) plt->bundle[1]) / 16)) 238 return 1; 239 return 0; 240 } 241 242 unsigned long 243 plt_target (struct plt_entry *plt) 244 { 245 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[1]; 246 long off; 247 248 b0 = b[0]; b1 = b[1]; 249 off = ( ((b1 & 0x00fffff000000000UL) >> 36) /* imm20b -> bit 0 */ 250 | ((b0 >> 48) << 20) | ((b1 & 0x7fffffUL) << 36) /* imm39 -> bit 20 */ 251 | ((b1 & 0x0800000000000000UL) << 0)); /* i -> bit 59 */ 252 return (long) plt->bundle[1] + 16*off; 253 } 254 255 #else /* !USE_BRL */ 256 257 struct plt_entry { 258 /* Three instruction bundles in PLT. */ 259 unsigned char bundle[3][16]; 260 }; 261 262 static const struct plt_entry ia64_plt_template = { 263 { 264 { 265 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 266 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* movl r16=TARGET_IP */ 267 0x02, 0x00, 0x00, 0x60 268 }, 269 { 270 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 271 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */ 272 0x00, 0x00, 0x00, 0x60 273 }, 274 { 275 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MIB] nop.m 0 */ 276 0x60, 0x80, 0x04, 0x80, 0x03, 0x00, /* mov b6=r16 */ 277 0x60, 0x00, 0x80, 0x00 /* br.few b6 */ 278 } 279 } 280 }; 281 282 static int 283 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp) 284 { 285 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_ip) 286 && apply_imm64(mod, (struct insn *) (plt->bundle[1] + 2), target_gp)) 287 return 1; 288 return 0; 289 } 290 291 unsigned long 292 plt_target (struct plt_entry *plt) 293 { 294 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[0]; 295 296 b0 = b[0]; b1 = b[1]; 297 return ( ((b1 & 0x000007f000000000) >> 36) /* imm7b -> bit 0 */ 298 | ((b1 & 0x07fc000000000000) >> 43) /* imm9d -> bit 7 */ 299 | ((b1 & 0x0003e00000000000) >> 29) /* imm5c -> bit 16 */ 300 | ((b1 & 0x0000100000000000) >> 23) /* ic -> bit 21 */ 301 | ((b0 >> 46) << 22) | ((b1 & 0x7fffff) << 40) /* imm41 -> bit 22 */ 302 | ((b1 & 0x0800000000000000) << 4)); /* i -> bit 63 */ 303 } 304 305 #endif /* !USE_BRL */ 306 307 void * 308 module_alloc (unsigned long size) 309 { 310 if (!size) 311 return NULL; 312 return vmalloc(size); 313 } 314 315 void 316 module_free (struct module *mod, void *module_region) 317 { 318 if (mod && mod->arch.init_unw_table && 319 module_region == mod->module_init) { 320 unw_remove_unwind_table(mod->arch.init_unw_table); 321 mod->arch.init_unw_table = NULL; 322 } 323 vfree(module_region); 324 } 325 326 /* Have we already seen one of these relocations? */ 327 /* FIXME: we could look in other sections, too --RR */ 328 static int 329 duplicate_reloc (const Elf64_Rela *rela, unsigned int num) 330 { 331 unsigned int i; 332 333 for (i = 0; i < num; i++) { 334 if (rela[i].r_info == rela[num].r_info && rela[i].r_addend == rela[num].r_addend) 335 return 1; 336 } 337 return 0; 338 } 339 340 /* Count how many GOT entries we may need */ 341 static unsigned int 342 count_gots (const Elf64_Rela *rela, unsigned int num) 343 { 344 unsigned int i, ret = 0; 345 346 /* Sure, this is order(n^2), but it's usually short, and not 347 time critical */ 348 for (i = 0; i < num; i++) { 349 switch (ELF64_R_TYPE(rela[i].r_info)) { 350 case R_IA64_LTOFF22: 351 case R_IA64_LTOFF22X: 352 case R_IA64_LTOFF64I: 353 case R_IA64_LTOFF_FPTR22: 354 case R_IA64_LTOFF_FPTR64I: 355 case R_IA64_LTOFF_FPTR32MSB: 356 case R_IA64_LTOFF_FPTR32LSB: 357 case R_IA64_LTOFF_FPTR64MSB: 358 case R_IA64_LTOFF_FPTR64LSB: 359 if (!duplicate_reloc(rela, i)) 360 ret++; 361 break; 362 } 363 } 364 return ret; 365 } 366 367 /* Count how many PLT entries we may need */ 368 static unsigned int 369 count_plts (const Elf64_Rela *rela, unsigned int num) 370 { 371 unsigned int i, ret = 0; 372 373 /* Sure, this is order(n^2), but it's usually short, and not 374 time critical */ 375 for (i = 0; i < num; i++) { 376 switch (ELF64_R_TYPE(rela[i].r_info)) { 377 case R_IA64_PCREL21B: 378 case R_IA64_PLTOFF22: 379 case R_IA64_PLTOFF64I: 380 case R_IA64_PLTOFF64MSB: 381 case R_IA64_PLTOFF64LSB: 382 case R_IA64_IPLTMSB: 383 case R_IA64_IPLTLSB: 384 if (!duplicate_reloc(rela, i)) 385 ret++; 386 break; 387 } 388 } 389 return ret; 390 } 391 392 /* We need to create an function-descriptors for any internal function 393 which is referenced. */ 394 static unsigned int 395 count_fdescs (const Elf64_Rela *rela, unsigned int num) 396 { 397 unsigned int i, ret = 0; 398 399 /* Sure, this is order(n^2), but it's usually short, and not time critical. */ 400 for (i = 0; i < num; i++) { 401 switch (ELF64_R_TYPE(rela[i].r_info)) { 402 case R_IA64_FPTR64I: 403 case R_IA64_FPTR32LSB: 404 case R_IA64_FPTR32MSB: 405 case R_IA64_FPTR64LSB: 406 case R_IA64_FPTR64MSB: 407 case R_IA64_LTOFF_FPTR22: 408 case R_IA64_LTOFF_FPTR32LSB: 409 case R_IA64_LTOFF_FPTR32MSB: 410 case R_IA64_LTOFF_FPTR64I: 411 case R_IA64_LTOFF_FPTR64LSB: 412 case R_IA64_LTOFF_FPTR64MSB: 413 case R_IA64_IPLTMSB: 414 case R_IA64_IPLTLSB: 415 /* 416 * Jumps to static functions sometimes go straight to their 417 * offset. Of course, that may not be possible if the jump is 418 * from init -> core or vice. versa, so we need to generate an 419 * FDESC (and PLT etc) for that. 420 */ 421 case R_IA64_PCREL21B: 422 if (!duplicate_reloc(rela, i)) 423 ret++; 424 break; 425 } 426 } 427 return ret; 428 } 429 430 int 431 module_frob_arch_sections (Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings, 432 struct module *mod) 433 { 434 unsigned long core_plts = 0, init_plts = 0, gots = 0, fdescs = 0; 435 Elf64_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum; 436 437 /* 438 * To store the PLTs and function-descriptors, we expand the .text section for 439 * core module-code and the .init.text section for initialization code. 440 */ 441 for (s = sechdrs; s < sechdrs_end; ++s) 442 if (strcmp(".core.plt", secstrings + s->sh_name) == 0) 443 mod->arch.core_plt = s; 444 else if (strcmp(".init.plt", secstrings + s->sh_name) == 0) 445 mod->arch.init_plt = s; 446 else if (strcmp(".got", secstrings + s->sh_name) == 0) 447 mod->arch.got = s; 448 else if (strcmp(".opd", secstrings + s->sh_name) == 0) 449 mod->arch.opd = s; 450 else if (strcmp(".IA_64.unwind", secstrings + s->sh_name) == 0) 451 mod->arch.unwind = s; 452 #ifdef CONFIG_PARAVIRT 453 else if (strcmp(".paravirt_bundles", 454 secstrings + s->sh_name) == 0) 455 mod->arch.paravirt_bundles = s; 456 else if (strcmp(".paravirt_insts", 457 secstrings + s->sh_name) == 0) 458 mod->arch.paravirt_insts = s; 459 #endif 460 461 if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) { 462 printk(KERN_ERR "%s: sections missing\n", mod->name); 463 return -ENOEXEC; 464 } 465 466 /* GOT and PLTs can occur in any relocated section... */ 467 for (s = sechdrs + 1; s < sechdrs_end; ++s) { 468 const Elf64_Rela *rels = (void *)ehdr + s->sh_offset; 469 unsigned long numrels = s->sh_size/sizeof(Elf64_Rela); 470 471 if (s->sh_type != SHT_RELA) 472 continue; 473 474 gots += count_gots(rels, numrels); 475 fdescs += count_fdescs(rels, numrels); 476 if (strstr(secstrings + s->sh_name, ".init")) 477 init_plts += count_plts(rels, numrels); 478 else 479 core_plts += count_plts(rels, numrels); 480 } 481 482 mod->arch.core_plt->sh_type = SHT_NOBITS; 483 mod->arch.core_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC; 484 mod->arch.core_plt->sh_addralign = 16; 485 mod->arch.core_plt->sh_size = core_plts * sizeof(struct plt_entry); 486 mod->arch.init_plt->sh_type = SHT_NOBITS; 487 mod->arch.init_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC; 488 mod->arch.init_plt->sh_addralign = 16; 489 mod->arch.init_plt->sh_size = init_plts * sizeof(struct plt_entry); 490 mod->arch.got->sh_type = SHT_NOBITS; 491 mod->arch.got->sh_flags = ARCH_SHF_SMALL | SHF_ALLOC; 492 mod->arch.got->sh_addralign = 8; 493 mod->arch.got->sh_size = gots * sizeof(struct got_entry); 494 mod->arch.opd->sh_type = SHT_NOBITS; 495 mod->arch.opd->sh_flags = SHF_ALLOC; 496 mod->arch.opd->sh_addralign = 8; 497 mod->arch.opd->sh_size = fdescs * sizeof(struct fdesc); 498 DEBUGP("%s: core.plt=%lx, init.plt=%lx, got=%lx, fdesc=%lx\n", 499 __func__, mod->arch.core_plt->sh_size, mod->arch.init_plt->sh_size, 500 mod->arch.got->sh_size, mod->arch.opd->sh_size); 501 return 0; 502 } 503 504 static inline int 505 in_init (const struct module *mod, uint64_t addr) 506 { 507 return addr - (uint64_t) mod->module_init < mod->init_size; 508 } 509 510 static inline int 511 in_core (const struct module *mod, uint64_t addr) 512 { 513 return addr - (uint64_t) mod->module_core < mod->core_size; 514 } 515 516 static inline int 517 is_internal (const struct module *mod, uint64_t value) 518 { 519 return in_init(mod, value) || in_core(mod, value); 520 } 521 522 /* 523 * Get gp-relative offset for the linkage-table entry of VALUE. 524 */ 525 static uint64_t 526 get_ltoff (struct module *mod, uint64_t value, int *okp) 527 { 528 struct got_entry *got, *e; 529 530 if (!*okp) 531 return 0; 532 533 got = (void *) mod->arch.got->sh_addr; 534 for (e = got; e < got + mod->arch.next_got_entry; ++e) 535 if (e->val == value) 536 goto found; 537 538 /* Not enough GOT entries? */ 539 BUG_ON(e >= (struct got_entry *) (mod->arch.got->sh_addr + mod->arch.got->sh_size)); 540 541 e->val = value; 542 ++mod->arch.next_got_entry; 543 found: 544 return (uint64_t) e - mod->arch.gp; 545 } 546 547 static inline int 548 gp_addressable (struct module *mod, uint64_t value) 549 { 550 return value - mod->arch.gp + MAX_LTOFF/2 < MAX_LTOFF; 551 } 552 553 /* Get PC-relative PLT entry for this value. Returns 0 on failure. */ 554 static uint64_t 555 get_plt (struct module *mod, const struct insn *insn, uint64_t value, int *okp) 556 { 557 struct plt_entry *plt, *plt_end; 558 uint64_t target_ip, target_gp; 559 560 if (!*okp) 561 return 0; 562 563 if (in_init(mod, (uint64_t) insn)) { 564 plt = (void *) mod->arch.init_plt->sh_addr; 565 plt_end = (void *) plt + mod->arch.init_plt->sh_size; 566 } else { 567 plt = (void *) mod->arch.core_plt->sh_addr; 568 plt_end = (void *) plt + mod->arch.core_plt->sh_size; 569 } 570 571 /* "value" is a pointer to a function-descriptor; fetch the target ip/gp from it: */ 572 target_ip = ((uint64_t *) value)[0]; 573 target_gp = ((uint64_t *) value)[1]; 574 575 /* Look for existing PLT entry. */ 576 while (plt->bundle[0][0]) { 577 if (plt_target(plt) == target_ip) 578 goto found; 579 if (++plt >= plt_end) 580 BUG(); 581 } 582 *plt = ia64_plt_template; 583 if (!patch_plt(mod, plt, target_ip, target_gp)) { 584 *okp = 0; 585 return 0; 586 } 587 #if ARCH_MODULE_DEBUG 588 if (plt_target(plt) != target_ip) { 589 printk("%s: mistargeted PLT: wanted %lx, got %lx\n", 590 __func__, target_ip, plt_target(plt)); 591 *okp = 0; 592 return 0; 593 } 594 #endif 595 found: 596 return (uint64_t) plt; 597 } 598 599 /* Get function descriptor for VALUE. */ 600 static uint64_t 601 get_fdesc (struct module *mod, uint64_t value, int *okp) 602 { 603 struct fdesc *fdesc = (void *) mod->arch.opd->sh_addr; 604 605 if (!*okp) 606 return 0; 607 608 if (!value) { 609 printk(KERN_ERR "%s: fdesc for zero requested!\n", mod->name); 610 return 0; 611 } 612 613 if (!is_internal(mod, value)) 614 /* 615 * If it's not a module-local entry-point, "value" already points to a 616 * function-descriptor. 617 */ 618 return value; 619 620 /* Look for existing function descriptor. */ 621 while (fdesc->ip) { 622 if (fdesc->ip == value) 623 return (uint64_t)fdesc; 624 if ((uint64_t) ++fdesc >= mod->arch.opd->sh_addr + mod->arch.opd->sh_size) 625 BUG(); 626 } 627 628 /* Create new one */ 629 fdesc->ip = value; 630 fdesc->gp = mod->arch.gp; 631 return (uint64_t) fdesc; 632 } 633 634 static inline int 635 do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend, 636 Elf64_Shdr *sec, void *location) 637 { 638 enum reloc_target_format format = (r_type >> FORMAT_SHIFT) & FORMAT_MASK; 639 enum reloc_value_formula formula = (r_type >> VALUE_SHIFT) & VALUE_MASK; 640 uint64_t val; 641 int ok = 1; 642 643 val = sym->st_value + addend; 644 645 switch (formula) { 646 case RV_SEGREL: /* segment base is arbitrarily chosen to be 0 for kernel modules */ 647 case RV_DIRECT: 648 break; 649 650 case RV_GPREL: val -= mod->arch.gp; break; 651 case RV_LTREL: val = get_ltoff(mod, val, &ok); break; 652 case RV_PLTREL: val = get_plt(mod, location, val, &ok); break; 653 case RV_FPTR: val = get_fdesc(mod, val, &ok); break; 654 case RV_SECREL: val -= sec->sh_addr; break; 655 case RV_LTREL_FPTR: val = get_ltoff(mod, get_fdesc(mod, val, &ok), &ok); break; 656 657 case RV_PCREL: 658 switch (r_type) { 659 case R_IA64_PCREL21B: 660 if ((in_init(mod, val) && in_core(mod, (uint64_t)location)) || 661 (in_core(mod, val) && in_init(mod, (uint64_t)location))) { 662 /* 663 * Init section may have been allocated far away from core, 664 * if the branch won't reach, then allocate a plt for it. 665 */ 666 uint64_t delta = ((int64_t)val - (int64_t)location) / 16; 667 if (delta + (1 << 20) >= (1 << 21)) { 668 val = get_fdesc(mod, val, &ok); 669 val = get_plt(mod, location, val, &ok); 670 } 671 } else if (!is_internal(mod, val)) 672 val = get_plt(mod, location, val, &ok); 673 /* FALL THROUGH */ 674 default: 675 val -= bundle(location); 676 break; 677 678 case R_IA64_PCREL32MSB: 679 case R_IA64_PCREL32LSB: 680 case R_IA64_PCREL64MSB: 681 case R_IA64_PCREL64LSB: 682 val -= (uint64_t) location; 683 break; 684 685 } 686 switch (r_type) { 687 case R_IA64_PCREL60B: format = RF_INSN60; break; 688 case R_IA64_PCREL21B: format = RF_INSN21B; break; 689 case R_IA64_PCREL21M: format = RF_INSN21M; break; 690 case R_IA64_PCREL21F: format = RF_INSN21F; break; 691 default: break; 692 } 693 break; 694 695 case RV_BDREL: 696 val -= (uint64_t) (in_init(mod, val) ? mod->module_init : mod->module_core); 697 break; 698 699 case RV_LTV: 700 /* can link-time value relocs happen here? */ 701 BUG(); 702 break; 703 704 case RV_PCREL2: 705 if (r_type == R_IA64_PCREL21BI) { 706 if (!is_internal(mod, val)) { 707 printk(KERN_ERR "%s: %s reloc against " 708 "non-local symbol (%lx)\n", __func__, 709 reloc_name[r_type], (unsigned long)val); 710 return -ENOEXEC; 711 } 712 format = RF_INSN21B; 713 } 714 val -= bundle(location); 715 break; 716 717 case RV_SPECIAL: 718 switch (r_type) { 719 case R_IA64_IPLTMSB: 720 case R_IA64_IPLTLSB: 721 val = get_fdesc(mod, get_plt(mod, location, val, &ok), &ok); 722 format = RF_64LSB; 723 if (r_type == R_IA64_IPLTMSB) 724 format = RF_64MSB; 725 break; 726 727 case R_IA64_SUB: 728 val = addend - sym->st_value; 729 format = RF_INSN64; 730 break; 731 732 case R_IA64_LTOFF22X: 733 if (gp_addressable(mod, val)) 734 val -= mod->arch.gp; 735 else 736 val = get_ltoff(mod, val, &ok); 737 format = RF_INSN22; 738 break; 739 740 case R_IA64_LDXMOV: 741 if (gp_addressable(mod, val)) { 742 /* turn "ld8" into "mov": */ 743 DEBUGP("%s: patching ld8 at %p to mov\n", __func__, location); 744 ia64_patch((u64) location, 0x1fff80fe000UL, 0x10000000000UL); 745 } 746 return 0; 747 748 default: 749 if (reloc_name[r_type]) 750 printk(KERN_ERR "%s: special reloc %s not supported", 751 mod->name, reloc_name[r_type]); 752 else 753 printk(KERN_ERR "%s: unknown special reloc %x\n", 754 mod->name, r_type); 755 return -ENOEXEC; 756 } 757 break; 758 759 case RV_TPREL: 760 case RV_LTREL_TPREL: 761 case RV_DTPMOD: 762 case RV_LTREL_DTPMOD: 763 case RV_DTPREL: 764 case RV_LTREL_DTPREL: 765 printk(KERN_ERR "%s: %s reloc not supported\n", 766 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?"); 767 return -ENOEXEC; 768 769 default: 770 printk(KERN_ERR "%s: unknown reloc %x\n", mod->name, r_type); 771 return -ENOEXEC; 772 } 773 774 if (!ok) 775 return -ENOEXEC; 776 777 DEBUGP("%s: [%p]<-%016lx = %s(%lx)\n", __func__, location, val, 778 reloc_name[r_type] ? reloc_name[r_type] : "?", sym->st_value + addend); 779 780 switch (format) { 781 case RF_INSN21B: ok = apply_imm21b(mod, location, (int64_t) val / 16); break; 782 case RF_INSN22: ok = apply_imm22(mod, location, val); break; 783 case RF_INSN64: ok = apply_imm64(mod, location, val); break; 784 case RF_INSN60: ok = apply_imm60(mod, location, (int64_t) val / 16); break; 785 case RF_32LSB: put_unaligned(val, (uint32_t *) location); break; 786 case RF_64LSB: put_unaligned(val, (uint64_t *) location); break; 787 case RF_32MSB: /* ia64 Linux is little-endian... */ 788 case RF_64MSB: /* ia64 Linux is little-endian... */ 789 case RF_INSN14: /* must be within-module, i.e., resolved by "ld -r" */ 790 case RF_INSN21M: /* must be within-module, i.e., resolved by "ld -r" */ 791 case RF_INSN21F: /* must be within-module, i.e., resolved by "ld -r" */ 792 printk(KERN_ERR "%s: format %u needed by %s reloc is not supported\n", 793 mod->name, format, reloc_name[r_type] ? reloc_name[r_type] : "?"); 794 return -ENOEXEC; 795 796 default: 797 printk(KERN_ERR "%s: relocation %s resulted in unknown format %u\n", 798 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?", format); 799 return -ENOEXEC; 800 } 801 return ok ? 0 : -ENOEXEC; 802 } 803 804 int 805 apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, 806 unsigned int relsec, struct module *mod) 807 { 808 unsigned int i, n = sechdrs[relsec].sh_size / sizeof(Elf64_Rela); 809 Elf64_Rela *rela = (void *) sechdrs[relsec].sh_addr; 810 Elf64_Shdr *target_sec; 811 int ret; 812 813 DEBUGP("%s: applying section %u (%u relocs) to %u\n", __func__, 814 relsec, n, sechdrs[relsec].sh_info); 815 816 target_sec = sechdrs + sechdrs[relsec].sh_info; 817 818 if (target_sec->sh_entsize == ~0UL) 819 /* 820 * If target section wasn't allocated, we don't need to relocate it. 821 * Happens, e.g., for debug sections. 822 */ 823 return 0; 824 825 if (!mod->arch.gp) { 826 /* 827 * XXX Should have an arch-hook for running this after final section 828 * addresses have been selected... 829 */ 830 uint64_t gp; 831 if (mod->core_size > MAX_LTOFF) 832 /* 833 * This takes advantage of fact that SHF_ARCH_SMALL gets allocated 834 * at the end of the module. 835 */ 836 gp = mod->core_size - MAX_LTOFF / 2; 837 else 838 gp = mod->core_size / 2; 839 gp = (uint64_t) mod->module_core + ((gp + 7) & -8); 840 mod->arch.gp = gp; 841 DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp); 842 } 843 844 for (i = 0; i < n; i++) { 845 ret = do_reloc(mod, ELF64_R_TYPE(rela[i].r_info), 846 ((Elf64_Sym *) sechdrs[symindex].sh_addr 847 + ELF64_R_SYM(rela[i].r_info)), 848 rela[i].r_addend, target_sec, 849 (void *) target_sec->sh_addr + rela[i].r_offset); 850 if (ret < 0) 851 return ret; 852 } 853 return 0; 854 } 855 856 int 857 apply_relocate (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, 858 unsigned int relsec, struct module *mod) 859 { 860 printk(KERN_ERR "module %s: REL relocs in section %u unsupported\n", mod->name, relsec); 861 return -ENOEXEC; 862 } 863 864 /* 865 * Modules contain a single unwind table which covers both the core and the init text 866 * sections but since the two are not contiguous, we need to split this table up such that 867 * we can register (and unregister) each "segment" separately. Fortunately, this sounds 868 * more complicated than it really is. 869 */ 870 static void 871 register_unwind_table (struct module *mod) 872 { 873 struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr; 874 struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start); 875 struct unw_table_entry tmp, *e1, *e2, *core, *init; 876 unsigned long num_init = 0, num_core = 0; 877 878 /* First, count how many init and core unwind-table entries there are. */ 879 for (e1 = start; e1 < end; ++e1) 880 if (in_init(mod, e1->start_offset)) 881 ++num_init; 882 else 883 ++num_core; 884 /* 885 * Second, sort the table such that all unwind-table entries for the init and core 886 * text sections are nicely separated. We do this with a stupid bubble sort 887 * (unwind tables don't get ridiculously huge). 888 */ 889 for (e1 = start; e1 < end; ++e1) { 890 for (e2 = e1 + 1; e2 < end; ++e2) { 891 if (e2->start_offset < e1->start_offset) { 892 tmp = *e1; 893 *e1 = *e2; 894 *e2 = tmp; 895 } 896 } 897 } 898 /* 899 * Third, locate the init and core segments in the unwind table: 900 */ 901 if (in_init(mod, start->start_offset)) { 902 init = start; 903 core = start + num_init; 904 } else { 905 core = start; 906 init = start + num_core; 907 } 908 909 DEBUGP("%s: name=%s, gp=%lx, num_init=%lu, num_core=%lu\n", __func__, 910 mod->name, mod->arch.gp, num_init, num_core); 911 912 /* 913 * Fourth, register both tables (if not empty). 914 */ 915 if (num_core > 0) { 916 mod->arch.core_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp, 917 core, core + num_core); 918 DEBUGP("%s: core: handle=%p [%p-%p)\n", __func__, 919 mod->arch.core_unw_table, core, core + num_core); 920 } 921 if (num_init > 0) { 922 mod->arch.init_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp, 923 init, init + num_init); 924 DEBUGP("%s: init: handle=%p [%p-%p)\n", __func__, 925 mod->arch.init_unw_table, init, init + num_init); 926 } 927 } 928 929 int 930 module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) 931 { 932 DEBUGP("%s: init: entry=%p\n", __func__, mod->init); 933 if (mod->arch.unwind) 934 register_unwind_table(mod); 935 #ifdef CONFIG_PARAVIRT 936 if (mod->arch.paravirt_bundles) { 937 struct paravirt_patch_site_bundle *start = 938 (struct paravirt_patch_site_bundle *) 939 mod->arch.paravirt_bundles->sh_addr; 940 struct paravirt_patch_site_bundle *end = 941 (struct paravirt_patch_site_bundle *) 942 (mod->arch.paravirt_bundles->sh_addr + 943 mod->arch.paravirt_bundles->sh_size); 944 945 paravirt_patch_apply_bundle(start, end); 946 } 947 if (mod->arch.paravirt_insts) { 948 struct paravirt_patch_site_inst *start = 949 (struct paravirt_patch_site_inst *) 950 mod->arch.paravirt_insts->sh_addr; 951 struct paravirt_patch_site_inst *end = 952 (struct paravirt_patch_site_inst *) 953 (mod->arch.paravirt_insts->sh_addr + 954 mod->arch.paravirt_insts->sh_size); 955 956 paravirt_patch_apply_inst(start, end); 957 } 958 #endif 959 return 0; 960 } 961 962 void 963 module_arch_cleanup (struct module *mod) 964 { 965 if (mod->arch.init_unw_table) 966 unw_remove_unwind_table(mod->arch.init_unw_table); 967 if (mod->arch.core_unw_table) 968 unw_remove_unwind_table(mod->arch.core_unw_table); 969 } 970