1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * sorttable.h 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <shile.zhang@linux.alibaba.com> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by: 12 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com> 13 * 14 * Some of this code was taken out of recordmcount.h written by: 15 * 16 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. 17 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. 18 */ 19 20 #undef extable_ent_size 21 #undef compare_extable 22 #undef get_mcount_loc 23 #undef sort_mcount_loc 24 #undef elf_mcount_loc 25 #undef do_sort 26 #undef Elf_Addr 27 #undef Elf_Ehdr 28 #undef Elf_Shdr 29 #undef Elf_Rel 30 #undef Elf_Rela 31 #undef Elf_Sym 32 #undef ELF_R_SYM 33 #undef Elf_r_sym 34 #undef ELF_R_INFO 35 #undef Elf_r_info 36 #undef ELF_ST_BIND 37 #undef ELF_ST_TYPE 38 #undef fn_ELF_R_SYM 39 #undef fn_ELF_R_INFO 40 #undef uint_t 41 #undef _r 42 #undef _w 43 44 #ifdef SORTTABLE_64 45 # define extable_ent_size 16 46 # define compare_extable compare_extable_64 47 # define get_mcount_loc get_mcount_loc_64 48 # define sort_mcount_loc sort_mcount_loc_64 49 # define elf_mcount_loc elf_mcount_loc_64 50 # define do_sort do_sort_64 51 # define Elf_Addr Elf64_Addr 52 # define Elf_Ehdr Elf64_Ehdr 53 # define Elf_Shdr Elf64_Shdr 54 # define Elf_Rel Elf64_Rel 55 # define Elf_Rela Elf64_Rela 56 # define Elf_Sym Elf64_Sym 57 # define ELF_R_SYM ELF64_R_SYM 58 # define Elf_r_sym Elf64_r_sym 59 # define ELF_R_INFO ELF64_R_INFO 60 # define Elf_r_info Elf64_r_info 61 # define ELF_ST_BIND ELF64_ST_BIND 62 # define ELF_ST_TYPE ELF64_ST_TYPE 63 # define fn_ELF_R_SYM fn_ELF64_R_SYM 64 # define fn_ELF_R_INFO fn_ELF64_R_INFO 65 # define uint_t uint64_t 66 # define _r r8 67 # define _w w8 68 #else 69 # define extable_ent_size 8 70 # define compare_extable compare_extable_32 71 # define get_mcount_loc get_mcount_loc_32 72 # define sort_mcount_loc sort_mcount_loc_32 73 # define elf_mcount_loc elf_mcount_loc_32 74 # define do_sort do_sort_32 75 # define Elf_Addr Elf32_Addr 76 # define Elf_Ehdr Elf32_Ehdr 77 # define Elf_Shdr Elf32_Shdr 78 # define Elf_Rel Elf32_Rel 79 # define Elf_Rela Elf32_Rela 80 # define Elf_Sym Elf32_Sym 81 # define ELF_R_SYM ELF32_R_SYM 82 # define Elf_r_sym Elf32_r_sym 83 # define ELF_R_INFO ELF32_R_INFO 84 # define Elf_r_info Elf32_r_info 85 # define ELF_ST_BIND ELF32_ST_BIND 86 # define ELF_ST_TYPE ELF32_ST_TYPE 87 # define fn_ELF_R_SYM fn_ELF32_R_SYM 88 # define fn_ELF_R_INFO fn_ELF32_R_INFO 89 # define uint_t uint32_t 90 # define _r r 91 # define _w w 92 #endif 93 94 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 95 /* ORC unwinder only support X86_64 */ 96 #include <asm/orc_types.h> 97 98 #define ERRSTR_MAXSZ 256 99 100 char g_err[ERRSTR_MAXSZ]; 101 int *g_orc_ip_table; 102 struct orc_entry *g_orc_table; 103 104 pthread_t orc_sort_thread; 105 106 static inline unsigned long orc_ip(const int *ip) 107 { 108 return (unsigned long)ip + *ip; 109 } 110 111 static int orc_sort_cmp(const void *_a, const void *_b) 112 { 113 struct orc_entry *orc_a; 114 const int *a = g_orc_ip_table + *(int *)_a; 115 const int *b = g_orc_ip_table + *(int *)_b; 116 unsigned long a_val = orc_ip(a); 117 unsigned long b_val = orc_ip(b); 118 119 if (a_val > b_val) 120 return 1; 121 if (a_val < b_val) 122 return -1; 123 124 /* 125 * The "weak" section terminator entries need to always be on the left 126 * to ensure the lookup code skips them in favor of real entries. 127 * These terminator entries exist to handle any gaps created by 128 * whitelisted .o files which didn't get objtool generation. 129 */ 130 orc_a = g_orc_table + (a - g_orc_ip_table); 131 return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; 132 } 133 134 static void *sort_orctable(void *arg) 135 { 136 int i; 137 int *idxs = NULL; 138 int *tmp_orc_ip_table = NULL; 139 struct orc_entry *tmp_orc_table = NULL; 140 unsigned int *orc_ip_size = (unsigned int *)arg; 141 unsigned int num_entries = *orc_ip_size / sizeof(int); 142 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 143 144 idxs = (int *)malloc(*orc_ip_size); 145 if (!idxs) { 146 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 147 strerror(errno)); 148 pthread_exit(g_err); 149 } 150 151 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 152 if (!tmp_orc_ip_table) { 153 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 154 strerror(errno)); 155 pthread_exit(g_err); 156 } 157 158 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 159 if (!tmp_orc_table) { 160 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 161 strerror(errno)); 162 pthread_exit(g_err); 163 } 164 165 /* initialize indices array, convert ip_table to absolute address */ 166 for (i = 0; i < num_entries; i++) { 167 idxs[i] = i; 168 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 169 } 170 memcpy(tmp_orc_table, g_orc_table, orc_size); 171 172 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 173 174 for (i = 0; i < num_entries; i++) { 175 if (idxs[i] == i) 176 continue; 177 178 /* convert back to relative address */ 179 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 180 g_orc_table[i] = tmp_orc_table[idxs[i]]; 181 } 182 183 free(idxs); 184 free(tmp_orc_ip_table); 185 free(tmp_orc_table); 186 pthread_exit(NULL); 187 } 188 #endif 189 190 static int compare_extable(const void *a, const void *b) 191 { 192 Elf_Addr av = _r(a); 193 Elf_Addr bv = _r(b); 194 195 if (av < bv) 196 return -1; 197 if (av > bv) 198 return 1; 199 return 0; 200 } 201 #ifdef MCOUNT_SORT_ENABLED 202 struct elf_mcount_loc { 203 Elf_Ehdr *ehdr; 204 Elf_Shdr *init_data_sec; 205 uint_t start_mcount_loc; 206 uint_t stop_mcount_loc; 207 }; 208 209 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 210 static void *sort_mcount_loc(void *arg) 211 { 212 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 213 uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr) 214 + _r(&(emloc->init_data_sec)->sh_offset); 215 uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; 216 unsigned char *start_loc = (void *)emloc->ehdr + offset; 217 218 qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable); 219 return NULL; 220 } 221 222 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 223 static void get_mcount_loc(uint_t *_start, uint_t *_stop) 224 { 225 FILE *file_start, *file_stop; 226 char start_buff[20]; 227 char stop_buff[20]; 228 int len = 0; 229 230 file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r"); 231 if (!file_start) { 232 fprintf(stderr, "get start_mcount_loc error!"); 233 return; 234 } 235 236 file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r"); 237 if (!file_stop) { 238 fprintf(stderr, "get stop_mcount_loc error!"); 239 pclose(file_start); 240 return; 241 } 242 243 while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) { 244 len = strlen(start_buff); 245 start_buff[len - 1] = '\0'; 246 } 247 *_start = strtoul(start_buff, NULL, 16); 248 249 while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) { 250 len = strlen(stop_buff); 251 stop_buff[len - 1] = '\0'; 252 } 253 *_stop = strtoul(stop_buff, NULL, 16); 254 255 pclose(file_start); 256 pclose(file_stop); 257 } 258 #endif 259 static int do_sort(Elf_Ehdr *ehdr, 260 char const *const fname, 261 table_sort_t custom_sort) 262 { 263 int rc = -1; 264 Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff)); 265 Elf_Shdr *strtab_sec = NULL; 266 Elf_Shdr *symtab_sec = NULL; 267 Elf_Shdr *extab_sec = NULL; 268 Elf_Sym *sym; 269 const Elf_Sym *symtab; 270 Elf32_Word *symtab_shndx = NULL; 271 Elf_Sym *sort_needed_sym = NULL; 272 Elf_Shdr *sort_needed_sec; 273 Elf_Rel *relocs = NULL; 274 int relocs_size = 0; 275 uint32_t *sort_needed_loc; 276 const char *secstrings; 277 const char *strtab; 278 char *extab_image; 279 int extab_index = 0; 280 int i; 281 int idx; 282 unsigned int shnum; 283 unsigned int shstrndx; 284 #ifdef MCOUNT_SORT_ENABLED 285 struct elf_mcount_loc mstruct; 286 uint_t _start_mcount_loc = 0; 287 uint_t _stop_mcount_loc = 0; 288 pthread_t mcount_sort_thread; 289 #endif 290 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 291 unsigned int orc_ip_size = 0; 292 unsigned int orc_size = 0; 293 unsigned int orc_num_entries = 0; 294 #endif 295 296 shstrndx = r2(&ehdr->e_shstrndx); 297 if (shstrndx == SHN_XINDEX) 298 shstrndx = r(&shdr[0].sh_link); 299 secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset); 300 301 shnum = r2(&ehdr->e_shnum); 302 if (shnum == SHN_UNDEF) 303 shnum = _r(&shdr[0].sh_size); 304 305 for (i = 0, s = shdr; s < shdr + shnum; i++, s++) { 306 idx = r(&s->sh_name); 307 if (!strcmp(secstrings + idx, "__ex_table")) { 308 extab_sec = s; 309 extab_index = i; 310 } 311 if (!strcmp(secstrings + idx, ".symtab")) 312 symtab_sec = s; 313 if (!strcmp(secstrings + idx, ".strtab")) 314 strtab_sec = s; 315 316 if ((r(&s->sh_type) == SHT_REL || 317 r(&s->sh_type) == SHT_RELA) && 318 r(&s->sh_info) == extab_index) { 319 relocs = (void *)ehdr + _r(&s->sh_offset); 320 relocs_size = _r(&s->sh_size); 321 } 322 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX) 323 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 324 _r(&s->sh_offset)); 325 326 #ifdef MCOUNT_SORT_ENABLED 327 /* locate the .init.data section in vmlinux */ 328 if (!strcmp(secstrings + idx, ".init.data")) { 329 get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc); 330 mstruct.ehdr = ehdr; 331 mstruct.init_data_sec = s; 332 mstruct.start_mcount_loc = _start_mcount_loc; 333 mstruct.stop_mcount_loc = _stop_mcount_loc; 334 } 335 #endif 336 337 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 338 /* locate the ORC unwind tables */ 339 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 340 orc_ip_size = s->sh_size; 341 g_orc_ip_table = (int *)((void *)ehdr + 342 s->sh_offset); 343 } 344 if (!strcmp(secstrings + idx, ".orc_unwind")) { 345 orc_size = s->sh_size; 346 g_orc_table = (struct orc_entry *)((void *)ehdr + 347 s->sh_offset); 348 } 349 #endif 350 } /* for loop */ 351 352 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 353 if (!g_orc_ip_table || !g_orc_table) { 354 fprintf(stderr, 355 "incomplete ORC unwind tables in file: %s\n", fname); 356 goto out; 357 } 358 359 orc_num_entries = orc_ip_size / sizeof(int); 360 if (orc_ip_size % sizeof(int) != 0 || 361 orc_size % sizeof(struct orc_entry) != 0 || 362 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 363 fprintf(stderr, 364 "inconsistent ORC unwind table entries in file: %s\n", 365 fname); 366 goto out; 367 } 368 369 /* create thread to sort ORC unwind tables concurrently */ 370 if (pthread_create(&orc_sort_thread, NULL, 371 sort_orctable, &orc_ip_size)) { 372 fprintf(stderr, 373 "pthread_create orc_sort_thread failed '%s': %s\n", 374 strerror(errno), fname); 375 goto out; 376 } 377 #endif 378 379 #ifdef MCOUNT_SORT_ENABLED 380 if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) { 381 fprintf(stderr, 382 "incomplete mcount's sort in file: %s\n", 383 fname); 384 goto out; 385 } 386 387 /* create thread to sort mcount_loc concurrently */ 388 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 389 fprintf(stderr, 390 "pthread_create mcount_sort_thread failed '%s': %s\n", 391 strerror(errno), fname); 392 goto out; 393 } 394 #endif 395 if (!extab_sec) { 396 fprintf(stderr, "no __ex_table in file: %s\n", fname); 397 goto out; 398 } 399 400 if (!symtab_sec) { 401 fprintf(stderr, "no .symtab in file: %s\n", fname); 402 goto out; 403 } 404 405 if (!strtab_sec) { 406 fprintf(stderr, "no .strtab in file: %s\n", fname); 407 goto out; 408 } 409 410 extab_image = (void *)ehdr + _r(&extab_sec->sh_offset); 411 strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset); 412 symtab = (const Elf_Sym *)((const char *)ehdr + 413 _r(&symtab_sec->sh_offset)); 414 415 if (custom_sort) { 416 custom_sort(extab_image, _r(&extab_sec->sh_size)); 417 } else { 418 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size; 419 qsort(extab_image, num_entries, 420 extable_ent_size, compare_extable); 421 } 422 423 /* If there were relocations, we no longer need them. */ 424 if (relocs) 425 memset(relocs, 0, relocs_size); 426 427 /* find the flag main_extable_sort_needed */ 428 for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset); 429 sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym); 430 sym++) { 431 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT) 432 continue; 433 if (!strcmp(strtab + r(&sym->st_name), 434 "main_extable_sort_needed")) { 435 sort_needed_sym = sym; 436 break; 437 } 438 } 439 440 if (!sort_needed_sym) { 441 fprintf(stderr, 442 "no main_extable_sort_needed symbol in file: %s\n", 443 fname); 444 goto out; 445 } 446 447 sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx), 448 sort_needed_sym - symtab, 449 symtab_shndx)]; 450 sort_needed_loc = (void *)ehdr + 451 _r(&sort_needed_sec->sh_offset) + 452 _r(&sort_needed_sym->st_value) - 453 _r(&sort_needed_sec->sh_addr); 454 455 /* extable has been sorted, clear the flag */ 456 w(0, sort_needed_loc); 457 rc = 0; 458 459 out: 460 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 461 if (orc_sort_thread) { 462 void *retval = NULL; 463 /* wait for ORC tables sort done */ 464 rc = pthread_join(orc_sort_thread, &retval); 465 if (rc) { 466 fprintf(stderr, 467 "pthread_join failed '%s': %s\n", 468 strerror(errno), fname); 469 } else if (retval) { 470 rc = -1; 471 fprintf(stderr, 472 "failed to sort ORC tables '%s': %s\n", 473 (char *)retval, fname); 474 } 475 } 476 #endif 477 478 #ifdef MCOUNT_SORT_ENABLED 479 if (mcount_sort_thread) { 480 void *retval = NULL; 481 /* wait for mcount sort done */ 482 rc = pthread_join(mcount_sort_thread, &retval); 483 if (rc) { 484 fprintf(stderr, 485 "pthread_join failed '%s': %s\n", 486 strerror(errno), fname); 487 } else if (retval) { 488 rc = -1; 489 fprintf(stderr, 490 "failed to sort mcount '%s': %s\n", 491 (char *)retval, fname); 492 } 493 } 494 #endif 495 return rc; 496 } 497