1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * sorttable.h 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <shile.zhang@linux.alibaba.com> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by: 12 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com> 13 * 14 * Some of this code was taken out of recordmcount.h written by: 15 * 16 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. 17 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. 18 */ 19 20 #undef extable_ent_size 21 #undef compare_extable 22 #undef do_sort 23 #undef Elf_Addr 24 #undef Elf_Ehdr 25 #undef Elf_Shdr 26 #undef Elf_Rel 27 #undef Elf_Rela 28 #undef Elf_Sym 29 #undef ELF_R_SYM 30 #undef Elf_r_sym 31 #undef ELF_R_INFO 32 #undef Elf_r_info 33 #undef ELF_ST_BIND 34 #undef ELF_ST_TYPE 35 #undef fn_ELF_R_SYM 36 #undef fn_ELF_R_INFO 37 #undef uint_t 38 #undef _r 39 #undef _w 40 41 #ifdef SORTTABLE_64 42 # define extable_ent_size 16 43 # define compare_extable compare_extable_64 44 # define do_sort do_sort_64 45 # define Elf_Addr Elf64_Addr 46 # define Elf_Ehdr Elf64_Ehdr 47 # define Elf_Shdr Elf64_Shdr 48 # define Elf_Rel Elf64_Rel 49 # define Elf_Rela Elf64_Rela 50 # define Elf_Sym Elf64_Sym 51 # define ELF_R_SYM ELF64_R_SYM 52 # define Elf_r_sym Elf64_r_sym 53 # define ELF_R_INFO ELF64_R_INFO 54 # define Elf_r_info Elf64_r_info 55 # define ELF_ST_BIND ELF64_ST_BIND 56 # define ELF_ST_TYPE ELF64_ST_TYPE 57 # define fn_ELF_R_SYM fn_ELF64_R_SYM 58 # define fn_ELF_R_INFO fn_ELF64_R_INFO 59 # define uint_t uint64_t 60 # define _r r8 61 # define _w w8 62 #else 63 # define extable_ent_size 8 64 # define compare_extable compare_extable_32 65 # define do_sort do_sort_32 66 # define Elf_Addr Elf32_Addr 67 # define Elf_Ehdr Elf32_Ehdr 68 # define Elf_Shdr Elf32_Shdr 69 # define Elf_Rel Elf32_Rel 70 # define Elf_Rela Elf32_Rela 71 # define Elf_Sym Elf32_Sym 72 # define ELF_R_SYM ELF32_R_SYM 73 # define Elf_r_sym Elf32_r_sym 74 # define ELF_R_INFO ELF32_R_INFO 75 # define Elf_r_info Elf32_r_info 76 # define ELF_ST_BIND ELF32_ST_BIND 77 # define ELF_ST_TYPE ELF32_ST_TYPE 78 # define fn_ELF_R_SYM fn_ELF32_R_SYM 79 # define fn_ELF_R_INFO fn_ELF32_R_INFO 80 # define uint_t uint32_t 81 # define _r r 82 # define _w w 83 #endif 84 85 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 86 /* ORC unwinder only support X86_64 */ 87 #include <errno.h> 88 #include <pthread.h> 89 #include <asm/orc_types.h> 90 91 #define ERRSTR_MAXSZ 256 92 93 char g_err[ERRSTR_MAXSZ]; 94 int *g_orc_ip_table; 95 struct orc_entry *g_orc_table; 96 97 pthread_t orc_sort_thread; 98 99 static inline unsigned long orc_ip(const int *ip) 100 { 101 return (unsigned long)ip + *ip; 102 } 103 104 static int orc_sort_cmp(const void *_a, const void *_b) 105 { 106 struct orc_entry *orc_a; 107 const int *a = g_orc_ip_table + *(int *)_a; 108 const int *b = g_orc_ip_table + *(int *)_b; 109 unsigned long a_val = orc_ip(a); 110 unsigned long b_val = orc_ip(b); 111 112 if (a_val > b_val) 113 return 1; 114 if (a_val < b_val) 115 return -1; 116 117 /* 118 * The "weak" section terminator entries need to always be on the left 119 * to ensure the lookup code skips them in favor of real entries. 120 * These terminator entries exist to handle any gaps created by 121 * whitelisted .o files which didn't get objtool generation. 122 */ 123 orc_a = g_orc_table + (a - g_orc_ip_table); 124 return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; 125 } 126 127 static void *sort_orctable(void *arg) 128 { 129 int i; 130 int *idxs = NULL; 131 int *tmp_orc_ip_table = NULL; 132 struct orc_entry *tmp_orc_table = NULL; 133 unsigned int *orc_ip_size = (unsigned int *)arg; 134 unsigned int num_entries = *orc_ip_size / sizeof(int); 135 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 136 137 idxs = (int *)malloc(*orc_ip_size); 138 if (!idxs) { 139 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 140 strerror(errno)); 141 pthread_exit(g_err); 142 } 143 144 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 145 if (!tmp_orc_ip_table) { 146 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 147 strerror(errno)); 148 pthread_exit(g_err); 149 } 150 151 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 152 if (!tmp_orc_table) { 153 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 154 strerror(errno)); 155 pthread_exit(g_err); 156 } 157 158 /* initialize indices array, convert ip_table to absolute address */ 159 for (i = 0; i < num_entries; i++) { 160 idxs[i] = i; 161 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 162 } 163 memcpy(tmp_orc_table, g_orc_table, orc_size); 164 165 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 166 167 for (i = 0; i < num_entries; i++) { 168 if (idxs[i] == i) 169 continue; 170 171 /* convert back to relative address */ 172 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 173 g_orc_table[i] = tmp_orc_table[idxs[i]]; 174 } 175 176 free(idxs); 177 free(tmp_orc_ip_table); 178 free(tmp_orc_table); 179 pthread_exit(NULL); 180 } 181 #endif 182 183 static int compare_extable(const void *a, const void *b) 184 { 185 Elf_Addr av = _r(a); 186 Elf_Addr bv = _r(b); 187 188 if (av < bv) 189 return -1; 190 if (av > bv) 191 return 1; 192 return 0; 193 } 194 195 static int do_sort(Elf_Ehdr *ehdr, 196 char const *const fname, 197 table_sort_t custom_sort) 198 { 199 int rc = -1; 200 Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff)); 201 Elf_Shdr *strtab_sec = NULL; 202 Elf_Shdr *symtab_sec = NULL; 203 Elf_Shdr *extab_sec = NULL; 204 Elf_Sym *sym; 205 const Elf_Sym *symtab; 206 Elf32_Word *symtab_shndx = NULL; 207 Elf_Sym *sort_needed_sym = NULL; 208 Elf_Shdr *sort_needed_sec; 209 Elf_Rel *relocs = NULL; 210 int relocs_size = 0; 211 uint32_t *sort_needed_loc; 212 const char *secstrings; 213 const char *strtab; 214 char *extab_image; 215 int extab_index = 0; 216 int i; 217 int idx; 218 unsigned int shnum; 219 unsigned int shstrndx; 220 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 221 unsigned int orc_ip_size = 0; 222 unsigned int orc_size = 0; 223 unsigned int orc_num_entries = 0; 224 #endif 225 226 shstrndx = r2(&ehdr->e_shstrndx); 227 if (shstrndx == SHN_XINDEX) 228 shstrndx = r(&shdr[0].sh_link); 229 secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset); 230 231 shnum = r2(&ehdr->e_shnum); 232 if (shnum == SHN_UNDEF) 233 shnum = _r(&shdr[0].sh_size); 234 235 for (i = 0, s = shdr; s < shdr + shnum; i++, s++) { 236 idx = r(&s->sh_name); 237 if (!strcmp(secstrings + idx, "__ex_table")) { 238 extab_sec = s; 239 extab_index = i; 240 } 241 if (!strcmp(secstrings + idx, ".symtab")) 242 symtab_sec = s; 243 if (!strcmp(secstrings + idx, ".strtab")) 244 strtab_sec = s; 245 246 if ((r(&s->sh_type) == SHT_REL || 247 r(&s->sh_type) == SHT_RELA) && 248 r(&s->sh_info) == extab_index) { 249 relocs = (void *)ehdr + _r(&s->sh_offset); 250 relocs_size = _r(&s->sh_size); 251 } 252 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX) 253 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 254 _r(&s->sh_offset)); 255 256 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 257 /* locate the ORC unwind tables */ 258 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 259 orc_ip_size = s->sh_size; 260 g_orc_ip_table = (int *)((void *)ehdr + 261 s->sh_offset); 262 } 263 if (!strcmp(secstrings + idx, ".orc_unwind")) { 264 orc_size = s->sh_size; 265 g_orc_table = (struct orc_entry *)((void *)ehdr + 266 s->sh_offset); 267 } 268 #endif 269 } /* for loop */ 270 271 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 272 if (!g_orc_ip_table || !g_orc_table) { 273 fprintf(stderr, 274 "incomplete ORC unwind tables in file: %s\n", fname); 275 goto out; 276 } 277 278 orc_num_entries = orc_ip_size / sizeof(int); 279 if (orc_ip_size % sizeof(int) != 0 || 280 orc_size % sizeof(struct orc_entry) != 0 || 281 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 282 fprintf(stderr, 283 "inconsistent ORC unwind table entries in file: %s\n", 284 fname); 285 goto out; 286 } 287 288 /* create thread to sort ORC unwind tables concurrently */ 289 if (pthread_create(&orc_sort_thread, NULL, 290 sort_orctable, &orc_ip_size)) { 291 fprintf(stderr, 292 "pthread_create orc_sort_thread failed '%s': %s\n", 293 strerror(errno), fname); 294 goto out; 295 } 296 #endif 297 if (!extab_sec) { 298 fprintf(stderr, "no __ex_table in file: %s\n", fname); 299 goto out; 300 } 301 302 if (!symtab_sec) { 303 fprintf(stderr, "no .symtab in file: %s\n", fname); 304 goto out; 305 } 306 307 if (!strtab_sec) { 308 fprintf(stderr, "no .strtab in file: %s\n", fname); 309 goto out; 310 } 311 312 extab_image = (void *)ehdr + _r(&extab_sec->sh_offset); 313 strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset); 314 symtab = (const Elf_Sym *)((const char *)ehdr + 315 _r(&symtab_sec->sh_offset)); 316 317 if (custom_sort) { 318 custom_sort(extab_image, _r(&extab_sec->sh_size)); 319 } else { 320 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size; 321 qsort(extab_image, num_entries, 322 extable_ent_size, compare_extable); 323 } 324 325 /* If there were relocations, we no longer need them. */ 326 if (relocs) 327 memset(relocs, 0, relocs_size); 328 329 /* find the flag main_extable_sort_needed */ 330 for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset); 331 sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym); 332 sym++) { 333 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT) 334 continue; 335 if (!strcmp(strtab + r(&sym->st_name), 336 "main_extable_sort_needed")) { 337 sort_needed_sym = sym; 338 break; 339 } 340 } 341 342 if (!sort_needed_sym) { 343 fprintf(stderr, 344 "no main_extable_sort_needed symbol in file: %s\n", 345 fname); 346 goto out; 347 } 348 349 sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx), 350 sort_needed_sym - symtab, 351 symtab_shndx)]; 352 sort_needed_loc = (void *)ehdr + 353 _r(&sort_needed_sec->sh_offset) + 354 _r(&sort_needed_sym->st_value) - 355 _r(&sort_needed_sec->sh_addr); 356 357 /* extable has been sorted, clear the flag */ 358 w(0, sort_needed_loc); 359 rc = 0; 360 361 out: 362 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 363 if (orc_sort_thread) { 364 void *retval = NULL; 365 /* wait for ORC tables sort done */ 366 rc = pthread_join(orc_sort_thread, &retval); 367 if (rc) 368 fprintf(stderr, 369 "pthread_join failed '%s': %s\n", 370 strerror(errno), fname); 371 else if (retval) { 372 rc = -1; 373 fprintf(stderr, 374 "failed to sort ORC tables '%s': %s\n", 375 (char *)retval, fname); 376 } 377 } 378 #endif 379 return rc; 380 } 381