1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/cutils.h" 34 #include "qemu/host-utils.h" 35 #include "qemu/timer.h" 36 37 /* Note: the long term plan is to reduce the dependencies on the QEMU 38 CPU definitions. Currently they are used for qemu_ld/st 39 instructions */ 40 #define NO_CPU_IO_DEFS 41 #include "cpu.h" 42 43 #include "exec/cpu-common.h" 44 #include "exec/exec-all.h" 45 46 #include "tcg-op.h" 47 48 #if UINTPTR_MAX == UINT32_MAX 49 # define ELF_CLASS ELFCLASS32 50 #else 51 # define ELF_CLASS ELFCLASS64 52 #endif 53 #ifdef HOST_WORDS_BIGENDIAN 54 # define ELF_DATA ELFDATA2MSB 55 #else 56 # define ELF_DATA ELFDATA2LSB 57 #endif 58 59 #include "elf.h" 60 #include "exec/log.h" 61 62 /* Forward declarations for functions declared in tcg-target.inc.c and 63 used here. */ 64 static void tcg_target_init(TCGContext *s); 65 static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode); 66 static void tcg_target_qemu_prologue(TCGContext *s); 67 static void patch_reloc(tcg_insn_unit *code_ptr, int type, 68 intptr_t value, intptr_t addend); 69 70 /* The CIE and FDE header definitions will be common to all hosts. */ 71 typedef struct { 72 uint32_t len __attribute__((aligned((sizeof(void *))))); 73 uint32_t id; 74 uint8_t version; 75 char augmentation[1]; 76 uint8_t code_align; 77 uint8_t data_align; 78 uint8_t return_column; 79 } DebugFrameCIE; 80 81 typedef struct QEMU_PACKED { 82 uint32_t len __attribute__((aligned((sizeof(void *))))); 83 uint32_t cie_offset; 84 uintptr_t func_start; 85 uintptr_t func_len; 86 } DebugFrameFDEHeader; 87 88 typedef struct QEMU_PACKED { 89 DebugFrameCIE cie; 90 DebugFrameFDEHeader fde; 91 } DebugFrameHeader; 92 93 static void tcg_register_jit_int(void *buf, size_t size, 94 const void *debug_frame, 95 size_t debug_frame_size) 96 __attribute__((unused)); 97 98 /* Forward declarations for functions declared and used in tcg-target.inc.c. */ 99 static const char *target_parse_constraint(TCGArgConstraint *ct, 100 const char *ct_str, TCGType type); 101 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 102 intptr_t arg2); 103 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 104 static void tcg_out_movi(TCGContext *s, TCGType type, 105 TCGReg ret, tcg_target_long arg); 106 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 107 const int *const_args); 108 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 109 intptr_t arg2); 110 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 111 TCGReg base, intptr_t ofs); 112 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); 113 static int tcg_target_const_match(tcg_target_long val, TCGType type, 114 const TCGArgConstraint *arg_ct); 115 #ifdef TCG_TARGET_NEED_LDST_LABELS 116 static bool tcg_out_ldst_finalize(TCGContext *s); 117 #endif 118 119 #define TCG_HIGHWATER 1024 120 121 static TCGRegSet tcg_target_available_regs[2]; 122 static TCGRegSet tcg_target_call_clobber_regs; 123 124 #if TCG_TARGET_INSN_UNIT_SIZE == 1 125 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 126 { 127 *s->code_ptr++ = v; 128 } 129 130 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 131 uint8_t v) 132 { 133 *p = v; 134 } 135 #endif 136 137 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 138 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 139 { 140 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 141 *s->code_ptr++ = v; 142 } else { 143 tcg_insn_unit *p = s->code_ptr; 144 memcpy(p, &v, sizeof(v)); 145 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 146 } 147 } 148 149 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 150 uint16_t v) 151 { 152 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 153 *p = v; 154 } else { 155 memcpy(p, &v, sizeof(v)); 156 } 157 } 158 #endif 159 160 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 161 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 162 { 163 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 164 *s->code_ptr++ = v; 165 } else { 166 tcg_insn_unit *p = s->code_ptr; 167 memcpy(p, &v, sizeof(v)); 168 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 169 } 170 } 171 172 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 173 uint32_t v) 174 { 175 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 176 *p = v; 177 } else { 178 memcpy(p, &v, sizeof(v)); 179 } 180 } 181 #endif 182 183 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 184 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 185 { 186 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 187 *s->code_ptr++ = v; 188 } else { 189 tcg_insn_unit *p = s->code_ptr; 190 memcpy(p, &v, sizeof(v)); 191 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 192 } 193 } 194 195 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 196 uint64_t v) 197 { 198 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 199 *p = v; 200 } else { 201 memcpy(p, &v, sizeof(v)); 202 } 203 } 204 #endif 205 206 /* label relocation processing */ 207 208 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 209 TCGLabel *l, intptr_t addend) 210 { 211 TCGRelocation *r; 212 213 if (l->has_value) { 214 /* FIXME: This may break relocations on RISC targets that 215 modify instruction fields in place. The caller may not have 216 written the initial value. */ 217 patch_reloc(code_ptr, type, l->u.value, addend); 218 } else { 219 /* add a new relocation entry */ 220 r = tcg_malloc(sizeof(TCGRelocation)); 221 r->type = type; 222 r->ptr = code_ptr; 223 r->addend = addend; 224 r->next = l->u.first_reloc; 225 l->u.first_reloc = r; 226 } 227 } 228 229 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr) 230 { 231 intptr_t value = (intptr_t)ptr; 232 TCGRelocation *r; 233 234 tcg_debug_assert(!l->has_value); 235 236 for (r = l->u.first_reloc; r != NULL; r = r->next) { 237 patch_reloc(r->ptr, r->type, value, r->addend); 238 } 239 240 l->has_value = 1; 241 l->u.value_ptr = ptr; 242 } 243 244 TCGLabel *gen_new_label(void) 245 { 246 TCGContext *s = &tcg_ctx; 247 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 248 249 *l = (TCGLabel){ 250 .id = s->nb_labels++ 251 }; 252 253 return l; 254 } 255 256 #include "tcg-target.inc.c" 257 258 /* pool based memory allocation */ 259 void *tcg_malloc_internal(TCGContext *s, int size) 260 { 261 TCGPool *p; 262 int pool_size; 263 264 if (size > TCG_POOL_CHUNK_SIZE) { 265 /* big malloc: insert a new pool (XXX: could optimize) */ 266 p = g_malloc(sizeof(TCGPool) + size); 267 p->size = size; 268 p->next = s->pool_first_large; 269 s->pool_first_large = p; 270 return p->data; 271 } else { 272 p = s->pool_current; 273 if (!p) { 274 p = s->pool_first; 275 if (!p) 276 goto new_pool; 277 } else { 278 if (!p->next) { 279 new_pool: 280 pool_size = TCG_POOL_CHUNK_SIZE; 281 p = g_malloc(sizeof(TCGPool) + pool_size); 282 p->size = pool_size; 283 p->next = NULL; 284 if (s->pool_current) 285 s->pool_current->next = p; 286 else 287 s->pool_first = p; 288 } else { 289 p = p->next; 290 } 291 } 292 } 293 s->pool_current = p; 294 s->pool_cur = p->data + size; 295 s->pool_end = p->data + p->size; 296 return p->data; 297 } 298 299 void tcg_pool_reset(TCGContext *s) 300 { 301 TCGPool *p, *t; 302 for (p = s->pool_first_large; p; p = t) { 303 t = p->next; 304 g_free(p); 305 } 306 s->pool_first_large = NULL; 307 s->pool_cur = s->pool_end = NULL; 308 s->pool_current = NULL; 309 } 310 311 typedef struct TCGHelperInfo { 312 void *func; 313 const char *name; 314 unsigned flags; 315 unsigned sizemask; 316 } TCGHelperInfo; 317 318 #include "exec/helper-proto.h" 319 320 static const TCGHelperInfo all_helpers[] = { 321 #include "exec/helper-tcg.h" 322 }; 323 static GHashTable *helper_table; 324 325 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 326 static void process_op_defs(TCGContext *s); 327 328 void tcg_context_init(TCGContext *s) 329 { 330 int op, total_args, n, i; 331 TCGOpDef *def; 332 TCGArgConstraint *args_ct; 333 int *sorted_args; 334 335 memset(s, 0, sizeof(*s)); 336 s->nb_globals = 0; 337 338 /* Count total number of arguments and allocate the corresponding 339 space */ 340 total_args = 0; 341 for(op = 0; op < NB_OPS; op++) { 342 def = &tcg_op_defs[op]; 343 n = def->nb_iargs + def->nb_oargs; 344 total_args += n; 345 } 346 347 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 348 sorted_args = g_malloc(sizeof(int) * total_args); 349 350 for(op = 0; op < NB_OPS; op++) { 351 def = &tcg_op_defs[op]; 352 def->args_ct = args_ct; 353 def->sorted_args = sorted_args; 354 n = def->nb_iargs + def->nb_oargs; 355 sorted_args += n; 356 args_ct += n; 357 } 358 359 /* Register helpers. */ 360 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 361 helper_table = g_hash_table_new(NULL, NULL); 362 363 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 364 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 365 (gpointer)&all_helpers[i]); 366 } 367 368 tcg_target_init(s); 369 process_op_defs(s); 370 371 /* Reverse the order of the saved registers, assuming they're all at 372 the start of tcg_target_reg_alloc_order. */ 373 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 374 int r = tcg_target_reg_alloc_order[n]; 375 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 376 break; 377 } 378 } 379 for (i = 0; i < n; ++i) { 380 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 381 } 382 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 383 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 384 } 385 } 386 387 /* 388 * Allocate TBs right before their corresponding translated code, making 389 * sure that TBs and code are on different cache lines. 390 */ 391 TranslationBlock *tcg_tb_alloc(TCGContext *s) 392 { 393 uintptr_t align = qemu_icache_linesize; 394 TranslationBlock *tb; 395 void *next; 396 397 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 398 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 399 400 if (unlikely(next > s->code_gen_highwater)) { 401 return NULL; 402 } 403 s->code_gen_ptr = next; 404 s->data_gen_ptr = NULL; 405 return tb; 406 } 407 408 void tcg_prologue_init(TCGContext *s) 409 { 410 size_t prologue_size, total_size; 411 void *buf0, *buf1; 412 413 /* Put the prologue at the beginning of code_gen_buffer. */ 414 buf0 = s->code_gen_buffer; 415 s->code_ptr = buf0; 416 s->code_buf = buf0; 417 s->code_gen_prologue = buf0; 418 419 /* Generate the prologue. */ 420 tcg_target_qemu_prologue(s); 421 buf1 = s->code_ptr; 422 flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1); 423 424 /* Deduct the prologue from the buffer. */ 425 prologue_size = tcg_current_code_size(s); 426 s->code_gen_ptr = buf1; 427 s->code_gen_buffer = buf1; 428 s->code_buf = buf1; 429 total_size = s->code_gen_buffer_size - prologue_size; 430 s->code_gen_buffer_size = total_size; 431 432 /* Compute a high-water mark, at which we voluntarily flush the buffer 433 and start over. The size here is arbitrary, significantly larger 434 than we expect the code generation for any one opcode to require. */ 435 s->code_gen_highwater = s->code_gen_buffer + (total_size - TCG_HIGHWATER); 436 437 tcg_register_jit(s->code_gen_buffer, total_size); 438 439 #ifdef DEBUG_DISAS 440 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 441 qemu_log_lock(); 442 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 443 log_disas(buf0, prologue_size); 444 qemu_log("\n"); 445 qemu_log_flush(); 446 qemu_log_unlock(); 447 } 448 #endif 449 450 /* Assert that goto_ptr is implemented completely. */ 451 if (TCG_TARGET_HAS_goto_ptr) { 452 tcg_debug_assert(s->code_gen_epilogue != NULL); 453 } 454 } 455 456 void tcg_func_start(TCGContext *s) 457 { 458 tcg_pool_reset(s); 459 s->nb_temps = s->nb_globals; 460 461 /* No temps have been previously allocated for size or locality. */ 462 memset(s->free_temps, 0, sizeof(s->free_temps)); 463 464 s->nb_labels = 0; 465 s->current_frame_offset = s->frame_start; 466 467 #ifdef CONFIG_DEBUG_TCG 468 s->goto_tb_issue_mask = 0; 469 #endif 470 471 s->gen_op_buf[0].next = 1; 472 s->gen_op_buf[0].prev = 0; 473 s->gen_next_op_idx = 1; 474 s->gen_next_parm_idx = 0; 475 } 476 477 static inline int temp_idx(TCGContext *s, TCGTemp *ts) 478 { 479 ptrdiff_t n = ts - s->temps; 480 tcg_debug_assert(n >= 0 && n < s->nb_temps); 481 return n; 482 } 483 484 static inline TCGTemp *tcg_temp_alloc(TCGContext *s) 485 { 486 int n = s->nb_temps++; 487 tcg_debug_assert(n < TCG_MAX_TEMPS); 488 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 489 } 490 491 static inline TCGTemp *tcg_global_alloc(TCGContext *s) 492 { 493 tcg_debug_assert(s->nb_globals == s->nb_temps); 494 s->nb_globals++; 495 return tcg_temp_alloc(s); 496 } 497 498 static int tcg_global_reg_new_internal(TCGContext *s, TCGType type, 499 TCGReg reg, const char *name) 500 { 501 TCGTemp *ts; 502 503 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 504 tcg_abort(); 505 } 506 507 ts = tcg_global_alloc(s); 508 ts->base_type = type; 509 ts->type = type; 510 ts->fixed_reg = 1; 511 ts->reg = reg; 512 ts->name = name; 513 tcg_regset_set_reg(s->reserved_regs, reg); 514 515 return temp_idx(s, ts); 516 } 517 518 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 519 { 520 int idx; 521 s->frame_start = start; 522 s->frame_end = start + size; 523 idx = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 524 s->frame_temp = &s->temps[idx]; 525 } 526 527 TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name) 528 { 529 TCGContext *s = &tcg_ctx; 530 int idx; 531 532 if (tcg_regset_test_reg(s->reserved_regs, reg)) { 533 tcg_abort(); 534 } 535 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I32, reg, name); 536 return MAKE_TCGV_I32(idx); 537 } 538 539 TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name) 540 { 541 TCGContext *s = &tcg_ctx; 542 int idx; 543 544 if (tcg_regset_test_reg(s->reserved_regs, reg)) { 545 tcg_abort(); 546 } 547 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I64, reg, name); 548 return MAKE_TCGV_I64(idx); 549 } 550 551 int tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 552 intptr_t offset, const char *name) 553 { 554 TCGContext *s = &tcg_ctx; 555 TCGTemp *base_ts = &s->temps[GET_TCGV_PTR(base)]; 556 TCGTemp *ts = tcg_global_alloc(s); 557 int indirect_reg = 0, bigendian = 0; 558 #ifdef HOST_WORDS_BIGENDIAN 559 bigendian = 1; 560 #endif 561 562 if (!base_ts->fixed_reg) { 563 /* We do not support double-indirect registers. */ 564 tcg_debug_assert(!base_ts->indirect_reg); 565 base_ts->indirect_base = 1; 566 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 567 ? 2 : 1); 568 indirect_reg = 1; 569 } 570 571 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 572 TCGTemp *ts2 = tcg_global_alloc(s); 573 char buf[64]; 574 575 ts->base_type = TCG_TYPE_I64; 576 ts->type = TCG_TYPE_I32; 577 ts->indirect_reg = indirect_reg; 578 ts->mem_allocated = 1; 579 ts->mem_base = base_ts; 580 ts->mem_offset = offset + bigendian * 4; 581 pstrcpy(buf, sizeof(buf), name); 582 pstrcat(buf, sizeof(buf), "_0"); 583 ts->name = strdup(buf); 584 585 tcg_debug_assert(ts2 == ts + 1); 586 ts2->base_type = TCG_TYPE_I64; 587 ts2->type = TCG_TYPE_I32; 588 ts2->indirect_reg = indirect_reg; 589 ts2->mem_allocated = 1; 590 ts2->mem_base = base_ts; 591 ts2->mem_offset = offset + (1 - bigendian) * 4; 592 pstrcpy(buf, sizeof(buf), name); 593 pstrcat(buf, sizeof(buf), "_1"); 594 ts2->name = strdup(buf); 595 } else { 596 ts->base_type = type; 597 ts->type = type; 598 ts->indirect_reg = indirect_reg; 599 ts->mem_allocated = 1; 600 ts->mem_base = base_ts; 601 ts->mem_offset = offset; 602 ts->name = name; 603 } 604 return temp_idx(s, ts); 605 } 606 607 static int tcg_temp_new_internal(TCGType type, int temp_local) 608 { 609 TCGContext *s = &tcg_ctx; 610 TCGTemp *ts; 611 int idx, k; 612 613 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 614 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 615 if (idx < TCG_MAX_TEMPS) { 616 /* There is already an available temp with the right type. */ 617 clear_bit(idx, s->free_temps[k].l); 618 619 ts = &s->temps[idx]; 620 ts->temp_allocated = 1; 621 tcg_debug_assert(ts->base_type == type); 622 tcg_debug_assert(ts->temp_local == temp_local); 623 } else { 624 ts = tcg_temp_alloc(s); 625 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 626 TCGTemp *ts2 = tcg_temp_alloc(s); 627 628 ts->base_type = type; 629 ts->type = TCG_TYPE_I32; 630 ts->temp_allocated = 1; 631 ts->temp_local = temp_local; 632 633 tcg_debug_assert(ts2 == ts + 1); 634 ts2->base_type = TCG_TYPE_I64; 635 ts2->type = TCG_TYPE_I32; 636 ts2->temp_allocated = 1; 637 ts2->temp_local = temp_local; 638 } else { 639 ts->base_type = type; 640 ts->type = type; 641 ts->temp_allocated = 1; 642 ts->temp_local = temp_local; 643 } 644 idx = temp_idx(s, ts); 645 } 646 647 #if defined(CONFIG_DEBUG_TCG) 648 s->temps_in_use++; 649 #endif 650 return idx; 651 } 652 653 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 654 { 655 int idx; 656 657 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 658 return MAKE_TCGV_I32(idx); 659 } 660 661 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 662 { 663 int idx; 664 665 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 666 return MAKE_TCGV_I64(idx); 667 } 668 669 static void tcg_temp_free_internal(int idx) 670 { 671 TCGContext *s = &tcg_ctx; 672 TCGTemp *ts; 673 int k; 674 675 #if defined(CONFIG_DEBUG_TCG) 676 s->temps_in_use--; 677 if (s->temps_in_use < 0) { 678 fprintf(stderr, "More temporaries freed than allocated!\n"); 679 } 680 #endif 681 682 tcg_debug_assert(idx >= s->nb_globals && idx < s->nb_temps); 683 ts = &s->temps[idx]; 684 tcg_debug_assert(ts->temp_allocated != 0); 685 ts->temp_allocated = 0; 686 687 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 688 set_bit(idx, s->free_temps[k].l); 689 } 690 691 void tcg_temp_free_i32(TCGv_i32 arg) 692 { 693 tcg_temp_free_internal(GET_TCGV_I32(arg)); 694 } 695 696 void tcg_temp_free_i64(TCGv_i64 arg) 697 { 698 tcg_temp_free_internal(GET_TCGV_I64(arg)); 699 } 700 701 TCGv_i32 tcg_const_i32(int32_t val) 702 { 703 TCGv_i32 t0; 704 t0 = tcg_temp_new_i32(); 705 tcg_gen_movi_i32(t0, val); 706 return t0; 707 } 708 709 TCGv_i64 tcg_const_i64(int64_t val) 710 { 711 TCGv_i64 t0; 712 t0 = tcg_temp_new_i64(); 713 tcg_gen_movi_i64(t0, val); 714 return t0; 715 } 716 717 TCGv_i32 tcg_const_local_i32(int32_t val) 718 { 719 TCGv_i32 t0; 720 t0 = tcg_temp_local_new_i32(); 721 tcg_gen_movi_i32(t0, val); 722 return t0; 723 } 724 725 TCGv_i64 tcg_const_local_i64(int64_t val) 726 { 727 TCGv_i64 t0; 728 t0 = tcg_temp_local_new_i64(); 729 tcg_gen_movi_i64(t0, val); 730 return t0; 731 } 732 733 #if defined(CONFIG_DEBUG_TCG) 734 void tcg_clear_temp_count(void) 735 { 736 TCGContext *s = &tcg_ctx; 737 s->temps_in_use = 0; 738 } 739 740 int tcg_check_temp_count(void) 741 { 742 TCGContext *s = &tcg_ctx; 743 if (s->temps_in_use) { 744 /* Clear the count so that we don't give another 745 * warning immediately next time around. 746 */ 747 s->temps_in_use = 0; 748 return 1; 749 } 750 return 0; 751 } 752 #endif 753 754 /* Return true if OP may appear in the opcode stream. 755 Test the runtime variable that controls each opcode. */ 756 bool tcg_op_supported(TCGOpcode op) 757 { 758 switch (op) { 759 case INDEX_op_discard: 760 case INDEX_op_set_label: 761 case INDEX_op_call: 762 case INDEX_op_br: 763 case INDEX_op_mb: 764 case INDEX_op_insn_start: 765 case INDEX_op_exit_tb: 766 case INDEX_op_goto_tb: 767 case INDEX_op_qemu_ld_i32: 768 case INDEX_op_qemu_st_i32: 769 case INDEX_op_qemu_ld_i64: 770 case INDEX_op_qemu_st_i64: 771 return true; 772 773 case INDEX_op_goto_ptr: 774 return TCG_TARGET_HAS_goto_ptr; 775 776 case INDEX_op_mov_i32: 777 case INDEX_op_movi_i32: 778 case INDEX_op_setcond_i32: 779 case INDEX_op_brcond_i32: 780 case INDEX_op_ld8u_i32: 781 case INDEX_op_ld8s_i32: 782 case INDEX_op_ld16u_i32: 783 case INDEX_op_ld16s_i32: 784 case INDEX_op_ld_i32: 785 case INDEX_op_st8_i32: 786 case INDEX_op_st16_i32: 787 case INDEX_op_st_i32: 788 case INDEX_op_add_i32: 789 case INDEX_op_sub_i32: 790 case INDEX_op_mul_i32: 791 case INDEX_op_and_i32: 792 case INDEX_op_or_i32: 793 case INDEX_op_xor_i32: 794 case INDEX_op_shl_i32: 795 case INDEX_op_shr_i32: 796 case INDEX_op_sar_i32: 797 return true; 798 799 case INDEX_op_movcond_i32: 800 return TCG_TARGET_HAS_movcond_i32; 801 case INDEX_op_div_i32: 802 case INDEX_op_divu_i32: 803 return TCG_TARGET_HAS_div_i32; 804 case INDEX_op_rem_i32: 805 case INDEX_op_remu_i32: 806 return TCG_TARGET_HAS_rem_i32; 807 case INDEX_op_div2_i32: 808 case INDEX_op_divu2_i32: 809 return TCG_TARGET_HAS_div2_i32; 810 case INDEX_op_rotl_i32: 811 case INDEX_op_rotr_i32: 812 return TCG_TARGET_HAS_rot_i32; 813 case INDEX_op_deposit_i32: 814 return TCG_TARGET_HAS_deposit_i32; 815 case INDEX_op_extract_i32: 816 return TCG_TARGET_HAS_extract_i32; 817 case INDEX_op_sextract_i32: 818 return TCG_TARGET_HAS_sextract_i32; 819 case INDEX_op_add2_i32: 820 return TCG_TARGET_HAS_add2_i32; 821 case INDEX_op_sub2_i32: 822 return TCG_TARGET_HAS_sub2_i32; 823 case INDEX_op_mulu2_i32: 824 return TCG_TARGET_HAS_mulu2_i32; 825 case INDEX_op_muls2_i32: 826 return TCG_TARGET_HAS_muls2_i32; 827 case INDEX_op_muluh_i32: 828 return TCG_TARGET_HAS_muluh_i32; 829 case INDEX_op_mulsh_i32: 830 return TCG_TARGET_HAS_mulsh_i32; 831 case INDEX_op_ext8s_i32: 832 return TCG_TARGET_HAS_ext8s_i32; 833 case INDEX_op_ext16s_i32: 834 return TCG_TARGET_HAS_ext16s_i32; 835 case INDEX_op_ext8u_i32: 836 return TCG_TARGET_HAS_ext8u_i32; 837 case INDEX_op_ext16u_i32: 838 return TCG_TARGET_HAS_ext16u_i32; 839 case INDEX_op_bswap16_i32: 840 return TCG_TARGET_HAS_bswap16_i32; 841 case INDEX_op_bswap32_i32: 842 return TCG_TARGET_HAS_bswap32_i32; 843 case INDEX_op_not_i32: 844 return TCG_TARGET_HAS_not_i32; 845 case INDEX_op_neg_i32: 846 return TCG_TARGET_HAS_neg_i32; 847 case INDEX_op_andc_i32: 848 return TCG_TARGET_HAS_andc_i32; 849 case INDEX_op_orc_i32: 850 return TCG_TARGET_HAS_orc_i32; 851 case INDEX_op_eqv_i32: 852 return TCG_TARGET_HAS_eqv_i32; 853 case INDEX_op_nand_i32: 854 return TCG_TARGET_HAS_nand_i32; 855 case INDEX_op_nor_i32: 856 return TCG_TARGET_HAS_nor_i32; 857 case INDEX_op_clz_i32: 858 return TCG_TARGET_HAS_clz_i32; 859 case INDEX_op_ctz_i32: 860 return TCG_TARGET_HAS_ctz_i32; 861 case INDEX_op_ctpop_i32: 862 return TCG_TARGET_HAS_ctpop_i32; 863 864 case INDEX_op_brcond2_i32: 865 case INDEX_op_setcond2_i32: 866 return TCG_TARGET_REG_BITS == 32; 867 868 case INDEX_op_mov_i64: 869 case INDEX_op_movi_i64: 870 case INDEX_op_setcond_i64: 871 case INDEX_op_brcond_i64: 872 case INDEX_op_ld8u_i64: 873 case INDEX_op_ld8s_i64: 874 case INDEX_op_ld16u_i64: 875 case INDEX_op_ld16s_i64: 876 case INDEX_op_ld32u_i64: 877 case INDEX_op_ld32s_i64: 878 case INDEX_op_ld_i64: 879 case INDEX_op_st8_i64: 880 case INDEX_op_st16_i64: 881 case INDEX_op_st32_i64: 882 case INDEX_op_st_i64: 883 case INDEX_op_add_i64: 884 case INDEX_op_sub_i64: 885 case INDEX_op_mul_i64: 886 case INDEX_op_and_i64: 887 case INDEX_op_or_i64: 888 case INDEX_op_xor_i64: 889 case INDEX_op_shl_i64: 890 case INDEX_op_shr_i64: 891 case INDEX_op_sar_i64: 892 case INDEX_op_ext_i32_i64: 893 case INDEX_op_extu_i32_i64: 894 return TCG_TARGET_REG_BITS == 64; 895 896 case INDEX_op_movcond_i64: 897 return TCG_TARGET_HAS_movcond_i64; 898 case INDEX_op_div_i64: 899 case INDEX_op_divu_i64: 900 return TCG_TARGET_HAS_div_i64; 901 case INDEX_op_rem_i64: 902 case INDEX_op_remu_i64: 903 return TCG_TARGET_HAS_rem_i64; 904 case INDEX_op_div2_i64: 905 case INDEX_op_divu2_i64: 906 return TCG_TARGET_HAS_div2_i64; 907 case INDEX_op_rotl_i64: 908 case INDEX_op_rotr_i64: 909 return TCG_TARGET_HAS_rot_i64; 910 case INDEX_op_deposit_i64: 911 return TCG_TARGET_HAS_deposit_i64; 912 case INDEX_op_extract_i64: 913 return TCG_TARGET_HAS_extract_i64; 914 case INDEX_op_sextract_i64: 915 return TCG_TARGET_HAS_sextract_i64; 916 case INDEX_op_extrl_i64_i32: 917 return TCG_TARGET_HAS_extrl_i64_i32; 918 case INDEX_op_extrh_i64_i32: 919 return TCG_TARGET_HAS_extrh_i64_i32; 920 case INDEX_op_ext8s_i64: 921 return TCG_TARGET_HAS_ext8s_i64; 922 case INDEX_op_ext16s_i64: 923 return TCG_TARGET_HAS_ext16s_i64; 924 case INDEX_op_ext32s_i64: 925 return TCG_TARGET_HAS_ext32s_i64; 926 case INDEX_op_ext8u_i64: 927 return TCG_TARGET_HAS_ext8u_i64; 928 case INDEX_op_ext16u_i64: 929 return TCG_TARGET_HAS_ext16u_i64; 930 case INDEX_op_ext32u_i64: 931 return TCG_TARGET_HAS_ext32u_i64; 932 case INDEX_op_bswap16_i64: 933 return TCG_TARGET_HAS_bswap16_i64; 934 case INDEX_op_bswap32_i64: 935 return TCG_TARGET_HAS_bswap32_i64; 936 case INDEX_op_bswap64_i64: 937 return TCG_TARGET_HAS_bswap64_i64; 938 case INDEX_op_not_i64: 939 return TCG_TARGET_HAS_not_i64; 940 case INDEX_op_neg_i64: 941 return TCG_TARGET_HAS_neg_i64; 942 case INDEX_op_andc_i64: 943 return TCG_TARGET_HAS_andc_i64; 944 case INDEX_op_orc_i64: 945 return TCG_TARGET_HAS_orc_i64; 946 case INDEX_op_eqv_i64: 947 return TCG_TARGET_HAS_eqv_i64; 948 case INDEX_op_nand_i64: 949 return TCG_TARGET_HAS_nand_i64; 950 case INDEX_op_nor_i64: 951 return TCG_TARGET_HAS_nor_i64; 952 case INDEX_op_clz_i64: 953 return TCG_TARGET_HAS_clz_i64; 954 case INDEX_op_ctz_i64: 955 return TCG_TARGET_HAS_ctz_i64; 956 case INDEX_op_ctpop_i64: 957 return TCG_TARGET_HAS_ctpop_i64; 958 case INDEX_op_add2_i64: 959 return TCG_TARGET_HAS_add2_i64; 960 case INDEX_op_sub2_i64: 961 return TCG_TARGET_HAS_sub2_i64; 962 case INDEX_op_mulu2_i64: 963 return TCG_TARGET_HAS_mulu2_i64; 964 case INDEX_op_muls2_i64: 965 return TCG_TARGET_HAS_muls2_i64; 966 case INDEX_op_muluh_i64: 967 return TCG_TARGET_HAS_muluh_i64; 968 case INDEX_op_mulsh_i64: 969 return TCG_TARGET_HAS_mulsh_i64; 970 971 case NB_OPS: 972 break; 973 } 974 g_assert_not_reached(); 975 } 976 977 /* Note: we convert the 64 bit args to 32 bit and do some alignment 978 and endian swap. Maybe it would be better to do the alignment 979 and endian swap in tcg_reg_alloc_call(). */ 980 void tcg_gen_callN(TCGContext *s, void *func, TCGArg ret, 981 int nargs, TCGArg *args) 982 { 983 int i, real_args, nb_rets, pi, pi_first; 984 unsigned sizemask, flags; 985 TCGHelperInfo *info; 986 987 info = g_hash_table_lookup(helper_table, (gpointer)func); 988 flags = info->flags; 989 sizemask = info->sizemask; 990 991 #if defined(__sparc__) && !defined(__arch64__) \ 992 && !defined(CONFIG_TCG_INTERPRETER) 993 /* We have 64-bit values in one register, but need to pass as two 994 separate parameters. Split them. */ 995 int orig_sizemask = sizemask; 996 int orig_nargs = nargs; 997 TCGv_i64 retl, reth; 998 999 TCGV_UNUSED_I64(retl); 1000 TCGV_UNUSED_I64(reth); 1001 if (sizemask != 0) { 1002 TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2); 1003 for (i = real_args = 0; i < nargs; ++i) { 1004 int is_64bit = sizemask & (1 << (i+1)*2); 1005 if (is_64bit) { 1006 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 1007 TCGv_i32 h = tcg_temp_new_i32(); 1008 TCGv_i32 l = tcg_temp_new_i32(); 1009 tcg_gen_extr_i64_i32(l, h, orig); 1010 split_args[real_args++] = GET_TCGV_I32(h); 1011 split_args[real_args++] = GET_TCGV_I32(l); 1012 } else { 1013 split_args[real_args++] = args[i]; 1014 } 1015 } 1016 nargs = real_args; 1017 args = split_args; 1018 sizemask = 0; 1019 } 1020 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1021 for (i = 0; i < nargs; ++i) { 1022 int is_64bit = sizemask & (1 << (i+1)*2); 1023 int is_signed = sizemask & (2 << (i+1)*2); 1024 if (!is_64bit) { 1025 TCGv_i64 temp = tcg_temp_new_i64(); 1026 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 1027 if (is_signed) { 1028 tcg_gen_ext32s_i64(temp, orig); 1029 } else { 1030 tcg_gen_ext32u_i64(temp, orig); 1031 } 1032 args[i] = GET_TCGV_I64(temp); 1033 } 1034 } 1035 #endif /* TCG_TARGET_EXTEND_ARGS */ 1036 1037 pi_first = pi = s->gen_next_parm_idx; 1038 if (ret != TCG_CALL_DUMMY_ARG) { 1039 #if defined(__sparc__) && !defined(__arch64__) \ 1040 && !defined(CONFIG_TCG_INTERPRETER) 1041 if (orig_sizemask & 1) { 1042 /* The 32-bit ABI is going to return the 64-bit value in 1043 the %o0/%o1 register pair. Prepare for this by using 1044 two return temporaries, and reassemble below. */ 1045 retl = tcg_temp_new_i64(); 1046 reth = tcg_temp_new_i64(); 1047 s->gen_opparam_buf[pi++] = GET_TCGV_I64(reth); 1048 s->gen_opparam_buf[pi++] = GET_TCGV_I64(retl); 1049 nb_rets = 2; 1050 } else { 1051 s->gen_opparam_buf[pi++] = ret; 1052 nb_rets = 1; 1053 } 1054 #else 1055 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { 1056 #ifdef HOST_WORDS_BIGENDIAN 1057 s->gen_opparam_buf[pi++] = ret + 1; 1058 s->gen_opparam_buf[pi++] = ret; 1059 #else 1060 s->gen_opparam_buf[pi++] = ret; 1061 s->gen_opparam_buf[pi++] = ret + 1; 1062 #endif 1063 nb_rets = 2; 1064 } else { 1065 s->gen_opparam_buf[pi++] = ret; 1066 nb_rets = 1; 1067 } 1068 #endif 1069 } else { 1070 nb_rets = 0; 1071 } 1072 real_args = 0; 1073 for (i = 0; i < nargs; i++) { 1074 int is_64bit = sizemask & (1 << (i+1)*2); 1075 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 1076 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 1077 /* some targets want aligned 64 bit args */ 1078 if (real_args & 1) { 1079 s->gen_opparam_buf[pi++] = TCG_CALL_DUMMY_ARG; 1080 real_args++; 1081 } 1082 #endif 1083 /* If stack grows up, then we will be placing successive 1084 arguments at lower addresses, which means we need to 1085 reverse the order compared to how we would normally 1086 treat either big or little-endian. For those arguments 1087 that will wind up in registers, this still works for 1088 HPPA (the only current STACK_GROWSUP target) since the 1089 argument registers are *also* allocated in decreasing 1090 order. If another such target is added, this logic may 1091 have to get more complicated to differentiate between 1092 stack arguments and register arguments. */ 1093 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 1094 s->gen_opparam_buf[pi++] = args[i] + 1; 1095 s->gen_opparam_buf[pi++] = args[i]; 1096 #else 1097 s->gen_opparam_buf[pi++] = args[i]; 1098 s->gen_opparam_buf[pi++] = args[i] + 1; 1099 #endif 1100 real_args += 2; 1101 continue; 1102 } 1103 1104 s->gen_opparam_buf[pi++] = args[i]; 1105 real_args++; 1106 } 1107 s->gen_opparam_buf[pi++] = (uintptr_t)func; 1108 s->gen_opparam_buf[pi++] = flags; 1109 1110 i = s->gen_next_op_idx; 1111 tcg_debug_assert(i < OPC_BUF_SIZE); 1112 tcg_debug_assert(pi <= OPPARAM_BUF_SIZE); 1113 1114 /* Set links for sequential allocation during translation. */ 1115 s->gen_op_buf[i] = (TCGOp){ 1116 .opc = INDEX_op_call, 1117 .callo = nb_rets, 1118 .calli = real_args, 1119 .args = pi_first, 1120 .prev = i - 1, 1121 .next = i + 1 1122 }; 1123 1124 /* Make sure the calli field didn't overflow. */ 1125 tcg_debug_assert(s->gen_op_buf[i].calli == real_args); 1126 1127 s->gen_op_buf[0].prev = i; 1128 s->gen_next_op_idx = i + 1; 1129 s->gen_next_parm_idx = pi; 1130 1131 #if defined(__sparc__) && !defined(__arch64__) \ 1132 && !defined(CONFIG_TCG_INTERPRETER) 1133 /* Free all of the parts we allocated above. */ 1134 for (i = real_args = 0; i < orig_nargs; ++i) { 1135 int is_64bit = orig_sizemask & (1 << (i+1)*2); 1136 if (is_64bit) { 1137 TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]); 1138 TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]); 1139 tcg_temp_free_i32(h); 1140 tcg_temp_free_i32(l); 1141 } else { 1142 real_args++; 1143 } 1144 } 1145 if (orig_sizemask & 1) { 1146 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 1147 Note that describing these as TCGv_i64 eliminates an unnecessary 1148 zero-extension that tcg_gen_concat_i32_i64 would create. */ 1149 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth); 1150 tcg_temp_free_i64(retl); 1151 tcg_temp_free_i64(reth); 1152 } 1153 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1154 for (i = 0; i < nargs; ++i) { 1155 int is_64bit = sizemask & (1 << (i+1)*2); 1156 if (!is_64bit) { 1157 TCGv_i64 temp = MAKE_TCGV_I64(args[i]); 1158 tcg_temp_free_i64(temp); 1159 } 1160 } 1161 #endif /* TCG_TARGET_EXTEND_ARGS */ 1162 } 1163 1164 static void tcg_reg_alloc_start(TCGContext *s) 1165 { 1166 int i; 1167 TCGTemp *ts; 1168 for(i = 0; i < s->nb_globals; i++) { 1169 ts = &s->temps[i]; 1170 if (ts->fixed_reg) { 1171 ts->val_type = TEMP_VAL_REG; 1172 } else { 1173 ts->val_type = TEMP_VAL_MEM; 1174 } 1175 } 1176 for(i = s->nb_globals; i < s->nb_temps; i++) { 1177 ts = &s->temps[i]; 1178 if (ts->temp_local) { 1179 ts->val_type = TEMP_VAL_MEM; 1180 } else { 1181 ts->val_type = TEMP_VAL_DEAD; 1182 } 1183 ts->mem_allocated = 0; 1184 ts->fixed_reg = 0; 1185 } 1186 1187 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1188 } 1189 1190 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1191 TCGTemp *ts) 1192 { 1193 int idx = temp_idx(s, ts); 1194 1195 if (idx < s->nb_globals) { 1196 pstrcpy(buf, buf_size, ts->name); 1197 } else if (ts->temp_local) { 1198 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1199 } else { 1200 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1201 } 1202 return buf; 1203 } 1204 1205 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, 1206 int buf_size, int idx) 1207 { 1208 tcg_debug_assert(idx >= 0 && idx < s->nb_temps); 1209 return tcg_get_arg_str_ptr(s, buf, buf_size, &s->temps[idx]); 1210 } 1211 1212 /* Find helper name. */ 1213 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 1214 { 1215 const char *ret = NULL; 1216 if (helper_table) { 1217 TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val); 1218 if (info) { 1219 ret = info->name; 1220 } 1221 } 1222 return ret; 1223 } 1224 1225 static const char * const cond_name[] = 1226 { 1227 [TCG_COND_NEVER] = "never", 1228 [TCG_COND_ALWAYS] = "always", 1229 [TCG_COND_EQ] = "eq", 1230 [TCG_COND_NE] = "ne", 1231 [TCG_COND_LT] = "lt", 1232 [TCG_COND_GE] = "ge", 1233 [TCG_COND_LE] = "le", 1234 [TCG_COND_GT] = "gt", 1235 [TCG_COND_LTU] = "ltu", 1236 [TCG_COND_GEU] = "geu", 1237 [TCG_COND_LEU] = "leu", 1238 [TCG_COND_GTU] = "gtu" 1239 }; 1240 1241 static const char * const ldst_name[] = 1242 { 1243 [MO_UB] = "ub", 1244 [MO_SB] = "sb", 1245 [MO_LEUW] = "leuw", 1246 [MO_LESW] = "lesw", 1247 [MO_LEUL] = "leul", 1248 [MO_LESL] = "lesl", 1249 [MO_LEQ] = "leq", 1250 [MO_BEUW] = "beuw", 1251 [MO_BESW] = "besw", 1252 [MO_BEUL] = "beul", 1253 [MO_BESL] = "besl", 1254 [MO_BEQ] = "beq", 1255 }; 1256 1257 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1258 #ifdef ALIGNED_ONLY 1259 [MO_UNALN >> MO_ASHIFT] = "un+", 1260 [MO_ALIGN >> MO_ASHIFT] = "", 1261 #else 1262 [MO_UNALN >> MO_ASHIFT] = "", 1263 [MO_ALIGN >> MO_ASHIFT] = "al+", 1264 #endif 1265 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1266 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1267 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1268 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1269 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1270 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1271 }; 1272 1273 void tcg_dump_ops(TCGContext *s) 1274 { 1275 char buf[128]; 1276 TCGOp *op; 1277 int oi; 1278 1279 for (oi = s->gen_op_buf[0].next; oi != 0; oi = op->next) { 1280 int i, k, nb_oargs, nb_iargs, nb_cargs; 1281 const TCGOpDef *def; 1282 const TCGArg *args; 1283 TCGOpcode c; 1284 int col = 0; 1285 1286 op = &s->gen_op_buf[oi]; 1287 c = op->opc; 1288 def = &tcg_op_defs[c]; 1289 args = &s->gen_opparam_buf[op->args]; 1290 1291 if (c == INDEX_op_insn_start) { 1292 col += qemu_log("%s ----", oi != s->gen_op_buf[0].next ? "\n" : ""); 1293 1294 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1295 target_ulong a; 1296 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1297 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2]; 1298 #else 1299 a = args[i]; 1300 #endif 1301 col += qemu_log(" " TARGET_FMT_lx, a); 1302 } 1303 } else if (c == INDEX_op_call) { 1304 /* variable number of arguments */ 1305 nb_oargs = op->callo; 1306 nb_iargs = op->calli; 1307 nb_cargs = def->nb_cargs; 1308 1309 /* function name, flags, out args */ 1310 col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, 1311 tcg_find_helper(s, args[nb_oargs + nb_iargs]), 1312 args[nb_oargs + nb_iargs + 1], nb_oargs); 1313 for (i = 0; i < nb_oargs; i++) { 1314 col += qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1315 args[i])); 1316 } 1317 for (i = 0; i < nb_iargs; i++) { 1318 TCGArg arg = args[nb_oargs + i]; 1319 const char *t = "<dummy>"; 1320 if (arg != TCG_CALL_DUMMY_ARG) { 1321 t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg); 1322 } 1323 col += qemu_log(",%s", t); 1324 } 1325 } else { 1326 col += qemu_log(" %s ", def->name); 1327 1328 nb_oargs = def->nb_oargs; 1329 nb_iargs = def->nb_iargs; 1330 nb_cargs = def->nb_cargs; 1331 1332 k = 0; 1333 for (i = 0; i < nb_oargs; i++) { 1334 if (k != 0) { 1335 col += qemu_log(","); 1336 } 1337 col += qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1338 args[k++])); 1339 } 1340 for (i = 0; i < nb_iargs; i++) { 1341 if (k != 0) { 1342 col += qemu_log(","); 1343 } 1344 col += qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1345 args[k++])); 1346 } 1347 switch (c) { 1348 case INDEX_op_brcond_i32: 1349 case INDEX_op_setcond_i32: 1350 case INDEX_op_movcond_i32: 1351 case INDEX_op_brcond2_i32: 1352 case INDEX_op_setcond2_i32: 1353 case INDEX_op_brcond_i64: 1354 case INDEX_op_setcond_i64: 1355 case INDEX_op_movcond_i64: 1356 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) { 1357 col += qemu_log(",%s", cond_name[args[k++]]); 1358 } else { 1359 col += qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1360 } 1361 i = 1; 1362 break; 1363 case INDEX_op_qemu_ld_i32: 1364 case INDEX_op_qemu_st_i32: 1365 case INDEX_op_qemu_ld_i64: 1366 case INDEX_op_qemu_st_i64: 1367 { 1368 TCGMemOpIdx oi = args[k++]; 1369 TCGMemOp op = get_memop(oi); 1370 unsigned ix = get_mmuidx(oi); 1371 1372 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1373 col += qemu_log(",$0x%x,%u", op, ix); 1374 } else { 1375 const char *s_al, *s_op; 1376 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1377 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1378 col += qemu_log(",%s%s,%u", s_al, s_op, ix); 1379 } 1380 i = 1; 1381 } 1382 break; 1383 default: 1384 i = 0; 1385 break; 1386 } 1387 switch (c) { 1388 case INDEX_op_set_label: 1389 case INDEX_op_br: 1390 case INDEX_op_brcond_i32: 1391 case INDEX_op_brcond_i64: 1392 case INDEX_op_brcond2_i32: 1393 col += qemu_log("%s$L%d", k ? "," : "", arg_label(args[k])->id); 1394 i++, k++; 1395 break; 1396 default: 1397 break; 1398 } 1399 for (; i < nb_cargs; i++, k++) { 1400 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", args[k]); 1401 } 1402 } 1403 if (op->life) { 1404 unsigned life = op->life; 1405 1406 for (; col < 48; ++col) { 1407 putc(' ', qemu_logfile); 1408 } 1409 1410 if (life & (SYNC_ARG * 3)) { 1411 qemu_log(" sync:"); 1412 for (i = 0; i < 2; ++i) { 1413 if (life & (SYNC_ARG << i)) { 1414 qemu_log(" %d", i); 1415 } 1416 } 1417 } 1418 life /= DEAD_ARG; 1419 if (life) { 1420 qemu_log(" dead:"); 1421 for (i = 0; life; ++i, life >>= 1) { 1422 if (life & 1) { 1423 qemu_log(" %d", i); 1424 } 1425 } 1426 } 1427 } 1428 qemu_log("\n"); 1429 } 1430 } 1431 1432 /* we give more priority to constraints with less registers */ 1433 static int get_constraint_priority(const TCGOpDef *def, int k) 1434 { 1435 const TCGArgConstraint *arg_ct; 1436 1437 int i, n; 1438 arg_ct = &def->args_ct[k]; 1439 if (arg_ct->ct & TCG_CT_ALIAS) { 1440 /* an alias is equivalent to a single register */ 1441 n = 1; 1442 } else { 1443 if (!(arg_ct->ct & TCG_CT_REG)) 1444 return 0; 1445 n = 0; 1446 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1447 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1448 n++; 1449 } 1450 } 1451 return TCG_TARGET_NB_REGS - n + 1; 1452 } 1453 1454 /* sort from highest priority to lowest */ 1455 static void sort_constraints(TCGOpDef *def, int start, int n) 1456 { 1457 int i, j, p1, p2, tmp; 1458 1459 for(i = 0; i < n; i++) 1460 def->sorted_args[start + i] = start + i; 1461 if (n <= 1) 1462 return; 1463 for(i = 0; i < n - 1; i++) { 1464 for(j = i + 1; j < n; j++) { 1465 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1466 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1467 if (p1 < p2) { 1468 tmp = def->sorted_args[start + i]; 1469 def->sorted_args[start + i] = def->sorted_args[start + j]; 1470 def->sorted_args[start + j] = tmp; 1471 } 1472 } 1473 } 1474 } 1475 1476 static void process_op_defs(TCGContext *s) 1477 { 1478 TCGOpcode op; 1479 1480 for (op = 0; op < NB_OPS; op++) { 1481 TCGOpDef *def = &tcg_op_defs[op]; 1482 const TCGTargetOpDef *tdefs; 1483 TCGType type; 1484 int i, nb_args; 1485 1486 if (def->flags & TCG_OPF_NOT_PRESENT) { 1487 continue; 1488 } 1489 1490 nb_args = def->nb_iargs + def->nb_oargs; 1491 if (nb_args == 0) { 1492 continue; 1493 } 1494 1495 tdefs = tcg_target_op_def(op); 1496 /* Missing TCGTargetOpDef entry. */ 1497 tcg_debug_assert(tdefs != NULL); 1498 1499 type = (def->flags & TCG_OPF_64BIT ? TCG_TYPE_I64 : TCG_TYPE_I32); 1500 for (i = 0; i < nb_args; i++) { 1501 const char *ct_str = tdefs->args_ct_str[i]; 1502 /* Incomplete TCGTargetOpDef entry. */ 1503 tcg_debug_assert(ct_str != NULL); 1504 1505 def->args_ct[i].u.regs = 0; 1506 def->args_ct[i].ct = 0; 1507 while (*ct_str != '\0') { 1508 switch(*ct_str) { 1509 case '0' ... '9': 1510 { 1511 int oarg = *ct_str - '0'; 1512 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]); 1513 tcg_debug_assert(oarg < def->nb_oargs); 1514 tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG); 1515 /* TCG_CT_ALIAS is for the output arguments. 1516 The input is tagged with TCG_CT_IALIAS. */ 1517 def->args_ct[i] = def->args_ct[oarg]; 1518 def->args_ct[oarg].ct |= TCG_CT_ALIAS; 1519 def->args_ct[oarg].alias_index = i; 1520 def->args_ct[i].ct |= TCG_CT_IALIAS; 1521 def->args_ct[i].alias_index = oarg; 1522 } 1523 ct_str++; 1524 break; 1525 case '&': 1526 def->args_ct[i].ct |= TCG_CT_NEWREG; 1527 ct_str++; 1528 break; 1529 case 'i': 1530 def->args_ct[i].ct |= TCG_CT_CONST; 1531 ct_str++; 1532 break; 1533 default: 1534 ct_str = target_parse_constraint(&def->args_ct[i], 1535 ct_str, type); 1536 /* Typo in TCGTargetOpDef constraint. */ 1537 tcg_debug_assert(ct_str != NULL); 1538 } 1539 } 1540 } 1541 1542 /* TCGTargetOpDef entry with too much information? */ 1543 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1544 1545 /* sort the constraints (XXX: this is just an heuristic) */ 1546 sort_constraints(def, 0, def->nb_oargs); 1547 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1548 } 1549 } 1550 1551 void tcg_op_remove(TCGContext *s, TCGOp *op) 1552 { 1553 int next = op->next; 1554 int prev = op->prev; 1555 1556 /* We should never attempt to remove the list terminator. */ 1557 tcg_debug_assert(op != &s->gen_op_buf[0]); 1558 1559 s->gen_op_buf[next].prev = prev; 1560 s->gen_op_buf[prev].next = next; 1561 1562 memset(op, 0, sizeof(*op)); 1563 1564 #ifdef CONFIG_PROFILER 1565 s->del_op_count++; 1566 #endif 1567 } 1568 1569 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 1570 TCGOpcode opc, int nargs) 1571 { 1572 int oi = s->gen_next_op_idx; 1573 int pi = s->gen_next_parm_idx; 1574 int prev = old_op->prev; 1575 int next = old_op - s->gen_op_buf; 1576 TCGOp *new_op; 1577 1578 tcg_debug_assert(oi < OPC_BUF_SIZE); 1579 tcg_debug_assert(pi + nargs <= OPPARAM_BUF_SIZE); 1580 s->gen_next_op_idx = oi + 1; 1581 s->gen_next_parm_idx = pi + nargs; 1582 1583 new_op = &s->gen_op_buf[oi]; 1584 *new_op = (TCGOp){ 1585 .opc = opc, 1586 .args = pi, 1587 .prev = prev, 1588 .next = next 1589 }; 1590 s->gen_op_buf[prev].next = oi; 1591 old_op->prev = oi; 1592 1593 return new_op; 1594 } 1595 1596 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 1597 TCGOpcode opc, int nargs) 1598 { 1599 int oi = s->gen_next_op_idx; 1600 int pi = s->gen_next_parm_idx; 1601 int prev = old_op - s->gen_op_buf; 1602 int next = old_op->next; 1603 TCGOp *new_op; 1604 1605 tcg_debug_assert(oi < OPC_BUF_SIZE); 1606 tcg_debug_assert(pi + nargs <= OPPARAM_BUF_SIZE); 1607 s->gen_next_op_idx = oi + 1; 1608 s->gen_next_parm_idx = pi + nargs; 1609 1610 new_op = &s->gen_op_buf[oi]; 1611 *new_op = (TCGOp){ 1612 .opc = opc, 1613 .args = pi, 1614 .prev = prev, 1615 .next = next 1616 }; 1617 s->gen_op_buf[next].prev = oi; 1618 old_op->next = oi; 1619 1620 return new_op; 1621 } 1622 1623 #define TS_DEAD 1 1624 #define TS_MEM 2 1625 1626 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 1627 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 1628 1629 /* liveness analysis: end of function: all temps are dead, and globals 1630 should be in memory. */ 1631 static inline void tcg_la_func_end(TCGContext *s, uint8_t *temp_state) 1632 { 1633 memset(temp_state, TS_DEAD | TS_MEM, s->nb_globals); 1634 memset(temp_state + s->nb_globals, TS_DEAD, s->nb_temps - s->nb_globals); 1635 } 1636 1637 /* liveness analysis: end of basic block: all temps are dead, globals 1638 and local temps should be in memory. */ 1639 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *temp_state) 1640 { 1641 int i, n; 1642 1643 tcg_la_func_end(s, temp_state); 1644 for (i = s->nb_globals, n = s->nb_temps; i < n; i++) { 1645 if (s->temps[i].temp_local) { 1646 temp_state[i] |= TS_MEM; 1647 } 1648 } 1649 } 1650 1651 /* Liveness analysis : update the opc_arg_life array to tell if a 1652 given input arguments is dead. Instructions updating dead 1653 temporaries are removed. */ 1654 static void liveness_pass_1(TCGContext *s, uint8_t *temp_state) 1655 { 1656 int nb_globals = s->nb_globals; 1657 int oi, oi_prev; 1658 1659 tcg_la_func_end(s, temp_state); 1660 1661 for (oi = s->gen_op_buf[0].prev; oi != 0; oi = oi_prev) { 1662 int i, nb_iargs, nb_oargs; 1663 TCGOpcode opc_new, opc_new2; 1664 bool have_opc_new2; 1665 TCGLifeData arg_life = 0; 1666 TCGArg arg; 1667 1668 TCGOp * const op = &s->gen_op_buf[oi]; 1669 TCGArg * const args = &s->gen_opparam_buf[op->args]; 1670 TCGOpcode opc = op->opc; 1671 const TCGOpDef *def = &tcg_op_defs[opc]; 1672 1673 oi_prev = op->prev; 1674 1675 switch (opc) { 1676 case INDEX_op_call: 1677 { 1678 int call_flags; 1679 1680 nb_oargs = op->callo; 1681 nb_iargs = op->calli; 1682 call_flags = args[nb_oargs + nb_iargs + 1]; 1683 1684 /* pure functions can be removed if their result is unused */ 1685 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 1686 for (i = 0; i < nb_oargs; i++) { 1687 arg = args[i]; 1688 if (temp_state[arg] != TS_DEAD) { 1689 goto do_not_remove_call; 1690 } 1691 } 1692 goto do_remove; 1693 } else { 1694 do_not_remove_call: 1695 1696 /* output args are dead */ 1697 for (i = 0; i < nb_oargs; i++) { 1698 arg = args[i]; 1699 if (temp_state[arg] & TS_DEAD) { 1700 arg_life |= DEAD_ARG << i; 1701 } 1702 if (temp_state[arg] & TS_MEM) { 1703 arg_life |= SYNC_ARG << i; 1704 } 1705 temp_state[arg] = TS_DEAD; 1706 } 1707 1708 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 1709 TCG_CALL_NO_READ_GLOBALS))) { 1710 /* globals should go back to memory */ 1711 memset(temp_state, TS_DEAD | TS_MEM, nb_globals); 1712 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 1713 /* globals should be synced to memory */ 1714 for (i = 0; i < nb_globals; i++) { 1715 temp_state[i] |= TS_MEM; 1716 } 1717 } 1718 1719 /* record arguments that die in this helper */ 1720 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1721 arg = args[i]; 1722 if (arg != TCG_CALL_DUMMY_ARG) { 1723 if (temp_state[arg] & TS_DEAD) { 1724 arg_life |= DEAD_ARG << i; 1725 } 1726 } 1727 } 1728 /* input arguments are live for preceding opcodes */ 1729 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1730 arg = args[i]; 1731 if (arg != TCG_CALL_DUMMY_ARG) { 1732 temp_state[arg] &= ~TS_DEAD; 1733 } 1734 } 1735 } 1736 } 1737 break; 1738 case INDEX_op_insn_start: 1739 break; 1740 case INDEX_op_discard: 1741 /* mark the temporary as dead */ 1742 temp_state[args[0]] = TS_DEAD; 1743 break; 1744 1745 case INDEX_op_add2_i32: 1746 opc_new = INDEX_op_add_i32; 1747 goto do_addsub2; 1748 case INDEX_op_sub2_i32: 1749 opc_new = INDEX_op_sub_i32; 1750 goto do_addsub2; 1751 case INDEX_op_add2_i64: 1752 opc_new = INDEX_op_add_i64; 1753 goto do_addsub2; 1754 case INDEX_op_sub2_i64: 1755 opc_new = INDEX_op_sub_i64; 1756 do_addsub2: 1757 nb_iargs = 4; 1758 nb_oargs = 2; 1759 /* Test if the high part of the operation is dead, but not 1760 the low part. The result can be optimized to a simple 1761 add or sub. This happens often for x86_64 guest when the 1762 cpu mode is set to 32 bit. */ 1763 if (temp_state[args[1]] == TS_DEAD) { 1764 if (temp_state[args[0]] == TS_DEAD) { 1765 goto do_remove; 1766 } 1767 /* Replace the opcode and adjust the args in place, 1768 leaving 3 unused args at the end. */ 1769 op->opc = opc = opc_new; 1770 args[1] = args[2]; 1771 args[2] = args[4]; 1772 /* Fall through and mark the single-word operation live. */ 1773 nb_iargs = 2; 1774 nb_oargs = 1; 1775 } 1776 goto do_not_remove; 1777 1778 case INDEX_op_mulu2_i32: 1779 opc_new = INDEX_op_mul_i32; 1780 opc_new2 = INDEX_op_muluh_i32; 1781 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 1782 goto do_mul2; 1783 case INDEX_op_muls2_i32: 1784 opc_new = INDEX_op_mul_i32; 1785 opc_new2 = INDEX_op_mulsh_i32; 1786 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 1787 goto do_mul2; 1788 case INDEX_op_mulu2_i64: 1789 opc_new = INDEX_op_mul_i64; 1790 opc_new2 = INDEX_op_muluh_i64; 1791 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 1792 goto do_mul2; 1793 case INDEX_op_muls2_i64: 1794 opc_new = INDEX_op_mul_i64; 1795 opc_new2 = INDEX_op_mulsh_i64; 1796 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 1797 goto do_mul2; 1798 do_mul2: 1799 nb_iargs = 2; 1800 nb_oargs = 2; 1801 if (temp_state[args[1]] == TS_DEAD) { 1802 if (temp_state[args[0]] == TS_DEAD) { 1803 /* Both parts of the operation are dead. */ 1804 goto do_remove; 1805 } 1806 /* The high part of the operation is dead; generate the low. */ 1807 op->opc = opc = opc_new; 1808 args[1] = args[2]; 1809 args[2] = args[3]; 1810 } else if (temp_state[args[0]] == TS_DEAD && have_opc_new2) { 1811 /* The low part of the operation is dead; generate the high. */ 1812 op->opc = opc = opc_new2; 1813 args[0] = args[1]; 1814 args[1] = args[2]; 1815 args[2] = args[3]; 1816 } else { 1817 goto do_not_remove; 1818 } 1819 /* Mark the single-word operation live. */ 1820 nb_oargs = 1; 1821 goto do_not_remove; 1822 1823 default: 1824 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 1825 nb_iargs = def->nb_iargs; 1826 nb_oargs = def->nb_oargs; 1827 1828 /* Test if the operation can be removed because all 1829 its outputs are dead. We assume that nb_oargs == 0 1830 implies side effects */ 1831 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 1832 for (i = 0; i < nb_oargs; i++) { 1833 if (temp_state[args[i]] != TS_DEAD) { 1834 goto do_not_remove; 1835 } 1836 } 1837 do_remove: 1838 tcg_op_remove(s, op); 1839 } else { 1840 do_not_remove: 1841 /* output args are dead */ 1842 for (i = 0; i < nb_oargs; i++) { 1843 arg = args[i]; 1844 if (temp_state[arg] & TS_DEAD) { 1845 arg_life |= DEAD_ARG << i; 1846 } 1847 if (temp_state[arg] & TS_MEM) { 1848 arg_life |= SYNC_ARG << i; 1849 } 1850 temp_state[arg] = TS_DEAD; 1851 } 1852 1853 /* if end of basic block, update */ 1854 if (def->flags & TCG_OPF_BB_END) { 1855 tcg_la_bb_end(s, temp_state); 1856 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 1857 /* globals should be synced to memory */ 1858 for (i = 0; i < nb_globals; i++) { 1859 temp_state[i] |= TS_MEM; 1860 } 1861 } 1862 1863 /* record arguments that die in this opcode */ 1864 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1865 arg = args[i]; 1866 if (temp_state[arg] & TS_DEAD) { 1867 arg_life |= DEAD_ARG << i; 1868 } 1869 } 1870 /* input arguments are live for preceding opcodes */ 1871 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1872 temp_state[args[i]] &= ~TS_DEAD; 1873 } 1874 } 1875 break; 1876 } 1877 op->life = arg_life; 1878 } 1879 } 1880 1881 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 1882 static bool liveness_pass_2(TCGContext *s, uint8_t *temp_state) 1883 { 1884 int nb_globals = s->nb_globals; 1885 int16_t *dir_temps; 1886 int i, oi, oi_next; 1887 bool changes = false; 1888 1889 dir_temps = tcg_malloc(nb_globals * sizeof(int16_t)); 1890 memset(dir_temps, 0, nb_globals * sizeof(int16_t)); 1891 1892 /* Create a temporary for each indirect global. */ 1893 for (i = 0; i < nb_globals; ++i) { 1894 TCGTemp *its = &s->temps[i]; 1895 if (its->indirect_reg) { 1896 TCGTemp *dts = tcg_temp_alloc(s); 1897 dts->type = its->type; 1898 dts->base_type = its->base_type; 1899 dir_temps[i] = temp_idx(s, dts); 1900 } 1901 } 1902 1903 memset(temp_state, TS_DEAD, nb_globals); 1904 1905 for (oi = s->gen_op_buf[0].next; oi != 0; oi = oi_next) { 1906 TCGOp *op = &s->gen_op_buf[oi]; 1907 TCGArg *args = &s->gen_opparam_buf[op->args]; 1908 TCGOpcode opc = op->opc; 1909 const TCGOpDef *def = &tcg_op_defs[opc]; 1910 TCGLifeData arg_life = op->life; 1911 int nb_iargs, nb_oargs, call_flags; 1912 TCGArg arg, dir; 1913 1914 oi_next = op->next; 1915 1916 if (opc == INDEX_op_call) { 1917 nb_oargs = op->callo; 1918 nb_iargs = op->calli; 1919 call_flags = args[nb_oargs + nb_iargs + 1]; 1920 } else { 1921 nb_iargs = def->nb_iargs; 1922 nb_oargs = def->nb_oargs; 1923 1924 /* Set flags similar to how calls require. */ 1925 if (def->flags & TCG_OPF_BB_END) { 1926 /* Like writing globals: save_globals */ 1927 call_flags = 0; 1928 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 1929 /* Like reading globals: sync_globals */ 1930 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 1931 } else { 1932 /* No effect on globals. */ 1933 call_flags = (TCG_CALL_NO_READ_GLOBALS | 1934 TCG_CALL_NO_WRITE_GLOBALS); 1935 } 1936 } 1937 1938 /* Make sure that input arguments are available. */ 1939 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1940 arg = args[i]; 1941 /* Note this unsigned test catches TCG_CALL_ARG_DUMMY too. */ 1942 if (arg < nb_globals) { 1943 dir = dir_temps[arg]; 1944 if (dir != 0 && temp_state[arg] == TS_DEAD) { 1945 TCGTemp *its = &s->temps[arg]; 1946 TCGOpcode lopc = (its->type == TCG_TYPE_I32 1947 ? INDEX_op_ld_i32 1948 : INDEX_op_ld_i64); 1949 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 1950 TCGArg *largs = &s->gen_opparam_buf[lop->args]; 1951 1952 largs[0] = dir; 1953 largs[1] = temp_idx(s, its->mem_base); 1954 largs[2] = its->mem_offset; 1955 1956 /* Loaded, but synced with memory. */ 1957 temp_state[arg] = TS_MEM; 1958 } 1959 } 1960 } 1961 1962 /* Perform input replacement, and mark inputs that became dead. 1963 No action is required except keeping temp_state up to date 1964 so that we reload when needed. */ 1965 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1966 arg = args[i]; 1967 if (arg < nb_globals) { 1968 dir = dir_temps[arg]; 1969 if (dir != 0) { 1970 args[i] = dir; 1971 changes = true; 1972 if (IS_DEAD_ARG(i)) { 1973 temp_state[arg] = TS_DEAD; 1974 } 1975 } 1976 } 1977 } 1978 1979 /* Liveness analysis should ensure that the following are 1980 all correct, for call sites and basic block end points. */ 1981 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 1982 /* Nothing to do */ 1983 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 1984 for (i = 0; i < nb_globals; ++i) { 1985 /* Liveness should see that globals are synced back, 1986 that is, either TS_DEAD or TS_MEM. */ 1987 tcg_debug_assert(dir_temps[i] == 0 1988 || temp_state[i] != 0); 1989 } 1990 } else { 1991 for (i = 0; i < nb_globals; ++i) { 1992 /* Liveness should see that globals are saved back, 1993 that is, TS_DEAD, waiting to be reloaded. */ 1994 tcg_debug_assert(dir_temps[i] == 0 1995 || temp_state[i] == TS_DEAD); 1996 } 1997 } 1998 1999 /* Outputs become available. */ 2000 for (i = 0; i < nb_oargs; i++) { 2001 arg = args[i]; 2002 if (arg >= nb_globals) { 2003 continue; 2004 } 2005 dir = dir_temps[arg]; 2006 if (dir == 0) { 2007 continue; 2008 } 2009 args[i] = dir; 2010 changes = true; 2011 2012 /* The output is now live and modified. */ 2013 temp_state[arg] = 0; 2014 2015 /* Sync outputs upon their last write. */ 2016 if (NEED_SYNC_ARG(i)) { 2017 TCGTemp *its = &s->temps[arg]; 2018 TCGOpcode sopc = (its->type == TCG_TYPE_I32 2019 ? INDEX_op_st_i32 2020 : INDEX_op_st_i64); 2021 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 2022 TCGArg *sargs = &s->gen_opparam_buf[sop->args]; 2023 2024 sargs[0] = dir; 2025 sargs[1] = temp_idx(s, its->mem_base); 2026 sargs[2] = its->mem_offset; 2027 2028 temp_state[arg] = TS_MEM; 2029 } 2030 /* Drop outputs that are dead. */ 2031 if (IS_DEAD_ARG(i)) { 2032 temp_state[arg] = TS_DEAD; 2033 } 2034 } 2035 } 2036 2037 return changes; 2038 } 2039 2040 #ifdef CONFIG_DEBUG_TCG 2041 static void dump_regs(TCGContext *s) 2042 { 2043 TCGTemp *ts; 2044 int i; 2045 char buf[64]; 2046 2047 for(i = 0; i < s->nb_temps; i++) { 2048 ts = &s->temps[i]; 2049 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i)); 2050 switch(ts->val_type) { 2051 case TEMP_VAL_REG: 2052 printf("%s", tcg_target_reg_names[ts->reg]); 2053 break; 2054 case TEMP_VAL_MEM: 2055 printf("%d(%s)", (int)ts->mem_offset, 2056 tcg_target_reg_names[ts->mem_base->reg]); 2057 break; 2058 case TEMP_VAL_CONST: 2059 printf("$0x%" TCG_PRIlx, ts->val); 2060 break; 2061 case TEMP_VAL_DEAD: 2062 printf("D"); 2063 break; 2064 default: 2065 printf("???"); 2066 break; 2067 } 2068 printf("\n"); 2069 } 2070 2071 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 2072 if (s->reg_to_temp[i] != NULL) { 2073 printf("%s: %s\n", 2074 tcg_target_reg_names[i], 2075 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 2076 } 2077 } 2078 } 2079 2080 static void check_regs(TCGContext *s) 2081 { 2082 int reg; 2083 int k; 2084 TCGTemp *ts; 2085 char buf[64]; 2086 2087 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2088 ts = s->reg_to_temp[reg]; 2089 if (ts != NULL) { 2090 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 2091 printf("Inconsistency for register %s:\n", 2092 tcg_target_reg_names[reg]); 2093 goto fail; 2094 } 2095 } 2096 } 2097 for (k = 0; k < s->nb_temps; k++) { 2098 ts = &s->temps[k]; 2099 if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg 2100 && s->reg_to_temp[ts->reg] != ts) { 2101 printf("Inconsistency for temp %s:\n", 2102 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2103 fail: 2104 printf("reg state:\n"); 2105 dump_regs(s); 2106 tcg_abort(); 2107 } 2108 } 2109 } 2110 #endif 2111 2112 static void temp_allocate_frame(TCGContext *s, int temp) 2113 { 2114 TCGTemp *ts; 2115 ts = &s->temps[temp]; 2116 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 2117 /* Sparc64 stack is accessed with offset of 2047 */ 2118 s->current_frame_offset = (s->current_frame_offset + 2119 (tcg_target_long)sizeof(tcg_target_long) - 1) & 2120 ~(sizeof(tcg_target_long) - 1); 2121 #endif 2122 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 2123 s->frame_end) { 2124 tcg_abort(); 2125 } 2126 ts->mem_offset = s->current_frame_offset; 2127 ts->mem_base = s->frame_temp; 2128 ts->mem_allocated = 1; 2129 s->current_frame_offset += sizeof(tcg_target_long); 2130 } 2131 2132 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet); 2133 2134 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 2135 mark it free; otherwise mark it dead. */ 2136 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 2137 { 2138 if (ts->fixed_reg) { 2139 return; 2140 } 2141 if (ts->val_type == TEMP_VAL_REG) { 2142 s->reg_to_temp[ts->reg] = NULL; 2143 } 2144 ts->val_type = (free_or_dead < 0 2145 || ts->temp_local 2146 || temp_idx(s, ts) < s->nb_globals 2147 ? TEMP_VAL_MEM : TEMP_VAL_DEAD); 2148 } 2149 2150 /* Mark a temporary as dead. */ 2151 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 2152 { 2153 temp_free_or_dead(s, ts, 1); 2154 } 2155 2156 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 2157 registers needs to be allocated to store a constant. If 'free_or_dead' 2158 is non-zero, subsequently release the temporary; if it is positive, the 2159 temp is dead; if it is negative, the temp is free. */ 2160 static void temp_sync(TCGContext *s, TCGTemp *ts, 2161 TCGRegSet allocated_regs, int free_or_dead) 2162 { 2163 if (ts->fixed_reg) { 2164 return; 2165 } 2166 if (!ts->mem_coherent) { 2167 if (!ts->mem_allocated) { 2168 temp_allocate_frame(s, temp_idx(s, ts)); 2169 } 2170 switch (ts->val_type) { 2171 case TEMP_VAL_CONST: 2172 /* If we're going to free the temp immediately, then we won't 2173 require it later in a register, so attempt to store the 2174 constant to memory directly. */ 2175 if (free_or_dead 2176 && tcg_out_sti(s, ts->type, ts->val, 2177 ts->mem_base->reg, ts->mem_offset)) { 2178 break; 2179 } 2180 temp_load(s, ts, tcg_target_available_regs[ts->type], 2181 allocated_regs); 2182 /* fallthrough */ 2183 2184 case TEMP_VAL_REG: 2185 tcg_out_st(s, ts->type, ts->reg, 2186 ts->mem_base->reg, ts->mem_offset); 2187 break; 2188 2189 case TEMP_VAL_MEM: 2190 break; 2191 2192 case TEMP_VAL_DEAD: 2193 default: 2194 tcg_abort(); 2195 } 2196 ts->mem_coherent = 1; 2197 } 2198 if (free_or_dead) { 2199 temp_free_or_dead(s, ts, free_or_dead); 2200 } 2201 } 2202 2203 /* free register 'reg' by spilling the corresponding temporary if necessary */ 2204 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 2205 { 2206 TCGTemp *ts = s->reg_to_temp[reg]; 2207 if (ts != NULL) { 2208 temp_sync(s, ts, allocated_regs, -1); 2209 } 2210 } 2211 2212 /* Allocate a register belonging to reg1 & ~reg2 */ 2213 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs, 2214 TCGRegSet allocated_regs, bool rev) 2215 { 2216 int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 2217 const int *order; 2218 TCGReg reg; 2219 TCGRegSet reg_ct; 2220 2221 reg_ct = desired_regs & ~allocated_regs; 2222 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 2223 2224 /* first try free registers */ 2225 for(i = 0; i < n; i++) { 2226 reg = order[i]; 2227 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL) 2228 return reg; 2229 } 2230 2231 /* XXX: do better spill choice */ 2232 for(i = 0; i < n; i++) { 2233 reg = order[i]; 2234 if (tcg_regset_test_reg(reg_ct, reg)) { 2235 tcg_reg_free(s, reg, allocated_regs); 2236 return reg; 2237 } 2238 } 2239 2240 tcg_abort(); 2241 } 2242 2243 /* Make sure the temporary is in a register. If needed, allocate the register 2244 from DESIRED while avoiding ALLOCATED. */ 2245 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 2246 TCGRegSet allocated_regs) 2247 { 2248 TCGReg reg; 2249 2250 switch (ts->val_type) { 2251 case TEMP_VAL_REG: 2252 return; 2253 case TEMP_VAL_CONST: 2254 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 2255 tcg_out_movi(s, ts->type, reg, ts->val); 2256 ts->mem_coherent = 0; 2257 break; 2258 case TEMP_VAL_MEM: 2259 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 2260 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 2261 ts->mem_coherent = 1; 2262 break; 2263 case TEMP_VAL_DEAD: 2264 default: 2265 tcg_abort(); 2266 } 2267 ts->reg = reg; 2268 ts->val_type = TEMP_VAL_REG; 2269 s->reg_to_temp[reg] = ts; 2270 } 2271 2272 /* Save a temporary to memory. 'allocated_regs' is used in case a 2273 temporary registers needs to be allocated to store a constant. */ 2274 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 2275 { 2276 /* The liveness analysis already ensures that globals are back 2277 in memory. Keep an tcg_debug_assert for safety. */ 2278 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg); 2279 } 2280 2281 /* save globals to their canonical location and assume they can be 2282 modified be the following code. 'allocated_regs' is used in case a 2283 temporary registers needs to be allocated to store a constant. */ 2284 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 2285 { 2286 int i; 2287 2288 for (i = 0; i < s->nb_globals; i++) { 2289 temp_save(s, &s->temps[i], allocated_regs); 2290 } 2291 } 2292 2293 /* sync globals to their canonical location and assume they can be 2294 read by the following code. 'allocated_regs' is used in case a 2295 temporary registers needs to be allocated to store a constant. */ 2296 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 2297 { 2298 int i; 2299 2300 for (i = 0; i < s->nb_globals; i++) { 2301 TCGTemp *ts = &s->temps[i]; 2302 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 2303 || ts->fixed_reg 2304 || ts->mem_coherent); 2305 } 2306 } 2307 2308 /* at the end of a basic block, we assume all temporaries are dead and 2309 all globals are stored at their canonical location. */ 2310 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 2311 { 2312 int i; 2313 2314 for (i = s->nb_globals; i < s->nb_temps; i++) { 2315 TCGTemp *ts = &s->temps[i]; 2316 if (ts->temp_local) { 2317 temp_save(s, ts, allocated_regs); 2318 } else { 2319 /* The liveness analysis already ensures that temps are dead. 2320 Keep an tcg_debug_assert for safety. */ 2321 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 2322 } 2323 } 2324 2325 save_globals(s, allocated_regs); 2326 } 2327 2328 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 2329 tcg_target_ulong val, TCGLifeData arg_life) 2330 { 2331 if (ots->fixed_reg) { 2332 /* For fixed registers, we do not do any constant propagation. */ 2333 tcg_out_movi(s, ots->type, ots->reg, val); 2334 return; 2335 } 2336 2337 /* The movi is not explicitly generated here. */ 2338 if (ots->val_type == TEMP_VAL_REG) { 2339 s->reg_to_temp[ots->reg] = NULL; 2340 } 2341 ots->val_type = TEMP_VAL_CONST; 2342 ots->val = val; 2343 ots->mem_coherent = 0; 2344 if (NEED_SYNC_ARG(0)) { 2345 temp_sync(s, ots, s->reserved_regs, IS_DEAD_ARG(0)); 2346 } else if (IS_DEAD_ARG(0)) { 2347 temp_dead(s, ots); 2348 } 2349 } 2350 2351 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args, 2352 TCGLifeData arg_life) 2353 { 2354 TCGTemp *ots = &s->temps[args[0]]; 2355 tcg_target_ulong val = args[1]; 2356 2357 tcg_reg_alloc_do_movi(s, ots, val, arg_life); 2358 } 2359 2360 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, 2361 const TCGArg *args, TCGLifeData arg_life) 2362 { 2363 TCGRegSet allocated_regs; 2364 TCGTemp *ts, *ots; 2365 TCGType otype, itype; 2366 2367 allocated_regs = s->reserved_regs; 2368 ots = &s->temps[args[0]]; 2369 ts = &s->temps[args[1]]; 2370 2371 /* Note that otype != itype for no-op truncation. */ 2372 otype = ots->type; 2373 itype = ts->type; 2374 2375 if (ts->val_type == TEMP_VAL_CONST) { 2376 /* propagate constant or generate sti */ 2377 tcg_target_ulong val = ts->val; 2378 if (IS_DEAD_ARG(1)) { 2379 temp_dead(s, ts); 2380 } 2381 tcg_reg_alloc_do_movi(s, ots, val, arg_life); 2382 return; 2383 } 2384 2385 /* If the source value is in memory we're going to be forced 2386 to have it in a register in order to perform the copy. Copy 2387 the SOURCE value into its own register first, that way we 2388 don't have to reload SOURCE the next time it is used. */ 2389 if (ts->val_type == TEMP_VAL_MEM) { 2390 temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs); 2391 } 2392 2393 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 2394 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 2395 /* mov to a non-saved dead register makes no sense (even with 2396 liveness analysis disabled). */ 2397 tcg_debug_assert(NEED_SYNC_ARG(0)); 2398 if (!ots->mem_allocated) { 2399 temp_allocate_frame(s, args[0]); 2400 } 2401 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 2402 if (IS_DEAD_ARG(1)) { 2403 temp_dead(s, ts); 2404 } 2405 temp_dead(s, ots); 2406 } else { 2407 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 2408 /* the mov can be suppressed */ 2409 if (ots->val_type == TEMP_VAL_REG) { 2410 s->reg_to_temp[ots->reg] = NULL; 2411 } 2412 ots->reg = ts->reg; 2413 temp_dead(s, ts); 2414 } else { 2415 if (ots->val_type != TEMP_VAL_REG) { 2416 /* When allocating a new register, make sure to not spill the 2417 input one. */ 2418 tcg_regset_set_reg(allocated_regs, ts->reg); 2419 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 2420 allocated_regs, ots->indirect_base); 2421 } 2422 tcg_out_mov(s, otype, ots->reg, ts->reg); 2423 } 2424 ots->val_type = TEMP_VAL_REG; 2425 ots->mem_coherent = 0; 2426 s->reg_to_temp[ots->reg] = ots; 2427 if (NEED_SYNC_ARG(0)) { 2428 temp_sync(s, ots, allocated_regs, 0); 2429 } 2430 } 2431 } 2432 2433 static void tcg_reg_alloc_op(TCGContext *s, 2434 const TCGOpDef *def, TCGOpcode opc, 2435 const TCGArg *args, TCGLifeData arg_life) 2436 { 2437 TCGRegSet i_allocated_regs; 2438 TCGRegSet o_allocated_regs; 2439 int i, k, nb_iargs, nb_oargs; 2440 TCGReg reg; 2441 TCGArg arg; 2442 const TCGArgConstraint *arg_ct; 2443 TCGTemp *ts; 2444 TCGArg new_args[TCG_MAX_OP_ARGS]; 2445 int const_args[TCG_MAX_OP_ARGS]; 2446 2447 nb_oargs = def->nb_oargs; 2448 nb_iargs = def->nb_iargs; 2449 2450 /* copy constants */ 2451 memcpy(new_args + nb_oargs + nb_iargs, 2452 args + nb_oargs + nb_iargs, 2453 sizeof(TCGArg) * def->nb_cargs); 2454 2455 i_allocated_regs = s->reserved_regs; 2456 o_allocated_regs = s->reserved_regs; 2457 2458 /* satisfy input constraints */ 2459 for(k = 0; k < nb_iargs; k++) { 2460 i = def->sorted_args[nb_oargs + k]; 2461 arg = args[i]; 2462 arg_ct = &def->args_ct[i]; 2463 ts = &s->temps[arg]; 2464 2465 if (ts->val_type == TEMP_VAL_CONST 2466 && tcg_target_const_match(ts->val, ts->type, arg_ct)) { 2467 /* constant is OK for instruction */ 2468 const_args[i] = 1; 2469 new_args[i] = ts->val; 2470 goto iarg_end; 2471 } 2472 2473 temp_load(s, ts, arg_ct->u.regs, i_allocated_regs); 2474 2475 if (arg_ct->ct & TCG_CT_IALIAS) { 2476 if (ts->fixed_reg) { 2477 /* if fixed register, we must allocate a new register 2478 if the alias is not the same register */ 2479 if (arg != args[arg_ct->alias_index]) 2480 goto allocate_in_reg; 2481 } else { 2482 /* if the input is aliased to an output and if it is 2483 not dead after the instruction, we must allocate 2484 a new register and move it */ 2485 if (!IS_DEAD_ARG(i)) { 2486 goto allocate_in_reg; 2487 } 2488 /* check if the current register has already been allocated 2489 for another input aliased to an output */ 2490 int k2, i2; 2491 for (k2 = 0 ; k2 < k ; k2++) { 2492 i2 = def->sorted_args[nb_oargs + k2]; 2493 if ((def->args_ct[i2].ct & TCG_CT_IALIAS) && 2494 (new_args[i2] == ts->reg)) { 2495 goto allocate_in_reg; 2496 } 2497 } 2498 } 2499 } 2500 reg = ts->reg; 2501 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2502 /* nothing to do : the constraint is satisfied */ 2503 } else { 2504 allocate_in_reg: 2505 /* allocate a new register matching the constraint 2506 and move the temporary register into it */ 2507 reg = tcg_reg_alloc(s, arg_ct->u.regs, i_allocated_regs, 2508 ts->indirect_base); 2509 tcg_out_mov(s, ts->type, reg, ts->reg); 2510 } 2511 new_args[i] = reg; 2512 const_args[i] = 0; 2513 tcg_regset_set_reg(i_allocated_regs, reg); 2514 iarg_end: ; 2515 } 2516 2517 /* mark dead temporaries and free the associated registers */ 2518 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2519 if (IS_DEAD_ARG(i)) { 2520 temp_dead(s, &s->temps[args[i]]); 2521 } 2522 } 2523 2524 if (def->flags & TCG_OPF_BB_END) { 2525 tcg_reg_alloc_bb_end(s, i_allocated_regs); 2526 } else { 2527 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2528 /* XXX: permit generic clobber register list ? */ 2529 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 2530 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 2531 tcg_reg_free(s, i, i_allocated_regs); 2532 } 2533 } 2534 } 2535 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2536 /* sync globals if the op has side effects and might trigger 2537 an exception. */ 2538 sync_globals(s, i_allocated_regs); 2539 } 2540 2541 /* satisfy the output constraints */ 2542 for(k = 0; k < nb_oargs; k++) { 2543 i = def->sorted_args[k]; 2544 arg = args[i]; 2545 arg_ct = &def->args_ct[i]; 2546 ts = &s->temps[arg]; 2547 if ((arg_ct->ct & TCG_CT_ALIAS) 2548 && !const_args[arg_ct->alias_index]) { 2549 reg = new_args[arg_ct->alias_index]; 2550 } else if (arg_ct->ct & TCG_CT_NEWREG) { 2551 reg = tcg_reg_alloc(s, arg_ct->u.regs, 2552 i_allocated_regs | o_allocated_regs, 2553 ts->indirect_base); 2554 } else { 2555 /* if fixed register, we try to use it */ 2556 reg = ts->reg; 2557 if (ts->fixed_reg && 2558 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2559 goto oarg_end; 2560 } 2561 reg = tcg_reg_alloc(s, arg_ct->u.regs, o_allocated_regs, 2562 ts->indirect_base); 2563 } 2564 tcg_regset_set_reg(o_allocated_regs, reg); 2565 /* if a fixed register is used, then a move will be done afterwards */ 2566 if (!ts->fixed_reg) { 2567 if (ts->val_type == TEMP_VAL_REG) { 2568 s->reg_to_temp[ts->reg] = NULL; 2569 } 2570 ts->val_type = TEMP_VAL_REG; 2571 ts->reg = reg; 2572 /* temp value is modified, so the value kept in memory is 2573 potentially not the same */ 2574 ts->mem_coherent = 0; 2575 s->reg_to_temp[reg] = ts; 2576 } 2577 oarg_end: 2578 new_args[i] = reg; 2579 } 2580 } 2581 2582 /* emit instruction */ 2583 tcg_out_op(s, opc, new_args, const_args); 2584 2585 /* move the outputs in the correct register if needed */ 2586 for(i = 0; i < nb_oargs; i++) { 2587 ts = &s->temps[args[i]]; 2588 reg = new_args[i]; 2589 if (ts->fixed_reg && ts->reg != reg) { 2590 tcg_out_mov(s, ts->type, ts->reg, reg); 2591 } 2592 if (NEED_SYNC_ARG(i)) { 2593 temp_sync(s, ts, o_allocated_regs, IS_DEAD_ARG(i)); 2594 } else if (IS_DEAD_ARG(i)) { 2595 temp_dead(s, ts); 2596 } 2597 } 2598 } 2599 2600 #ifdef TCG_TARGET_STACK_GROWSUP 2601 #define STACK_DIR(x) (-(x)) 2602 #else 2603 #define STACK_DIR(x) (x) 2604 #endif 2605 2606 static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs, 2607 const TCGArg * const args, TCGLifeData arg_life) 2608 { 2609 int flags, nb_regs, i; 2610 TCGReg reg; 2611 TCGArg arg; 2612 TCGTemp *ts; 2613 intptr_t stack_offset; 2614 size_t call_stack_size; 2615 tcg_insn_unit *func_addr; 2616 int allocate_args; 2617 TCGRegSet allocated_regs; 2618 2619 func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs]; 2620 flags = args[nb_oargs + nb_iargs + 1]; 2621 2622 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2623 if (nb_regs > nb_iargs) { 2624 nb_regs = nb_iargs; 2625 } 2626 2627 /* assign stack slots first */ 2628 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 2629 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 2630 ~(TCG_TARGET_STACK_ALIGN - 1); 2631 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 2632 if (allocate_args) { 2633 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 2634 preallocate call stack */ 2635 tcg_abort(); 2636 } 2637 2638 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 2639 for(i = nb_regs; i < nb_iargs; i++) { 2640 arg = args[nb_oargs + i]; 2641 #ifdef TCG_TARGET_STACK_GROWSUP 2642 stack_offset -= sizeof(tcg_target_long); 2643 #endif 2644 if (arg != TCG_CALL_DUMMY_ARG) { 2645 ts = &s->temps[arg]; 2646 temp_load(s, ts, tcg_target_available_regs[ts->type], 2647 s->reserved_regs); 2648 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 2649 } 2650 #ifndef TCG_TARGET_STACK_GROWSUP 2651 stack_offset += sizeof(tcg_target_long); 2652 #endif 2653 } 2654 2655 /* assign input registers */ 2656 allocated_regs = s->reserved_regs; 2657 for(i = 0; i < nb_regs; i++) { 2658 arg = args[nb_oargs + i]; 2659 if (arg != TCG_CALL_DUMMY_ARG) { 2660 ts = &s->temps[arg]; 2661 reg = tcg_target_call_iarg_regs[i]; 2662 tcg_reg_free(s, reg, allocated_regs); 2663 2664 if (ts->val_type == TEMP_VAL_REG) { 2665 if (ts->reg != reg) { 2666 tcg_out_mov(s, ts->type, reg, ts->reg); 2667 } 2668 } else { 2669 TCGRegSet arg_set = 0; 2670 2671 tcg_regset_set_reg(arg_set, reg); 2672 temp_load(s, ts, arg_set, allocated_regs); 2673 } 2674 2675 tcg_regset_set_reg(allocated_regs, reg); 2676 } 2677 } 2678 2679 /* mark dead temporaries and free the associated registers */ 2680 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2681 if (IS_DEAD_ARG(i)) { 2682 temp_dead(s, &s->temps[args[i]]); 2683 } 2684 } 2685 2686 /* clobber call registers */ 2687 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 2688 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 2689 tcg_reg_free(s, i, allocated_regs); 2690 } 2691 } 2692 2693 /* Save globals if they might be written by the helper, sync them if 2694 they might be read. */ 2695 if (flags & TCG_CALL_NO_READ_GLOBALS) { 2696 /* Nothing to do */ 2697 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 2698 sync_globals(s, allocated_regs); 2699 } else { 2700 save_globals(s, allocated_regs); 2701 } 2702 2703 tcg_out_call(s, func_addr); 2704 2705 /* assign output registers and emit moves if needed */ 2706 for(i = 0; i < nb_oargs; i++) { 2707 arg = args[i]; 2708 ts = &s->temps[arg]; 2709 reg = tcg_target_call_oarg_regs[i]; 2710 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 2711 2712 if (ts->fixed_reg) { 2713 if (ts->reg != reg) { 2714 tcg_out_mov(s, ts->type, ts->reg, reg); 2715 } 2716 } else { 2717 if (ts->val_type == TEMP_VAL_REG) { 2718 s->reg_to_temp[ts->reg] = NULL; 2719 } 2720 ts->val_type = TEMP_VAL_REG; 2721 ts->reg = reg; 2722 ts->mem_coherent = 0; 2723 s->reg_to_temp[reg] = ts; 2724 if (NEED_SYNC_ARG(i)) { 2725 temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i)); 2726 } else if (IS_DEAD_ARG(i)) { 2727 temp_dead(s, ts); 2728 } 2729 } 2730 } 2731 } 2732 2733 #ifdef CONFIG_PROFILER 2734 2735 static int64_t tcg_table_op_count[NB_OPS]; 2736 2737 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 2738 { 2739 int i; 2740 2741 for (i = 0; i < NB_OPS; i++) { 2742 cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, 2743 tcg_table_op_count[i]); 2744 } 2745 } 2746 #else 2747 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 2748 { 2749 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2750 } 2751 #endif 2752 2753 2754 int tcg_gen_code(TCGContext *s, TranslationBlock *tb) 2755 { 2756 int i, oi, oi_next, num_insns; 2757 2758 #ifdef CONFIG_PROFILER 2759 { 2760 int n; 2761 2762 n = s->gen_op_buf[0].prev + 1; 2763 s->op_count += n; 2764 if (n > s->op_count_max) { 2765 s->op_count_max = n; 2766 } 2767 2768 n = s->nb_temps; 2769 s->temp_count += n; 2770 if (n > s->temp_count_max) { 2771 s->temp_count_max = n; 2772 } 2773 } 2774 #endif 2775 2776 #ifdef DEBUG_DISAS 2777 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 2778 && qemu_log_in_addr_range(tb->pc))) { 2779 qemu_log_lock(); 2780 qemu_log("OP:\n"); 2781 tcg_dump_ops(s); 2782 qemu_log("\n"); 2783 qemu_log_unlock(); 2784 } 2785 #endif 2786 2787 #ifdef CONFIG_PROFILER 2788 s->opt_time -= profile_getclock(); 2789 #endif 2790 2791 #ifdef USE_TCG_OPTIMIZATIONS 2792 tcg_optimize(s); 2793 #endif 2794 2795 #ifdef CONFIG_PROFILER 2796 s->opt_time += profile_getclock(); 2797 s->la_time -= profile_getclock(); 2798 #endif 2799 2800 { 2801 uint8_t *temp_state = tcg_malloc(s->nb_temps + s->nb_indirects); 2802 2803 liveness_pass_1(s, temp_state); 2804 2805 if (s->nb_indirects > 0) { 2806 #ifdef DEBUG_DISAS 2807 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 2808 && qemu_log_in_addr_range(tb->pc))) { 2809 qemu_log_lock(); 2810 qemu_log("OP before indirect lowering:\n"); 2811 tcg_dump_ops(s); 2812 qemu_log("\n"); 2813 qemu_log_unlock(); 2814 } 2815 #endif 2816 /* Replace indirect temps with direct temps. */ 2817 if (liveness_pass_2(s, temp_state)) { 2818 /* If changes were made, re-run liveness. */ 2819 liveness_pass_1(s, temp_state); 2820 } 2821 } 2822 } 2823 2824 #ifdef CONFIG_PROFILER 2825 s->la_time += profile_getclock(); 2826 #endif 2827 2828 #ifdef DEBUG_DISAS 2829 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 2830 && qemu_log_in_addr_range(tb->pc))) { 2831 qemu_log_lock(); 2832 qemu_log("OP after optimization and liveness analysis:\n"); 2833 tcg_dump_ops(s); 2834 qemu_log("\n"); 2835 qemu_log_unlock(); 2836 } 2837 #endif 2838 2839 tcg_reg_alloc_start(s); 2840 2841 s->code_buf = tb->tc.ptr; 2842 s->code_ptr = tb->tc.ptr; 2843 2844 #ifdef TCG_TARGET_NEED_LDST_LABELS 2845 s->ldst_labels = NULL; 2846 #endif 2847 #ifdef TCG_TARGET_NEED_POOL_LABELS 2848 s->pool_labels = NULL; 2849 #endif 2850 2851 num_insns = -1; 2852 for (oi = s->gen_op_buf[0].next; oi != 0; oi = oi_next) { 2853 TCGOp * const op = &s->gen_op_buf[oi]; 2854 TCGArg * const args = &s->gen_opparam_buf[op->args]; 2855 TCGOpcode opc = op->opc; 2856 const TCGOpDef *def = &tcg_op_defs[opc]; 2857 TCGLifeData arg_life = op->life; 2858 2859 oi_next = op->next; 2860 #ifdef CONFIG_PROFILER 2861 tcg_table_op_count[opc]++; 2862 #endif 2863 2864 switch (opc) { 2865 case INDEX_op_mov_i32: 2866 case INDEX_op_mov_i64: 2867 tcg_reg_alloc_mov(s, def, args, arg_life); 2868 break; 2869 case INDEX_op_movi_i32: 2870 case INDEX_op_movi_i64: 2871 tcg_reg_alloc_movi(s, args, arg_life); 2872 break; 2873 case INDEX_op_insn_start: 2874 if (num_insns >= 0) { 2875 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 2876 } 2877 num_insns++; 2878 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 2879 target_ulong a; 2880 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 2881 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2]; 2882 #else 2883 a = args[i]; 2884 #endif 2885 s->gen_insn_data[num_insns][i] = a; 2886 } 2887 break; 2888 case INDEX_op_discard: 2889 temp_dead(s, &s->temps[args[0]]); 2890 break; 2891 case INDEX_op_set_label: 2892 tcg_reg_alloc_bb_end(s, s->reserved_regs); 2893 tcg_out_label(s, arg_label(args[0]), s->code_ptr); 2894 break; 2895 case INDEX_op_call: 2896 tcg_reg_alloc_call(s, op->callo, op->calli, args, arg_life); 2897 break; 2898 default: 2899 /* Sanity check that we've not introduced any unhandled opcodes. */ 2900 tcg_debug_assert(tcg_op_supported(opc)); 2901 /* Note: in order to speed up the code, it would be much 2902 faster to have specialized register allocator functions for 2903 some common argument patterns */ 2904 tcg_reg_alloc_op(s, def, opc, args, arg_life); 2905 break; 2906 } 2907 #ifdef CONFIG_DEBUG_TCG 2908 check_regs(s); 2909 #endif 2910 /* Test for (pending) buffer overflow. The assumption is that any 2911 one operation beginning below the high water mark cannot overrun 2912 the buffer completely. Thus we can test for overflow after 2913 generating code without having to check during generation. */ 2914 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 2915 return -1; 2916 } 2917 } 2918 tcg_debug_assert(num_insns >= 0); 2919 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 2920 2921 /* Generate TB finalization at the end of block */ 2922 #ifdef TCG_TARGET_NEED_LDST_LABELS 2923 if (!tcg_out_ldst_finalize(s)) { 2924 return -1; 2925 } 2926 #endif 2927 #ifdef TCG_TARGET_NEED_POOL_LABELS 2928 if (!tcg_out_pool_finalize(s)) { 2929 return -1; 2930 } 2931 #endif 2932 2933 /* flush instruction cache */ 2934 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 2935 2936 return tcg_current_code_size(s); 2937 } 2938 2939 #ifdef CONFIG_PROFILER 2940 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2941 { 2942 TCGContext *s = &tcg_ctx; 2943 int64_t tb_count = s->tb_count; 2944 int64_t tb_div_count = tb_count ? tb_count : 1; 2945 int64_t tot = s->interm_time + s->code_time; 2946 2947 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 2948 tot, tot / 2.4e9); 2949 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 2950 tb_count, s->tb_count1 - tb_count, 2951 (double)(s->tb_count1 - s->tb_count) 2952 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 2953 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 2954 (double)s->op_count / tb_div_count, s->op_count_max); 2955 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 2956 (double)s->del_op_count / tb_div_count); 2957 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 2958 (double)s->temp_count / tb_div_count, s->temp_count_max); 2959 cpu_fprintf(f, "avg host code/TB %0.1f\n", 2960 (double)s->code_out_len / tb_div_count); 2961 cpu_fprintf(f, "avg search data/TB %0.1f\n", 2962 (double)s->search_out_len / tb_div_count); 2963 2964 cpu_fprintf(f, "cycles/op %0.1f\n", 2965 s->op_count ? (double)tot / s->op_count : 0); 2966 cpu_fprintf(f, "cycles/in byte %0.1f\n", 2967 s->code_in_len ? (double)tot / s->code_in_len : 0); 2968 cpu_fprintf(f, "cycles/out byte %0.1f\n", 2969 s->code_out_len ? (double)tot / s->code_out_len : 0); 2970 cpu_fprintf(f, "cycles/search byte %0.1f\n", 2971 s->search_out_len ? (double)tot / s->search_out_len : 0); 2972 if (tot == 0) { 2973 tot = 1; 2974 } 2975 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 2976 (double)s->interm_time / tot * 100.0); 2977 cpu_fprintf(f, " gen_code time %0.1f%%\n", 2978 (double)s->code_time / tot * 100.0); 2979 cpu_fprintf(f, "optim./code time %0.1f%%\n", 2980 (double)s->opt_time / (s->code_time ? s->code_time : 1) 2981 * 100.0); 2982 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 2983 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 2984 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 2985 s->restore_count); 2986 cpu_fprintf(f, " avg cycles %0.1f\n", 2987 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 2988 } 2989 #else 2990 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2991 { 2992 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2993 } 2994 #endif 2995 2996 #ifdef ELF_HOST_MACHINE 2997 /* In order to use this feature, the backend needs to do three things: 2998 2999 (1) Define ELF_HOST_MACHINE to indicate both what value to 3000 put into the ELF image and to indicate support for the feature. 3001 3002 (2) Define tcg_register_jit. This should create a buffer containing 3003 the contents of a .debug_frame section that describes the post- 3004 prologue unwind info for the tcg machine. 3005 3006 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 3007 */ 3008 3009 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 3010 typedef enum { 3011 JIT_NOACTION = 0, 3012 JIT_REGISTER_FN, 3013 JIT_UNREGISTER_FN 3014 } jit_actions_t; 3015 3016 struct jit_code_entry { 3017 struct jit_code_entry *next_entry; 3018 struct jit_code_entry *prev_entry; 3019 const void *symfile_addr; 3020 uint64_t symfile_size; 3021 }; 3022 3023 struct jit_descriptor { 3024 uint32_t version; 3025 uint32_t action_flag; 3026 struct jit_code_entry *relevant_entry; 3027 struct jit_code_entry *first_entry; 3028 }; 3029 3030 void __jit_debug_register_code(void) __attribute__((noinline)); 3031 void __jit_debug_register_code(void) 3032 { 3033 asm(""); 3034 } 3035 3036 /* Must statically initialize the version, because GDB may check 3037 the version before we can set it. */ 3038 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 3039 3040 /* End GDB interface. */ 3041 3042 static int find_string(const char *strtab, const char *str) 3043 { 3044 const char *p = strtab + 1; 3045 3046 while (1) { 3047 if (strcmp(p, str) == 0) { 3048 return p - strtab; 3049 } 3050 p += strlen(p) + 1; 3051 } 3052 } 3053 3054 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 3055 const void *debug_frame, 3056 size_t debug_frame_size) 3057 { 3058 struct __attribute__((packed)) DebugInfo { 3059 uint32_t len; 3060 uint16_t version; 3061 uint32_t abbrev; 3062 uint8_t ptr_size; 3063 uint8_t cu_die; 3064 uint16_t cu_lang; 3065 uintptr_t cu_low_pc; 3066 uintptr_t cu_high_pc; 3067 uint8_t fn_die; 3068 char fn_name[16]; 3069 uintptr_t fn_low_pc; 3070 uintptr_t fn_high_pc; 3071 uint8_t cu_eoc; 3072 }; 3073 3074 struct ElfImage { 3075 ElfW(Ehdr) ehdr; 3076 ElfW(Phdr) phdr; 3077 ElfW(Shdr) shdr[7]; 3078 ElfW(Sym) sym[2]; 3079 struct DebugInfo di; 3080 uint8_t da[24]; 3081 char str[80]; 3082 }; 3083 3084 struct ElfImage *img; 3085 3086 static const struct ElfImage img_template = { 3087 .ehdr = { 3088 .e_ident[EI_MAG0] = ELFMAG0, 3089 .e_ident[EI_MAG1] = ELFMAG1, 3090 .e_ident[EI_MAG2] = ELFMAG2, 3091 .e_ident[EI_MAG3] = ELFMAG3, 3092 .e_ident[EI_CLASS] = ELF_CLASS, 3093 .e_ident[EI_DATA] = ELF_DATA, 3094 .e_ident[EI_VERSION] = EV_CURRENT, 3095 .e_type = ET_EXEC, 3096 .e_machine = ELF_HOST_MACHINE, 3097 .e_version = EV_CURRENT, 3098 .e_phoff = offsetof(struct ElfImage, phdr), 3099 .e_shoff = offsetof(struct ElfImage, shdr), 3100 .e_ehsize = sizeof(ElfW(Shdr)), 3101 .e_phentsize = sizeof(ElfW(Phdr)), 3102 .e_phnum = 1, 3103 .e_shentsize = sizeof(ElfW(Shdr)), 3104 .e_shnum = ARRAY_SIZE(img->shdr), 3105 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 3106 #ifdef ELF_HOST_FLAGS 3107 .e_flags = ELF_HOST_FLAGS, 3108 #endif 3109 #ifdef ELF_OSABI 3110 .e_ident[EI_OSABI] = ELF_OSABI, 3111 #endif 3112 }, 3113 .phdr = { 3114 .p_type = PT_LOAD, 3115 .p_flags = PF_X, 3116 }, 3117 .shdr = { 3118 [0] = { .sh_type = SHT_NULL }, 3119 /* Trick: The contents of code_gen_buffer are not present in 3120 this fake ELF file; that got allocated elsewhere. Therefore 3121 we mark .text as SHT_NOBITS (similar to .bss) so that readers 3122 will not look for contents. We can record any address. */ 3123 [1] = { /* .text */ 3124 .sh_type = SHT_NOBITS, 3125 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 3126 }, 3127 [2] = { /* .debug_info */ 3128 .sh_type = SHT_PROGBITS, 3129 .sh_offset = offsetof(struct ElfImage, di), 3130 .sh_size = sizeof(struct DebugInfo), 3131 }, 3132 [3] = { /* .debug_abbrev */ 3133 .sh_type = SHT_PROGBITS, 3134 .sh_offset = offsetof(struct ElfImage, da), 3135 .sh_size = sizeof(img->da), 3136 }, 3137 [4] = { /* .debug_frame */ 3138 .sh_type = SHT_PROGBITS, 3139 .sh_offset = sizeof(struct ElfImage), 3140 }, 3141 [5] = { /* .symtab */ 3142 .sh_type = SHT_SYMTAB, 3143 .sh_offset = offsetof(struct ElfImage, sym), 3144 .sh_size = sizeof(img->sym), 3145 .sh_info = 1, 3146 .sh_link = ARRAY_SIZE(img->shdr) - 1, 3147 .sh_entsize = sizeof(ElfW(Sym)), 3148 }, 3149 [6] = { /* .strtab */ 3150 .sh_type = SHT_STRTAB, 3151 .sh_offset = offsetof(struct ElfImage, str), 3152 .sh_size = sizeof(img->str), 3153 } 3154 }, 3155 .sym = { 3156 [1] = { /* code_gen_buffer */ 3157 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 3158 .st_shndx = 1, 3159 } 3160 }, 3161 .di = { 3162 .len = sizeof(struct DebugInfo) - 4, 3163 .version = 2, 3164 .ptr_size = sizeof(void *), 3165 .cu_die = 1, 3166 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 3167 .fn_die = 2, 3168 .fn_name = "code_gen_buffer" 3169 }, 3170 .da = { 3171 1, /* abbrev number (the cu) */ 3172 0x11, 1, /* DW_TAG_compile_unit, has children */ 3173 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 3174 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 3175 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 3176 0, 0, /* end of abbrev */ 3177 2, /* abbrev number (the fn) */ 3178 0x2e, 0, /* DW_TAG_subprogram, no children */ 3179 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 3180 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 3181 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 3182 0, 0, /* end of abbrev */ 3183 0 /* no more abbrev */ 3184 }, 3185 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 3186 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 3187 }; 3188 3189 /* We only need a single jit entry; statically allocate it. */ 3190 static struct jit_code_entry one_entry; 3191 3192 uintptr_t buf = (uintptr_t)buf_ptr; 3193 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 3194 DebugFrameHeader *dfh; 3195 3196 img = g_malloc(img_size); 3197 *img = img_template; 3198 3199 img->phdr.p_vaddr = buf; 3200 img->phdr.p_paddr = buf; 3201 img->phdr.p_memsz = buf_size; 3202 3203 img->shdr[1].sh_name = find_string(img->str, ".text"); 3204 img->shdr[1].sh_addr = buf; 3205 img->shdr[1].sh_size = buf_size; 3206 3207 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 3208 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 3209 3210 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 3211 img->shdr[4].sh_size = debug_frame_size; 3212 3213 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 3214 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 3215 3216 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 3217 img->sym[1].st_value = buf; 3218 img->sym[1].st_size = buf_size; 3219 3220 img->di.cu_low_pc = buf; 3221 img->di.cu_high_pc = buf + buf_size; 3222 img->di.fn_low_pc = buf; 3223 img->di.fn_high_pc = buf + buf_size; 3224 3225 dfh = (DebugFrameHeader *)(img + 1); 3226 memcpy(dfh, debug_frame, debug_frame_size); 3227 dfh->fde.func_start = buf; 3228 dfh->fde.func_len = buf_size; 3229 3230 #ifdef DEBUG_JIT 3231 /* Enable this block to be able to debug the ELF image file creation. 3232 One can use readelf, objdump, or other inspection utilities. */ 3233 { 3234 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 3235 if (f) { 3236 if (fwrite(img, img_size, 1, f) != img_size) { 3237 /* Avoid stupid unused return value warning for fwrite. */ 3238 } 3239 fclose(f); 3240 } 3241 } 3242 #endif 3243 3244 one_entry.symfile_addr = img; 3245 one_entry.symfile_size = img_size; 3246 3247 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 3248 __jit_debug_descriptor.relevant_entry = &one_entry; 3249 __jit_debug_descriptor.first_entry = &one_entry; 3250 __jit_debug_register_code(); 3251 } 3252 #else 3253 /* No support for the feature. Provide the entry point expected by exec.c, 3254 and implement the internal function we declared earlier. */ 3255 3256 static void tcg_register_jit_int(void *buf, size_t size, 3257 const void *debug_frame, 3258 size_t debug_frame_size) 3259 { 3260 } 3261 3262 void tcg_register_jit(void *buf, size_t buf_size) 3263 { 3264 } 3265 #endif /* ELF_HOST_MACHINE */ 3266