1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_LIVENESS_ANALYSIS 27 #define USE_TCG_OPTIMIZATIONS 28 29 #include "qemu/osdep.h" 30 31 /* Define to jump the ELF file used to communicate with GDB. */ 32 #undef DEBUG_JIT 33 34 #if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG) 35 /* define it to suppress various consistency checks (faster) */ 36 #define NDEBUG 37 #endif 38 39 #include "qemu-common.h" 40 #include "qemu/host-utils.h" 41 #include "qemu/timer.h" 42 43 /* Note: the long term plan is to reduce the dependencies on the QEMU 44 CPU definitions. Currently they are used for qemu_ld/st 45 instructions */ 46 #define NO_CPU_IO_DEFS 47 #include "cpu.h" 48 49 #include "tcg-op.h" 50 51 #if UINTPTR_MAX == UINT32_MAX 52 # define ELF_CLASS ELFCLASS32 53 #else 54 # define ELF_CLASS ELFCLASS64 55 #endif 56 #ifdef HOST_WORDS_BIGENDIAN 57 # define ELF_DATA ELFDATA2MSB 58 #else 59 # define ELF_DATA ELFDATA2LSB 60 #endif 61 62 #include "elf.h" 63 #include "exec/log.h" 64 65 /* Forward declarations for functions declared in tcg-target.c and used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static void patch_reloc(tcg_insn_unit *code_ptr, int type, 69 intptr_t value, intptr_t addend); 70 71 /* The CIE and FDE header definitions will be common to all hosts. */ 72 typedef struct { 73 uint32_t len __attribute__((aligned((sizeof(void *))))); 74 uint32_t id; 75 uint8_t version; 76 char augmentation[1]; 77 uint8_t code_align; 78 uint8_t data_align; 79 uint8_t return_column; 80 } DebugFrameCIE; 81 82 typedef struct QEMU_PACKED { 83 uint32_t len __attribute__((aligned((sizeof(void *))))); 84 uint32_t cie_offset; 85 uintptr_t func_start; 86 uintptr_t func_len; 87 } DebugFrameFDEHeader; 88 89 typedef struct QEMU_PACKED { 90 DebugFrameCIE cie; 91 DebugFrameFDEHeader fde; 92 } DebugFrameHeader; 93 94 static void tcg_register_jit_int(void *buf, size_t size, 95 const void *debug_frame, 96 size_t debug_frame_size) 97 __attribute__((unused)); 98 99 /* Forward declarations for functions declared and used in tcg-target.c. */ 100 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str); 101 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 102 intptr_t arg2); 103 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 104 static void tcg_out_movi(TCGContext *s, TCGType type, 105 TCGReg ret, tcg_target_long arg); 106 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 107 const int *const_args); 108 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 109 intptr_t arg2); 110 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); 111 static int tcg_target_const_match(tcg_target_long val, TCGType type, 112 const TCGArgConstraint *arg_ct); 113 static void tcg_out_tb_init(TCGContext *s); 114 static bool tcg_out_tb_finalize(TCGContext *s); 115 116 117 118 static TCGRegSet tcg_target_available_regs[2]; 119 static TCGRegSet tcg_target_call_clobber_regs; 120 121 #if TCG_TARGET_INSN_UNIT_SIZE == 1 122 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 123 { 124 *s->code_ptr++ = v; 125 } 126 127 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 128 uint8_t v) 129 { 130 *p = v; 131 } 132 #endif 133 134 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 135 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 136 { 137 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 138 *s->code_ptr++ = v; 139 } else { 140 tcg_insn_unit *p = s->code_ptr; 141 memcpy(p, &v, sizeof(v)); 142 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 143 } 144 } 145 146 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 147 uint16_t v) 148 { 149 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 150 *p = v; 151 } else { 152 memcpy(p, &v, sizeof(v)); 153 } 154 } 155 #endif 156 157 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 158 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 159 { 160 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 161 *s->code_ptr++ = v; 162 } else { 163 tcg_insn_unit *p = s->code_ptr; 164 memcpy(p, &v, sizeof(v)); 165 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 166 } 167 } 168 169 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 170 uint32_t v) 171 { 172 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 173 *p = v; 174 } else { 175 memcpy(p, &v, sizeof(v)); 176 } 177 } 178 #endif 179 180 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 181 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 182 { 183 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 184 *s->code_ptr++ = v; 185 } else { 186 tcg_insn_unit *p = s->code_ptr; 187 memcpy(p, &v, sizeof(v)); 188 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 189 } 190 } 191 192 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 193 uint64_t v) 194 { 195 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 196 *p = v; 197 } else { 198 memcpy(p, &v, sizeof(v)); 199 } 200 } 201 #endif 202 203 /* label relocation processing */ 204 205 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 206 TCGLabel *l, intptr_t addend) 207 { 208 TCGRelocation *r; 209 210 if (l->has_value) { 211 /* FIXME: This may break relocations on RISC targets that 212 modify instruction fields in place. The caller may not have 213 written the initial value. */ 214 patch_reloc(code_ptr, type, l->u.value, addend); 215 } else { 216 /* add a new relocation entry */ 217 r = tcg_malloc(sizeof(TCGRelocation)); 218 r->type = type; 219 r->ptr = code_ptr; 220 r->addend = addend; 221 r->next = l->u.first_reloc; 222 l->u.first_reloc = r; 223 } 224 } 225 226 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr) 227 { 228 intptr_t value = (intptr_t)ptr; 229 TCGRelocation *r; 230 231 assert(!l->has_value); 232 233 for (r = l->u.first_reloc; r != NULL; r = r->next) { 234 patch_reloc(r->ptr, r->type, value, r->addend); 235 } 236 237 l->has_value = 1; 238 l->u.value_ptr = ptr; 239 } 240 241 TCGLabel *gen_new_label(void) 242 { 243 TCGContext *s = &tcg_ctx; 244 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 245 246 *l = (TCGLabel){ 247 .id = s->nb_labels++ 248 }; 249 250 return l; 251 } 252 253 #include "tcg-target.c" 254 255 /* pool based memory allocation */ 256 void *tcg_malloc_internal(TCGContext *s, int size) 257 { 258 TCGPool *p; 259 int pool_size; 260 261 if (size > TCG_POOL_CHUNK_SIZE) { 262 /* big malloc: insert a new pool (XXX: could optimize) */ 263 p = g_malloc(sizeof(TCGPool) + size); 264 p->size = size; 265 p->next = s->pool_first_large; 266 s->pool_first_large = p; 267 return p->data; 268 } else { 269 p = s->pool_current; 270 if (!p) { 271 p = s->pool_first; 272 if (!p) 273 goto new_pool; 274 } else { 275 if (!p->next) { 276 new_pool: 277 pool_size = TCG_POOL_CHUNK_SIZE; 278 p = g_malloc(sizeof(TCGPool) + pool_size); 279 p->size = pool_size; 280 p->next = NULL; 281 if (s->pool_current) 282 s->pool_current->next = p; 283 else 284 s->pool_first = p; 285 } else { 286 p = p->next; 287 } 288 } 289 } 290 s->pool_current = p; 291 s->pool_cur = p->data + size; 292 s->pool_end = p->data + p->size; 293 return p->data; 294 } 295 296 void tcg_pool_reset(TCGContext *s) 297 { 298 TCGPool *p, *t; 299 for (p = s->pool_first_large; p; p = t) { 300 t = p->next; 301 g_free(p); 302 } 303 s->pool_first_large = NULL; 304 s->pool_cur = s->pool_end = NULL; 305 s->pool_current = NULL; 306 } 307 308 typedef struct TCGHelperInfo { 309 void *func; 310 const char *name; 311 unsigned flags; 312 unsigned sizemask; 313 } TCGHelperInfo; 314 315 #include "exec/helper-proto.h" 316 317 static const TCGHelperInfo all_helpers[] = { 318 #include "exec/helper-tcg.h" 319 }; 320 321 void tcg_context_init(TCGContext *s) 322 { 323 int op, total_args, n, i; 324 TCGOpDef *def; 325 TCGArgConstraint *args_ct; 326 int *sorted_args; 327 GHashTable *helper_table; 328 329 memset(s, 0, sizeof(*s)); 330 s->nb_globals = 0; 331 332 /* Count total number of arguments and allocate the corresponding 333 space */ 334 total_args = 0; 335 for(op = 0; op < NB_OPS; op++) { 336 def = &tcg_op_defs[op]; 337 n = def->nb_iargs + def->nb_oargs; 338 total_args += n; 339 } 340 341 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 342 sorted_args = g_malloc(sizeof(int) * total_args); 343 344 for(op = 0; op < NB_OPS; op++) { 345 def = &tcg_op_defs[op]; 346 def->args_ct = args_ct; 347 def->sorted_args = sorted_args; 348 n = def->nb_iargs + def->nb_oargs; 349 sorted_args += n; 350 args_ct += n; 351 } 352 353 /* Register helpers. */ 354 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 355 s->helpers = helper_table = g_hash_table_new(NULL, NULL); 356 357 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 358 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 359 (gpointer)&all_helpers[i]); 360 } 361 362 tcg_target_init(s); 363 } 364 365 void tcg_prologue_init(TCGContext *s) 366 { 367 size_t prologue_size, total_size; 368 void *buf0, *buf1; 369 370 /* Put the prologue at the beginning of code_gen_buffer. */ 371 buf0 = s->code_gen_buffer; 372 s->code_ptr = buf0; 373 s->code_buf = buf0; 374 s->code_gen_prologue = buf0; 375 376 /* Generate the prologue. */ 377 tcg_target_qemu_prologue(s); 378 buf1 = s->code_ptr; 379 flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1); 380 381 /* Deduct the prologue from the buffer. */ 382 prologue_size = tcg_current_code_size(s); 383 s->code_gen_ptr = buf1; 384 s->code_gen_buffer = buf1; 385 s->code_buf = buf1; 386 total_size = s->code_gen_buffer_size - prologue_size; 387 s->code_gen_buffer_size = total_size; 388 389 /* Compute a high-water mark, at which we voluntarily flush the buffer 390 and start over. The size here is arbitrary, significantly larger 391 than we expect the code generation for any one opcode to require. */ 392 s->code_gen_highwater = s->code_gen_buffer + (total_size - 1024); 393 394 tcg_register_jit(s->code_gen_buffer, total_size); 395 396 #ifdef DEBUG_DISAS 397 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 398 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 399 log_disas(buf0, prologue_size); 400 qemu_log("\n"); 401 qemu_log_flush(); 402 } 403 #endif 404 } 405 406 void tcg_func_start(TCGContext *s) 407 { 408 tcg_pool_reset(s); 409 s->nb_temps = s->nb_globals; 410 411 /* No temps have been previously allocated for size or locality. */ 412 memset(s->free_temps, 0, sizeof(s->free_temps)); 413 414 s->nb_labels = 0; 415 s->current_frame_offset = s->frame_start; 416 417 #ifdef CONFIG_DEBUG_TCG 418 s->goto_tb_issue_mask = 0; 419 #endif 420 421 s->gen_first_op_idx = 0; 422 s->gen_last_op_idx = -1; 423 s->gen_next_op_idx = 0; 424 s->gen_next_parm_idx = 0; 425 426 s->be = tcg_malloc(sizeof(TCGBackendData)); 427 } 428 429 static inline int temp_idx(TCGContext *s, TCGTemp *ts) 430 { 431 ptrdiff_t n = ts - s->temps; 432 tcg_debug_assert(n >= 0 && n < s->nb_temps); 433 return n; 434 } 435 436 static inline TCGTemp *tcg_temp_alloc(TCGContext *s) 437 { 438 int n = s->nb_temps++; 439 tcg_debug_assert(n < TCG_MAX_TEMPS); 440 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 441 } 442 443 static inline TCGTemp *tcg_global_alloc(TCGContext *s) 444 { 445 tcg_debug_assert(s->nb_globals == s->nb_temps); 446 s->nb_globals++; 447 return tcg_temp_alloc(s); 448 } 449 450 static int tcg_global_reg_new_internal(TCGContext *s, TCGType type, 451 TCGReg reg, const char *name) 452 { 453 TCGTemp *ts; 454 455 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 456 tcg_abort(); 457 } 458 459 ts = tcg_global_alloc(s); 460 ts->base_type = type; 461 ts->type = type; 462 ts->fixed_reg = 1; 463 ts->reg = reg; 464 ts->name = name; 465 tcg_regset_set_reg(s->reserved_regs, reg); 466 467 return temp_idx(s, ts); 468 } 469 470 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 471 { 472 int idx; 473 s->frame_start = start; 474 s->frame_end = start + size; 475 idx = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 476 s->frame_temp = &s->temps[idx]; 477 } 478 479 TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name) 480 { 481 TCGContext *s = &tcg_ctx; 482 int idx; 483 484 if (tcg_regset_test_reg(s->reserved_regs, reg)) { 485 tcg_abort(); 486 } 487 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I32, reg, name); 488 return MAKE_TCGV_I32(idx); 489 } 490 491 TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name) 492 { 493 TCGContext *s = &tcg_ctx; 494 int idx; 495 496 if (tcg_regset_test_reg(s->reserved_regs, reg)) { 497 tcg_abort(); 498 } 499 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I64, reg, name); 500 return MAKE_TCGV_I64(idx); 501 } 502 503 int tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 504 intptr_t offset, const char *name) 505 { 506 TCGContext *s = &tcg_ctx; 507 TCGTemp *base_ts = &s->temps[GET_TCGV_PTR(base)]; 508 TCGTemp *ts = tcg_global_alloc(s); 509 int bigendian = 0; 510 #ifdef HOST_WORDS_BIGENDIAN 511 bigendian = 1; 512 #endif 513 514 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 515 TCGTemp *ts2 = tcg_global_alloc(s); 516 char buf[64]; 517 518 ts->base_type = TCG_TYPE_I64; 519 ts->type = TCG_TYPE_I32; 520 ts->mem_allocated = 1; 521 ts->mem_base = base_ts; 522 ts->mem_offset = offset + bigendian * 4; 523 pstrcpy(buf, sizeof(buf), name); 524 pstrcat(buf, sizeof(buf), "_0"); 525 ts->name = strdup(buf); 526 527 tcg_debug_assert(ts2 == ts + 1); 528 ts2->base_type = TCG_TYPE_I64; 529 ts2->type = TCG_TYPE_I32; 530 ts2->mem_allocated = 1; 531 ts2->mem_base = base_ts; 532 ts2->mem_offset = offset + (1 - bigendian) * 4; 533 pstrcpy(buf, sizeof(buf), name); 534 pstrcat(buf, sizeof(buf), "_1"); 535 ts->name = strdup(buf); 536 } else { 537 ts->base_type = type; 538 ts->type = type; 539 ts->mem_allocated = 1; 540 ts->mem_base = base_ts; 541 ts->mem_offset = offset; 542 ts->name = name; 543 } 544 return temp_idx(s, ts); 545 } 546 547 static int tcg_temp_new_internal(TCGType type, int temp_local) 548 { 549 TCGContext *s = &tcg_ctx; 550 TCGTemp *ts; 551 int idx, k; 552 553 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 554 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 555 if (idx < TCG_MAX_TEMPS) { 556 /* There is already an available temp with the right type. */ 557 clear_bit(idx, s->free_temps[k].l); 558 559 ts = &s->temps[idx]; 560 ts->temp_allocated = 1; 561 tcg_debug_assert(ts->base_type == type); 562 tcg_debug_assert(ts->temp_local == temp_local); 563 } else { 564 ts = tcg_temp_alloc(s); 565 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 566 TCGTemp *ts2 = tcg_temp_alloc(s); 567 568 ts->base_type = type; 569 ts->type = TCG_TYPE_I32; 570 ts->temp_allocated = 1; 571 ts->temp_local = temp_local; 572 573 tcg_debug_assert(ts2 == ts + 1); 574 ts2->base_type = TCG_TYPE_I64; 575 ts2->type = TCG_TYPE_I32; 576 ts2->temp_allocated = 1; 577 ts2->temp_local = temp_local; 578 } else { 579 ts->base_type = type; 580 ts->type = type; 581 ts->temp_allocated = 1; 582 ts->temp_local = temp_local; 583 } 584 idx = temp_idx(s, ts); 585 } 586 587 #if defined(CONFIG_DEBUG_TCG) 588 s->temps_in_use++; 589 #endif 590 return idx; 591 } 592 593 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 594 { 595 int idx; 596 597 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 598 return MAKE_TCGV_I32(idx); 599 } 600 601 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 602 { 603 int idx; 604 605 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 606 return MAKE_TCGV_I64(idx); 607 } 608 609 static void tcg_temp_free_internal(int idx) 610 { 611 TCGContext *s = &tcg_ctx; 612 TCGTemp *ts; 613 int k; 614 615 #if defined(CONFIG_DEBUG_TCG) 616 s->temps_in_use--; 617 if (s->temps_in_use < 0) { 618 fprintf(stderr, "More temporaries freed than allocated!\n"); 619 } 620 #endif 621 622 assert(idx >= s->nb_globals && idx < s->nb_temps); 623 ts = &s->temps[idx]; 624 assert(ts->temp_allocated != 0); 625 ts->temp_allocated = 0; 626 627 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 628 set_bit(idx, s->free_temps[k].l); 629 } 630 631 void tcg_temp_free_i32(TCGv_i32 arg) 632 { 633 tcg_temp_free_internal(GET_TCGV_I32(arg)); 634 } 635 636 void tcg_temp_free_i64(TCGv_i64 arg) 637 { 638 tcg_temp_free_internal(GET_TCGV_I64(arg)); 639 } 640 641 TCGv_i32 tcg_const_i32(int32_t val) 642 { 643 TCGv_i32 t0; 644 t0 = tcg_temp_new_i32(); 645 tcg_gen_movi_i32(t0, val); 646 return t0; 647 } 648 649 TCGv_i64 tcg_const_i64(int64_t val) 650 { 651 TCGv_i64 t0; 652 t0 = tcg_temp_new_i64(); 653 tcg_gen_movi_i64(t0, val); 654 return t0; 655 } 656 657 TCGv_i32 tcg_const_local_i32(int32_t val) 658 { 659 TCGv_i32 t0; 660 t0 = tcg_temp_local_new_i32(); 661 tcg_gen_movi_i32(t0, val); 662 return t0; 663 } 664 665 TCGv_i64 tcg_const_local_i64(int64_t val) 666 { 667 TCGv_i64 t0; 668 t0 = tcg_temp_local_new_i64(); 669 tcg_gen_movi_i64(t0, val); 670 return t0; 671 } 672 673 #if defined(CONFIG_DEBUG_TCG) 674 void tcg_clear_temp_count(void) 675 { 676 TCGContext *s = &tcg_ctx; 677 s->temps_in_use = 0; 678 } 679 680 int tcg_check_temp_count(void) 681 { 682 TCGContext *s = &tcg_ctx; 683 if (s->temps_in_use) { 684 /* Clear the count so that we don't give another 685 * warning immediately next time around. 686 */ 687 s->temps_in_use = 0; 688 return 1; 689 } 690 return 0; 691 } 692 #endif 693 694 /* Note: we convert the 64 bit args to 32 bit and do some alignment 695 and endian swap. Maybe it would be better to do the alignment 696 and endian swap in tcg_reg_alloc_call(). */ 697 void tcg_gen_callN(TCGContext *s, void *func, TCGArg ret, 698 int nargs, TCGArg *args) 699 { 700 int i, real_args, nb_rets, pi, pi_first; 701 unsigned sizemask, flags; 702 TCGHelperInfo *info; 703 704 info = g_hash_table_lookup(s->helpers, (gpointer)func); 705 flags = info->flags; 706 sizemask = info->sizemask; 707 708 #if defined(__sparc__) && !defined(__arch64__) \ 709 && !defined(CONFIG_TCG_INTERPRETER) 710 /* We have 64-bit values in one register, but need to pass as two 711 separate parameters. Split them. */ 712 int orig_sizemask = sizemask; 713 int orig_nargs = nargs; 714 TCGv_i64 retl, reth; 715 716 TCGV_UNUSED_I64(retl); 717 TCGV_UNUSED_I64(reth); 718 if (sizemask != 0) { 719 TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2); 720 for (i = real_args = 0; i < nargs; ++i) { 721 int is_64bit = sizemask & (1 << (i+1)*2); 722 if (is_64bit) { 723 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 724 TCGv_i32 h = tcg_temp_new_i32(); 725 TCGv_i32 l = tcg_temp_new_i32(); 726 tcg_gen_extr_i64_i32(l, h, orig); 727 split_args[real_args++] = GET_TCGV_I32(h); 728 split_args[real_args++] = GET_TCGV_I32(l); 729 } else { 730 split_args[real_args++] = args[i]; 731 } 732 } 733 nargs = real_args; 734 args = split_args; 735 sizemask = 0; 736 } 737 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 738 for (i = 0; i < nargs; ++i) { 739 int is_64bit = sizemask & (1 << (i+1)*2); 740 int is_signed = sizemask & (2 << (i+1)*2); 741 if (!is_64bit) { 742 TCGv_i64 temp = tcg_temp_new_i64(); 743 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 744 if (is_signed) { 745 tcg_gen_ext32s_i64(temp, orig); 746 } else { 747 tcg_gen_ext32u_i64(temp, orig); 748 } 749 args[i] = GET_TCGV_I64(temp); 750 } 751 } 752 #endif /* TCG_TARGET_EXTEND_ARGS */ 753 754 pi_first = pi = s->gen_next_parm_idx; 755 if (ret != TCG_CALL_DUMMY_ARG) { 756 #if defined(__sparc__) && !defined(__arch64__) \ 757 && !defined(CONFIG_TCG_INTERPRETER) 758 if (orig_sizemask & 1) { 759 /* The 32-bit ABI is going to return the 64-bit value in 760 the %o0/%o1 register pair. Prepare for this by using 761 two return temporaries, and reassemble below. */ 762 retl = tcg_temp_new_i64(); 763 reth = tcg_temp_new_i64(); 764 s->gen_opparam_buf[pi++] = GET_TCGV_I64(reth); 765 s->gen_opparam_buf[pi++] = GET_TCGV_I64(retl); 766 nb_rets = 2; 767 } else { 768 s->gen_opparam_buf[pi++] = ret; 769 nb_rets = 1; 770 } 771 #else 772 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { 773 #ifdef HOST_WORDS_BIGENDIAN 774 s->gen_opparam_buf[pi++] = ret + 1; 775 s->gen_opparam_buf[pi++] = ret; 776 #else 777 s->gen_opparam_buf[pi++] = ret; 778 s->gen_opparam_buf[pi++] = ret + 1; 779 #endif 780 nb_rets = 2; 781 } else { 782 s->gen_opparam_buf[pi++] = ret; 783 nb_rets = 1; 784 } 785 #endif 786 } else { 787 nb_rets = 0; 788 } 789 real_args = 0; 790 for (i = 0; i < nargs; i++) { 791 int is_64bit = sizemask & (1 << (i+1)*2); 792 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 793 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 794 /* some targets want aligned 64 bit args */ 795 if (real_args & 1) { 796 s->gen_opparam_buf[pi++] = TCG_CALL_DUMMY_ARG; 797 real_args++; 798 } 799 #endif 800 /* If stack grows up, then we will be placing successive 801 arguments at lower addresses, which means we need to 802 reverse the order compared to how we would normally 803 treat either big or little-endian. For those arguments 804 that will wind up in registers, this still works for 805 HPPA (the only current STACK_GROWSUP target) since the 806 argument registers are *also* allocated in decreasing 807 order. If another such target is added, this logic may 808 have to get more complicated to differentiate between 809 stack arguments and register arguments. */ 810 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 811 s->gen_opparam_buf[pi++] = args[i] + 1; 812 s->gen_opparam_buf[pi++] = args[i]; 813 #else 814 s->gen_opparam_buf[pi++] = args[i]; 815 s->gen_opparam_buf[pi++] = args[i] + 1; 816 #endif 817 real_args += 2; 818 continue; 819 } 820 821 s->gen_opparam_buf[pi++] = args[i]; 822 real_args++; 823 } 824 s->gen_opparam_buf[pi++] = (uintptr_t)func; 825 s->gen_opparam_buf[pi++] = flags; 826 827 i = s->gen_next_op_idx; 828 tcg_debug_assert(i < OPC_BUF_SIZE); 829 tcg_debug_assert(pi <= OPPARAM_BUF_SIZE); 830 831 /* Set links for sequential allocation during translation. */ 832 s->gen_op_buf[i] = (TCGOp){ 833 .opc = INDEX_op_call, 834 .callo = nb_rets, 835 .calli = real_args, 836 .args = pi_first, 837 .prev = i - 1, 838 .next = i + 1 839 }; 840 841 /* Make sure the calli field didn't overflow. */ 842 tcg_debug_assert(s->gen_op_buf[i].calli == real_args); 843 844 s->gen_last_op_idx = i; 845 s->gen_next_op_idx = i + 1; 846 s->gen_next_parm_idx = pi; 847 848 #if defined(__sparc__) && !defined(__arch64__) \ 849 && !defined(CONFIG_TCG_INTERPRETER) 850 /* Free all of the parts we allocated above. */ 851 for (i = real_args = 0; i < orig_nargs; ++i) { 852 int is_64bit = orig_sizemask & (1 << (i+1)*2); 853 if (is_64bit) { 854 TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]); 855 TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]); 856 tcg_temp_free_i32(h); 857 tcg_temp_free_i32(l); 858 } else { 859 real_args++; 860 } 861 } 862 if (orig_sizemask & 1) { 863 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 864 Note that describing these as TCGv_i64 eliminates an unnecessary 865 zero-extension that tcg_gen_concat_i32_i64 would create. */ 866 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth); 867 tcg_temp_free_i64(retl); 868 tcg_temp_free_i64(reth); 869 } 870 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 871 for (i = 0; i < nargs; ++i) { 872 int is_64bit = sizemask & (1 << (i+1)*2); 873 if (!is_64bit) { 874 TCGv_i64 temp = MAKE_TCGV_I64(args[i]); 875 tcg_temp_free_i64(temp); 876 } 877 } 878 #endif /* TCG_TARGET_EXTEND_ARGS */ 879 } 880 881 static void tcg_reg_alloc_start(TCGContext *s) 882 { 883 int i; 884 TCGTemp *ts; 885 for(i = 0; i < s->nb_globals; i++) { 886 ts = &s->temps[i]; 887 if (ts->fixed_reg) { 888 ts->val_type = TEMP_VAL_REG; 889 } else { 890 ts->val_type = TEMP_VAL_MEM; 891 } 892 } 893 for(i = s->nb_globals; i < s->nb_temps; i++) { 894 ts = &s->temps[i]; 895 if (ts->temp_local) { 896 ts->val_type = TEMP_VAL_MEM; 897 } else { 898 ts->val_type = TEMP_VAL_DEAD; 899 } 900 ts->mem_allocated = 0; 901 ts->fixed_reg = 0; 902 } 903 904 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 905 } 906 907 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 908 TCGTemp *ts) 909 { 910 int idx = temp_idx(s, ts); 911 912 if (idx < s->nb_globals) { 913 pstrcpy(buf, buf_size, ts->name); 914 } else if (ts->temp_local) { 915 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 916 } else { 917 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 918 } 919 return buf; 920 } 921 922 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, 923 int buf_size, int idx) 924 { 925 assert(idx >= 0 && idx < s->nb_temps); 926 return tcg_get_arg_str_ptr(s, buf, buf_size, &s->temps[idx]); 927 } 928 929 /* Find helper name. */ 930 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 931 { 932 const char *ret = NULL; 933 if (s->helpers) { 934 TCGHelperInfo *info = g_hash_table_lookup(s->helpers, (gpointer)val); 935 if (info) { 936 ret = info->name; 937 } 938 } 939 return ret; 940 } 941 942 static const char * const cond_name[] = 943 { 944 [TCG_COND_NEVER] = "never", 945 [TCG_COND_ALWAYS] = "always", 946 [TCG_COND_EQ] = "eq", 947 [TCG_COND_NE] = "ne", 948 [TCG_COND_LT] = "lt", 949 [TCG_COND_GE] = "ge", 950 [TCG_COND_LE] = "le", 951 [TCG_COND_GT] = "gt", 952 [TCG_COND_LTU] = "ltu", 953 [TCG_COND_GEU] = "geu", 954 [TCG_COND_LEU] = "leu", 955 [TCG_COND_GTU] = "gtu" 956 }; 957 958 static const char * const ldst_name[] = 959 { 960 [MO_UB] = "ub", 961 [MO_SB] = "sb", 962 [MO_LEUW] = "leuw", 963 [MO_LESW] = "lesw", 964 [MO_LEUL] = "leul", 965 [MO_LESL] = "lesl", 966 [MO_LEQ] = "leq", 967 [MO_BEUW] = "beuw", 968 [MO_BESW] = "besw", 969 [MO_BEUL] = "beul", 970 [MO_BESL] = "besl", 971 [MO_BEQ] = "beq", 972 }; 973 974 void tcg_dump_ops(TCGContext *s) 975 { 976 char buf[128]; 977 TCGOp *op; 978 int oi; 979 980 for (oi = s->gen_first_op_idx; oi >= 0; oi = op->next) { 981 int i, k, nb_oargs, nb_iargs, nb_cargs; 982 const TCGOpDef *def; 983 const TCGArg *args; 984 TCGOpcode c; 985 986 op = &s->gen_op_buf[oi]; 987 c = op->opc; 988 def = &tcg_op_defs[c]; 989 args = &s->gen_opparam_buf[op->args]; 990 991 if (c == INDEX_op_insn_start) { 992 qemu_log("%s ----", oi != s->gen_first_op_idx ? "\n" : ""); 993 994 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 995 target_ulong a; 996 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 997 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2]; 998 #else 999 a = args[i]; 1000 #endif 1001 qemu_log(" " TARGET_FMT_lx, a); 1002 } 1003 } else if (c == INDEX_op_call) { 1004 /* variable number of arguments */ 1005 nb_oargs = op->callo; 1006 nb_iargs = op->calli; 1007 nb_cargs = def->nb_cargs; 1008 1009 /* function name, flags, out args */ 1010 qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, 1011 tcg_find_helper(s, args[nb_oargs + nb_iargs]), 1012 args[nb_oargs + nb_iargs + 1], nb_oargs); 1013 for (i = 0; i < nb_oargs; i++) { 1014 qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1015 args[i])); 1016 } 1017 for (i = 0; i < nb_iargs; i++) { 1018 TCGArg arg = args[nb_oargs + i]; 1019 const char *t = "<dummy>"; 1020 if (arg != TCG_CALL_DUMMY_ARG) { 1021 t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg); 1022 } 1023 qemu_log(",%s", t); 1024 } 1025 } else { 1026 qemu_log(" %s ", def->name); 1027 1028 nb_oargs = def->nb_oargs; 1029 nb_iargs = def->nb_iargs; 1030 nb_cargs = def->nb_cargs; 1031 1032 k = 0; 1033 for (i = 0; i < nb_oargs; i++) { 1034 if (k != 0) { 1035 qemu_log(","); 1036 } 1037 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1038 args[k++])); 1039 } 1040 for (i = 0; i < nb_iargs; i++) { 1041 if (k != 0) { 1042 qemu_log(","); 1043 } 1044 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1045 args[k++])); 1046 } 1047 switch (c) { 1048 case INDEX_op_brcond_i32: 1049 case INDEX_op_setcond_i32: 1050 case INDEX_op_movcond_i32: 1051 case INDEX_op_brcond2_i32: 1052 case INDEX_op_setcond2_i32: 1053 case INDEX_op_brcond_i64: 1054 case INDEX_op_setcond_i64: 1055 case INDEX_op_movcond_i64: 1056 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) { 1057 qemu_log(",%s", cond_name[args[k++]]); 1058 } else { 1059 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1060 } 1061 i = 1; 1062 break; 1063 case INDEX_op_qemu_ld_i32: 1064 case INDEX_op_qemu_st_i32: 1065 case INDEX_op_qemu_ld_i64: 1066 case INDEX_op_qemu_st_i64: 1067 { 1068 TCGMemOpIdx oi = args[k++]; 1069 TCGMemOp op = get_memop(oi); 1070 unsigned ix = get_mmuidx(oi); 1071 1072 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1073 qemu_log(",$0x%x,%u", op, ix); 1074 } else { 1075 const char *s_al = "", *s_op; 1076 if (op & MO_AMASK) { 1077 if ((op & MO_AMASK) == MO_ALIGN) { 1078 s_al = "al+"; 1079 } else { 1080 s_al = "un+"; 1081 } 1082 } 1083 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1084 qemu_log(",%s%s,%u", s_al, s_op, ix); 1085 } 1086 i = 1; 1087 } 1088 break; 1089 default: 1090 i = 0; 1091 break; 1092 } 1093 switch (c) { 1094 case INDEX_op_set_label: 1095 case INDEX_op_br: 1096 case INDEX_op_brcond_i32: 1097 case INDEX_op_brcond_i64: 1098 case INDEX_op_brcond2_i32: 1099 qemu_log("%s$L%d", k ? "," : "", arg_label(args[k])->id); 1100 i++, k++; 1101 break; 1102 default: 1103 break; 1104 } 1105 for (; i < nb_cargs; i++, k++) { 1106 qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", args[k]); 1107 } 1108 } 1109 qemu_log("\n"); 1110 } 1111 } 1112 1113 /* we give more priority to constraints with less registers */ 1114 static int get_constraint_priority(const TCGOpDef *def, int k) 1115 { 1116 const TCGArgConstraint *arg_ct; 1117 1118 int i, n; 1119 arg_ct = &def->args_ct[k]; 1120 if (arg_ct->ct & TCG_CT_ALIAS) { 1121 /* an alias is equivalent to a single register */ 1122 n = 1; 1123 } else { 1124 if (!(arg_ct->ct & TCG_CT_REG)) 1125 return 0; 1126 n = 0; 1127 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1128 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1129 n++; 1130 } 1131 } 1132 return TCG_TARGET_NB_REGS - n + 1; 1133 } 1134 1135 /* sort from highest priority to lowest */ 1136 static void sort_constraints(TCGOpDef *def, int start, int n) 1137 { 1138 int i, j, p1, p2, tmp; 1139 1140 for(i = 0; i < n; i++) 1141 def->sorted_args[start + i] = start + i; 1142 if (n <= 1) 1143 return; 1144 for(i = 0; i < n - 1; i++) { 1145 for(j = i + 1; j < n; j++) { 1146 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1147 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1148 if (p1 < p2) { 1149 tmp = def->sorted_args[start + i]; 1150 def->sorted_args[start + i] = def->sorted_args[start + j]; 1151 def->sorted_args[start + j] = tmp; 1152 } 1153 } 1154 } 1155 } 1156 1157 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs) 1158 { 1159 TCGOpcode op; 1160 TCGOpDef *def; 1161 const char *ct_str; 1162 int i, nb_args; 1163 1164 for(;;) { 1165 if (tdefs->op == (TCGOpcode)-1) 1166 break; 1167 op = tdefs->op; 1168 assert((unsigned)op < NB_OPS); 1169 def = &tcg_op_defs[op]; 1170 #if defined(CONFIG_DEBUG_TCG) 1171 /* Duplicate entry in op definitions? */ 1172 assert(!def->used); 1173 def->used = 1; 1174 #endif 1175 nb_args = def->nb_iargs + def->nb_oargs; 1176 for(i = 0; i < nb_args; i++) { 1177 ct_str = tdefs->args_ct_str[i]; 1178 /* Incomplete TCGTargetOpDef entry? */ 1179 assert(ct_str != NULL); 1180 tcg_regset_clear(def->args_ct[i].u.regs); 1181 def->args_ct[i].ct = 0; 1182 if (ct_str[0] >= '0' && ct_str[0] <= '9') { 1183 int oarg; 1184 oarg = ct_str[0] - '0'; 1185 assert(oarg < def->nb_oargs); 1186 assert(def->args_ct[oarg].ct & TCG_CT_REG); 1187 /* TCG_CT_ALIAS is for the output arguments. The input 1188 argument is tagged with TCG_CT_IALIAS. */ 1189 def->args_ct[i] = def->args_ct[oarg]; 1190 def->args_ct[oarg].ct = TCG_CT_ALIAS; 1191 def->args_ct[oarg].alias_index = i; 1192 def->args_ct[i].ct |= TCG_CT_IALIAS; 1193 def->args_ct[i].alias_index = oarg; 1194 } else { 1195 for(;;) { 1196 if (*ct_str == '\0') 1197 break; 1198 switch(*ct_str) { 1199 case 'i': 1200 def->args_ct[i].ct |= TCG_CT_CONST; 1201 ct_str++; 1202 break; 1203 default: 1204 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) { 1205 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n", 1206 ct_str, i, def->name); 1207 exit(1); 1208 } 1209 } 1210 } 1211 } 1212 } 1213 1214 /* TCGTargetOpDef entry with too much information? */ 1215 assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1216 1217 /* sort the constraints (XXX: this is just an heuristic) */ 1218 sort_constraints(def, 0, def->nb_oargs); 1219 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1220 1221 #if 0 1222 { 1223 int i; 1224 1225 printf("%s: sorted=", def->name); 1226 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++) 1227 printf(" %d", def->sorted_args[i]); 1228 printf("\n"); 1229 } 1230 #endif 1231 tdefs++; 1232 } 1233 1234 #if defined(CONFIG_DEBUG_TCG) 1235 i = 0; 1236 for (op = 0; op < tcg_op_defs_max; op++) { 1237 const TCGOpDef *def = &tcg_op_defs[op]; 1238 if (def->flags & TCG_OPF_NOT_PRESENT) { 1239 /* Wrong entry in op definitions? */ 1240 if (def->used) { 1241 fprintf(stderr, "Invalid op definition for %s\n", def->name); 1242 i = 1; 1243 } 1244 } else { 1245 /* Missing entry in op definitions? */ 1246 if (!def->used) { 1247 fprintf(stderr, "Missing op definition for %s\n", def->name); 1248 i = 1; 1249 } 1250 } 1251 } 1252 if (i == 1) { 1253 tcg_abort(); 1254 } 1255 #endif 1256 } 1257 1258 void tcg_op_remove(TCGContext *s, TCGOp *op) 1259 { 1260 int next = op->next; 1261 int prev = op->prev; 1262 1263 if (next >= 0) { 1264 s->gen_op_buf[next].prev = prev; 1265 } else { 1266 s->gen_last_op_idx = prev; 1267 } 1268 if (prev >= 0) { 1269 s->gen_op_buf[prev].next = next; 1270 } else { 1271 s->gen_first_op_idx = next; 1272 } 1273 1274 memset(op, -1, sizeof(*op)); 1275 1276 #ifdef CONFIG_PROFILER 1277 s->del_op_count++; 1278 #endif 1279 } 1280 1281 #ifdef USE_LIVENESS_ANALYSIS 1282 /* liveness analysis: end of function: all temps are dead, and globals 1283 should be in memory. */ 1284 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps, 1285 uint8_t *mem_temps) 1286 { 1287 memset(dead_temps, 1, s->nb_temps); 1288 memset(mem_temps, 1, s->nb_globals); 1289 memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals); 1290 } 1291 1292 /* liveness analysis: end of basic block: all temps are dead, globals 1293 and local temps should be in memory. */ 1294 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps, 1295 uint8_t *mem_temps) 1296 { 1297 int i; 1298 1299 memset(dead_temps, 1, s->nb_temps); 1300 memset(mem_temps, 1, s->nb_globals); 1301 for(i = s->nb_globals; i < s->nb_temps; i++) { 1302 mem_temps[i] = s->temps[i].temp_local; 1303 } 1304 } 1305 1306 /* Liveness analysis : update the opc_dead_args array to tell if a 1307 given input arguments is dead. Instructions updating dead 1308 temporaries are removed. */ 1309 static void tcg_liveness_analysis(TCGContext *s) 1310 { 1311 uint8_t *dead_temps, *mem_temps; 1312 int oi, oi_prev, nb_ops; 1313 1314 nb_ops = s->gen_next_op_idx; 1315 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1316 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1317 1318 dead_temps = tcg_malloc(s->nb_temps); 1319 mem_temps = tcg_malloc(s->nb_temps); 1320 tcg_la_func_end(s, dead_temps, mem_temps); 1321 1322 for (oi = s->gen_last_op_idx; oi >= 0; oi = oi_prev) { 1323 int i, nb_iargs, nb_oargs; 1324 TCGOpcode opc_new, opc_new2; 1325 bool have_opc_new2; 1326 uint16_t dead_args; 1327 uint8_t sync_args; 1328 TCGArg arg; 1329 1330 TCGOp * const op = &s->gen_op_buf[oi]; 1331 TCGArg * const args = &s->gen_opparam_buf[op->args]; 1332 TCGOpcode opc = op->opc; 1333 const TCGOpDef *def = &tcg_op_defs[opc]; 1334 1335 oi_prev = op->prev; 1336 1337 switch (opc) { 1338 case INDEX_op_call: 1339 { 1340 int call_flags; 1341 1342 nb_oargs = op->callo; 1343 nb_iargs = op->calli; 1344 call_flags = args[nb_oargs + nb_iargs + 1]; 1345 1346 /* pure functions can be removed if their result is unused */ 1347 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 1348 for (i = 0; i < nb_oargs; i++) { 1349 arg = args[i]; 1350 if (!dead_temps[arg] || mem_temps[arg]) { 1351 goto do_not_remove_call; 1352 } 1353 } 1354 goto do_remove; 1355 } else { 1356 do_not_remove_call: 1357 1358 /* output args are dead */ 1359 dead_args = 0; 1360 sync_args = 0; 1361 for (i = 0; i < nb_oargs; i++) { 1362 arg = args[i]; 1363 if (dead_temps[arg]) { 1364 dead_args |= (1 << i); 1365 } 1366 if (mem_temps[arg]) { 1367 sync_args |= (1 << i); 1368 } 1369 dead_temps[arg] = 1; 1370 mem_temps[arg] = 0; 1371 } 1372 1373 if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 1374 /* globals should be synced to memory */ 1375 memset(mem_temps, 1, s->nb_globals); 1376 } 1377 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 1378 TCG_CALL_NO_READ_GLOBALS))) { 1379 /* globals should go back to memory */ 1380 memset(dead_temps, 1, s->nb_globals); 1381 } 1382 1383 /* record arguments that die in this helper */ 1384 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1385 arg = args[i]; 1386 if (arg != TCG_CALL_DUMMY_ARG) { 1387 if (dead_temps[arg]) { 1388 dead_args |= (1 << i); 1389 } 1390 } 1391 } 1392 /* input arguments are live for preceding opcodes */ 1393 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1394 arg = args[i]; 1395 dead_temps[arg] = 0; 1396 } 1397 s->op_dead_args[oi] = dead_args; 1398 s->op_sync_args[oi] = sync_args; 1399 } 1400 } 1401 break; 1402 case INDEX_op_insn_start: 1403 break; 1404 case INDEX_op_discard: 1405 /* mark the temporary as dead */ 1406 dead_temps[args[0]] = 1; 1407 mem_temps[args[0]] = 0; 1408 break; 1409 1410 case INDEX_op_add2_i32: 1411 opc_new = INDEX_op_add_i32; 1412 goto do_addsub2; 1413 case INDEX_op_sub2_i32: 1414 opc_new = INDEX_op_sub_i32; 1415 goto do_addsub2; 1416 case INDEX_op_add2_i64: 1417 opc_new = INDEX_op_add_i64; 1418 goto do_addsub2; 1419 case INDEX_op_sub2_i64: 1420 opc_new = INDEX_op_sub_i64; 1421 do_addsub2: 1422 nb_iargs = 4; 1423 nb_oargs = 2; 1424 /* Test if the high part of the operation is dead, but not 1425 the low part. The result can be optimized to a simple 1426 add or sub. This happens often for x86_64 guest when the 1427 cpu mode is set to 32 bit. */ 1428 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1429 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1430 goto do_remove; 1431 } 1432 /* Replace the opcode and adjust the args in place, 1433 leaving 3 unused args at the end. */ 1434 op->opc = opc = opc_new; 1435 args[1] = args[2]; 1436 args[2] = args[4]; 1437 /* Fall through and mark the single-word operation live. */ 1438 nb_iargs = 2; 1439 nb_oargs = 1; 1440 } 1441 goto do_not_remove; 1442 1443 case INDEX_op_mulu2_i32: 1444 opc_new = INDEX_op_mul_i32; 1445 opc_new2 = INDEX_op_muluh_i32; 1446 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 1447 goto do_mul2; 1448 case INDEX_op_muls2_i32: 1449 opc_new = INDEX_op_mul_i32; 1450 opc_new2 = INDEX_op_mulsh_i32; 1451 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 1452 goto do_mul2; 1453 case INDEX_op_mulu2_i64: 1454 opc_new = INDEX_op_mul_i64; 1455 opc_new2 = INDEX_op_muluh_i64; 1456 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 1457 goto do_mul2; 1458 case INDEX_op_muls2_i64: 1459 opc_new = INDEX_op_mul_i64; 1460 opc_new2 = INDEX_op_mulsh_i64; 1461 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 1462 goto do_mul2; 1463 do_mul2: 1464 nb_iargs = 2; 1465 nb_oargs = 2; 1466 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1467 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1468 /* Both parts of the operation are dead. */ 1469 goto do_remove; 1470 } 1471 /* The high part of the operation is dead; generate the low. */ 1472 op->opc = opc = opc_new; 1473 args[1] = args[2]; 1474 args[2] = args[3]; 1475 } else if (have_opc_new2 && dead_temps[args[0]] 1476 && !mem_temps[args[0]]) { 1477 /* The low part of the operation is dead; generate the high. */ 1478 op->opc = opc = opc_new2; 1479 args[0] = args[1]; 1480 args[1] = args[2]; 1481 args[2] = args[3]; 1482 } else { 1483 goto do_not_remove; 1484 } 1485 /* Mark the single-word operation live. */ 1486 nb_oargs = 1; 1487 goto do_not_remove; 1488 1489 default: 1490 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 1491 nb_iargs = def->nb_iargs; 1492 nb_oargs = def->nb_oargs; 1493 1494 /* Test if the operation can be removed because all 1495 its outputs are dead. We assume that nb_oargs == 0 1496 implies side effects */ 1497 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 1498 for (i = 0; i < nb_oargs; i++) { 1499 arg = args[i]; 1500 if (!dead_temps[arg] || mem_temps[arg]) { 1501 goto do_not_remove; 1502 } 1503 } 1504 do_remove: 1505 tcg_op_remove(s, op); 1506 } else { 1507 do_not_remove: 1508 /* output args are dead */ 1509 dead_args = 0; 1510 sync_args = 0; 1511 for (i = 0; i < nb_oargs; i++) { 1512 arg = args[i]; 1513 if (dead_temps[arg]) { 1514 dead_args |= (1 << i); 1515 } 1516 if (mem_temps[arg]) { 1517 sync_args |= (1 << i); 1518 } 1519 dead_temps[arg] = 1; 1520 mem_temps[arg] = 0; 1521 } 1522 1523 /* if end of basic block, update */ 1524 if (def->flags & TCG_OPF_BB_END) { 1525 tcg_la_bb_end(s, dead_temps, mem_temps); 1526 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 1527 /* globals should be synced to memory */ 1528 memset(mem_temps, 1, s->nb_globals); 1529 } 1530 1531 /* record arguments that die in this opcode */ 1532 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1533 arg = args[i]; 1534 if (dead_temps[arg]) { 1535 dead_args |= (1 << i); 1536 } 1537 } 1538 /* input arguments are live for preceding opcodes */ 1539 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1540 arg = args[i]; 1541 dead_temps[arg] = 0; 1542 } 1543 s->op_dead_args[oi] = dead_args; 1544 s->op_sync_args[oi] = sync_args; 1545 } 1546 break; 1547 } 1548 } 1549 } 1550 #else 1551 /* dummy liveness analysis */ 1552 static void tcg_liveness_analysis(TCGContext *s) 1553 { 1554 int nb_ops = s->gen_next_op_idx; 1555 1556 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1557 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t)); 1558 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1559 memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t)); 1560 } 1561 #endif 1562 1563 #ifndef NDEBUG 1564 static void dump_regs(TCGContext *s) 1565 { 1566 TCGTemp *ts; 1567 int i; 1568 char buf[64]; 1569 1570 for(i = 0; i < s->nb_temps; i++) { 1571 ts = &s->temps[i]; 1572 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i)); 1573 switch(ts->val_type) { 1574 case TEMP_VAL_REG: 1575 printf("%s", tcg_target_reg_names[ts->reg]); 1576 break; 1577 case TEMP_VAL_MEM: 1578 printf("%d(%s)", (int)ts->mem_offset, 1579 tcg_target_reg_names[ts->mem_base->reg]); 1580 break; 1581 case TEMP_VAL_CONST: 1582 printf("$0x%" TCG_PRIlx, ts->val); 1583 break; 1584 case TEMP_VAL_DEAD: 1585 printf("D"); 1586 break; 1587 default: 1588 printf("???"); 1589 break; 1590 } 1591 printf("\n"); 1592 } 1593 1594 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1595 if (s->reg_to_temp[i] != NULL) { 1596 printf("%s: %s\n", 1597 tcg_target_reg_names[i], 1598 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 1599 } 1600 } 1601 } 1602 1603 static void check_regs(TCGContext *s) 1604 { 1605 TCGReg reg; 1606 int k; 1607 TCGTemp *ts; 1608 char buf[64]; 1609 1610 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 1611 ts = s->reg_to_temp[reg]; 1612 if (ts != NULL) { 1613 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 1614 printf("Inconsistency for register %s:\n", 1615 tcg_target_reg_names[reg]); 1616 goto fail; 1617 } 1618 } 1619 } 1620 for (k = 0; k < s->nb_temps; k++) { 1621 ts = &s->temps[k]; 1622 if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg 1623 && s->reg_to_temp[ts->reg] != ts) { 1624 printf("Inconsistency for temp %s:\n", 1625 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 1626 fail: 1627 printf("reg state:\n"); 1628 dump_regs(s); 1629 tcg_abort(); 1630 } 1631 } 1632 } 1633 #endif 1634 1635 static void temp_allocate_frame(TCGContext *s, int temp) 1636 { 1637 TCGTemp *ts; 1638 ts = &s->temps[temp]; 1639 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 1640 /* Sparc64 stack is accessed with offset of 2047 */ 1641 s->current_frame_offset = (s->current_frame_offset + 1642 (tcg_target_long)sizeof(tcg_target_long) - 1) & 1643 ~(sizeof(tcg_target_long) - 1); 1644 #endif 1645 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 1646 s->frame_end) { 1647 tcg_abort(); 1648 } 1649 ts->mem_offset = s->current_frame_offset; 1650 ts->mem_base = s->frame_temp; 1651 ts->mem_allocated = 1; 1652 s->current_frame_offset += sizeof(tcg_target_long); 1653 } 1654 1655 /* sync register 'reg' by saving it to the corresponding temporary */ 1656 static inline void tcg_reg_sync(TCGContext *s, TCGReg reg) 1657 { 1658 TCGTemp *ts = s->reg_to_temp[reg]; 1659 1660 assert(ts->val_type == TEMP_VAL_REG); 1661 if (!ts->mem_coherent && !ts->fixed_reg) { 1662 if (!ts->mem_allocated) { 1663 temp_allocate_frame(s, temp_idx(s, ts)); 1664 } 1665 tcg_out_st(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 1666 } 1667 ts->mem_coherent = 1; 1668 } 1669 1670 /* free register 'reg' by spilling the corresponding temporary if necessary */ 1671 static void tcg_reg_free(TCGContext *s, TCGReg reg) 1672 { 1673 TCGTemp *ts = s->reg_to_temp[reg]; 1674 1675 if (ts != NULL) { 1676 tcg_reg_sync(s, reg); 1677 ts->val_type = TEMP_VAL_MEM; 1678 s->reg_to_temp[reg] = NULL; 1679 } 1680 } 1681 1682 /* Allocate a register belonging to reg1 & ~reg2 */ 1683 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2) 1684 { 1685 int i; 1686 TCGReg reg; 1687 TCGRegSet reg_ct; 1688 1689 tcg_regset_andnot(reg_ct, reg1, reg2); 1690 1691 /* first try free registers */ 1692 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1693 reg = tcg_target_reg_alloc_order[i]; 1694 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL) 1695 return reg; 1696 } 1697 1698 /* XXX: do better spill choice */ 1699 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1700 reg = tcg_target_reg_alloc_order[i]; 1701 if (tcg_regset_test_reg(reg_ct, reg)) { 1702 tcg_reg_free(s, reg); 1703 return reg; 1704 } 1705 } 1706 1707 tcg_abort(); 1708 } 1709 1710 /* Make sure the temporary is in a register. If needed, allocate the register 1711 from DESIRED while avoiding ALLOCATED. */ 1712 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 1713 TCGRegSet allocated_regs) 1714 { 1715 TCGReg reg; 1716 1717 switch (ts->val_type) { 1718 case TEMP_VAL_REG: 1719 return; 1720 case TEMP_VAL_CONST: 1721 reg = tcg_reg_alloc(s, desired_regs, allocated_regs); 1722 tcg_out_movi(s, ts->type, reg, ts->val); 1723 ts->mem_coherent = 0; 1724 break; 1725 case TEMP_VAL_MEM: 1726 reg = tcg_reg_alloc(s, desired_regs, allocated_regs); 1727 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 1728 ts->mem_coherent = 1; 1729 break; 1730 case TEMP_VAL_DEAD: 1731 default: 1732 tcg_abort(); 1733 } 1734 ts->reg = reg; 1735 ts->val_type = TEMP_VAL_REG; 1736 s->reg_to_temp[reg] = ts; 1737 } 1738 1739 /* mark a temporary as dead. */ 1740 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 1741 { 1742 if (ts->fixed_reg) { 1743 return; 1744 } 1745 if (ts->val_type == TEMP_VAL_REG) { 1746 s->reg_to_temp[ts->reg] = NULL; 1747 } 1748 ts->val_type = (temp_idx(s, ts) < s->nb_globals || ts->temp_local 1749 ? TEMP_VAL_MEM : TEMP_VAL_DEAD); 1750 } 1751 1752 /* sync a temporary to memory. 'allocated_regs' is used in case a 1753 temporary registers needs to be allocated to store a constant. */ 1754 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 1755 { 1756 if (ts->fixed_reg) { 1757 return; 1758 } 1759 switch (ts->val_type) { 1760 case TEMP_VAL_CONST: 1761 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs); 1762 /* fallthrough */ 1763 case TEMP_VAL_REG: 1764 tcg_reg_sync(s, ts->reg); 1765 break; 1766 case TEMP_VAL_DEAD: 1767 case TEMP_VAL_MEM: 1768 break; 1769 default: 1770 tcg_abort(); 1771 } 1772 } 1773 1774 /* save a temporary to memory. 'allocated_regs' is used in case a 1775 temporary registers needs to be allocated to store a constant. */ 1776 static inline void temp_save(TCGContext *s, TCGTemp *ts, 1777 TCGRegSet allocated_regs) 1778 { 1779 #ifdef USE_LIVENESS_ANALYSIS 1780 /* The liveness analysis already ensures that globals are back 1781 in memory. Keep an assert for safety. */ 1782 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg); 1783 #else 1784 temp_sync(s, ts, allocated_regs); 1785 temp_dead(s, ts); 1786 #endif 1787 } 1788 1789 /* save globals to their canonical location and assume they can be 1790 modified be the following code. 'allocated_regs' is used in case a 1791 temporary registers needs to be allocated to store a constant. */ 1792 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 1793 { 1794 int i; 1795 1796 for (i = 0; i < s->nb_globals; i++) { 1797 temp_save(s, &s->temps[i], allocated_regs); 1798 } 1799 } 1800 1801 /* sync globals to their canonical location and assume they can be 1802 read by the following code. 'allocated_regs' is used in case a 1803 temporary registers needs to be allocated to store a constant. */ 1804 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 1805 { 1806 int i; 1807 1808 for (i = 0; i < s->nb_globals; i++) { 1809 TCGTemp *ts = &s->temps[i]; 1810 #ifdef USE_LIVENESS_ANALYSIS 1811 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 1812 || ts->fixed_reg 1813 || ts->mem_coherent); 1814 #else 1815 temp_sync(s, ts, allocated_regs); 1816 #endif 1817 } 1818 } 1819 1820 /* at the end of a basic block, we assume all temporaries are dead and 1821 all globals are stored at their canonical location. */ 1822 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 1823 { 1824 int i; 1825 1826 for (i = s->nb_globals; i < s->nb_temps; i++) { 1827 TCGTemp *ts = &s->temps[i]; 1828 if (ts->temp_local) { 1829 temp_save(s, ts, allocated_regs); 1830 } else { 1831 #ifdef USE_LIVENESS_ANALYSIS 1832 /* The liveness analysis already ensures that temps are dead. 1833 Keep an assert for safety. */ 1834 assert(ts->val_type == TEMP_VAL_DEAD); 1835 #else 1836 temp_dead(s, ts); 1837 #endif 1838 } 1839 } 1840 1841 save_globals(s, allocated_regs); 1842 } 1843 1844 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1) 1845 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1) 1846 1847 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args, 1848 uint16_t dead_args, uint8_t sync_args) 1849 { 1850 TCGTemp *ots; 1851 tcg_target_ulong val; 1852 1853 ots = &s->temps[args[0]]; 1854 val = args[1]; 1855 1856 if (ots->fixed_reg) { 1857 /* for fixed registers, we do not do any constant 1858 propagation */ 1859 tcg_out_movi(s, ots->type, ots->reg, val); 1860 } else { 1861 /* The movi is not explicitly generated here */ 1862 if (ots->val_type == TEMP_VAL_REG) { 1863 s->reg_to_temp[ots->reg] = NULL; 1864 } 1865 ots->val_type = TEMP_VAL_CONST; 1866 ots->val = val; 1867 } 1868 if (NEED_SYNC_ARG(0)) { 1869 temp_sync(s, ots, s->reserved_regs); 1870 } 1871 if (IS_DEAD_ARG(0)) { 1872 temp_dead(s, ots); 1873 } 1874 } 1875 1876 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, 1877 const TCGArg *args, uint16_t dead_args, 1878 uint8_t sync_args) 1879 { 1880 TCGRegSet allocated_regs; 1881 TCGTemp *ts, *ots; 1882 TCGType otype, itype; 1883 1884 tcg_regset_set(allocated_regs, s->reserved_regs); 1885 ots = &s->temps[args[0]]; 1886 ts = &s->temps[args[1]]; 1887 1888 /* Note that otype != itype for no-op truncation. */ 1889 otype = ots->type; 1890 itype = ts->type; 1891 1892 /* If the source value is not in a register, and we're going to be 1893 forced to have it in a register in order to perform the copy, 1894 then copy the SOURCE value into its own register first. That way 1895 we don't have to reload SOURCE the next time it is used. */ 1896 if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG) 1897 || ts->val_type == TEMP_VAL_MEM) { 1898 temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs); 1899 } 1900 1901 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 1902 /* mov to a non-saved dead register makes no sense (even with 1903 liveness analysis disabled). */ 1904 assert(NEED_SYNC_ARG(0)); 1905 /* The code above should have moved the temp to a register. */ 1906 assert(ts->val_type == TEMP_VAL_REG); 1907 if (!ots->mem_allocated) { 1908 temp_allocate_frame(s, args[0]); 1909 } 1910 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 1911 if (IS_DEAD_ARG(1)) { 1912 temp_dead(s, ts); 1913 } 1914 temp_dead(s, ots); 1915 } else if (ts->val_type == TEMP_VAL_CONST) { 1916 /* propagate constant */ 1917 if (ots->val_type == TEMP_VAL_REG) { 1918 s->reg_to_temp[ots->reg] = NULL; 1919 } 1920 ots->val_type = TEMP_VAL_CONST; 1921 ots->val = ts->val; 1922 if (IS_DEAD_ARG(1)) { 1923 temp_dead(s, ts); 1924 } 1925 } else { 1926 /* The code in the first if block should have moved the 1927 temp to a register. */ 1928 assert(ts->val_type == TEMP_VAL_REG); 1929 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 1930 /* the mov can be suppressed */ 1931 if (ots->val_type == TEMP_VAL_REG) { 1932 s->reg_to_temp[ots->reg] = NULL; 1933 } 1934 ots->reg = ts->reg; 1935 temp_dead(s, ts); 1936 } else { 1937 if (ots->val_type != TEMP_VAL_REG) { 1938 /* When allocating a new register, make sure to not spill the 1939 input one. */ 1940 tcg_regset_set_reg(allocated_regs, ts->reg); 1941 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 1942 allocated_regs); 1943 } 1944 tcg_out_mov(s, otype, ots->reg, ts->reg); 1945 } 1946 ots->val_type = TEMP_VAL_REG; 1947 ots->mem_coherent = 0; 1948 s->reg_to_temp[ots->reg] = ots; 1949 if (NEED_SYNC_ARG(0)) { 1950 tcg_reg_sync(s, ots->reg); 1951 } 1952 } 1953 } 1954 1955 static void tcg_reg_alloc_op(TCGContext *s, 1956 const TCGOpDef *def, TCGOpcode opc, 1957 const TCGArg *args, uint16_t dead_args, 1958 uint8_t sync_args) 1959 { 1960 TCGRegSet allocated_regs; 1961 int i, k, nb_iargs, nb_oargs; 1962 TCGReg reg; 1963 TCGArg arg; 1964 const TCGArgConstraint *arg_ct; 1965 TCGTemp *ts; 1966 TCGArg new_args[TCG_MAX_OP_ARGS]; 1967 int const_args[TCG_MAX_OP_ARGS]; 1968 1969 nb_oargs = def->nb_oargs; 1970 nb_iargs = def->nb_iargs; 1971 1972 /* copy constants */ 1973 memcpy(new_args + nb_oargs + nb_iargs, 1974 args + nb_oargs + nb_iargs, 1975 sizeof(TCGArg) * def->nb_cargs); 1976 1977 /* satisfy input constraints */ 1978 tcg_regset_set(allocated_regs, s->reserved_regs); 1979 for(k = 0; k < nb_iargs; k++) { 1980 i = def->sorted_args[nb_oargs + k]; 1981 arg = args[i]; 1982 arg_ct = &def->args_ct[i]; 1983 ts = &s->temps[arg]; 1984 1985 if (ts->val_type == TEMP_VAL_CONST 1986 && tcg_target_const_match(ts->val, ts->type, arg_ct)) { 1987 /* constant is OK for instruction */ 1988 const_args[i] = 1; 1989 new_args[i] = ts->val; 1990 goto iarg_end; 1991 } 1992 1993 temp_load(s, ts, arg_ct->u.regs, allocated_regs); 1994 1995 if (arg_ct->ct & TCG_CT_IALIAS) { 1996 if (ts->fixed_reg) { 1997 /* if fixed register, we must allocate a new register 1998 if the alias is not the same register */ 1999 if (arg != args[arg_ct->alias_index]) 2000 goto allocate_in_reg; 2001 } else { 2002 /* if the input is aliased to an output and if it is 2003 not dead after the instruction, we must allocate 2004 a new register and move it */ 2005 if (!IS_DEAD_ARG(i)) { 2006 goto allocate_in_reg; 2007 } 2008 /* check if the current register has already been allocated 2009 for another input aliased to an output */ 2010 int k2, i2; 2011 for (k2 = 0 ; k2 < k ; k2++) { 2012 i2 = def->sorted_args[nb_oargs + k2]; 2013 if ((def->args_ct[i2].ct & TCG_CT_IALIAS) && 2014 (new_args[i2] == ts->reg)) { 2015 goto allocate_in_reg; 2016 } 2017 } 2018 } 2019 } 2020 reg = ts->reg; 2021 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2022 /* nothing to do : the constraint is satisfied */ 2023 } else { 2024 allocate_in_reg: 2025 /* allocate a new register matching the constraint 2026 and move the temporary register into it */ 2027 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2028 tcg_out_mov(s, ts->type, reg, ts->reg); 2029 } 2030 new_args[i] = reg; 2031 const_args[i] = 0; 2032 tcg_regset_set_reg(allocated_regs, reg); 2033 iarg_end: ; 2034 } 2035 2036 /* mark dead temporaries and free the associated registers */ 2037 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2038 if (IS_DEAD_ARG(i)) { 2039 temp_dead(s, &s->temps[args[i]]); 2040 } 2041 } 2042 2043 if (def->flags & TCG_OPF_BB_END) { 2044 tcg_reg_alloc_bb_end(s, allocated_regs); 2045 } else { 2046 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2047 /* XXX: permit generic clobber register list ? */ 2048 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 2049 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 2050 tcg_reg_free(s, i); 2051 } 2052 } 2053 } 2054 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2055 /* sync globals if the op has side effects and might trigger 2056 an exception. */ 2057 sync_globals(s, allocated_regs); 2058 } 2059 2060 /* satisfy the output constraints */ 2061 tcg_regset_set(allocated_regs, s->reserved_regs); 2062 for(k = 0; k < nb_oargs; k++) { 2063 i = def->sorted_args[k]; 2064 arg = args[i]; 2065 arg_ct = &def->args_ct[i]; 2066 ts = &s->temps[arg]; 2067 if (arg_ct->ct & TCG_CT_ALIAS) { 2068 reg = new_args[arg_ct->alias_index]; 2069 } else { 2070 /* if fixed register, we try to use it */ 2071 reg = ts->reg; 2072 if (ts->fixed_reg && 2073 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2074 goto oarg_end; 2075 } 2076 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2077 } 2078 tcg_regset_set_reg(allocated_regs, reg); 2079 /* if a fixed register is used, then a move will be done afterwards */ 2080 if (!ts->fixed_reg) { 2081 if (ts->val_type == TEMP_VAL_REG) { 2082 s->reg_to_temp[ts->reg] = NULL; 2083 } 2084 ts->val_type = TEMP_VAL_REG; 2085 ts->reg = reg; 2086 /* temp value is modified, so the value kept in memory is 2087 potentially not the same */ 2088 ts->mem_coherent = 0; 2089 s->reg_to_temp[reg] = ts; 2090 } 2091 oarg_end: 2092 new_args[i] = reg; 2093 } 2094 } 2095 2096 /* emit instruction */ 2097 tcg_out_op(s, opc, new_args, const_args); 2098 2099 /* move the outputs in the correct register if needed */ 2100 for(i = 0; i < nb_oargs; i++) { 2101 ts = &s->temps[args[i]]; 2102 reg = new_args[i]; 2103 if (ts->fixed_reg && ts->reg != reg) { 2104 tcg_out_mov(s, ts->type, ts->reg, reg); 2105 } 2106 if (NEED_SYNC_ARG(i)) { 2107 tcg_reg_sync(s, reg); 2108 } 2109 if (IS_DEAD_ARG(i)) { 2110 temp_dead(s, ts); 2111 } 2112 } 2113 } 2114 2115 #ifdef TCG_TARGET_STACK_GROWSUP 2116 #define STACK_DIR(x) (-(x)) 2117 #else 2118 #define STACK_DIR(x) (x) 2119 #endif 2120 2121 static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs, 2122 const TCGArg * const args, uint16_t dead_args, 2123 uint8_t sync_args) 2124 { 2125 int flags, nb_regs, i; 2126 TCGReg reg; 2127 TCGArg arg; 2128 TCGTemp *ts; 2129 intptr_t stack_offset; 2130 size_t call_stack_size; 2131 tcg_insn_unit *func_addr; 2132 int allocate_args; 2133 TCGRegSet allocated_regs; 2134 2135 func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs]; 2136 flags = args[nb_oargs + nb_iargs + 1]; 2137 2138 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2139 if (nb_regs > nb_iargs) { 2140 nb_regs = nb_iargs; 2141 } 2142 2143 /* assign stack slots first */ 2144 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 2145 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 2146 ~(TCG_TARGET_STACK_ALIGN - 1); 2147 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 2148 if (allocate_args) { 2149 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 2150 preallocate call stack */ 2151 tcg_abort(); 2152 } 2153 2154 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 2155 for(i = nb_regs; i < nb_iargs; i++) { 2156 arg = args[nb_oargs + i]; 2157 #ifdef TCG_TARGET_STACK_GROWSUP 2158 stack_offset -= sizeof(tcg_target_long); 2159 #endif 2160 if (arg != TCG_CALL_DUMMY_ARG) { 2161 ts = &s->temps[arg]; 2162 temp_load(s, ts, tcg_target_available_regs[ts->type], 2163 s->reserved_regs); 2164 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 2165 } 2166 #ifndef TCG_TARGET_STACK_GROWSUP 2167 stack_offset += sizeof(tcg_target_long); 2168 #endif 2169 } 2170 2171 /* assign input registers */ 2172 tcg_regset_set(allocated_regs, s->reserved_regs); 2173 for(i = 0; i < nb_regs; i++) { 2174 arg = args[nb_oargs + i]; 2175 if (arg != TCG_CALL_DUMMY_ARG) { 2176 ts = &s->temps[arg]; 2177 reg = tcg_target_call_iarg_regs[i]; 2178 tcg_reg_free(s, reg); 2179 2180 if (ts->val_type == TEMP_VAL_REG) { 2181 if (ts->reg != reg) { 2182 tcg_out_mov(s, ts->type, reg, ts->reg); 2183 } 2184 } else { 2185 TCGRegSet arg_set; 2186 2187 tcg_regset_clear(arg_set); 2188 tcg_regset_set_reg(arg_set, reg); 2189 temp_load(s, ts, arg_set, allocated_regs); 2190 } 2191 2192 tcg_regset_set_reg(allocated_regs, reg); 2193 } 2194 } 2195 2196 /* mark dead temporaries and free the associated registers */ 2197 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2198 if (IS_DEAD_ARG(i)) { 2199 temp_dead(s, &s->temps[args[i]]); 2200 } 2201 } 2202 2203 /* clobber call registers */ 2204 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 2205 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 2206 tcg_reg_free(s, i); 2207 } 2208 } 2209 2210 /* Save globals if they might be written by the helper, sync them if 2211 they might be read. */ 2212 if (flags & TCG_CALL_NO_READ_GLOBALS) { 2213 /* Nothing to do */ 2214 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 2215 sync_globals(s, allocated_regs); 2216 } else { 2217 save_globals(s, allocated_regs); 2218 } 2219 2220 tcg_out_call(s, func_addr); 2221 2222 /* assign output registers and emit moves if needed */ 2223 for(i = 0; i < nb_oargs; i++) { 2224 arg = args[i]; 2225 ts = &s->temps[arg]; 2226 reg = tcg_target_call_oarg_regs[i]; 2227 assert(s->reg_to_temp[reg] == NULL); 2228 2229 if (ts->fixed_reg) { 2230 if (ts->reg != reg) { 2231 tcg_out_mov(s, ts->type, ts->reg, reg); 2232 } 2233 } else { 2234 if (ts->val_type == TEMP_VAL_REG) { 2235 s->reg_to_temp[ts->reg] = NULL; 2236 } 2237 ts->val_type = TEMP_VAL_REG; 2238 ts->reg = reg; 2239 ts->mem_coherent = 0; 2240 s->reg_to_temp[reg] = ts; 2241 if (NEED_SYNC_ARG(i)) { 2242 tcg_reg_sync(s, reg); 2243 } 2244 if (IS_DEAD_ARG(i)) { 2245 temp_dead(s, ts); 2246 } 2247 } 2248 } 2249 } 2250 2251 #ifdef CONFIG_PROFILER 2252 2253 static int64_t tcg_table_op_count[NB_OPS]; 2254 2255 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 2256 { 2257 int i; 2258 2259 for (i = 0; i < NB_OPS; i++) { 2260 cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, 2261 tcg_table_op_count[i]); 2262 } 2263 } 2264 #else 2265 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 2266 { 2267 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2268 } 2269 #endif 2270 2271 2272 int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf) 2273 { 2274 int i, oi, oi_next, num_insns; 2275 2276 #ifdef CONFIG_PROFILER 2277 { 2278 int n; 2279 2280 n = s->gen_last_op_idx + 1; 2281 s->op_count += n; 2282 if (n > s->op_count_max) { 2283 s->op_count_max = n; 2284 } 2285 2286 n = s->nb_temps; 2287 s->temp_count += n; 2288 if (n > s->temp_count_max) { 2289 s->temp_count_max = n; 2290 } 2291 } 2292 #endif 2293 2294 #ifdef DEBUG_DISAS 2295 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) { 2296 qemu_log("OP:\n"); 2297 tcg_dump_ops(s); 2298 qemu_log("\n"); 2299 } 2300 #endif 2301 2302 #ifdef CONFIG_PROFILER 2303 s->opt_time -= profile_getclock(); 2304 #endif 2305 2306 #ifdef USE_TCG_OPTIMIZATIONS 2307 tcg_optimize(s); 2308 #endif 2309 2310 #ifdef CONFIG_PROFILER 2311 s->opt_time += profile_getclock(); 2312 s->la_time -= profile_getclock(); 2313 #endif 2314 2315 tcg_liveness_analysis(s); 2316 2317 #ifdef CONFIG_PROFILER 2318 s->la_time += profile_getclock(); 2319 #endif 2320 2321 #ifdef DEBUG_DISAS 2322 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) { 2323 qemu_log("OP after optimization and liveness analysis:\n"); 2324 tcg_dump_ops(s); 2325 qemu_log("\n"); 2326 } 2327 #endif 2328 2329 tcg_reg_alloc_start(s); 2330 2331 s->code_buf = gen_code_buf; 2332 s->code_ptr = gen_code_buf; 2333 2334 tcg_out_tb_init(s); 2335 2336 num_insns = -1; 2337 for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) { 2338 TCGOp * const op = &s->gen_op_buf[oi]; 2339 TCGArg * const args = &s->gen_opparam_buf[op->args]; 2340 TCGOpcode opc = op->opc; 2341 const TCGOpDef *def = &tcg_op_defs[opc]; 2342 uint16_t dead_args = s->op_dead_args[oi]; 2343 uint8_t sync_args = s->op_sync_args[oi]; 2344 2345 oi_next = op->next; 2346 #ifdef CONFIG_PROFILER 2347 tcg_table_op_count[opc]++; 2348 #endif 2349 2350 switch (opc) { 2351 case INDEX_op_mov_i32: 2352 case INDEX_op_mov_i64: 2353 tcg_reg_alloc_mov(s, def, args, dead_args, sync_args); 2354 break; 2355 case INDEX_op_movi_i32: 2356 case INDEX_op_movi_i64: 2357 tcg_reg_alloc_movi(s, args, dead_args, sync_args); 2358 break; 2359 case INDEX_op_insn_start: 2360 if (num_insns >= 0) { 2361 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 2362 } 2363 num_insns++; 2364 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 2365 target_ulong a; 2366 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 2367 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2]; 2368 #else 2369 a = args[i]; 2370 #endif 2371 s->gen_insn_data[num_insns][i] = a; 2372 } 2373 break; 2374 case INDEX_op_discard: 2375 temp_dead(s, &s->temps[args[0]]); 2376 break; 2377 case INDEX_op_set_label: 2378 tcg_reg_alloc_bb_end(s, s->reserved_regs); 2379 tcg_out_label(s, arg_label(args[0]), s->code_ptr); 2380 break; 2381 case INDEX_op_call: 2382 tcg_reg_alloc_call(s, op->callo, op->calli, args, 2383 dead_args, sync_args); 2384 break; 2385 default: 2386 /* Sanity check that we've not introduced any unhandled opcodes. */ 2387 if (def->flags & TCG_OPF_NOT_PRESENT) { 2388 tcg_abort(); 2389 } 2390 /* Note: in order to speed up the code, it would be much 2391 faster to have specialized register allocator functions for 2392 some common argument patterns */ 2393 tcg_reg_alloc_op(s, def, opc, args, dead_args, sync_args); 2394 break; 2395 } 2396 #ifndef NDEBUG 2397 check_regs(s); 2398 #endif 2399 /* Test for (pending) buffer overflow. The assumption is that any 2400 one operation beginning below the high water mark cannot overrun 2401 the buffer completely. Thus we can test for overflow after 2402 generating code without having to check during generation. */ 2403 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 2404 return -1; 2405 } 2406 } 2407 tcg_debug_assert(num_insns >= 0); 2408 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 2409 2410 /* Generate TB finalization at the end of block */ 2411 if (!tcg_out_tb_finalize(s)) { 2412 return -1; 2413 } 2414 2415 /* flush instruction cache */ 2416 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 2417 2418 return tcg_current_code_size(s); 2419 } 2420 2421 #ifdef CONFIG_PROFILER 2422 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2423 { 2424 TCGContext *s = &tcg_ctx; 2425 int64_t tb_count = s->tb_count; 2426 int64_t tb_div_count = tb_count ? tb_count : 1; 2427 int64_t tot = s->interm_time + s->code_time; 2428 2429 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 2430 tot, tot / 2.4e9); 2431 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 2432 tb_count, s->tb_count1 - tb_count, 2433 (double)(s->tb_count1 - s->tb_count) 2434 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 2435 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 2436 (double)s->op_count / tb_div_count, s->op_count_max); 2437 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 2438 (double)s->del_op_count / tb_div_count); 2439 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 2440 (double)s->temp_count / tb_div_count, s->temp_count_max); 2441 cpu_fprintf(f, "avg host code/TB %0.1f\n", 2442 (double)s->code_out_len / tb_div_count); 2443 cpu_fprintf(f, "avg search data/TB %0.1f\n", 2444 (double)s->search_out_len / tb_div_count); 2445 2446 cpu_fprintf(f, "cycles/op %0.1f\n", 2447 s->op_count ? (double)tot / s->op_count : 0); 2448 cpu_fprintf(f, "cycles/in byte %0.1f\n", 2449 s->code_in_len ? (double)tot / s->code_in_len : 0); 2450 cpu_fprintf(f, "cycles/out byte %0.1f\n", 2451 s->code_out_len ? (double)tot / s->code_out_len : 0); 2452 cpu_fprintf(f, "cycles/search byte %0.1f\n", 2453 s->search_out_len ? (double)tot / s->search_out_len : 0); 2454 if (tot == 0) { 2455 tot = 1; 2456 } 2457 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 2458 (double)s->interm_time / tot * 100.0); 2459 cpu_fprintf(f, " gen_code time %0.1f%%\n", 2460 (double)s->code_time / tot * 100.0); 2461 cpu_fprintf(f, "optim./code time %0.1f%%\n", 2462 (double)s->opt_time / (s->code_time ? s->code_time : 1) 2463 * 100.0); 2464 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 2465 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 2466 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 2467 s->restore_count); 2468 cpu_fprintf(f, " avg cycles %0.1f\n", 2469 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 2470 } 2471 #else 2472 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2473 { 2474 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2475 } 2476 #endif 2477 2478 #ifdef ELF_HOST_MACHINE 2479 /* In order to use this feature, the backend needs to do three things: 2480 2481 (1) Define ELF_HOST_MACHINE to indicate both what value to 2482 put into the ELF image and to indicate support for the feature. 2483 2484 (2) Define tcg_register_jit. This should create a buffer containing 2485 the contents of a .debug_frame section that describes the post- 2486 prologue unwind info for the tcg machine. 2487 2488 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 2489 */ 2490 2491 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 2492 typedef enum { 2493 JIT_NOACTION = 0, 2494 JIT_REGISTER_FN, 2495 JIT_UNREGISTER_FN 2496 } jit_actions_t; 2497 2498 struct jit_code_entry { 2499 struct jit_code_entry *next_entry; 2500 struct jit_code_entry *prev_entry; 2501 const void *symfile_addr; 2502 uint64_t symfile_size; 2503 }; 2504 2505 struct jit_descriptor { 2506 uint32_t version; 2507 uint32_t action_flag; 2508 struct jit_code_entry *relevant_entry; 2509 struct jit_code_entry *first_entry; 2510 }; 2511 2512 void __jit_debug_register_code(void) __attribute__((noinline)); 2513 void __jit_debug_register_code(void) 2514 { 2515 asm(""); 2516 } 2517 2518 /* Must statically initialize the version, because GDB may check 2519 the version before we can set it. */ 2520 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 2521 2522 /* End GDB interface. */ 2523 2524 static int find_string(const char *strtab, const char *str) 2525 { 2526 const char *p = strtab + 1; 2527 2528 while (1) { 2529 if (strcmp(p, str) == 0) { 2530 return p - strtab; 2531 } 2532 p += strlen(p) + 1; 2533 } 2534 } 2535 2536 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 2537 const void *debug_frame, 2538 size_t debug_frame_size) 2539 { 2540 struct __attribute__((packed)) DebugInfo { 2541 uint32_t len; 2542 uint16_t version; 2543 uint32_t abbrev; 2544 uint8_t ptr_size; 2545 uint8_t cu_die; 2546 uint16_t cu_lang; 2547 uintptr_t cu_low_pc; 2548 uintptr_t cu_high_pc; 2549 uint8_t fn_die; 2550 char fn_name[16]; 2551 uintptr_t fn_low_pc; 2552 uintptr_t fn_high_pc; 2553 uint8_t cu_eoc; 2554 }; 2555 2556 struct ElfImage { 2557 ElfW(Ehdr) ehdr; 2558 ElfW(Phdr) phdr; 2559 ElfW(Shdr) shdr[7]; 2560 ElfW(Sym) sym[2]; 2561 struct DebugInfo di; 2562 uint8_t da[24]; 2563 char str[80]; 2564 }; 2565 2566 struct ElfImage *img; 2567 2568 static const struct ElfImage img_template = { 2569 .ehdr = { 2570 .e_ident[EI_MAG0] = ELFMAG0, 2571 .e_ident[EI_MAG1] = ELFMAG1, 2572 .e_ident[EI_MAG2] = ELFMAG2, 2573 .e_ident[EI_MAG3] = ELFMAG3, 2574 .e_ident[EI_CLASS] = ELF_CLASS, 2575 .e_ident[EI_DATA] = ELF_DATA, 2576 .e_ident[EI_VERSION] = EV_CURRENT, 2577 .e_type = ET_EXEC, 2578 .e_machine = ELF_HOST_MACHINE, 2579 .e_version = EV_CURRENT, 2580 .e_phoff = offsetof(struct ElfImage, phdr), 2581 .e_shoff = offsetof(struct ElfImage, shdr), 2582 .e_ehsize = sizeof(ElfW(Shdr)), 2583 .e_phentsize = sizeof(ElfW(Phdr)), 2584 .e_phnum = 1, 2585 .e_shentsize = sizeof(ElfW(Shdr)), 2586 .e_shnum = ARRAY_SIZE(img->shdr), 2587 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 2588 #ifdef ELF_HOST_FLAGS 2589 .e_flags = ELF_HOST_FLAGS, 2590 #endif 2591 #ifdef ELF_OSABI 2592 .e_ident[EI_OSABI] = ELF_OSABI, 2593 #endif 2594 }, 2595 .phdr = { 2596 .p_type = PT_LOAD, 2597 .p_flags = PF_X, 2598 }, 2599 .shdr = { 2600 [0] = { .sh_type = SHT_NULL }, 2601 /* Trick: The contents of code_gen_buffer are not present in 2602 this fake ELF file; that got allocated elsewhere. Therefore 2603 we mark .text as SHT_NOBITS (similar to .bss) so that readers 2604 will not look for contents. We can record any address. */ 2605 [1] = { /* .text */ 2606 .sh_type = SHT_NOBITS, 2607 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 2608 }, 2609 [2] = { /* .debug_info */ 2610 .sh_type = SHT_PROGBITS, 2611 .sh_offset = offsetof(struct ElfImage, di), 2612 .sh_size = sizeof(struct DebugInfo), 2613 }, 2614 [3] = { /* .debug_abbrev */ 2615 .sh_type = SHT_PROGBITS, 2616 .sh_offset = offsetof(struct ElfImage, da), 2617 .sh_size = sizeof(img->da), 2618 }, 2619 [4] = { /* .debug_frame */ 2620 .sh_type = SHT_PROGBITS, 2621 .sh_offset = sizeof(struct ElfImage), 2622 }, 2623 [5] = { /* .symtab */ 2624 .sh_type = SHT_SYMTAB, 2625 .sh_offset = offsetof(struct ElfImage, sym), 2626 .sh_size = sizeof(img->sym), 2627 .sh_info = 1, 2628 .sh_link = ARRAY_SIZE(img->shdr) - 1, 2629 .sh_entsize = sizeof(ElfW(Sym)), 2630 }, 2631 [6] = { /* .strtab */ 2632 .sh_type = SHT_STRTAB, 2633 .sh_offset = offsetof(struct ElfImage, str), 2634 .sh_size = sizeof(img->str), 2635 } 2636 }, 2637 .sym = { 2638 [1] = { /* code_gen_buffer */ 2639 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 2640 .st_shndx = 1, 2641 } 2642 }, 2643 .di = { 2644 .len = sizeof(struct DebugInfo) - 4, 2645 .version = 2, 2646 .ptr_size = sizeof(void *), 2647 .cu_die = 1, 2648 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 2649 .fn_die = 2, 2650 .fn_name = "code_gen_buffer" 2651 }, 2652 .da = { 2653 1, /* abbrev number (the cu) */ 2654 0x11, 1, /* DW_TAG_compile_unit, has children */ 2655 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 2656 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2657 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2658 0, 0, /* end of abbrev */ 2659 2, /* abbrev number (the fn) */ 2660 0x2e, 0, /* DW_TAG_subprogram, no children */ 2661 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 2662 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2663 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2664 0, 0, /* end of abbrev */ 2665 0 /* no more abbrev */ 2666 }, 2667 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 2668 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 2669 }; 2670 2671 /* We only need a single jit entry; statically allocate it. */ 2672 static struct jit_code_entry one_entry; 2673 2674 uintptr_t buf = (uintptr_t)buf_ptr; 2675 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 2676 DebugFrameHeader *dfh; 2677 2678 img = g_malloc(img_size); 2679 *img = img_template; 2680 2681 img->phdr.p_vaddr = buf; 2682 img->phdr.p_paddr = buf; 2683 img->phdr.p_memsz = buf_size; 2684 2685 img->shdr[1].sh_name = find_string(img->str, ".text"); 2686 img->shdr[1].sh_addr = buf; 2687 img->shdr[1].sh_size = buf_size; 2688 2689 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 2690 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 2691 2692 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 2693 img->shdr[4].sh_size = debug_frame_size; 2694 2695 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 2696 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 2697 2698 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 2699 img->sym[1].st_value = buf; 2700 img->sym[1].st_size = buf_size; 2701 2702 img->di.cu_low_pc = buf; 2703 img->di.cu_high_pc = buf + buf_size; 2704 img->di.fn_low_pc = buf; 2705 img->di.fn_high_pc = buf + buf_size; 2706 2707 dfh = (DebugFrameHeader *)(img + 1); 2708 memcpy(dfh, debug_frame, debug_frame_size); 2709 dfh->fde.func_start = buf; 2710 dfh->fde.func_len = buf_size; 2711 2712 #ifdef DEBUG_JIT 2713 /* Enable this block to be able to debug the ELF image file creation. 2714 One can use readelf, objdump, or other inspection utilities. */ 2715 { 2716 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 2717 if (f) { 2718 if (fwrite(img, img_size, 1, f) != img_size) { 2719 /* Avoid stupid unused return value warning for fwrite. */ 2720 } 2721 fclose(f); 2722 } 2723 } 2724 #endif 2725 2726 one_entry.symfile_addr = img; 2727 one_entry.symfile_size = img_size; 2728 2729 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 2730 __jit_debug_descriptor.relevant_entry = &one_entry; 2731 __jit_debug_descriptor.first_entry = &one_entry; 2732 __jit_debug_register_code(); 2733 } 2734 #else 2735 /* No support for the feature. Provide the entry point expected by exec.c, 2736 and implement the internal function we declared earlier. */ 2737 2738 static void tcg_register_jit_int(void *buf, size_t size, 2739 const void *debug_frame, 2740 size_t debug_frame_size) 2741 { 2742 } 2743 2744 void tcg_register_jit(void *buf, size_t buf_size) 2745 { 2746 } 2747 #endif /* ELF_HOST_MACHINE */ 2748