1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_LIVENESS_ANALYSIS 27 #define USE_TCG_OPTIMIZATIONS 28 29 #include "config.h" 30 31 /* Define to jump the ELF file used to communicate with GDB. */ 32 #undef DEBUG_JIT 33 34 #if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG) 35 /* define it to suppress various consistency checks (faster) */ 36 #define NDEBUG 37 #endif 38 39 #include "qemu-common.h" 40 #include "qemu/cache-utils.h" 41 #include "qemu/host-utils.h" 42 #include "qemu/timer.h" 43 44 /* Note: the long term plan is to reduce the dependencies on the QEMU 45 CPU definitions. Currently they are used for qemu_ld/st 46 instructions */ 47 #define NO_CPU_IO_DEFS 48 #include "cpu.h" 49 50 #include "tcg-op.h" 51 52 #if UINTPTR_MAX == UINT32_MAX 53 # define ELF_CLASS ELFCLASS32 54 #else 55 # define ELF_CLASS ELFCLASS64 56 #endif 57 #ifdef HOST_WORDS_BIGENDIAN 58 # define ELF_DATA ELFDATA2MSB 59 #else 60 # define ELF_DATA ELFDATA2LSB 61 #endif 62 63 #include "elf.h" 64 65 /* Forward declarations for functions declared in tcg-target.c and used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static void patch_reloc(tcg_insn_unit *code_ptr, int type, 69 intptr_t value, intptr_t addend); 70 71 /* The CIE and FDE header definitions will be common to all hosts. */ 72 typedef struct { 73 uint32_t len __attribute__((aligned((sizeof(void *))))); 74 uint32_t id; 75 uint8_t version; 76 char augmentation[1]; 77 uint8_t code_align; 78 uint8_t data_align; 79 uint8_t return_column; 80 } DebugFrameCIE; 81 82 typedef struct QEMU_PACKED { 83 uint32_t len __attribute__((aligned((sizeof(void *))))); 84 uint32_t cie_offset; 85 uintptr_t func_start; 86 uintptr_t func_len; 87 } DebugFrameFDEHeader; 88 89 static void tcg_register_jit_int(void *buf, size_t size, 90 void *debug_frame, size_t debug_frame_size) 91 __attribute__((unused)); 92 93 /* Forward declarations for functions declared and used in tcg-target.c. */ 94 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str); 95 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 96 intptr_t arg2); 97 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 98 static void tcg_out_movi(TCGContext *s, TCGType type, 99 TCGReg ret, tcg_target_long arg); 100 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 101 const int *const_args); 102 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 103 intptr_t arg2); 104 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); 105 static int tcg_target_const_match(tcg_target_long val, TCGType type, 106 const TCGArgConstraint *arg_ct); 107 static void tcg_out_tb_init(TCGContext *s); 108 static void tcg_out_tb_finalize(TCGContext *s); 109 110 111 TCGOpDef tcg_op_defs[] = { 112 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags }, 113 #include "tcg-opc.h" 114 #undef DEF 115 }; 116 const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs); 117 118 static TCGRegSet tcg_target_available_regs[2]; 119 static TCGRegSet tcg_target_call_clobber_regs; 120 121 #if TCG_TARGET_INSN_UNIT_SIZE == 1 122 static inline void tcg_out8(TCGContext *s, uint8_t v) 123 { 124 *s->code_ptr++ = v; 125 } 126 127 static inline void tcg_patch8(tcg_insn_unit *p, uint8_t v) 128 { 129 *p = v; 130 } 131 #endif 132 133 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 134 static inline void tcg_out16(TCGContext *s, uint16_t v) 135 { 136 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 137 *s->code_ptr++ = v; 138 } else { 139 tcg_insn_unit *p = s->code_ptr; 140 memcpy(p, &v, sizeof(v)); 141 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 142 } 143 } 144 145 static inline void tcg_patch16(tcg_insn_unit *p, uint16_t v) 146 { 147 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 148 *p = v; 149 } else { 150 memcpy(p, &v, sizeof(v)); 151 } 152 } 153 #endif 154 155 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 156 static inline void tcg_out32(TCGContext *s, uint32_t v) 157 { 158 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 159 *s->code_ptr++ = v; 160 } else { 161 tcg_insn_unit *p = s->code_ptr; 162 memcpy(p, &v, sizeof(v)); 163 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 164 } 165 } 166 167 static inline void tcg_patch32(tcg_insn_unit *p, uint32_t v) 168 { 169 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 170 *p = v; 171 } else { 172 memcpy(p, &v, sizeof(v)); 173 } 174 } 175 #endif 176 177 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 178 static inline void tcg_out64(TCGContext *s, uint64_t v) 179 { 180 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 181 *s->code_ptr++ = v; 182 } else { 183 tcg_insn_unit *p = s->code_ptr; 184 memcpy(p, &v, sizeof(v)); 185 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 186 } 187 } 188 189 static inline void tcg_patch64(tcg_insn_unit *p, uint64_t v) 190 { 191 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 192 *p = v; 193 } else { 194 memcpy(p, &v, sizeof(v)); 195 } 196 } 197 #endif 198 199 /* label relocation processing */ 200 201 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 202 int label_index, intptr_t addend) 203 { 204 TCGLabel *l; 205 TCGRelocation *r; 206 207 l = &s->labels[label_index]; 208 if (l->has_value) { 209 /* FIXME: This may break relocations on RISC targets that 210 modify instruction fields in place. The caller may not have 211 written the initial value. */ 212 patch_reloc(code_ptr, type, l->u.value, addend); 213 } else { 214 /* add a new relocation entry */ 215 r = tcg_malloc(sizeof(TCGRelocation)); 216 r->type = type; 217 r->ptr = code_ptr; 218 r->addend = addend; 219 r->next = l->u.first_reloc; 220 l->u.first_reloc = r; 221 } 222 } 223 224 static void tcg_out_label(TCGContext *s, int label_index, tcg_insn_unit *ptr) 225 { 226 TCGLabel *l = &s->labels[label_index]; 227 intptr_t value = (intptr_t)ptr; 228 TCGRelocation *r; 229 230 assert(!l->has_value); 231 232 for (r = l->u.first_reloc; r != NULL; r = r->next) { 233 patch_reloc(r->ptr, r->type, value, r->addend); 234 } 235 236 l->has_value = 1; 237 l->u.value_ptr = ptr; 238 } 239 240 int gen_new_label(void) 241 { 242 TCGContext *s = &tcg_ctx; 243 int idx; 244 TCGLabel *l; 245 246 if (s->nb_labels >= TCG_MAX_LABELS) 247 tcg_abort(); 248 idx = s->nb_labels++; 249 l = &s->labels[idx]; 250 l->has_value = 0; 251 l->u.first_reloc = NULL; 252 return idx; 253 } 254 255 #include "tcg-target.c" 256 257 /* pool based memory allocation */ 258 void *tcg_malloc_internal(TCGContext *s, int size) 259 { 260 TCGPool *p; 261 int pool_size; 262 263 if (size > TCG_POOL_CHUNK_SIZE) { 264 /* big malloc: insert a new pool (XXX: could optimize) */ 265 p = g_malloc(sizeof(TCGPool) + size); 266 p->size = size; 267 p->next = s->pool_first_large; 268 s->pool_first_large = p; 269 return p->data; 270 } else { 271 p = s->pool_current; 272 if (!p) { 273 p = s->pool_first; 274 if (!p) 275 goto new_pool; 276 } else { 277 if (!p->next) { 278 new_pool: 279 pool_size = TCG_POOL_CHUNK_SIZE; 280 p = g_malloc(sizeof(TCGPool) + pool_size); 281 p->size = pool_size; 282 p->next = NULL; 283 if (s->pool_current) 284 s->pool_current->next = p; 285 else 286 s->pool_first = p; 287 } else { 288 p = p->next; 289 } 290 } 291 } 292 s->pool_current = p; 293 s->pool_cur = p->data + size; 294 s->pool_end = p->data + p->size; 295 return p->data; 296 } 297 298 void tcg_pool_reset(TCGContext *s) 299 { 300 TCGPool *p, *t; 301 for (p = s->pool_first_large; p; p = t) { 302 t = p->next; 303 g_free(p); 304 } 305 s->pool_first_large = NULL; 306 s->pool_cur = s->pool_end = NULL; 307 s->pool_current = NULL; 308 } 309 310 #include "helper.h" 311 312 typedef struct TCGHelperInfo { 313 void *func; 314 const char *name; 315 } TCGHelperInfo; 316 317 static const TCGHelperInfo all_helpers[] = { 318 #define GEN_HELPER 2 319 #include "helper.h" 320 321 /* Include tcg-runtime.c functions. */ 322 { tcg_helper_div_i32, "div_i32" }, 323 { tcg_helper_rem_i32, "rem_i32" }, 324 { tcg_helper_divu_i32, "divu_i32" }, 325 { tcg_helper_remu_i32, "remu_i32" }, 326 327 { tcg_helper_shl_i64, "shl_i64" }, 328 { tcg_helper_shr_i64, "shr_i64" }, 329 { tcg_helper_sar_i64, "sar_i64" }, 330 { tcg_helper_div_i64, "div_i64" }, 331 { tcg_helper_rem_i64, "rem_i64" }, 332 { tcg_helper_divu_i64, "divu_i64" }, 333 { tcg_helper_remu_i64, "remu_i64" }, 334 { tcg_helper_mulsh_i64, "mulsh_i64" }, 335 { tcg_helper_muluh_i64, "muluh_i64" }, 336 }; 337 338 void tcg_context_init(TCGContext *s) 339 { 340 int op, total_args, n, i; 341 TCGOpDef *def; 342 TCGArgConstraint *args_ct; 343 int *sorted_args; 344 GHashTable *helper_table; 345 346 memset(s, 0, sizeof(*s)); 347 s->nb_globals = 0; 348 349 /* Count total number of arguments and allocate the corresponding 350 space */ 351 total_args = 0; 352 for(op = 0; op < NB_OPS; op++) { 353 def = &tcg_op_defs[op]; 354 n = def->nb_iargs + def->nb_oargs; 355 total_args += n; 356 } 357 358 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 359 sorted_args = g_malloc(sizeof(int) * total_args); 360 361 for(op = 0; op < NB_OPS; op++) { 362 def = &tcg_op_defs[op]; 363 def->args_ct = args_ct; 364 def->sorted_args = sorted_args; 365 n = def->nb_iargs + def->nb_oargs; 366 sorted_args += n; 367 args_ct += n; 368 } 369 370 /* Register helpers. */ 371 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 372 s->helpers = helper_table = g_hash_table_new(NULL, NULL); 373 374 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 375 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 376 (gpointer)all_helpers[i].name); 377 } 378 379 tcg_target_init(s); 380 } 381 382 void tcg_prologue_init(TCGContext *s) 383 { 384 /* init global prologue and epilogue */ 385 s->code_buf = s->code_gen_prologue; 386 s->code_ptr = s->code_buf; 387 tcg_target_qemu_prologue(s); 388 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 389 390 #ifdef DEBUG_DISAS 391 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 392 size_t size = tcg_current_code_size(s); 393 qemu_log("PROLOGUE: [size=%zu]\n", size); 394 log_disas(s->code_buf, size); 395 qemu_log("\n"); 396 qemu_log_flush(); 397 } 398 #endif 399 } 400 401 void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size) 402 { 403 s->frame_start = start; 404 s->frame_end = start + size; 405 s->frame_reg = reg; 406 } 407 408 void tcg_func_start(TCGContext *s) 409 { 410 tcg_pool_reset(s); 411 s->nb_temps = s->nb_globals; 412 413 /* No temps have been previously allocated for size or locality. */ 414 memset(s->free_temps, 0, sizeof(s->free_temps)); 415 416 s->labels = tcg_malloc(sizeof(TCGLabel) * TCG_MAX_LABELS); 417 s->nb_labels = 0; 418 s->current_frame_offset = s->frame_start; 419 420 #ifdef CONFIG_DEBUG_TCG 421 s->goto_tb_issue_mask = 0; 422 #endif 423 424 s->gen_opc_ptr = s->gen_opc_buf; 425 s->gen_opparam_ptr = s->gen_opparam_buf; 426 427 s->be = tcg_malloc(sizeof(TCGBackendData)); 428 } 429 430 static inline void tcg_temp_alloc(TCGContext *s, int n) 431 { 432 if (n > TCG_MAX_TEMPS) 433 tcg_abort(); 434 } 435 436 static inline int tcg_global_reg_new_internal(TCGType type, int reg, 437 const char *name) 438 { 439 TCGContext *s = &tcg_ctx; 440 TCGTemp *ts; 441 int idx; 442 443 #if TCG_TARGET_REG_BITS == 32 444 if (type != TCG_TYPE_I32) 445 tcg_abort(); 446 #endif 447 if (tcg_regset_test_reg(s->reserved_regs, reg)) 448 tcg_abort(); 449 idx = s->nb_globals; 450 tcg_temp_alloc(s, s->nb_globals + 1); 451 ts = &s->temps[s->nb_globals]; 452 ts->base_type = type; 453 ts->type = type; 454 ts->fixed_reg = 1; 455 ts->reg = reg; 456 ts->name = name; 457 s->nb_globals++; 458 tcg_regset_set_reg(s->reserved_regs, reg); 459 return idx; 460 } 461 462 TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name) 463 { 464 int idx; 465 466 idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name); 467 return MAKE_TCGV_I32(idx); 468 } 469 470 TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name) 471 { 472 int idx; 473 474 idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name); 475 return MAKE_TCGV_I64(idx); 476 } 477 478 static inline int tcg_global_mem_new_internal(TCGType type, int reg, 479 intptr_t offset, 480 const char *name) 481 { 482 TCGContext *s = &tcg_ctx; 483 TCGTemp *ts; 484 int idx; 485 486 idx = s->nb_globals; 487 #if TCG_TARGET_REG_BITS == 32 488 if (type == TCG_TYPE_I64) { 489 char buf[64]; 490 tcg_temp_alloc(s, s->nb_globals + 2); 491 ts = &s->temps[s->nb_globals]; 492 ts->base_type = type; 493 ts->type = TCG_TYPE_I32; 494 ts->fixed_reg = 0; 495 ts->mem_allocated = 1; 496 ts->mem_reg = reg; 497 #ifdef HOST_WORDS_BIGENDIAN 498 ts->mem_offset = offset + 4; 499 #else 500 ts->mem_offset = offset; 501 #endif 502 pstrcpy(buf, sizeof(buf), name); 503 pstrcat(buf, sizeof(buf), "_0"); 504 ts->name = strdup(buf); 505 ts++; 506 507 ts->base_type = type; 508 ts->type = TCG_TYPE_I32; 509 ts->fixed_reg = 0; 510 ts->mem_allocated = 1; 511 ts->mem_reg = reg; 512 #ifdef HOST_WORDS_BIGENDIAN 513 ts->mem_offset = offset; 514 #else 515 ts->mem_offset = offset + 4; 516 #endif 517 pstrcpy(buf, sizeof(buf), name); 518 pstrcat(buf, sizeof(buf), "_1"); 519 ts->name = strdup(buf); 520 521 s->nb_globals += 2; 522 } else 523 #endif 524 { 525 tcg_temp_alloc(s, s->nb_globals + 1); 526 ts = &s->temps[s->nb_globals]; 527 ts->base_type = type; 528 ts->type = type; 529 ts->fixed_reg = 0; 530 ts->mem_allocated = 1; 531 ts->mem_reg = reg; 532 ts->mem_offset = offset; 533 ts->name = name; 534 s->nb_globals++; 535 } 536 return idx; 537 } 538 539 TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name) 540 { 541 int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name); 542 return MAKE_TCGV_I32(idx); 543 } 544 545 TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name) 546 { 547 int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name); 548 return MAKE_TCGV_I64(idx); 549 } 550 551 static inline int tcg_temp_new_internal(TCGType type, int temp_local) 552 { 553 TCGContext *s = &tcg_ctx; 554 TCGTemp *ts; 555 int idx, k; 556 557 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 558 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 559 if (idx < TCG_MAX_TEMPS) { 560 /* There is already an available temp with the right type. */ 561 clear_bit(idx, s->free_temps[k].l); 562 563 ts = &s->temps[idx]; 564 ts->temp_allocated = 1; 565 assert(ts->base_type == type); 566 assert(ts->temp_local == temp_local); 567 } else { 568 idx = s->nb_temps; 569 #if TCG_TARGET_REG_BITS == 32 570 if (type == TCG_TYPE_I64) { 571 tcg_temp_alloc(s, s->nb_temps + 2); 572 ts = &s->temps[s->nb_temps]; 573 ts->base_type = type; 574 ts->type = TCG_TYPE_I32; 575 ts->temp_allocated = 1; 576 ts->temp_local = temp_local; 577 ts->name = NULL; 578 ts++; 579 ts->base_type = type; 580 ts->type = TCG_TYPE_I32; 581 ts->temp_allocated = 1; 582 ts->temp_local = temp_local; 583 ts->name = NULL; 584 s->nb_temps += 2; 585 } else 586 #endif 587 { 588 tcg_temp_alloc(s, s->nb_temps + 1); 589 ts = &s->temps[s->nb_temps]; 590 ts->base_type = type; 591 ts->type = type; 592 ts->temp_allocated = 1; 593 ts->temp_local = temp_local; 594 ts->name = NULL; 595 s->nb_temps++; 596 } 597 } 598 599 #if defined(CONFIG_DEBUG_TCG) 600 s->temps_in_use++; 601 #endif 602 return idx; 603 } 604 605 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 606 { 607 int idx; 608 609 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 610 return MAKE_TCGV_I32(idx); 611 } 612 613 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 614 { 615 int idx; 616 617 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 618 return MAKE_TCGV_I64(idx); 619 } 620 621 static void tcg_temp_free_internal(int idx) 622 { 623 TCGContext *s = &tcg_ctx; 624 TCGTemp *ts; 625 int k; 626 627 #if defined(CONFIG_DEBUG_TCG) 628 s->temps_in_use--; 629 if (s->temps_in_use < 0) { 630 fprintf(stderr, "More temporaries freed than allocated!\n"); 631 } 632 #endif 633 634 assert(idx >= s->nb_globals && idx < s->nb_temps); 635 ts = &s->temps[idx]; 636 assert(ts->temp_allocated != 0); 637 ts->temp_allocated = 0; 638 639 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 640 set_bit(idx, s->free_temps[k].l); 641 } 642 643 void tcg_temp_free_i32(TCGv_i32 arg) 644 { 645 tcg_temp_free_internal(GET_TCGV_I32(arg)); 646 } 647 648 void tcg_temp_free_i64(TCGv_i64 arg) 649 { 650 tcg_temp_free_internal(GET_TCGV_I64(arg)); 651 } 652 653 TCGv_i32 tcg_const_i32(int32_t val) 654 { 655 TCGv_i32 t0; 656 t0 = tcg_temp_new_i32(); 657 tcg_gen_movi_i32(t0, val); 658 return t0; 659 } 660 661 TCGv_i64 tcg_const_i64(int64_t val) 662 { 663 TCGv_i64 t0; 664 t0 = tcg_temp_new_i64(); 665 tcg_gen_movi_i64(t0, val); 666 return t0; 667 } 668 669 TCGv_i32 tcg_const_local_i32(int32_t val) 670 { 671 TCGv_i32 t0; 672 t0 = tcg_temp_local_new_i32(); 673 tcg_gen_movi_i32(t0, val); 674 return t0; 675 } 676 677 TCGv_i64 tcg_const_local_i64(int64_t val) 678 { 679 TCGv_i64 t0; 680 t0 = tcg_temp_local_new_i64(); 681 tcg_gen_movi_i64(t0, val); 682 return t0; 683 } 684 685 #if defined(CONFIG_DEBUG_TCG) 686 void tcg_clear_temp_count(void) 687 { 688 TCGContext *s = &tcg_ctx; 689 s->temps_in_use = 0; 690 } 691 692 int tcg_check_temp_count(void) 693 { 694 TCGContext *s = &tcg_ctx; 695 if (s->temps_in_use) { 696 /* Clear the count so that we don't give another 697 * warning immediately next time around. 698 */ 699 s->temps_in_use = 0; 700 return 1; 701 } 702 return 0; 703 } 704 #endif 705 706 /* Note: we convert the 64 bit args to 32 bit and do some alignment 707 and endian swap. Maybe it would be better to do the alignment 708 and endian swap in tcg_reg_alloc_call(). */ 709 void tcg_gen_callN(TCGContext *s, void *func, unsigned int flags, 710 int sizemask, TCGArg ret, int nargs, TCGArg *args) 711 { 712 int i; 713 int real_args; 714 int nb_rets; 715 TCGArg *nparam; 716 717 #if defined(__sparc__) && !defined(__arch64__) \ 718 && !defined(CONFIG_TCG_INTERPRETER) 719 /* We have 64-bit values in one register, but need to pass as two 720 separate parameters. Split them. */ 721 int orig_sizemask = sizemask; 722 int orig_nargs = nargs; 723 TCGv_i64 retl, reth; 724 725 TCGV_UNUSED_I64(retl); 726 TCGV_UNUSED_I64(reth); 727 if (sizemask != 0) { 728 TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2); 729 for (i = real_args = 0; i < nargs; ++i) { 730 int is_64bit = sizemask & (1 << (i+1)*2); 731 if (is_64bit) { 732 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 733 TCGv_i32 h = tcg_temp_new_i32(); 734 TCGv_i32 l = tcg_temp_new_i32(); 735 tcg_gen_extr_i64_i32(l, h, orig); 736 split_args[real_args++] = GET_TCGV_I32(h); 737 split_args[real_args++] = GET_TCGV_I32(l); 738 } else { 739 split_args[real_args++] = args[i]; 740 } 741 } 742 nargs = real_args; 743 args = split_args; 744 sizemask = 0; 745 } 746 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 747 for (i = 0; i < nargs; ++i) { 748 int is_64bit = sizemask & (1 << (i+1)*2); 749 int is_signed = sizemask & (2 << (i+1)*2); 750 if (!is_64bit) { 751 TCGv_i64 temp = tcg_temp_new_i64(); 752 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 753 if (is_signed) { 754 tcg_gen_ext32s_i64(temp, orig); 755 } else { 756 tcg_gen_ext32u_i64(temp, orig); 757 } 758 args[i] = GET_TCGV_I64(temp); 759 } 760 } 761 #endif /* TCG_TARGET_EXTEND_ARGS */ 762 763 *s->gen_opc_ptr++ = INDEX_op_call; 764 nparam = s->gen_opparam_ptr++; 765 if (ret != TCG_CALL_DUMMY_ARG) { 766 #if defined(__sparc__) && !defined(__arch64__) \ 767 && !defined(CONFIG_TCG_INTERPRETER) 768 if (orig_sizemask & 1) { 769 /* The 32-bit ABI is going to return the 64-bit value in 770 the %o0/%o1 register pair. Prepare for this by using 771 two return temporaries, and reassemble below. */ 772 retl = tcg_temp_new_i64(); 773 reth = tcg_temp_new_i64(); 774 *s->gen_opparam_ptr++ = GET_TCGV_I64(reth); 775 *s->gen_opparam_ptr++ = GET_TCGV_I64(retl); 776 nb_rets = 2; 777 } else { 778 *s->gen_opparam_ptr++ = ret; 779 nb_rets = 1; 780 } 781 #else 782 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { 783 #ifdef HOST_WORDS_BIGENDIAN 784 *s->gen_opparam_ptr++ = ret + 1; 785 *s->gen_opparam_ptr++ = ret; 786 #else 787 *s->gen_opparam_ptr++ = ret; 788 *s->gen_opparam_ptr++ = ret + 1; 789 #endif 790 nb_rets = 2; 791 } else { 792 *s->gen_opparam_ptr++ = ret; 793 nb_rets = 1; 794 } 795 #endif 796 } else { 797 nb_rets = 0; 798 } 799 real_args = 0; 800 for (i = 0; i < nargs; i++) { 801 #if TCG_TARGET_REG_BITS < 64 802 int is_64bit = sizemask & (1 << (i+1)*2); 803 if (is_64bit) { 804 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 805 /* some targets want aligned 64 bit args */ 806 if (real_args & 1) { 807 *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG; 808 real_args++; 809 } 810 #endif 811 /* If stack grows up, then we will be placing successive 812 arguments at lower addresses, which means we need to 813 reverse the order compared to how we would normally 814 treat either big or little-endian. For those arguments 815 that will wind up in registers, this still works for 816 HPPA (the only current STACK_GROWSUP target) since the 817 argument registers are *also* allocated in decreasing 818 order. If another such target is added, this logic may 819 have to get more complicated to differentiate between 820 stack arguments and register arguments. */ 821 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 822 *s->gen_opparam_ptr++ = args[i] + 1; 823 *s->gen_opparam_ptr++ = args[i]; 824 #else 825 *s->gen_opparam_ptr++ = args[i]; 826 *s->gen_opparam_ptr++ = args[i] + 1; 827 #endif 828 real_args += 2; 829 continue; 830 } 831 #endif /* TCG_TARGET_REG_BITS < 64 */ 832 833 *s->gen_opparam_ptr++ = args[i]; 834 real_args++; 835 } 836 *s->gen_opparam_ptr++ = (uintptr_t)func; 837 *s->gen_opparam_ptr++ = flags; 838 839 *nparam = (nb_rets << 16) | real_args; 840 841 /* total parameters, needed to go backward in the instruction stream */ 842 *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3; 843 844 #if defined(__sparc__) && !defined(__arch64__) \ 845 && !defined(CONFIG_TCG_INTERPRETER) 846 /* Free all of the parts we allocated above. */ 847 for (i = real_args = 0; i < orig_nargs; ++i) { 848 int is_64bit = orig_sizemask & (1 << (i+1)*2); 849 if (is_64bit) { 850 TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]); 851 TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]); 852 tcg_temp_free_i32(h); 853 tcg_temp_free_i32(l); 854 } else { 855 real_args++; 856 } 857 } 858 if (orig_sizemask & 1) { 859 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 860 Note that describing these as TCGv_i64 eliminates an unnecessary 861 zero-extension that tcg_gen_concat_i32_i64 would create. */ 862 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth); 863 tcg_temp_free_i64(retl); 864 tcg_temp_free_i64(reth); 865 } 866 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 867 for (i = 0; i < nargs; ++i) { 868 int is_64bit = sizemask & (1 << (i+1)*2); 869 if (!is_64bit) { 870 TCGv_i64 temp = MAKE_TCGV_I64(args[i]); 871 tcg_temp_free_i64(temp); 872 } 873 } 874 #endif /* TCG_TARGET_EXTEND_ARGS */ 875 } 876 877 #if TCG_TARGET_REG_BITS == 32 878 void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 879 int c, int right, int arith) 880 { 881 if (c == 0) { 882 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 883 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 884 } else if (c >= 32) { 885 c -= 32; 886 if (right) { 887 if (arith) { 888 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 889 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 890 } else { 891 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 892 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 893 } 894 } else { 895 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 896 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 897 } 898 } else { 899 TCGv_i32 t0, t1; 900 901 t0 = tcg_temp_new_i32(); 902 t1 = tcg_temp_new_i32(); 903 if (right) { 904 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 905 if (arith) 906 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 907 else 908 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 909 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 910 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 911 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 912 } else { 913 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 914 /* Note: ret can be the same as arg1, so we use t1 */ 915 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 916 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 917 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 918 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 919 } 920 tcg_temp_free_i32(t0); 921 tcg_temp_free_i32(t1); 922 } 923 } 924 #endif 925 926 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 927 { 928 switch (op & MO_SIZE) { 929 case MO_8: 930 op &= ~MO_BSWAP; 931 break; 932 case MO_16: 933 break; 934 case MO_32: 935 if (!is64) { 936 op &= ~MO_SIGN; 937 } 938 break; 939 case MO_64: 940 if (!is64) { 941 tcg_abort(); 942 } 943 break; 944 } 945 if (st) { 946 op &= ~MO_SIGN; 947 } 948 return op; 949 } 950 951 static const TCGOpcode old_ld_opc[8] = { 952 [MO_UB] = INDEX_op_qemu_ld8u, 953 [MO_SB] = INDEX_op_qemu_ld8s, 954 [MO_UW] = INDEX_op_qemu_ld16u, 955 [MO_SW] = INDEX_op_qemu_ld16s, 956 #if TCG_TARGET_REG_BITS == 32 957 [MO_UL] = INDEX_op_qemu_ld32, 958 [MO_SL] = INDEX_op_qemu_ld32, 959 #else 960 [MO_UL] = INDEX_op_qemu_ld32u, 961 [MO_SL] = INDEX_op_qemu_ld32s, 962 #endif 963 [MO_Q] = INDEX_op_qemu_ld64, 964 }; 965 966 static const TCGOpcode old_st_opc[4] = { 967 [MO_UB] = INDEX_op_qemu_st8, 968 [MO_UW] = INDEX_op_qemu_st16, 969 [MO_UL] = INDEX_op_qemu_st32, 970 [MO_Q] = INDEX_op_qemu_st64, 971 }; 972 973 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 974 { 975 memop = tcg_canonicalize_memop(memop, 0, 0); 976 977 if (TCG_TARGET_HAS_new_ldst) { 978 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32; 979 tcg_add_param_i32(val); 980 tcg_add_param_tl(addr); 981 *tcg_ctx.gen_opparam_ptr++ = memop; 982 *tcg_ctx.gen_opparam_ptr++ = idx; 983 return; 984 } 985 986 /* The old opcodes only support target-endian memory operations. */ 987 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 988 assert(old_ld_opc[memop & MO_SSIZE] != 0); 989 990 if (TCG_TARGET_REG_BITS == 32) { 991 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 992 tcg_add_param_i32(val); 993 tcg_add_param_tl(addr); 994 *tcg_ctx.gen_opparam_ptr++ = idx; 995 } else { 996 TCGv_i64 val64 = tcg_temp_new_i64(); 997 998 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 999 tcg_add_param_i64(val64); 1000 tcg_add_param_tl(addr); 1001 *tcg_ctx.gen_opparam_ptr++ = idx; 1002 1003 tcg_gen_trunc_i64_i32(val, val64); 1004 tcg_temp_free_i64(val64); 1005 } 1006 } 1007 1008 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1009 { 1010 memop = tcg_canonicalize_memop(memop, 0, 1); 1011 1012 if (TCG_TARGET_HAS_new_ldst) { 1013 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32; 1014 tcg_add_param_i32(val); 1015 tcg_add_param_tl(addr); 1016 *tcg_ctx.gen_opparam_ptr++ = memop; 1017 *tcg_ctx.gen_opparam_ptr++ = idx; 1018 return; 1019 } 1020 1021 /* The old opcodes only support target-endian memory operations. */ 1022 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 1023 assert(old_st_opc[memop & MO_SIZE] != 0); 1024 1025 if (TCG_TARGET_REG_BITS == 32) { 1026 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 1027 tcg_add_param_i32(val); 1028 tcg_add_param_tl(addr); 1029 *tcg_ctx.gen_opparam_ptr++ = idx; 1030 } else { 1031 TCGv_i64 val64 = tcg_temp_new_i64(); 1032 1033 tcg_gen_extu_i32_i64(val64, val); 1034 1035 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 1036 tcg_add_param_i64(val64); 1037 tcg_add_param_tl(addr); 1038 *tcg_ctx.gen_opparam_ptr++ = idx; 1039 1040 tcg_temp_free_i64(val64); 1041 } 1042 } 1043 1044 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1045 { 1046 memop = tcg_canonicalize_memop(memop, 1, 0); 1047 1048 #if TCG_TARGET_REG_BITS == 32 1049 if ((memop & MO_SIZE) < MO_64) { 1050 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 1051 if (memop & MO_SIGN) { 1052 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 1053 } else { 1054 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 1055 } 1056 return; 1057 } 1058 #endif 1059 1060 if (TCG_TARGET_HAS_new_ldst) { 1061 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64; 1062 tcg_add_param_i64(val); 1063 tcg_add_param_tl(addr); 1064 *tcg_ctx.gen_opparam_ptr++ = memop; 1065 *tcg_ctx.gen_opparam_ptr++ = idx; 1066 return; 1067 } 1068 1069 /* The old opcodes only support target-endian memory operations. */ 1070 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 1071 assert(old_ld_opc[memop & MO_SSIZE] != 0); 1072 1073 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 1074 tcg_add_param_i64(val); 1075 tcg_add_param_tl(addr); 1076 *tcg_ctx.gen_opparam_ptr++ = idx; 1077 } 1078 1079 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1080 { 1081 memop = tcg_canonicalize_memop(memop, 1, 1); 1082 1083 #if TCG_TARGET_REG_BITS == 32 1084 if ((memop & MO_SIZE) < MO_64) { 1085 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 1086 return; 1087 } 1088 #endif 1089 1090 if (TCG_TARGET_HAS_new_ldst) { 1091 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64; 1092 tcg_add_param_i64(val); 1093 tcg_add_param_tl(addr); 1094 *tcg_ctx.gen_opparam_ptr++ = memop; 1095 *tcg_ctx.gen_opparam_ptr++ = idx; 1096 return; 1097 } 1098 1099 /* The old opcodes only support target-endian memory operations. */ 1100 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 1101 assert(old_st_opc[memop & MO_SIZE] != 0); 1102 1103 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 1104 tcg_add_param_i64(val); 1105 tcg_add_param_tl(addr); 1106 *tcg_ctx.gen_opparam_ptr++ = idx; 1107 } 1108 1109 static void tcg_reg_alloc_start(TCGContext *s) 1110 { 1111 int i; 1112 TCGTemp *ts; 1113 for(i = 0; i < s->nb_globals; i++) { 1114 ts = &s->temps[i]; 1115 if (ts->fixed_reg) { 1116 ts->val_type = TEMP_VAL_REG; 1117 } else { 1118 ts->val_type = TEMP_VAL_MEM; 1119 } 1120 } 1121 for(i = s->nb_globals; i < s->nb_temps; i++) { 1122 ts = &s->temps[i]; 1123 if (ts->temp_local) { 1124 ts->val_type = TEMP_VAL_MEM; 1125 } else { 1126 ts->val_type = TEMP_VAL_DEAD; 1127 } 1128 ts->mem_allocated = 0; 1129 ts->fixed_reg = 0; 1130 } 1131 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1132 s->reg_to_temp[i] = -1; 1133 } 1134 } 1135 1136 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size, 1137 int idx) 1138 { 1139 TCGTemp *ts; 1140 1141 assert(idx >= 0 && idx < s->nb_temps); 1142 ts = &s->temps[idx]; 1143 if (idx < s->nb_globals) { 1144 pstrcpy(buf, buf_size, ts->name); 1145 } else { 1146 if (ts->temp_local) 1147 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1148 else 1149 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1150 } 1151 return buf; 1152 } 1153 1154 char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg) 1155 { 1156 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg)); 1157 } 1158 1159 char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg) 1160 { 1161 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg)); 1162 } 1163 1164 /* Find helper name. */ 1165 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 1166 { 1167 const char *ret = NULL; 1168 if (s->helpers) { 1169 ret = g_hash_table_lookup(s->helpers, (gpointer)val); 1170 } 1171 return ret; 1172 } 1173 1174 static const char * const cond_name[] = 1175 { 1176 [TCG_COND_NEVER] = "never", 1177 [TCG_COND_ALWAYS] = "always", 1178 [TCG_COND_EQ] = "eq", 1179 [TCG_COND_NE] = "ne", 1180 [TCG_COND_LT] = "lt", 1181 [TCG_COND_GE] = "ge", 1182 [TCG_COND_LE] = "le", 1183 [TCG_COND_GT] = "gt", 1184 [TCG_COND_LTU] = "ltu", 1185 [TCG_COND_GEU] = "geu", 1186 [TCG_COND_LEU] = "leu", 1187 [TCG_COND_GTU] = "gtu" 1188 }; 1189 1190 static const char * const ldst_name[] = 1191 { 1192 [MO_UB] = "ub", 1193 [MO_SB] = "sb", 1194 [MO_LEUW] = "leuw", 1195 [MO_LESW] = "lesw", 1196 [MO_LEUL] = "leul", 1197 [MO_LESL] = "lesl", 1198 [MO_LEQ] = "leq", 1199 [MO_BEUW] = "beuw", 1200 [MO_BESW] = "besw", 1201 [MO_BEUL] = "beul", 1202 [MO_BESL] = "besl", 1203 [MO_BEQ] = "beq", 1204 }; 1205 1206 void tcg_dump_ops(TCGContext *s) 1207 { 1208 const uint16_t *opc_ptr; 1209 const TCGArg *args; 1210 TCGArg arg; 1211 TCGOpcode c; 1212 int i, k, nb_oargs, nb_iargs, nb_cargs, first_insn; 1213 const TCGOpDef *def; 1214 char buf[128]; 1215 1216 first_insn = 1; 1217 opc_ptr = s->gen_opc_buf; 1218 args = s->gen_opparam_buf; 1219 while (opc_ptr < s->gen_opc_ptr) { 1220 c = *opc_ptr++; 1221 def = &tcg_op_defs[c]; 1222 if (c == INDEX_op_debug_insn_start) { 1223 uint64_t pc; 1224 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1225 pc = ((uint64_t)args[1] << 32) | args[0]; 1226 #else 1227 pc = args[0]; 1228 #endif 1229 if (!first_insn) { 1230 qemu_log("\n"); 1231 } 1232 qemu_log(" ---- 0x%" PRIx64, pc); 1233 first_insn = 0; 1234 nb_oargs = def->nb_oargs; 1235 nb_iargs = def->nb_iargs; 1236 nb_cargs = def->nb_cargs; 1237 } else if (c == INDEX_op_call) { 1238 TCGArg arg; 1239 1240 /* variable number of arguments */ 1241 arg = *args++; 1242 nb_oargs = arg >> 16; 1243 nb_iargs = arg & 0xffff; 1244 nb_cargs = def->nb_cargs; 1245 1246 /* function name, flags, out args */ 1247 qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, 1248 tcg_find_helper(s, args[nb_oargs + nb_iargs]), 1249 args[nb_oargs + nb_iargs + 1], nb_oargs); 1250 for (i = 0; i < nb_oargs; i++) { 1251 qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1252 args[i])); 1253 } 1254 for (i = 0; i < nb_iargs; i++) { 1255 TCGArg arg = args[nb_oargs + i]; 1256 const char *t = "<dummy>"; 1257 if (arg != TCG_CALL_DUMMY_ARG) { 1258 t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg); 1259 } 1260 qemu_log(",%s", t); 1261 } 1262 } else { 1263 qemu_log(" %s ", def->name); 1264 if (c == INDEX_op_nopn) { 1265 /* variable number of arguments */ 1266 nb_cargs = *args; 1267 nb_oargs = 0; 1268 nb_iargs = 0; 1269 } else { 1270 nb_oargs = def->nb_oargs; 1271 nb_iargs = def->nb_iargs; 1272 nb_cargs = def->nb_cargs; 1273 } 1274 1275 k = 0; 1276 for(i = 0; i < nb_oargs; i++) { 1277 if (k != 0) { 1278 qemu_log(","); 1279 } 1280 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1281 args[k++])); 1282 } 1283 for(i = 0; i < nb_iargs; i++) { 1284 if (k != 0) { 1285 qemu_log(","); 1286 } 1287 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1288 args[k++])); 1289 } 1290 switch (c) { 1291 case INDEX_op_brcond_i32: 1292 case INDEX_op_setcond_i32: 1293 case INDEX_op_movcond_i32: 1294 case INDEX_op_brcond2_i32: 1295 case INDEX_op_setcond2_i32: 1296 case INDEX_op_brcond_i64: 1297 case INDEX_op_setcond_i64: 1298 case INDEX_op_movcond_i64: 1299 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) { 1300 qemu_log(",%s", cond_name[args[k++]]); 1301 } else { 1302 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1303 } 1304 i = 1; 1305 break; 1306 case INDEX_op_qemu_ld_i32: 1307 case INDEX_op_qemu_st_i32: 1308 case INDEX_op_qemu_ld_i64: 1309 case INDEX_op_qemu_st_i64: 1310 if (args[k] < ARRAY_SIZE(ldst_name) && ldst_name[args[k]]) { 1311 qemu_log(",%s", ldst_name[args[k++]]); 1312 } else { 1313 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1314 } 1315 i = 1; 1316 break; 1317 default: 1318 i = 0; 1319 break; 1320 } 1321 for(; i < nb_cargs; i++) { 1322 if (k != 0) { 1323 qemu_log(","); 1324 } 1325 arg = args[k++]; 1326 qemu_log("$0x%" TCG_PRIlx, arg); 1327 } 1328 } 1329 qemu_log("\n"); 1330 args += nb_iargs + nb_oargs + nb_cargs; 1331 } 1332 } 1333 1334 /* we give more priority to constraints with less registers */ 1335 static int get_constraint_priority(const TCGOpDef *def, int k) 1336 { 1337 const TCGArgConstraint *arg_ct; 1338 1339 int i, n; 1340 arg_ct = &def->args_ct[k]; 1341 if (arg_ct->ct & TCG_CT_ALIAS) { 1342 /* an alias is equivalent to a single register */ 1343 n = 1; 1344 } else { 1345 if (!(arg_ct->ct & TCG_CT_REG)) 1346 return 0; 1347 n = 0; 1348 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1349 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1350 n++; 1351 } 1352 } 1353 return TCG_TARGET_NB_REGS - n + 1; 1354 } 1355 1356 /* sort from highest priority to lowest */ 1357 static void sort_constraints(TCGOpDef *def, int start, int n) 1358 { 1359 int i, j, p1, p2, tmp; 1360 1361 for(i = 0; i < n; i++) 1362 def->sorted_args[start + i] = start + i; 1363 if (n <= 1) 1364 return; 1365 for(i = 0; i < n - 1; i++) { 1366 for(j = i + 1; j < n; j++) { 1367 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1368 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1369 if (p1 < p2) { 1370 tmp = def->sorted_args[start + i]; 1371 def->sorted_args[start + i] = def->sorted_args[start + j]; 1372 def->sorted_args[start + j] = tmp; 1373 } 1374 } 1375 } 1376 } 1377 1378 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs) 1379 { 1380 TCGOpcode op; 1381 TCGOpDef *def; 1382 const char *ct_str; 1383 int i, nb_args; 1384 1385 for(;;) { 1386 if (tdefs->op == (TCGOpcode)-1) 1387 break; 1388 op = tdefs->op; 1389 assert((unsigned)op < NB_OPS); 1390 def = &tcg_op_defs[op]; 1391 #if defined(CONFIG_DEBUG_TCG) 1392 /* Duplicate entry in op definitions? */ 1393 assert(!def->used); 1394 def->used = 1; 1395 #endif 1396 nb_args = def->nb_iargs + def->nb_oargs; 1397 for(i = 0; i < nb_args; i++) { 1398 ct_str = tdefs->args_ct_str[i]; 1399 /* Incomplete TCGTargetOpDef entry? */ 1400 assert(ct_str != NULL); 1401 tcg_regset_clear(def->args_ct[i].u.regs); 1402 def->args_ct[i].ct = 0; 1403 if (ct_str[0] >= '0' && ct_str[0] <= '9') { 1404 int oarg; 1405 oarg = ct_str[0] - '0'; 1406 assert(oarg < def->nb_oargs); 1407 assert(def->args_ct[oarg].ct & TCG_CT_REG); 1408 /* TCG_CT_ALIAS is for the output arguments. The input 1409 argument is tagged with TCG_CT_IALIAS. */ 1410 def->args_ct[i] = def->args_ct[oarg]; 1411 def->args_ct[oarg].ct = TCG_CT_ALIAS; 1412 def->args_ct[oarg].alias_index = i; 1413 def->args_ct[i].ct |= TCG_CT_IALIAS; 1414 def->args_ct[i].alias_index = oarg; 1415 } else { 1416 for(;;) { 1417 if (*ct_str == '\0') 1418 break; 1419 switch(*ct_str) { 1420 case 'i': 1421 def->args_ct[i].ct |= TCG_CT_CONST; 1422 ct_str++; 1423 break; 1424 default: 1425 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) { 1426 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n", 1427 ct_str, i, def->name); 1428 exit(1); 1429 } 1430 } 1431 } 1432 } 1433 } 1434 1435 /* TCGTargetOpDef entry with too much information? */ 1436 assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1437 1438 /* sort the constraints (XXX: this is just an heuristic) */ 1439 sort_constraints(def, 0, def->nb_oargs); 1440 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1441 1442 #if 0 1443 { 1444 int i; 1445 1446 printf("%s: sorted=", def->name); 1447 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++) 1448 printf(" %d", def->sorted_args[i]); 1449 printf("\n"); 1450 } 1451 #endif 1452 tdefs++; 1453 } 1454 1455 #if defined(CONFIG_DEBUG_TCG) 1456 i = 0; 1457 for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) { 1458 const TCGOpDef *def = &tcg_op_defs[op]; 1459 if (def->flags & TCG_OPF_NOT_PRESENT) { 1460 /* Wrong entry in op definitions? */ 1461 if (def->used) { 1462 fprintf(stderr, "Invalid op definition for %s\n", def->name); 1463 i = 1; 1464 } 1465 } else { 1466 /* Missing entry in op definitions? */ 1467 if (!def->used) { 1468 fprintf(stderr, "Missing op definition for %s\n", def->name); 1469 i = 1; 1470 } 1471 } 1472 } 1473 if (i == 1) { 1474 tcg_abort(); 1475 } 1476 #endif 1477 } 1478 1479 #ifdef USE_LIVENESS_ANALYSIS 1480 1481 /* set a nop for an operation using 'nb_args' */ 1482 static inline void tcg_set_nop(TCGContext *s, uint16_t *opc_ptr, 1483 TCGArg *args, int nb_args) 1484 { 1485 if (nb_args == 0) { 1486 *opc_ptr = INDEX_op_nop; 1487 } else { 1488 *opc_ptr = INDEX_op_nopn; 1489 args[0] = nb_args; 1490 args[nb_args - 1] = nb_args; 1491 } 1492 } 1493 1494 /* liveness analysis: end of function: all temps are dead, and globals 1495 should be in memory. */ 1496 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps, 1497 uint8_t *mem_temps) 1498 { 1499 memset(dead_temps, 1, s->nb_temps); 1500 memset(mem_temps, 1, s->nb_globals); 1501 memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals); 1502 } 1503 1504 /* liveness analysis: end of basic block: all temps are dead, globals 1505 and local temps should be in memory. */ 1506 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps, 1507 uint8_t *mem_temps) 1508 { 1509 int i; 1510 1511 memset(dead_temps, 1, s->nb_temps); 1512 memset(mem_temps, 1, s->nb_globals); 1513 for(i = s->nb_globals; i < s->nb_temps; i++) { 1514 mem_temps[i] = s->temps[i].temp_local; 1515 } 1516 } 1517 1518 /* Liveness analysis : update the opc_dead_args array to tell if a 1519 given input arguments is dead. Instructions updating dead 1520 temporaries are removed. */ 1521 static void tcg_liveness_analysis(TCGContext *s) 1522 { 1523 int i, op_index, nb_args, nb_iargs, nb_oargs, nb_ops; 1524 TCGOpcode op, op_new, op_new2; 1525 TCGArg *args, arg; 1526 const TCGOpDef *def; 1527 uint8_t *dead_temps, *mem_temps; 1528 uint16_t dead_args; 1529 uint8_t sync_args; 1530 bool have_op_new2; 1531 1532 s->gen_opc_ptr++; /* skip end */ 1533 1534 nb_ops = s->gen_opc_ptr - s->gen_opc_buf; 1535 1536 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1537 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1538 1539 dead_temps = tcg_malloc(s->nb_temps); 1540 mem_temps = tcg_malloc(s->nb_temps); 1541 tcg_la_func_end(s, dead_temps, mem_temps); 1542 1543 args = s->gen_opparam_ptr; 1544 op_index = nb_ops - 1; 1545 while (op_index >= 0) { 1546 op = s->gen_opc_buf[op_index]; 1547 def = &tcg_op_defs[op]; 1548 switch(op) { 1549 case INDEX_op_call: 1550 { 1551 int call_flags; 1552 1553 nb_args = args[-1]; 1554 args -= nb_args; 1555 arg = *args++; 1556 nb_iargs = arg & 0xffff; 1557 nb_oargs = arg >> 16; 1558 call_flags = args[nb_oargs + nb_iargs + 1]; 1559 1560 /* pure functions can be removed if their result is not 1561 used */ 1562 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 1563 for (i = 0; i < nb_oargs; i++) { 1564 arg = args[i]; 1565 if (!dead_temps[arg] || mem_temps[arg]) { 1566 goto do_not_remove_call; 1567 } 1568 } 1569 tcg_set_nop(s, s->gen_opc_buf + op_index, 1570 args - 1, nb_args); 1571 } else { 1572 do_not_remove_call: 1573 1574 /* output args are dead */ 1575 dead_args = 0; 1576 sync_args = 0; 1577 for (i = 0; i < nb_oargs; i++) { 1578 arg = args[i]; 1579 if (dead_temps[arg]) { 1580 dead_args |= (1 << i); 1581 } 1582 if (mem_temps[arg]) { 1583 sync_args |= (1 << i); 1584 } 1585 dead_temps[arg] = 1; 1586 mem_temps[arg] = 0; 1587 } 1588 1589 if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 1590 /* globals should be synced to memory */ 1591 memset(mem_temps, 1, s->nb_globals); 1592 } 1593 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 1594 TCG_CALL_NO_READ_GLOBALS))) { 1595 /* globals should go back to memory */ 1596 memset(dead_temps, 1, s->nb_globals); 1597 } 1598 1599 /* input args are live */ 1600 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1601 arg = args[i]; 1602 if (arg != TCG_CALL_DUMMY_ARG) { 1603 if (dead_temps[arg]) { 1604 dead_args |= (1 << i); 1605 } 1606 dead_temps[arg] = 0; 1607 } 1608 } 1609 s->op_dead_args[op_index] = dead_args; 1610 s->op_sync_args[op_index] = sync_args; 1611 } 1612 args--; 1613 } 1614 break; 1615 case INDEX_op_debug_insn_start: 1616 args -= def->nb_args; 1617 break; 1618 case INDEX_op_nopn: 1619 nb_args = args[-1]; 1620 args -= nb_args; 1621 break; 1622 case INDEX_op_discard: 1623 args--; 1624 /* mark the temporary as dead */ 1625 dead_temps[args[0]] = 1; 1626 mem_temps[args[0]] = 0; 1627 break; 1628 case INDEX_op_end: 1629 break; 1630 1631 case INDEX_op_add2_i32: 1632 op_new = INDEX_op_add_i32; 1633 goto do_addsub2; 1634 case INDEX_op_sub2_i32: 1635 op_new = INDEX_op_sub_i32; 1636 goto do_addsub2; 1637 case INDEX_op_add2_i64: 1638 op_new = INDEX_op_add_i64; 1639 goto do_addsub2; 1640 case INDEX_op_sub2_i64: 1641 op_new = INDEX_op_sub_i64; 1642 do_addsub2: 1643 args -= 6; 1644 nb_iargs = 4; 1645 nb_oargs = 2; 1646 /* Test if the high part of the operation is dead, but not 1647 the low part. The result can be optimized to a simple 1648 add or sub. This happens often for x86_64 guest when the 1649 cpu mode is set to 32 bit. */ 1650 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1651 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1652 goto do_remove; 1653 } 1654 /* Create the single operation plus nop. */ 1655 s->gen_opc_buf[op_index] = op = op_new; 1656 args[1] = args[2]; 1657 args[2] = args[4]; 1658 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop); 1659 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 3); 1660 /* Fall through and mark the single-word operation live. */ 1661 nb_iargs = 2; 1662 nb_oargs = 1; 1663 } 1664 goto do_not_remove; 1665 1666 case INDEX_op_mulu2_i32: 1667 op_new = INDEX_op_mul_i32; 1668 op_new2 = INDEX_op_muluh_i32; 1669 have_op_new2 = TCG_TARGET_HAS_muluh_i32; 1670 goto do_mul2; 1671 case INDEX_op_muls2_i32: 1672 op_new = INDEX_op_mul_i32; 1673 op_new2 = INDEX_op_mulsh_i32; 1674 have_op_new2 = TCG_TARGET_HAS_mulsh_i32; 1675 goto do_mul2; 1676 case INDEX_op_mulu2_i64: 1677 op_new = INDEX_op_mul_i64; 1678 op_new2 = INDEX_op_muluh_i64; 1679 have_op_new2 = TCG_TARGET_HAS_muluh_i64; 1680 goto do_mul2; 1681 case INDEX_op_muls2_i64: 1682 op_new = INDEX_op_mul_i64; 1683 op_new2 = INDEX_op_mulsh_i64; 1684 have_op_new2 = TCG_TARGET_HAS_mulsh_i64; 1685 goto do_mul2; 1686 do_mul2: 1687 args -= 4; 1688 nb_iargs = 2; 1689 nb_oargs = 2; 1690 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1691 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1692 /* Both parts of the operation are dead. */ 1693 goto do_remove; 1694 } 1695 /* The high part of the operation is dead; generate the low. */ 1696 s->gen_opc_buf[op_index] = op = op_new; 1697 args[1] = args[2]; 1698 args[2] = args[3]; 1699 } else if (have_op_new2 && dead_temps[args[0]] 1700 && !mem_temps[args[0]]) { 1701 /* The low part of the operation is dead; generate the high. */ 1702 s->gen_opc_buf[op_index] = op = op_new2; 1703 args[0] = args[1]; 1704 args[1] = args[2]; 1705 args[2] = args[3]; 1706 } else { 1707 goto do_not_remove; 1708 } 1709 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop); 1710 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1); 1711 /* Mark the single-word operation live. */ 1712 nb_oargs = 1; 1713 goto do_not_remove; 1714 1715 default: 1716 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 1717 args -= def->nb_args; 1718 nb_iargs = def->nb_iargs; 1719 nb_oargs = def->nb_oargs; 1720 1721 /* Test if the operation can be removed because all 1722 its outputs are dead. We assume that nb_oargs == 0 1723 implies side effects */ 1724 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 1725 for(i = 0; i < nb_oargs; i++) { 1726 arg = args[i]; 1727 if (!dead_temps[arg] || mem_temps[arg]) { 1728 goto do_not_remove; 1729 } 1730 } 1731 do_remove: 1732 tcg_set_nop(s, s->gen_opc_buf + op_index, args, def->nb_args); 1733 #ifdef CONFIG_PROFILER 1734 s->del_op_count++; 1735 #endif 1736 } else { 1737 do_not_remove: 1738 1739 /* output args are dead */ 1740 dead_args = 0; 1741 sync_args = 0; 1742 for(i = 0; i < nb_oargs; i++) { 1743 arg = args[i]; 1744 if (dead_temps[arg]) { 1745 dead_args |= (1 << i); 1746 } 1747 if (mem_temps[arg]) { 1748 sync_args |= (1 << i); 1749 } 1750 dead_temps[arg] = 1; 1751 mem_temps[arg] = 0; 1752 } 1753 1754 /* if end of basic block, update */ 1755 if (def->flags & TCG_OPF_BB_END) { 1756 tcg_la_bb_end(s, dead_temps, mem_temps); 1757 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 1758 /* globals should be synced to memory */ 1759 memset(mem_temps, 1, s->nb_globals); 1760 } 1761 1762 /* input args are live */ 1763 for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1764 arg = args[i]; 1765 if (dead_temps[arg]) { 1766 dead_args |= (1 << i); 1767 } 1768 dead_temps[arg] = 0; 1769 } 1770 s->op_dead_args[op_index] = dead_args; 1771 s->op_sync_args[op_index] = sync_args; 1772 } 1773 break; 1774 } 1775 op_index--; 1776 } 1777 1778 if (args != s->gen_opparam_buf) { 1779 tcg_abort(); 1780 } 1781 } 1782 #else 1783 /* dummy liveness analysis */ 1784 static void tcg_liveness_analysis(TCGContext *s) 1785 { 1786 int nb_ops; 1787 nb_ops = s->gen_opc_ptr - s->gen_opc_buf; 1788 1789 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1790 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t)); 1791 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1792 memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t)); 1793 } 1794 #endif 1795 1796 #ifndef NDEBUG 1797 static void dump_regs(TCGContext *s) 1798 { 1799 TCGTemp *ts; 1800 int i; 1801 char buf[64]; 1802 1803 for(i = 0; i < s->nb_temps; i++) { 1804 ts = &s->temps[i]; 1805 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i)); 1806 switch(ts->val_type) { 1807 case TEMP_VAL_REG: 1808 printf("%s", tcg_target_reg_names[ts->reg]); 1809 break; 1810 case TEMP_VAL_MEM: 1811 printf("%d(%s)", (int)ts->mem_offset, tcg_target_reg_names[ts->mem_reg]); 1812 break; 1813 case TEMP_VAL_CONST: 1814 printf("$0x%" TCG_PRIlx, ts->val); 1815 break; 1816 case TEMP_VAL_DEAD: 1817 printf("D"); 1818 break; 1819 default: 1820 printf("???"); 1821 break; 1822 } 1823 printf("\n"); 1824 } 1825 1826 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1827 if (s->reg_to_temp[i] >= 0) { 1828 printf("%s: %s\n", 1829 tcg_target_reg_names[i], 1830 tcg_get_arg_str_idx(s, buf, sizeof(buf), s->reg_to_temp[i])); 1831 } 1832 } 1833 } 1834 1835 static void check_regs(TCGContext *s) 1836 { 1837 int reg, k; 1838 TCGTemp *ts; 1839 char buf[64]; 1840 1841 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 1842 k = s->reg_to_temp[reg]; 1843 if (k >= 0) { 1844 ts = &s->temps[k]; 1845 if (ts->val_type != TEMP_VAL_REG || 1846 ts->reg != reg) { 1847 printf("Inconsistency for register %s:\n", 1848 tcg_target_reg_names[reg]); 1849 goto fail; 1850 } 1851 } 1852 } 1853 for(k = 0; k < s->nb_temps; k++) { 1854 ts = &s->temps[k]; 1855 if (ts->val_type == TEMP_VAL_REG && 1856 !ts->fixed_reg && 1857 s->reg_to_temp[ts->reg] != k) { 1858 printf("Inconsistency for temp %s:\n", 1859 tcg_get_arg_str_idx(s, buf, sizeof(buf), k)); 1860 fail: 1861 printf("reg state:\n"); 1862 dump_regs(s); 1863 tcg_abort(); 1864 } 1865 } 1866 } 1867 #endif 1868 1869 static void temp_allocate_frame(TCGContext *s, int temp) 1870 { 1871 TCGTemp *ts; 1872 ts = &s->temps[temp]; 1873 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 1874 /* Sparc64 stack is accessed with offset of 2047 */ 1875 s->current_frame_offset = (s->current_frame_offset + 1876 (tcg_target_long)sizeof(tcg_target_long) - 1) & 1877 ~(sizeof(tcg_target_long) - 1); 1878 #endif 1879 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 1880 s->frame_end) { 1881 tcg_abort(); 1882 } 1883 ts->mem_offset = s->current_frame_offset; 1884 ts->mem_reg = s->frame_reg; 1885 ts->mem_allocated = 1; 1886 s->current_frame_offset += sizeof(tcg_target_long); 1887 } 1888 1889 /* sync register 'reg' by saving it to the corresponding temporary */ 1890 static inline void tcg_reg_sync(TCGContext *s, int reg) 1891 { 1892 TCGTemp *ts; 1893 int temp; 1894 1895 temp = s->reg_to_temp[reg]; 1896 ts = &s->temps[temp]; 1897 assert(ts->val_type == TEMP_VAL_REG); 1898 if (!ts->mem_coherent && !ts->fixed_reg) { 1899 if (!ts->mem_allocated) { 1900 temp_allocate_frame(s, temp); 1901 } 1902 tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1903 } 1904 ts->mem_coherent = 1; 1905 } 1906 1907 /* free register 'reg' by spilling the corresponding temporary if necessary */ 1908 static void tcg_reg_free(TCGContext *s, int reg) 1909 { 1910 int temp; 1911 1912 temp = s->reg_to_temp[reg]; 1913 if (temp != -1) { 1914 tcg_reg_sync(s, reg); 1915 s->temps[temp].val_type = TEMP_VAL_MEM; 1916 s->reg_to_temp[reg] = -1; 1917 } 1918 } 1919 1920 /* Allocate a register belonging to reg1 & ~reg2 */ 1921 static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2) 1922 { 1923 int i, reg; 1924 TCGRegSet reg_ct; 1925 1926 tcg_regset_andnot(reg_ct, reg1, reg2); 1927 1928 /* first try free registers */ 1929 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1930 reg = tcg_target_reg_alloc_order[i]; 1931 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == -1) 1932 return reg; 1933 } 1934 1935 /* XXX: do better spill choice */ 1936 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1937 reg = tcg_target_reg_alloc_order[i]; 1938 if (tcg_regset_test_reg(reg_ct, reg)) { 1939 tcg_reg_free(s, reg); 1940 return reg; 1941 } 1942 } 1943 1944 tcg_abort(); 1945 } 1946 1947 /* mark a temporary as dead. */ 1948 static inline void temp_dead(TCGContext *s, int temp) 1949 { 1950 TCGTemp *ts; 1951 1952 ts = &s->temps[temp]; 1953 if (!ts->fixed_reg) { 1954 if (ts->val_type == TEMP_VAL_REG) { 1955 s->reg_to_temp[ts->reg] = -1; 1956 } 1957 if (temp < s->nb_globals || ts->temp_local) { 1958 ts->val_type = TEMP_VAL_MEM; 1959 } else { 1960 ts->val_type = TEMP_VAL_DEAD; 1961 } 1962 } 1963 } 1964 1965 /* sync a temporary to memory. 'allocated_regs' is used in case a 1966 temporary registers needs to be allocated to store a constant. */ 1967 static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs) 1968 { 1969 TCGTemp *ts; 1970 1971 ts = &s->temps[temp]; 1972 if (!ts->fixed_reg) { 1973 switch(ts->val_type) { 1974 case TEMP_VAL_CONST: 1975 ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 1976 allocated_regs); 1977 ts->val_type = TEMP_VAL_REG; 1978 s->reg_to_temp[ts->reg] = temp; 1979 ts->mem_coherent = 0; 1980 tcg_out_movi(s, ts->type, ts->reg, ts->val); 1981 /* fallthrough*/ 1982 case TEMP_VAL_REG: 1983 tcg_reg_sync(s, ts->reg); 1984 break; 1985 case TEMP_VAL_DEAD: 1986 case TEMP_VAL_MEM: 1987 break; 1988 default: 1989 tcg_abort(); 1990 } 1991 } 1992 } 1993 1994 /* save a temporary to memory. 'allocated_regs' is used in case a 1995 temporary registers needs to be allocated to store a constant. */ 1996 static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs) 1997 { 1998 #ifdef USE_LIVENESS_ANALYSIS 1999 /* The liveness analysis already ensures that globals are back 2000 in memory. Keep an assert for safety. */ 2001 assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg); 2002 #else 2003 temp_sync(s, temp, allocated_regs); 2004 temp_dead(s, temp); 2005 #endif 2006 } 2007 2008 /* save globals to their canonical location and assume they can be 2009 modified be the following code. 'allocated_regs' is used in case a 2010 temporary registers needs to be allocated to store a constant. */ 2011 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 2012 { 2013 int i; 2014 2015 for(i = 0; i < s->nb_globals; i++) { 2016 temp_save(s, i, allocated_regs); 2017 } 2018 } 2019 2020 /* sync globals to their canonical location and assume they can be 2021 read by the following code. 'allocated_regs' is used in case a 2022 temporary registers needs to be allocated to store a constant. */ 2023 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 2024 { 2025 int i; 2026 2027 for (i = 0; i < s->nb_globals; i++) { 2028 #ifdef USE_LIVENESS_ANALYSIS 2029 assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg || 2030 s->temps[i].mem_coherent); 2031 #else 2032 temp_sync(s, i, allocated_regs); 2033 #endif 2034 } 2035 } 2036 2037 /* at the end of a basic block, we assume all temporaries are dead and 2038 all globals are stored at their canonical location. */ 2039 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 2040 { 2041 TCGTemp *ts; 2042 int i; 2043 2044 for(i = s->nb_globals; i < s->nb_temps; i++) { 2045 ts = &s->temps[i]; 2046 if (ts->temp_local) { 2047 temp_save(s, i, allocated_regs); 2048 } else { 2049 #ifdef USE_LIVENESS_ANALYSIS 2050 /* The liveness analysis already ensures that temps are dead. 2051 Keep an assert for safety. */ 2052 assert(ts->val_type == TEMP_VAL_DEAD); 2053 #else 2054 temp_dead(s, i); 2055 #endif 2056 } 2057 } 2058 2059 save_globals(s, allocated_regs); 2060 } 2061 2062 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1) 2063 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1) 2064 2065 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args, 2066 uint16_t dead_args, uint8_t sync_args) 2067 { 2068 TCGTemp *ots; 2069 tcg_target_ulong val; 2070 2071 ots = &s->temps[args[0]]; 2072 val = args[1]; 2073 2074 if (ots->fixed_reg) { 2075 /* for fixed registers, we do not do any constant 2076 propagation */ 2077 tcg_out_movi(s, ots->type, ots->reg, val); 2078 } else { 2079 /* The movi is not explicitly generated here */ 2080 if (ots->val_type == TEMP_VAL_REG) 2081 s->reg_to_temp[ots->reg] = -1; 2082 ots->val_type = TEMP_VAL_CONST; 2083 ots->val = val; 2084 } 2085 if (NEED_SYNC_ARG(0)) { 2086 temp_sync(s, args[0], s->reserved_regs); 2087 } 2088 if (IS_DEAD_ARG(0)) { 2089 temp_dead(s, args[0]); 2090 } 2091 } 2092 2093 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, 2094 const TCGArg *args, uint16_t dead_args, 2095 uint8_t sync_args) 2096 { 2097 TCGRegSet allocated_regs; 2098 TCGTemp *ts, *ots; 2099 TCGType type; 2100 2101 tcg_regset_set(allocated_regs, s->reserved_regs); 2102 ots = &s->temps[args[0]]; 2103 ts = &s->temps[args[1]]; 2104 type = ots->type; 2105 2106 /* If the source value is not in a register, and we're going to be 2107 forced to have it in a register in order to perform the copy, 2108 then copy the SOURCE value into its own register first. That way 2109 we don't have to reload SOURCE the next time it is used. */ 2110 if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG) 2111 || ts->val_type == TEMP_VAL_MEM) { 2112 ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[type], 2113 allocated_regs); 2114 if (ts->val_type == TEMP_VAL_MEM) { 2115 tcg_out_ld(s, type, ts->reg, ts->mem_reg, ts->mem_offset); 2116 ts->mem_coherent = 1; 2117 } else if (ts->val_type == TEMP_VAL_CONST) { 2118 tcg_out_movi(s, type, ts->reg, ts->val); 2119 } 2120 s->reg_to_temp[ts->reg] = args[1]; 2121 ts->val_type = TEMP_VAL_REG; 2122 } 2123 2124 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 2125 /* mov to a non-saved dead register makes no sense (even with 2126 liveness analysis disabled). */ 2127 assert(NEED_SYNC_ARG(0)); 2128 /* The code above should have moved the temp to a register. */ 2129 assert(ts->val_type == TEMP_VAL_REG); 2130 if (!ots->mem_allocated) { 2131 temp_allocate_frame(s, args[0]); 2132 } 2133 tcg_out_st(s, type, ts->reg, ots->mem_reg, ots->mem_offset); 2134 if (IS_DEAD_ARG(1)) { 2135 temp_dead(s, args[1]); 2136 } 2137 temp_dead(s, args[0]); 2138 } else if (ts->val_type == TEMP_VAL_CONST) { 2139 /* propagate constant */ 2140 if (ots->val_type == TEMP_VAL_REG) { 2141 s->reg_to_temp[ots->reg] = -1; 2142 } 2143 ots->val_type = TEMP_VAL_CONST; 2144 ots->val = ts->val; 2145 } else { 2146 /* The code in the first if block should have moved the 2147 temp to a register. */ 2148 assert(ts->val_type == TEMP_VAL_REG); 2149 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 2150 /* the mov can be suppressed */ 2151 if (ots->val_type == TEMP_VAL_REG) { 2152 s->reg_to_temp[ots->reg] = -1; 2153 } 2154 ots->reg = ts->reg; 2155 temp_dead(s, args[1]); 2156 } else { 2157 if (ots->val_type != TEMP_VAL_REG) { 2158 /* When allocating a new register, make sure to not spill the 2159 input one. */ 2160 tcg_regset_set_reg(allocated_regs, ts->reg); 2161 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[type], 2162 allocated_regs); 2163 } 2164 tcg_out_mov(s, type, ots->reg, ts->reg); 2165 } 2166 ots->val_type = TEMP_VAL_REG; 2167 ots->mem_coherent = 0; 2168 s->reg_to_temp[ots->reg] = args[0]; 2169 if (NEED_SYNC_ARG(0)) { 2170 tcg_reg_sync(s, ots->reg); 2171 } 2172 } 2173 } 2174 2175 static void tcg_reg_alloc_op(TCGContext *s, 2176 const TCGOpDef *def, TCGOpcode opc, 2177 const TCGArg *args, uint16_t dead_args, 2178 uint8_t sync_args) 2179 { 2180 TCGRegSet allocated_regs; 2181 int i, k, nb_iargs, nb_oargs, reg; 2182 TCGArg arg; 2183 const TCGArgConstraint *arg_ct; 2184 TCGTemp *ts; 2185 TCGArg new_args[TCG_MAX_OP_ARGS]; 2186 int const_args[TCG_MAX_OP_ARGS]; 2187 2188 nb_oargs = def->nb_oargs; 2189 nb_iargs = def->nb_iargs; 2190 2191 /* copy constants */ 2192 memcpy(new_args + nb_oargs + nb_iargs, 2193 args + nb_oargs + nb_iargs, 2194 sizeof(TCGArg) * def->nb_cargs); 2195 2196 /* satisfy input constraints */ 2197 tcg_regset_set(allocated_regs, s->reserved_regs); 2198 for(k = 0; k < nb_iargs; k++) { 2199 i = def->sorted_args[nb_oargs + k]; 2200 arg = args[i]; 2201 arg_ct = &def->args_ct[i]; 2202 ts = &s->temps[arg]; 2203 if (ts->val_type == TEMP_VAL_MEM) { 2204 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2205 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2206 ts->val_type = TEMP_VAL_REG; 2207 ts->reg = reg; 2208 ts->mem_coherent = 1; 2209 s->reg_to_temp[reg] = arg; 2210 } else if (ts->val_type == TEMP_VAL_CONST) { 2211 if (tcg_target_const_match(ts->val, ts->type, arg_ct)) { 2212 /* constant is OK for instruction */ 2213 const_args[i] = 1; 2214 new_args[i] = ts->val; 2215 goto iarg_end; 2216 } else { 2217 /* need to move to a register */ 2218 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2219 tcg_out_movi(s, ts->type, reg, ts->val); 2220 ts->val_type = TEMP_VAL_REG; 2221 ts->reg = reg; 2222 ts->mem_coherent = 0; 2223 s->reg_to_temp[reg] = arg; 2224 } 2225 } 2226 assert(ts->val_type == TEMP_VAL_REG); 2227 if (arg_ct->ct & TCG_CT_IALIAS) { 2228 if (ts->fixed_reg) { 2229 /* if fixed register, we must allocate a new register 2230 if the alias is not the same register */ 2231 if (arg != args[arg_ct->alias_index]) 2232 goto allocate_in_reg; 2233 } else { 2234 /* if the input is aliased to an output and if it is 2235 not dead after the instruction, we must allocate 2236 a new register and move it */ 2237 if (!IS_DEAD_ARG(i)) { 2238 goto allocate_in_reg; 2239 } 2240 } 2241 } 2242 reg = ts->reg; 2243 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2244 /* nothing to do : the constraint is satisfied */ 2245 } else { 2246 allocate_in_reg: 2247 /* allocate a new register matching the constraint 2248 and move the temporary register into it */ 2249 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2250 tcg_out_mov(s, ts->type, reg, ts->reg); 2251 } 2252 new_args[i] = reg; 2253 const_args[i] = 0; 2254 tcg_regset_set_reg(allocated_regs, reg); 2255 iarg_end: ; 2256 } 2257 2258 /* mark dead temporaries and free the associated registers */ 2259 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2260 if (IS_DEAD_ARG(i)) { 2261 temp_dead(s, args[i]); 2262 } 2263 } 2264 2265 if (def->flags & TCG_OPF_BB_END) { 2266 tcg_reg_alloc_bb_end(s, allocated_regs); 2267 } else { 2268 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2269 /* XXX: permit generic clobber register list ? */ 2270 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2271 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 2272 tcg_reg_free(s, reg); 2273 } 2274 } 2275 } 2276 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2277 /* sync globals if the op has side effects and might trigger 2278 an exception. */ 2279 sync_globals(s, allocated_regs); 2280 } 2281 2282 /* satisfy the output constraints */ 2283 tcg_regset_set(allocated_regs, s->reserved_regs); 2284 for(k = 0; k < nb_oargs; k++) { 2285 i = def->sorted_args[k]; 2286 arg = args[i]; 2287 arg_ct = &def->args_ct[i]; 2288 ts = &s->temps[arg]; 2289 if (arg_ct->ct & TCG_CT_ALIAS) { 2290 reg = new_args[arg_ct->alias_index]; 2291 } else { 2292 /* if fixed register, we try to use it */ 2293 reg = ts->reg; 2294 if (ts->fixed_reg && 2295 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2296 goto oarg_end; 2297 } 2298 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2299 } 2300 tcg_regset_set_reg(allocated_regs, reg); 2301 /* if a fixed register is used, then a move will be done afterwards */ 2302 if (!ts->fixed_reg) { 2303 if (ts->val_type == TEMP_VAL_REG) { 2304 s->reg_to_temp[ts->reg] = -1; 2305 } 2306 ts->val_type = TEMP_VAL_REG; 2307 ts->reg = reg; 2308 /* temp value is modified, so the value kept in memory is 2309 potentially not the same */ 2310 ts->mem_coherent = 0; 2311 s->reg_to_temp[reg] = arg; 2312 } 2313 oarg_end: 2314 new_args[i] = reg; 2315 } 2316 } 2317 2318 /* emit instruction */ 2319 tcg_out_op(s, opc, new_args, const_args); 2320 2321 /* move the outputs in the correct register if needed */ 2322 for(i = 0; i < nb_oargs; i++) { 2323 ts = &s->temps[args[i]]; 2324 reg = new_args[i]; 2325 if (ts->fixed_reg && ts->reg != reg) { 2326 tcg_out_mov(s, ts->type, ts->reg, reg); 2327 } 2328 if (NEED_SYNC_ARG(i)) { 2329 tcg_reg_sync(s, reg); 2330 } 2331 if (IS_DEAD_ARG(i)) { 2332 temp_dead(s, args[i]); 2333 } 2334 } 2335 } 2336 2337 #ifdef TCG_TARGET_STACK_GROWSUP 2338 #define STACK_DIR(x) (-(x)) 2339 #else 2340 #define STACK_DIR(x) (x) 2341 #endif 2342 2343 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, 2344 TCGOpcode opc, const TCGArg *args, 2345 uint16_t dead_args, uint8_t sync_args) 2346 { 2347 int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params; 2348 TCGArg arg; 2349 TCGTemp *ts; 2350 intptr_t stack_offset; 2351 size_t call_stack_size; 2352 tcg_insn_unit *func_addr; 2353 int allocate_args; 2354 TCGRegSet allocated_regs; 2355 2356 arg = *args++; 2357 2358 nb_oargs = arg >> 16; 2359 nb_iargs = arg & 0xffff; 2360 nb_params = nb_iargs; 2361 2362 func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs]; 2363 flags = args[nb_oargs + nb_iargs + 1]; 2364 2365 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2366 if (nb_regs > nb_params) { 2367 nb_regs = nb_params; 2368 } 2369 2370 /* assign stack slots first */ 2371 call_stack_size = (nb_params - nb_regs) * sizeof(tcg_target_long); 2372 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 2373 ~(TCG_TARGET_STACK_ALIGN - 1); 2374 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 2375 if (allocate_args) { 2376 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 2377 preallocate call stack */ 2378 tcg_abort(); 2379 } 2380 2381 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 2382 for(i = nb_regs; i < nb_params; i++) { 2383 arg = args[nb_oargs + i]; 2384 #ifdef TCG_TARGET_STACK_GROWSUP 2385 stack_offset -= sizeof(tcg_target_long); 2386 #endif 2387 if (arg != TCG_CALL_DUMMY_ARG) { 2388 ts = &s->temps[arg]; 2389 if (ts->val_type == TEMP_VAL_REG) { 2390 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 2391 } else if (ts->val_type == TEMP_VAL_MEM) { 2392 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 2393 s->reserved_regs); 2394 /* XXX: not correct if reading values from the stack */ 2395 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2396 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 2397 } else if (ts->val_type == TEMP_VAL_CONST) { 2398 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 2399 s->reserved_regs); 2400 /* XXX: sign extend may be needed on some targets */ 2401 tcg_out_movi(s, ts->type, reg, ts->val); 2402 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 2403 } else { 2404 tcg_abort(); 2405 } 2406 } 2407 #ifndef TCG_TARGET_STACK_GROWSUP 2408 stack_offset += sizeof(tcg_target_long); 2409 #endif 2410 } 2411 2412 /* assign input registers */ 2413 tcg_regset_set(allocated_regs, s->reserved_regs); 2414 for(i = 0; i < nb_regs; i++) { 2415 arg = args[nb_oargs + i]; 2416 if (arg != TCG_CALL_DUMMY_ARG) { 2417 ts = &s->temps[arg]; 2418 reg = tcg_target_call_iarg_regs[i]; 2419 tcg_reg_free(s, reg); 2420 if (ts->val_type == TEMP_VAL_REG) { 2421 if (ts->reg != reg) { 2422 tcg_out_mov(s, ts->type, reg, ts->reg); 2423 } 2424 } else if (ts->val_type == TEMP_VAL_MEM) { 2425 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2426 } else if (ts->val_type == TEMP_VAL_CONST) { 2427 /* XXX: sign extend ? */ 2428 tcg_out_movi(s, ts->type, reg, ts->val); 2429 } else { 2430 tcg_abort(); 2431 } 2432 tcg_regset_set_reg(allocated_regs, reg); 2433 } 2434 } 2435 2436 /* mark dead temporaries and free the associated registers */ 2437 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2438 if (IS_DEAD_ARG(i)) { 2439 temp_dead(s, args[i]); 2440 } 2441 } 2442 2443 /* clobber call registers */ 2444 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2445 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 2446 tcg_reg_free(s, reg); 2447 } 2448 } 2449 2450 /* Save globals if they might be written by the helper, sync them if 2451 they might be read. */ 2452 if (flags & TCG_CALL_NO_READ_GLOBALS) { 2453 /* Nothing to do */ 2454 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 2455 sync_globals(s, allocated_regs); 2456 } else { 2457 save_globals(s, allocated_regs); 2458 } 2459 2460 tcg_out_call(s, func_addr); 2461 2462 /* assign output registers and emit moves if needed */ 2463 for(i = 0; i < nb_oargs; i++) { 2464 arg = args[i]; 2465 ts = &s->temps[arg]; 2466 reg = tcg_target_call_oarg_regs[i]; 2467 assert(s->reg_to_temp[reg] == -1); 2468 2469 if (ts->fixed_reg) { 2470 if (ts->reg != reg) { 2471 tcg_out_mov(s, ts->type, ts->reg, reg); 2472 } 2473 } else { 2474 if (ts->val_type == TEMP_VAL_REG) { 2475 s->reg_to_temp[ts->reg] = -1; 2476 } 2477 ts->val_type = TEMP_VAL_REG; 2478 ts->reg = reg; 2479 ts->mem_coherent = 0; 2480 s->reg_to_temp[reg] = arg; 2481 if (NEED_SYNC_ARG(i)) { 2482 tcg_reg_sync(s, reg); 2483 } 2484 if (IS_DEAD_ARG(i)) { 2485 temp_dead(s, args[i]); 2486 } 2487 } 2488 } 2489 2490 return nb_iargs + nb_oargs + def->nb_cargs + 1; 2491 } 2492 2493 #ifdef CONFIG_PROFILER 2494 2495 static int64_t tcg_table_op_count[NB_OPS]; 2496 2497 static void dump_op_count(void) 2498 { 2499 int i; 2500 FILE *f; 2501 f = fopen("/tmp/op.log", "w"); 2502 for(i = INDEX_op_end; i < NB_OPS; i++) { 2503 fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]); 2504 } 2505 fclose(f); 2506 } 2507 #endif 2508 2509 2510 static inline int tcg_gen_code_common(TCGContext *s, 2511 tcg_insn_unit *gen_code_buf, 2512 long search_pc) 2513 { 2514 TCGOpcode opc; 2515 int op_index; 2516 const TCGOpDef *def; 2517 const TCGArg *args; 2518 2519 #ifdef DEBUG_DISAS 2520 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) { 2521 qemu_log("OP:\n"); 2522 tcg_dump_ops(s); 2523 qemu_log("\n"); 2524 } 2525 #endif 2526 2527 #ifdef CONFIG_PROFILER 2528 s->opt_time -= profile_getclock(); 2529 #endif 2530 2531 #ifdef USE_TCG_OPTIMIZATIONS 2532 s->gen_opparam_ptr = 2533 tcg_optimize(s, s->gen_opc_ptr, s->gen_opparam_buf, tcg_op_defs); 2534 #endif 2535 2536 #ifdef CONFIG_PROFILER 2537 s->opt_time += profile_getclock(); 2538 s->la_time -= profile_getclock(); 2539 #endif 2540 2541 tcg_liveness_analysis(s); 2542 2543 #ifdef CONFIG_PROFILER 2544 s->la_time += profile_getclock(); 2545 #endif 2546 2547 #ifdef DEBUG_DISAS 2548 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) { 2549 qemu_log("OP after optimization and liveness analysis:\n"); 2550 tcg_dump_ops(s); 2551 qemu_log("\n"); 2552 } 2553 #endif 2554 2555 tcg_reg_alloc_start(s); 2556 2557 s->code_buf = gen_code_buf; 2558 s->code_ptr = gen_code_buf; 2559 2560 tcg_out_tb_init(s); 2561 2562 args = s->gen_opparam_buf; 2563 op_index = 0; 2564 2565 for(;;) { 2566 opc = s->gen_opc_buf[op_index]; 2567 #ifdef CONFIG_PROFILER 2568 tcg_table_op_count[opc]++; 2569 #endif 2570 def = &tcg_op_defs[opc]; 2571 #if 0 2572 printf("%s: %d %d %d\n", def->name, 2573 def->nb_oargs, def->nb_iargs, def->nb_cargs); 2574 // dump_regs(s); 2575 #endif 2576 switch(opc) { 2577 case INDEX_op_mov_i32: 2578 case INDEX_op_mov_i64: 2579 tcg_reg_alloc_mov(s, def, args, s->op_dead_args[op_index], 2580 s->op_sync_args[op_index]); 2581 break; 2582 case INDEX_op_movi_i32: 2583 case INDEX_op_movi_i64: 2584 tcg_reg_alloc_movi(s, args, s->op_dead_args[op_index], 2585 s->op_sync_args[op_index]); 2586 break; 2587 case INDEX_op_debug_insn_start: 2588 /* debug instruction */ 2589 break; 2590 case INDEX_op_nop: 2591 case INDEX_op_nop1: 2592 case INDEX_op_nop2: 2593 case INDEX_op_nop3: 2594 break; 2595 case INDEX_op_nopn: 2596 args += args[0]; 2597 goto next; 2598 case INDEX_op_discard: 2599 temp_dead(s, args[0]); 2600 break; 2601 case INDEX_op_set_label: 2602 tcg_reg_alloc_bb_end(s, s->reserved_regs); 2603 tcg_out_label(s, args[0], s->code_ptr); 2604 break; 2605 case INDEX_op_call: 2606 args += tcg_reg_alloc_call(s, def, opc, args, 2607 s->op_dead_args[op_index], 2608 s->op_sync_args[op_index]); 2609 goto next; 2610 case INDEX_op_end: 2611 goto the_end; 2612 default: 2613 /* Sanity check that we've not introduced any unhandled opcodes. */ 2614 if (def->flags & TCG_OPF_NOT_PRESENT) { 2615 tcg_abort(); 2616 } 2617 /* Note: in order to speed up the code, it would be much 2618 faster to have specialized register allocator functions for 2619 some common argument patterns */ 2620 tcg_reg_alloc_op(s, def, opc, args, s->op_dead_args[op_index], 2621 s->op_sync_args[op_index]); 2622 break; 2623 } 2624 args += def->nb_args; 2625 next: 2626 if (search_pc >= 0 && search_pc < tcg_current_code_size(s)) { 2627 return op_index; 2628 } 2629 op_index++; 2630 #ifndef NDEBUG 2631 check_regs(s); 2632 #endif 2633 } 2634 the_end: 2635 /* Generate TB finalization at the end of block */ 2636 tcg_out_tb_finalize(s); 2637 return -1; 2638 } 2639 2640 int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf) 2641 { 2642 #ifdef CONFIG_PROFILER 2643 { 2644 int n; 2645 n = (s->gen_opc_ptr - s->gen_opc_buf); 2646 s->op_count += n; 2647 if (n > s->op_count_max) 2648 s->op_count_max = n; 2649 2650 s->temp_count += s->nb_temps; 2651 if (s->nb_temps > s->temp_count_max) 2652 s->temp_count_max = s->nb_temps; 2653 } 2654 #endif 2655 2656 tcg_gen_code_common(s, gen_code_buf, -1); 2657 2658 /* flush instruction cache */ 2659 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 2660 2661 return tcg_current_code_size(s); 2662 } 2663 2664 /* Return the index of the micro operation such as the pc after is < 2665 offset bytes from the start of the TB. The contents of gen_code_buf must 2666 not be changed, though writing the same values is ok. 2667 Return -1 if not found. */ 2668 int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf, 2669 long offset) 2670 { 2671 return tcg_gen_code_common(s, gen_code_buf, offset); 2672 } 2673 2674 #ifdef CONFIG_PROFILER 2675 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2676 { 2677 TCGContext *s = &tcg_ctx; 2678 int64_t tot; 2679 2680 tot = s->interm_time + s->code_time; 2681 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 2682 tot, tot / 2.4e9); 2683 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 2684 s->tb_count, 2685 s->tb_count1 - s->tb_count, 2686 s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0); 2687 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 2688 s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max); 2689 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 2690 s->tb_count ? 2691 (double)s->del_op_count / s->tb_count : 0); 2692 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 2693 s->tb_count ? 2694 (double)s->temp_count / s->tb_count : 0, 2695 s->temp_count_max); 2696 2697 cpu_fprintf(f, "cycles/op %0.1f\n", 2698 s->op_count ? (double)tot / s->op_count : 0); 2699 cpu_fprintf(f, "cycles/in byte %0.1f\n", 2700 s->code_in_len ? (double)tot / s->code_in_len : 0); 2701 cpu_fprintf(f, "cycles/out byte %0.1f\n", 2702 s->code_out_len ? (double)tot / s->code_out_len : 0); 2703 if (tot == 0) 2704 tot = 1; 2705 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 2706 (double)s->interm_time / tot * 100.0); 2707 cpu_fprintf(f, " gen_code time %0.1f%%\n", 2708 (double)s->code_time / tot * 100.0); 2709 cpu_fprintf(f, "optim./code time %0.1f%%\n", 2710 (double)s->opt_time / (s->code_time ? s->code_time : 1) 2711 * 100.0); 2712 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 2713 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 2714 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 2715 s->restore_count); 2716 cpu_fprintf(f, " avg cycles %0.1f\n", 2717 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 2718 2719 dump_op_count(); 2720 } 2721 #else 2722 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2723 { 2724 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2725 } 2726 #endif 2727 2728 #ifdef ELF_HOST_MACHINE 2729 /* In order to use this feature, the backend needs to do three things: 2730 2731 (1) Define ELF_HOST_MACHINE to indicate both what value to 2732 put into the ELF image and to indicate support for the feature. 2733 2734 (2) Define tcg_register_jit. This should create a buffer containing 2735 the contents of a .debug_frame section that describes the post- 2736 prologue unwind info for the tcg machine. 2737 2738 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 2739 */ 2740 2741 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 2742 typedef enum { 2743 JIT_NOACTION = 0, 2744 JIT_REGISTER_FN, 2745 JIT_UNREGISTER_FN 2746 } jit_actions_t; 2747 2748 struct jit_code_entry { 2749 struct jit_code_entry *next_entry; 2750 struct jit_code_entry *prev_entry; 2751 const void *symfile_addr; 2752 uint64_t symfile_size; 2753 }; 2754 2755 struct jit_descriptor { 2756 uint32_t version; 2757 uint32_t action_flag; 2758 struct jit_code_entry *relevant_entry; 2759 struct jit_code_entry *first_entry; 2760 }; 2761 2762 void __jit_debug_register_code(void) __attribute__((noinline)); 2763 void __jit_debug_register_code(void) 2764 { 2765 asm(""); 2766 } 2767 2768 /* Must statically initialize the version, because GDB may check 2769 the version before we can set it. */ 2770 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 2771 2772 /* End GDB interface. */ 2773 2774 static int find_string(const char *strtab, const char *str) 2775 { 2776 const char *p = strtab + 1; 2777 2778 while (1) { 2779 if (strcmp(p, str) == 0) { 2780 return p - strtab; 2781 } 2782 p += strlen(p) + 1; 2783 } 2784 } 2785 2786 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 2787 void *debug_frame, size_t debug_frame_size) 2788 { 2789 struct __attribute__((packed)) DebugInfo { 2790 uint32_t len; 2791 uint16_t version; 2792 uint32_t abbrev; 2793 uint8_t ptr_size; 2794 uint8_t cu_die; 2795 uint16_t cu_lang; 2796 uintptr_t cu_low_pc; 2797 uintptr_t cu_high_pc; 2798 uint8_t fn_die; 2799 char fn_name[16]; 2800 uintptr_t fn_low_pc; 2801 uintptr_t fn_high_pc; 2802 uint8_t cu_eoc; 2803 }; 2804 2805 struct ElfImage { 2806 ElfW(Ehdr) ehdr; 2807 ElfW(Phdr) phdr; 2808 ElfW(Shdr) shdr[7]; 2809 ElfW(Sym) sym[2]; 2810 struct DebugInfo di; 2811 uint8_t da[24]; 2812 char str[80]; 2813 }; 2814 2815 struct ElfImage *img; 2816 2817 static const struct ElfImage img_template = { 2818 .ehdr = { 2819 .e_ident[EI_MAG0] = ELFMAG0, 2820 .e_ident[EI_MAG1] = ELFMAG1, 2821 .e_ident[EI_MAG2] = ELFMAG2, 2822 .e_ident[EI_MAG3] = ELFMAG3, 2823 .e_ident[EI_CLASS] = ELF_CLASS, 2824 .e_ident[EI_DATA] = ELF_DATA, 2825 .e_ident[EI_VERSION] = EV_CURRENT, 2826 .e_type = ET_EXEC, 2827 .e_machine = ELF_HOST_MACHINE, 2828 .e_version = EV_CURRENT, 2829 .e_phoff = offsetof(struct ElfImage, phdr), 2830 .e_shoff = offsetof(struct ElfImage, shdr), 2831 .e_ehsize = sizeof(ElfW(Shdr)), 2832 .e_phentsize = sizeof(ElfW(Phdr)), 2833 .e_phnum = 1, 2834 .e_shentsize = sizeof(ElfW(Shdr)), 2835 .e_shnum = ARRAY_SIZE(img->shdr), 2836 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 2837 #ifdef ELF_HOST_FLAGS 2838 .e_flags = ELF_HOST_FLAGS, 2839 #endif 2840 #ifdef ELF_OSABI 2841 .e_ident[EI_OSABI] = ELF_OSABI, 2842 #endif 2843 }, 2844 .phdr = { 2845 .p_type = PT_LOAD, 2846 .p_flags = PF_X, 2847 }, 2848 .shdr = { 2849 [0] = { .sh_type = SHT_NULL }, 2850 /* Trick: The contents of code_gen_buffer are not present in 2851 this fake ELF file; that got allocated elsewhere. Therefore 2852 we mark .text as SHT_NOBITS (similar to .bss) so that readers 2853 will not look for contents. We can record any address. */ 2854 [1] = { /* .text */ 2855 .sh_type = SHT_NOBITS, 2856 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 2857 }, 2858 [2] = { /* .debug_info */ 2859 .sh_type = SHT_PROGBITS, 2860 .sh_offset = offsetof(struct ElfImage, di), 2861 .sh_size = sizeof(struct DebugInfo), 2862 }, 2863 [3] = { /* .debug_abbrev */ 2864 .sh_type = SHT_PROGBITS, 2865 .sh_offset = offsetof(struct ElfImage, da), 2866 .sh_size = sizeof(img->da), 2867 }, 2868 [4] = { /* .debug_frame */ 2869 .sh_type = SHT_PROGBITS, 2870 .sh_offset = sizeof(struct ElfImage), 2871 }, 2872 [5] = { /* .symtab */ 2873 .sh_type = SHT_SYMTAB, 2874 .sh_offset = offsetof(struct ElfImage, sym), 2875 .sh_size = sizeof(img->sym), 2876 .sh_info = 1, 2877 .sh_link = ARRAY_SIZE(img->shdr) - 1, 2878 .sh_entsize = sizeof(ElfW(Sym)), 2879 }, 2880 [6] = { /* .strtab */ 2881 .sh_type = SHT_STRTAB, 2882 .sh_offset = offsetof(struct ElfImage, str), 2883 .sh_size = sizeof(img->str), 2884 } 2885 }, 2886 .sym = { 2887 [1] = { /* code_gen_buffer */ 2888 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 2889 .st_shndx = 1, 2890 } 2891 }, 2892 .di = { 2893 .len = sizeof(struct DebugInfo) - 4, 2894 .version = 2, 2895 .ptr_size = sizeof(void *), 2896 .cu_die = 1, 2897 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 2898 .fn_die = 2, 2899 .fn_name = "code_gen_buffer" 2900 }, 2901 .da = { 2902 1, /* abbrev number (the cu) */ 2903 0x11, 1, /* DW_TAG_compile_unit, has children */ 2904 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 2905 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2906 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2907 0, 0, /* end of abbrev */ 2908 2, /* abbrev number (the fn) */ 2909 0x2e, 0, /* DW_TAG_subprogram, no children */ 2910 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 2911 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2912 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2913 0, 0, /* end of abbrev */ 2914 0 /* no more abbrev */ 2915 }, 2916 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 2917 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 2918 }; 2919 2920 /* We only need a single jit entry; statically allocate it. */ 2921 static struct jit_code_entry one_entry; 2922 2923 uintptr_t buf = (uintptr_t)buf_ptr; 2924 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 2925 2926 img = g_malloc(img_size); 2927 *img = img_template; 2928 memcpy(img + 1, debug_frame, debug_frame_size); 2929 2930 img->phdr.p_vaddr = buf; 2931 img->phdr.p_paddr = buf; 2932 img->phdr.p_memsz = buf_size; 2933 2934 img->shdr[1].sh_name = find_string(img->str, ".text"); 2935 img->shdr[1].sh_addr = buf; 2936 img->shdr[1].sh_size = buf_size; 2937 2938 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 2939 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 2940 2941 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 2942 img->shdr[4].sh_size = debug_frame_size; 2943 2944 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 2945 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 2946 2947 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 2948 img->sym[1].st_value = buf; 2949 img->sym[1].st_size = buf_size; 2950 2951 img->di.cu_low_pc = buf; 2952 img->di.cu_high_pc = buf + buf_size; 2953 img->di.fn_low_pc = buf; 2954 img->di.fn_high_pc = buf + buf_size; 2955 2956 #ifdef DEBUG_JIT 2957 /* Enable this block to be able to debug the ELF image file creation. 2958 One can use readelf, objdump, or other inspection utilities. */ 2959 { 2960 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 2961 if (f) { 2962 if (fwrite(img, img_size, 1, f) != img_size) { 2963 /* Avoid stupid unused return value warning for fwrite. */ 2964 } 2965 fclose(f); 2966 } 2967 } 2968 #endif 2969 2970 one_entry.symfile_addr = img; 2971 one_entry.symfile_size = img_size; 2972 2973 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 2974 __jit_debug_descriptor.relevant_entry = &one_entry; 2975 __jit_debug_descriptor.first_entry = &one_entry; 2976 __jit_debug_register_code(); 2977 } 2978 #else 2979 /* No support for the feature. Provide the entry point expected by exec.c, 2980 and implement the internal function we declared earlier. */ 2981 2982 static void tcg_register_jit_int(void *buf, size_t size, 2983 void *debug_frame, size_t debug_frame_size) 2984 { 2985 } 2986 2987 void tcg_register_jit(void *buf, size_t buf_size) 2988 { 2989 } 2990 #endif /* ELF_HOST_MACHINE */ 2991