1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_LIVENESS_ANALYSIS 27 #define USE_TCG_OPTIMIZATIONS 28 29 #include "config.h" 30 31 /* Define to jump the ELF file used to communicate with GDB. */ 32 #undef DEBUG_JIT 33 34 #if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG) 35 /* define it to suppress various consistency checks (faster) */ 36 #define NDEBUG 37 #endif 38 39 #include "qemu-common.h" 40 #include "qemu/cache-utils.h" 41 #include "qemu/host-utils.h" 42 #include "qemu/timer.h" 43 44 /* Note: the long term plan is to reduce the dependencies on the QEMU 45 CPU definitions. Currently they are used for qemu_ld/st 46 instructions */ 47 #define NO_CPU_IO_DEFS 48 #include "cpu.h" 49 50 #include "tcg-op.h" 51 52 #if UINTPTR_MAX == UINT32_MAX 53 # define ELF_CLASS ELFCLASS32 54 #else 55 # define ELF_CLASS ELFCLASS64 56 #endif 57 #ifdef HOST_WORDS_BIGENDIAN 58 # define ELF_DATA ELFDATA2MSB 59 #else 60 # define ELF_DATA ELFDATA2LSB 61 #endif 62 63 #include "elf.h" 64 65 /* Forward declarations for functions declared in tcg-target.c and used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static void patch_reloc(uint8_t *code_ptr, int type, 69 intptr_t value, intptr_t addend); 70 71 /* The CIE and FDE header definitions will be common to all hosts. */ 72 typedef struct { 73 uint32_t len __attribute__((aligned((sizeof(void *))))); 74 uint32_t id; 75 uint8_t version; 76 char augmentation[1]; 77 uint8_t code_align; 78 uint8_t data_align; 79 uint8_t return_column; 80 } DebugFrameCIE; 81 82 typedef struct QEMU_PACKED { 83 uint32_t len __attribute__((aligned((sizeof(void *))))); 84 uint32_t cie_offset; 85 uintptr_t func_start; 86 uintptr_t func_len; 87 } DebugFrameFDEHeader; 88 89 static void tcg_register_jit_int(void *buf, size_t size, 90 void *debug_frame, size_t debug_frame_size) 91 __attribute__((unused)); 92 93 /* Forward declarations for functions declared and used in tcg-target.c. */ 94 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str); 95 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 96 intptr_t arg2); 97 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 98 static void tcg_out_movi(TCGContext *s, TCGType type, 99 TCGReg ret, tcg_target_long arg); 100 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 101 const int *const_args); 102 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 103 intptr_t arg2); 104 static int tcg_target_const_match(tcg_target_long val, TCGType type, 105 const TCGArgConstraint *arg_ct); 106 static void tcg_out_tb_init(TCGContext *s); 107 static void tcg_out_tb_finalize(TCGContext *s); 108 109 110 TCGOpDef tcg_op_defs[] = { 111 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags }, 112 #include "tcg-opc.h" 113 #undef DEF 114 }; 115 const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs); 116 117 static TCGRegSet tcg_target_available_regs[2]; 118 static TCGRegSet tcg_target_call_clobber_regs; 119 120 static inline void tcg_out8(TCGContext *s, uint8_t v) 121 { 122 *s->code_ptr++ = v; 123 } 124 125 static inline void tcg_out16(TCGContext *s, uint16_t v) 126 { 127 uint8_t *p = s->code_ptr; 128 *(uint16_t *)p = v; 129 s->code_ptr = p + 2; 130 } 131 132 static inline void tcg_out32(TCGContext *s, uint32_t v) 133 { 134 uint8_t *p = s->code_ptr; 135 *(uint32_t *)p = v; 136 s->code_ptr = p + 4; 137 } 138 139 static inline void tcg_out64(TCGContext *s, uint64_t v) 140 { 141 uint8_t *p = s->code_ptr; 142 *(uint64_t *)p = v; 143 s->code_ptr = p + 8; 144 } 145 146 /* label relocation processing */ 147 148 static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type, 149 int label_index, intptr_t addend) 150 { 151 TCGLabel *l; 152 TCGRelocation *r; 153 154 l = &s->labels[label_index]; 155 if (l->has_value) { 156 /* FIXME: This may break relocations on RISC targets that 157 modify instruction fields in place. The caller may not have 158 written the initial value. */ 159 patch_reloc(code_ptr, type, l->u.value, addend); 160 } else { 161 /* add a new relocation entry */ 162 r = tcg_malloc(sizeof(TCGRelocation)); 163 r->type = type; 164 r->ptr = code_ptr; 165 r->addend = addend; 166 r->next = l->u.first_reloc; 167 l->u.first_reloc = r; 168 } 169 } 170 171 static void tcg_out_label(TCGContext *s, int label_index, void *ptr) 172 { 173 TCGLabel *l; 174 TCGRelocation *r; 175 intptr_t value = (intptr_t)ptr; 176 177 l = &s->labels[label_index]; 178 if (l->has_value) { 179 tcg_abort(); 180 } 181 r = l->u.first_reloc; 182 while (r != NULL) { 183 patch_reloc(r->ptr, r->type, value, r->addend); 184 r = r->next; 185 } 186 l->has_value = 1; 187 l->u.value = value; 188 } 189 190 int gen_new_label(void) 191 { 192 TCGContext *s = &tcg_ctx; 193 int idx; 194 TCGLabel *l; 195 196 if (s->nb_labels >= TCG_MAX_LABELS) 197 tcg_abort(); 198 idx = s->nb_labels++; 199 l = &s->labels[idx]; 200 l->has_value = 0; 201 l->u.first_reloc = NULL; 202 return idx; 203 } 204 205 #include "tcg-target.c" 206 207 /* pool based memory allocation */ 208 void *tcg_malloc_internal(TCGContext *s, int size) 209 { 210 TCGPool *p; 211 int pool_size; 212 213 if (size > TCG_POOL_CHUNK_SIZE) { 214 /* big malloc: insert a new pool (XXX: could optimize) */ 215 p = g_malloc(sizeof(TCGPool) + size); 216 p->size = size; 217 p->next = s->pool_first_large; 218 s->pool_first_large = p; 219 return p->data; 220 } else { 221 p = s->pool_current; 222 if (!p) { 223 p = s->pool_first; 224 if (!p) 225 goto new_pool; 226 } else { 227 if (!p->next) { 228 new_pool: 229 pool_size = TCG_POOL_CHUNK_SIZE; 230 p = g_malloc(sizeof(TCGPool) + pool_size); 231 p->size = pool_size; 232 p->next = NULL; 233 if (s->pool_current) 234 s->pool_current->next = p; 235 else 236 s->pool_first = p; 237 } else { 238 p = p->next; 239 } 240 } 241 } 242 s->pool_current = p; 243 s->pool_cur = p->data + size; 244 s->pool_end = p->data + p->size; 245 return p->data; 246 } 247 248 void tcg_pool_reset(TCGContext *s) 249 { 250 TCGPool *p, *t; 251 for (p = s->pool_first_large; p; p = t) { 252 t = p->next; 253 g_free(p); 254 } 255 s->pool_first_large = NULL; 256 s->pool_cur = s->pool_end = NULL; 257 s->pool_current = NULL; 258 } 259 260 #include "helper.h" 261 262 typedef struct TCGHelperInfo { 263 void *func; 264 const char *name; 265 } TCGHelperInfo; 266 267 static const TCGHelperInfo all_helpers[] = { 268 #define GEN_HELPER 2 269 #include "helper.h" 270 271 /* Include tcg-runtime.c functions. */ 272 { tcg_helper_div_i32, "div_i32" }, 273 { tcg_helper_rem_i32, "rem_i32" }, 274 { tcg_helper_divu_i32, "divu_i32" }, 275 { tcg_helper_remu_i32, "remu_i32" }, 276 277 { tcg_helper_shl_i64, "shl_i64" }, 278 { tcg_helper_shr_i64, "shr_i64" }, 279 { tcg_helper_sar_i64, "sar_i64" }, 280 { tcg_helper_div_i64, "div_i64" }, 281 { tcg_helper_rem_i64, "rem_i64" }, 282 { tcg_helper_divu_i64, "divu_i64" }, 283 { tcg_helper_remu_i64, "remu_i64" }, 284 { tcg_helper_mulsh_i64, "mulsh_i64" }, 285 { tcg_helper_muluh_i64, "muluh_i64" }, 286 }; 287 288 void tcg_context_init(TCGContext *s) 289 { 290 int op, total_args, n, i; 291 TCGOpDef *def; 292 TCGArgConstraint *args_ct; 293 int *sorted_args; 294 GHashTable *helper_table; 295 296 memset(s, 0, sizeof(*s)); 297 s->nb_globals = 0; 298 299 /* Count total number of arguments and allocate the corresponding 300 space */ 301 total_args = 0; 302 for(op = 0; op < NB_OPS; op++) { 303 def = &tcg_op_defs[op]; 304 n = def->nb_iargs + def->nb_oargs; 305 total_args += n; 306 } 307 308 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 309 sorted_args = g_malloc(sizeof(int) * total_args); 310 311 for(op = 0; op < NB_OPS; op++) { 312 def = &tcg_op_defs[op]; 313 def->args_ct = args_ct; 314 def->sorted_args = sorted_args; 315 n = def->nb_iargs + def->nb_oargs; 316 sorted_args += n; 317 args_ct += n; 318 } 319 320 /* Register helpers. */ 321 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 322 s->helpers = helper_table = g_hash_table_new(NULL, NULL); 323 324 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 325 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 326 (gpointer)all_helpers[i].name); 327 } 328 329 tcg_target_init(s); 330 } 331 332 void tcg_prologue_init(TCGContext *s) 333 { 334 /* init global prologue and epilogue */ 335 s->code_buf = s->code_gen_prologue; 336 s->code_ptr = s->code_buf; 337 tcg_target_qemu_prologue(s); 338 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 339 340 #ifdef DEBUG_DISAS 341 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 342 size_t size = s->code_ptr - s->code_buf; 343 qemu_log("PROLOGUE: [size=%zu]\n", size); 344 log_disas(s->code_buf, size); 345 qemu_log("\n"); 346 qemu_log_flush(); 347 } 348 #endif 349 } 350 351 void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size) 352 { 353 s->frame_start = start; 354 s->frame_end = start + size; 355 s->frame_reg = reg; 356 } 357 358 void tcg_func_start(TCGContext *s) 359 { 360 tcg_pool_reset(s); 361 s->nb_temps = s->nb_globals; 362 363 /* No temps have been previously allocated for size or locality. */ 364 memset(s->free_temps, 0, sizeof(s->free_temps)); 365 366 s->labels = tcg_malloc(sizeof(TCGLabel) * TCG_MAX_LABELS); 367 s->nb_labels = 0; 368 s->current_frame_offset = s->frame_start; 369 370 #ifdef CONFIG_DEBUG_TCG 371 s->goto_tb_issue_mask = 0; 372 #endif 373 374 s->gen_opc_ptr = s->gen_opc_buf; 375 s->gen_opparam_ptr = s->gen_opparam_buf; 376 377 s->be = tcg_malloc(sizeof(TCGBackendData)); 378 } 379 380 static inline void tcg_temp_alloc(TCGContext *s, int n) 381 { 382 if (n > TCG_MAX_TEMPS) 383 tcg_abort(); 384 } 385 386 static inline int tcg_global_reg_new_internal(TCGType type, int reg, 387 const char *name) 388 { 389 TCGContext *s = &tcg_ctx; 390 TCGTemp *ts; 391 int idx; 392 393 #if TCG_TARGET_REG_BITS == 32 394 if (type != TCG_TYPE_I32) 395 tcg_abort(); 396 #endif 397 if (tcg_regset_test_reg(s->reserved_regs, reg)) 398 tcg_abort(); 399 idx = s->nb_globals; 400 tcg_temp_alloc(s, s->nb_globals + 1); 401 ts = &s->temps[s->nb_globals]; 402 ts->base_type = type; 403 ts->type = type; 404 ts->fixed_reg = 1; 405 ts->reg = reg; 406 ts->name = name; 407 s->nb_globals++; 408 tcg_regset_set_reg(s->reserved_regs, reg); 409 return idx; 410 } 411 412 TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name) 413 { 414 int idx; 415 416 idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name); 417 return MAKE_TCGV_I32(idx); 418 } 419 420 TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name) 421 { 422 int idx; 423 424 idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name); 425 return MAKE_TCGV_I64(idx); 426 } 427 428 static inline int tcg_global_mem_new_internal(TCGType type, int reg, 429 intptr_t offset, 430 const char *name) 431 { 432 TCGContext *s = &tcg_ctx; 433 TCGTemp *ts; 434 int idx; 435 436 idx = s->nb_globals; 437 #if TCG_TARGET_REG_BITS == 32 438 if (type == TCG_TYPE_I64) { 439 char buf[64]; 440 tcg_temp_alloc(s, s->nb_globals + 2); 441 ts = &s->temps[s->nb_globals]; 442 ts->base_type = type; 443 ts->type = TCG_TYPE_I32; 444 ts->fixed_reg = 0; 445 ts->mem_allocated = 1; 446 ts->mem_reg = reg; 447 #ifdef HOST_WORDS_BIGENDIAN 448 ts->mem_offset = offset + 4; 449 #else 450 ts->mem_offset = offset; 451 #endif 452 pstrcpy(buf, sizeof(buf), name); 453 pstrcat(buf, sizeof(buf), "_0"); 454 ts->name = strdup(buf); 455 ts++; 456 457 ts->base_type = type; 458 ts->type = TCG_TYPE_I32; 459 ts->fixed_reg = 0; 460 ts->mem_allocated = 1; 461 ts->mem_reg = reg; 462 #ifdef HOST_WORDS_BIGENDIAN 463 ts->mem_offset = offset; 464 #else 465 ts->mem_offset = offset + 4; 466 #endif 467 pstrcpy(buf, sizeof(buf), name); 468 pstrcat(buf, sizeof(buf), "_1"); 469 ts->name = strdup(buf); 470 471 s->nb_globals += 2; 472 } else 473 #endif 474 { 475 tcg_temp_alloc(s, s->nb_globals + 1); 476 ts = &s->temps[s->nb_globals]; 477 ts->base_type = type; 478 ts->type = type; 479 ts->fixed_reg = 0; 480 ts->mem_allocated = 1; 481 ts->mem_reg = reg; 482 ts->mem_offset = offset; 483 ts->name = name; 484 s->nb_globals++; 485 } 486 return idx; 487 } 488 489 TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name) 490 { 491 int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name); 492 return MAKE_TCGV_I32(idx); 493 } 494 495 TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name) 496 { 497 int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name); 498 return MAKE_TCGV_I64(idx); 499 } 500 501 static inline int tcg_temp_new_internal(TCGType type, int temp_local) 502 { 503 TCGContext *s = &tcg_ctx; 504 TCGTemp *ts; 505 int idx, k; 506 507 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 508 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 509 if (idx < TCG_MAX_TEMPS) { 510 /* There is already an available temp with the right type. */ 511 clear_bit(idx, s->free_temps[k].l); 512 513 ts = &s->temps[idx]; 514 ts->temp_allocated = 1; 515 assert(ts->base_type == type); 516 assert(ts->temp_local == temp_local); 517 } else { 518 idx = s->nb_temps; 519 #if TCG_TARGET_REG_BITS == 32 520 if (type == TCG_TYPE_I64) { 521 tcg_temp_alloc(s, s->nb_temps + 2); 522 ts = &s->temps[s->nb_temps]; 523 ts->base_type = type; 524 ts->type = TCG_TYPE_I32; 525 ts->temp_allocated = 1; 526 ts->temp_local = temp_local; 527 ts->name = NULL; 528 ts++; 529 ts->base_type = type; 530 ts->type = TCG_TYPE_I32; 531 ts->temp_allocated = 1; 532 ts->temp_local = temp_local; 533 ts->name = NULL; 534 s->nb_temps += 2; 535 } else 536 #endif 537 { 538 tcg_temp_alloc(s, s->nb_temps + 1); 539 ts = &s->temps[s->nb_temps]; 540 ts->base_type = type; 541 ts->type = type; 542 ts->temp_allocated = 1; 543 ts->temp_local = temp_local; 544 ts->name = NULL; 545 s->nb_temps++; 546 } 547 } 548 549 #if defined(CONFIG_DEBUG_TCG) 550 s->temps_in_use++; 551 #endif 552 return idx; 553 } 554 555 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 556 { 557 int idx; 558 559 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 560 return MAKE_TCGV_I32(idx); 561 } 562 563 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 564 { 565 int idx; 566 567 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 568 return MAKE_TCGV_I64(idx); 569 } 570 571 static void tcg_temp_free_internal(int idx) 572 { 573 TCGContext *s = &tcg_ctx; 574 TCGTemp *ts; 575 int k; 576 577 #if defined(CONFIG_DEBUG_TCG) 578 s->temps_in_use--; 579 if (s->temps_in_use < 0) { 580 fprintf(stderr, "More temporaries freed than allocated!\n"); 581 } 582 #endif 583 584 assert(idx >= s->nb_globals && idx < s->nb_temps); 585 ts = &s->temps[idx]; 586 assert(ts->temp_allocated != 0); 587 ts->temp_allocated = 0; 588 589 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 590 set_bit(idx, s->free_temps[k].l); 591 } 592 593 void tcg_temp_free_i32(TCGv_i32 arg) 594 { 595 tcg_temp_free_internal(GET_TCGV_I32(arg)); 596 } 597 598 void tcg_temp_free_i64(TCGv_i64 arg) 599 { 600 tcg_temp_free_internal(GET_TCGV_I64(arg)); 601 } 602 603 TCGv_i32 tcg_const_i32(int32_t val) 604 { 605 TCGv_i32 t0; 606 t0 = tcg_temp_new_i32(); 607 tcg_gen_movi_i32(t0, val); 608 return t0; 609 } 610 611 TCGv_i64 tcg_const_i64(int64_t val) 612 { 613 TCGv_i64 t0; 614 t0 = tcg_temp_new_i64(); 615 tcg_gen_movi_i64(t0, val); 616 return t0; 617 } 618 619 TCGv_i32 tcg_const_local_i32(int32_t val) 620 { 621 TCGv_i32 t0; 622 t0 = tcg_temp_local_new_i32(); 623 tcg_gen_movi_i32(t0, val); 624 return t0; 625 } 626 627 TCGv_i64 tcg_const_local_i64(int64_t val) 628 { 629 TCGv_i64 t0; 630 t0 = tcg_temp_local_new_i64(); 631 tcg_gen_movi_i64(t0, val); 632 return t0; 633 } 634 635 #if defined(CONFIG_DEBUG_TCG) 636 void tcg_clear_temp_count(void) 637 { 638 TCGContext *s = &tcg_ctx; 639 s->temps_in_use = 0; 640 } 641 642 int tcg_check_temp_count(void) 643 { 644 TCGContext *s = &tcg_ctx; 645 if (s->temps_in_use) { 646 /* Clear the count so that we don't give another 647 * warning immediately next time around. 648 */ 649 s->temps_in_use = 0; 650 return 1; 651 } 652 return 0; 653 } 654 #endif 655 656 /* Note: we convert the 64 bit args to 32 bit and do some alignment 657 and endian swap. Maybe it would be better to do the alignment 658 and endian swap in tcg_reg_alloc_call(). */ 659 void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags, 660 int sizemask, TCGArg ret, int nargs, TCGArg *args) 661 { 662 int i; 663 int real_args; 664 int nb_rets; 665 TCGArg *nparam; 666 667 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 668 for (i = 0; i < nargs; ++i) { 669 int is_64bit = sizemask & (1 << (i+1)*2); 670 int is_signed = sizemask & (2 << (i+1)*2); 671 if (!is_64bit) { 672 TCGv_i64 temp = tcg_temp_new_i64(); 673 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 674 if (is_signed) { 675 tcg_gen_ext32s_i64(temp, orig); 676 } else { 677 tcg_gen_ext32u_i64(temp, orig); 678 } 679 args[i] = GET_TCGV_I64(temp); 680 } 681 } 682 #endif /* TCG_TARGET_EXTEND_ARGS */ 683 684 *s->gen_opc_ptr++ = INDEX_op_call; 685 nparam = s->gen_opparam_ptr++; 686 if (ret != TCG_CALL_DUMMY_ARG) { 687 #if TCG_TARGET_REG_BITS < 64 688 if (sizemask & 1) { 689 #ifdef HOST_WORDS_BIGENDIAN 690 *s->gen_opparam_ptr++ = ret + 1; 691 *s->gen_opparam_ptr++ = ret; 692 #else 693 *s->gen_opparam_ptr++ = ret; 694 *s->gen_opparam_ptr++ = ret + 1; 695 #endif 696 nb_rets = 2; 697 } else 698 #endif 699 { 700 *s->gen_opparam_ptr++ = ret; 701 nb_rets = 1; 702 } 703 } else { 704 nb_rets = 0; 705 } 706 real_args = 0; 707 for (i = 0; i < nargs; i++) { 708 #if TCG_TARGET_REG_BITS < 64 709 int is_64bit = sizemask & (1 << (i+1)*2); 710 if (is_64bit) { 711 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 712 /* some targets want aligned 64 bit args */ 713 if (real_args & 1) { 714 *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG; 715 real_args++; 716 } 717 #endif 718 /* If stack grows up, then we will be placing successive 719 arguments at lower addresses, which means we need to 720 reverse the order compared to how we would normally 721 treat either big or little-endian. For those arguments 722 that will wind up in registers, this still works for 723 HPPA (the only current STACK_GROWSUP target) since the 724 argument registers are *also* allocated in decreasing 725 order. If another such target is added, this logic may 726 have to get more complicated to differentiate between 727 stack arguments and register arguments. */ 728 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 729 *s->gen_opparam_ptr++ = args[i] + 1; 730 *s->gen_opparam_ptr++ = args[i]; 731 #else 732 *s->gen_opparam_ptr++ = args[i]; 733 *s->gen_opparam_ptr++ = args[i] + 1; 734 #endif 735 real_args += 2; 736 continue; 737 } 738 #endif /* TCG_TARGET_REG_BITS < 64 */ 739 740 *s->gen_opparam_ptr++ = args[i]; 741 real_args++; 742 } 743 *s->gen_opparam_ptr++ = GET_TCGV_PTR(func); 744 745 *s->gen_opparam_ptr++ = flags; 746 747 *nparam = (nb_rets << 16) | (real_args + 1); 748 749 /* total parameters, needed to go backward in the instruction stream */ 750 *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3; 751 752 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 753 for (i = 0; i < nargs; ++i) { 754 int is_64bit = sizemask & (1 << (i+1)*2); 755 if (!is_64bit) { 756 TCGv_i64 temp = MAKE_TCGV_I64(args[i]); 757 tcg_temp_free_i64(temp); 758 } 759 } 760 #endif /* TCG_TARGET_EXTEND_ARGS */ 761 } 762 763 #if TCG_TARGET_REG_BITS == 32 764 void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 765 int c, int right, int arith) 766 { 767 if (c == 0) { 768 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 769 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 770 } else if (c >= 32) { 771 c -= 32; 772 if (right) { 773 if (arith) { 774 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 775 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 776 } else { 777 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 778 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 779 } 780 } else { 781 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 782 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 783 } 784 } else { 785 TCGv_i32 t0, t1; 786 787 t0 = tcg_temp_new_i32(); 788 t1 = tcg_temp_new_i32(); 789 if (right) { 790 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 791 if (arith) 792 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 793 else 794 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 795 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 796 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 797 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 798 } else { 799 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 800 /* Note: ret can be the same as arg1, so we use t1 */ 801 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 802 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 803 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 804 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 805 } 806 tcg_temp_free_i32(t0); 807 tcg_temp_free_i32(t1); 808 } 809 } 810 #endif 811 812 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 813 { 814 switch (op & MO_SIZE) { 815 case MO_8: 816 op &= ~MO_BSWAP; 817 break; 818 case MO_16: 819 break; 820 case MO_32: 821 if (!is64) { 822 op &= ~MO_SIGN; 823 } 824 break; 825 case MO_64: 826 if (!is64) { 827 tcg_abort(); 828 } 829 break; 830 } 831 if (st) { 832 op &= ~MO_SIGN; 833 } 834 return op; 835 } 836 837 static const TCGOpcode old_ld_opc[8] = { 838 [MO_UB] = INDEX_op_qemu_ld8u, 839 [MO_SB] = INDEX_op_qemu_ld8s, 840 [MO_UW] = INDEX_op_qemu_ld16u, 841 [MO_SW] = INDEX_op_qemu_ld16s, 842 #if TCG_TARGET_REG_BITS == 32 843 [MO_UL] = INDEX_op_qemu_ld32, 844 [MO_SL] = INDEX_op_qemu_ld32, 845 #else 846 [MO_UL] = INDEX_op_qemu_ld32u, 847 [MO_SL] = INDEX_op_qemu_ld32s, 848 #endif 849 [MO_Q] = INDEX_op_qemu_ld64, 850 }; 851 852 static const TCGOpcode old_st_opc[4] = { 853 [MO_UB] = INDEX_op_qemu_st8, 854 [MO_UW] = INDEX_op_qemu_st16, 855 [MO_UL] = INDEX_op_qemu_st32, 856 [MO_Q] = INDEX_op_qemu_st64, 857 }; 858 859 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 860 { 861 memop = tcg_canonicalize_memop(memop, 0, 0); 862 863 if (TCG_TARGET_HAS_new_ldst) { 864 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32; 865 tcg_add_param_i32(val); 866 tcg_add_param_tl(addr); 867 *tcg_ctx.gen_opparam_ptr++ = memop; 868 *tcg_ctx.gen_opparam_ptr++ = idx; 869 return; 870 } 871 872 /* The old opcodes only support target-endian memory operations. */ 873 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 874 assert(old_ld_opc[memop & MO_SSIZE] != 0); 875 876 if (TCG_TARGET_REG_BITS == 32) { 877 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 878 tcg_add_param_i32(val); 879 tcg_add_param_tl(addr); 880 *tcg_ctx.gen_opparam_ptr++ = idx; 881 } else { 882 TCGv_i64 val64 = tcg_temp_new_i64(); 883 884 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 885 tcg_add_param_i64(val64); 886 tcg_add_param_tl(addr); 887 *tcg_ctx.gen_opparam_ptr++ = idx; 888 889 tcg_gen_trunc_i64_i32(val, val64); 890 tcg_temp_free_i64(val64); 891 } 892 } 893 894 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 895 { 896 memop = tcg_canonicalize_memop(memop, 0, 1); 897 898 if (TCG_TARGET_HAS_new_ldst) { 899 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32; 900 tcg_add_param_i32(val); 901 tcg_add_param_tl(addr); 902 *tcg_ctx.gen_opparam_ptr++ = memop; 903 *tcg_ctx.gen_opparam_ptr++ = idx; 904 return; 905 } 906 907 /* The old opcodes only support target-endian memory operations. */ 908 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 909 assert(old_st_opc[memop & MO_SIZE] != 0); 910 911 if (TCG_TARGET_REG_BITS == 32) { 912 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 913 tcg_add_param_i32(val); 914 tcg_add_param_tl(addr); 915 *tcg_ctx.gen_opparam_ptr++ = idx; 916 } else { 917 TCGv_i64 val64 = tcg_temp_new_i64(); 918 919 tcg_gen_extu_i32_i64(val64, val); 920 921 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 922 tcg_add_param_i64(val64); 923 tcg_add_param_tl(addr); 924 *tcg_ctx.gen_opparam_ptr++ = idx; 925 926 tcg_temp_free_i64(val64); 927 } 928 } 929 930 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 931 { 932 memop = tcg_canonicalize_memop(memop, 1, 0); 933 934 #if TCG_TARGET_REG_BITS == 32 935 if ((memop & MO_SIZE) < MO_64) { 936 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 937 if (memop & MO_SIGN) { 938 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 939 } else { 940 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 941 } 942 return; 943 } 944 #endif 945 946 if (TCG_TARGET_HAS_new_ldst) { 947 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64; 948 tcg_add_param_i64(val); 949 tcg_add_param_tl(addr); 950 *tcg_ctx.gen_opparam_ptr++ = memop; 951 *tcg_ctx.gen_opparam_ptr++ = idx; 952 return; 953 } 954 955 /* The old opcodes only support target-endian memory operations. */ 956 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 957 assert(old_ld_opc[memop & MO_SSIZE] != 0); 958 959 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 960 tcg_add_param_i64(val); 961 tcg_add_param_tl(addr); 962 *tcg_ctx.gen_opparam_ptr++ = idx; 963 } 964 965 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 966 { 967 memop = tcg_canonicalize_memop(memop, 1, 1); 968 969 #if TCG_TARGET_REG_BITS == 32 970 if ((memop & MO_SIZE) < MO_64) { 971 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 972 return; 973 } 974 #endif 975 976 if (TCG_TARGET_HAS_new_ldst) { 977 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64; 978 tcg_add_param_i64(val); 979 tcg_add_param_tl(addr); 980 *tcg_ctx.gen_opparam_ptr++ = memop; 981 *tcg_ctx.gen_opparam_ptr++ = idx; 982 return; 983 } 984 985 /* The old opcodes only support target-endian memory operations. */ 986 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 987 assert(old_st_opc[memop & MO_SIZE] != 0); 988 989 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 990 tcg_add_param_i64(val); 991 tcg_add_param_tl(addr); 992 *tcg_ctx.gen_opparam_ptr++ = idx; 993 } 994 995 static void tcg_reg_alloc_start(TCGContext *s) 996 { 997 int i; 998 TCGTemp *ts; 999 for(i = 0; i < s->nb_globals; i++) { 1000 ts = &s->temps[i]; 1001 if (ts->fixed_reg) { 1002 ts->val_type = TEMP_VAL_REG; 1003 } else { 1004 ts->val_type = TEMP_VAL_MEM; 1005 } 1006 } 1007 for(i = s->nb_globals; i < s->nb_temps; i++) { 1008 ts = &s->temps[i]; 1009 if (ts->temp_local) { 1010 ts->val_type = TEMP_VAL_MEM; 1011 } else { 1012 ts->val_type = TEMP_VAL_DEAD; 1013 } 1014 ts->mem_allocated = 0; 1015 ts->fixed_reg = 0; 1016 } 1017 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1018 s->reg_to_temp[i] = -1; 1019 } 1020 } 1021 1022 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size, 1023 int idx) 1024 { 1025 TCGTemp *ts; 1026 1027 assert(idx >= 0 && idx < s->nb_temps); 1028 ts = &s->temps[idx]; 1029 if (idx < s->nb_globals) { 1030 pstrcpy(buf, buf_size, ts->name); 1031 } else { 1032 if (ts->temp_local) 1033 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1034 else 1035 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1036 } 1037 return buf; 1038 } 1039 1040 char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg) 1041 { 1042 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg)); 1043 } 1044 1045 char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg) 1046 { 1047 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg)); 1048 } 1049 1050 /* Find helper name. */ 1051 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 1052 { 1053 const char *ret = NULL; 1054 if (s->helpers) { 1055 ret = g_hash_table_lookup(s->helpers, (gpointer)val); 1056 } 1057 return ret; 1058 } 1059 1060 static const char * const cond_name[] = 1061 { 1062 [TCG_COND_NEVER] = "never", 1063 [TCG_COND_ALWAYS] = "always", 1064 [TCG_COND_EQ] = "eq", 1065 [TCG_COND_NE] = "ne", 1066 [TCG_COND_LT] = "lt", 1067 [TCG_COND_GE] = "ge", 1068 [TCG_COND_LE] = "le", 1069 [TCG_COND_GT] = "gt", 1070 [TCG_COND_LTU] = "ltu", 1071 [TCG_COND_GEU] = "geu", 1072 [TCG_COND_LEU] = "leu", 1073 [TCG_COND_GTU] = "gtu" 1074 }; 1075 1076 static const char * const ldst_name[] = 1077 { 1078 [MO_UB] = "ub", 1079 [MO_SB] = "sb", 1080 [MO_LEUW] = "leuw", 1081 [MO_LESW] = "lesw", 1082 [MO_LEUL] = "leul", 1083 [MO_LESL] = "lesl", 1084 [MO_LEQ] = "leq", 1085 [MO_BEUW] = "beuw", 1086 [MO_BESW] = "besw", 1087 [MO_BEUL] = "beul", 1088 [MO_BESL] = "besl", 1089 [MO_BEQ] = "beq", 1090 }; 1091 1092 void tcg_dump_ops(TCGContext *s) 1093 { 1094 const uint16_t *opc_ptr; 1095 const TCGArg *args; 1096 TCGArg arg; 1097 TCGOpcode c; 1098 int i, k, nb_oargs, nb_iargs, nb_cargs, first_insn; 1099 const TCGOpDef *def; 1100 char buf[128]; 1101 1102 first_insn = 1; 1103 opc_ptr = s->gen_opc_buf; 1104 args = s->gen_opparam_buf; 1105 while (opc_ptr < s->gen_opc_ptr) { 1106 c = *opc_ptr++; 1107 def = &tcg_op_defs[c]; 1108 if (c == INDEX_op_debug_insn_start) { 1109 uint64_t pc; 1110 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1111 pc = ((uint64_t)args[1] << 32) | args[0]; 1112 #else 1113 pc = args[0]; 1114 #endif 1115 if (!first_insn) { 1116 qemu_log("\n"); 1117 } 1118 qemu_log(" ---- 0x%" PRIx64, pc); 1119 first_insn = 0; 1120 nb_oargs = def->nb_oargs; 1121 nb_iargs = def->nb_iargs; 1122 nb_cargs = def->nb_cargs; 1123 } else if (c == INDEX_op_call) { 1124 TCGArg arg; 1125 1126 /* variable number of arguments */ 1127 arg = *args++; 1128 nb_oargs = arg >> 16; 1129 nb_iargs = arg & 0xffff; 1130 nb_cargs = def->nb_cargs; 1131 1132 qemu_log(" %s ", def->name); 1133 1134 /* function name */ 1135 qemu_log("%s", 1136 tcg_get_arg_str_idx(s, buf, sizeof(buf), 1137 args[nb_oargs + nb_iargs - 1])); 1138 /* flags */ 1139 qemu_log(",$0x%" TCG_PRIlx, args[nb_oargs + nb_iargs]); 1140 /* nb out args */ 1141 qemu_log(",$%d", nb_oargs); 1142 for(i = 0; i < nb_oargs; i++) { 1143 qemu_log(","); 1144 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1145 args[i])); 1146 } 1147 for(i = 0; i < (nb_iargs - 1); i++) { 1148 qemu_log(","); 1149 if (args[nb_oargs + i] == TCG_CALL_DUMMY_ARG) { 1150 qemu_log("<dummy>"); 1151 } else { 1152 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1153 args[nb_oargs + i])); 1154 } 1155 } 1156 } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) { 1157 tcg_target_ulong val; 1158 const char *name; 1159 1160 nb_oargs = def->nb_oargs; 1161 nb_iargs = def->nb_iargs; 1162 nb_cargs = def->nb_cargs; 1163 qemu_log(" %s %s,$", def->name, 1164 tcg_get_arg_str_idx(s, buf, sizeof(buf), args[0])); 1165 val = args[1]; 1166 name = tcg_find_helper(s, val); 1167 if (name) { 1168 qemu_log("%s", name); 1169 } else { 1170 if (c == INDEX_op_movi_i32) { 1171 qemu_log("0x%x", (uint32_t)val); 1172 } else { 1173 qemu_log("0x%" PRIx64 , (uint64_t)val); 1174 } 1175 } 1176 } else { 1177 qemu_log(" %s ", def->name); 1178 if (c == INDEX_op_nopn) { 1179 /* variable number of arguments */ 1180 nb_cargs = *args; 1181 nb_oargs = 0; 1182 nb_iargs = 0; 1183 } else { 1184 nb_oargs = def->nb_oargs; 1185 nb_iargs = def->nb_iargs; 1186 nb_cargs = def->nb_cargs; 1187 } 1188 1189 k = 0; 1190 for(i = 0; i < nb_oargs; i++) { 1191 if (k != 0) { 1192 qemu_log(","); 1193 } 1194 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1195 args[k++])); 1196 } 1197 for(i = 0; i < nb_iargs; i++) { 1198 if (k != 0) { 1199 qemu_log(","); 1200 } 1201 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1202 args[k++])); 1203 } 1204 switch (c) { 1205 case INDEX_op_brcond_i32: 1206 case INDEX_op_setcond_i32: 1207 case INDEX_op_movcond_i32: 1208 case INDEX_op_brcond2_i32: 1209 case INDEX_op_setcond2_i32: 1210 case INDEX_op_brcond_i64: 1211 case INDEX_op_setcond_i64: 1212 case INDEX_op_movcond_i64: 1213 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) { 1214 qemu_log(",%s", cond_name[args[k++]]); 1215 } else { 1216 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1217 } 1218 i = 1; 1219 break; 1220 case INDEX_op_qemu_ld_i32: 1221 case INDEX_op_qemu_st_i32: 1222 case INDEX_op_qemu_ld_i64: 1223 case INDEX_op_qemu_st_i64: 1224 if (args[k] < ARRAY_SIZE(ldst_name) && ldst_name[args[k]]) { 1225 qemu_log(",%s", ldst_name[args[k++]]); 1226 } else { 1227 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1228 } 1229 i = 1; 1230 break; 1231 default: 1232 i = 0; 1233 break; 1234 } 1235 for(; i < nb_cargs; i++) { 1236 if (k != 0) { 1237 qemu_log(","); 1238 } 1239 arg = args[k++]; 1240 qemu_log("$0x%" TCG_PRIlx, arg); 1241 } 1242 } 1243 qemu_log("\n"); 1244 args += nb_iargs + nb_oargs + nb_cargs; 1245 } 1246 } 1247 1248 /* we give more priority to constraints with less registers */ 1249 static int get_constraint_priority(const TCGOpDef *def, int k) 1250 { 1251 const TCGArgConstraint *arg_ct; 1252 1253 int i, n; 1254 arg_ct = &def->args_ct[k]; 1255 if (arg_ct->ct & TCG_CT_ALIAS) { 1256 /* an alias is equivalent to a single register */ 1257 n = 1; 1258 } else { 1259 if (!(arg_ct->ct & TCG_CT_REG)) 1260 return 0; 1261 n = 0; 1262 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1263 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1264 n++; 1265 } 1266 } 1267 return TCG_TARGET_NB_REGS - n + 1; 1268 } 1269 1270 /* sort from highest priority to lowest */ 1271 static void sort_constraints(TCGOpDef *def, int start, int n) 1272 { 1273 int i, j, p1, p2, tmp; 1274 1275 for(i = 0; i < n; i++) 1276 def->sorted_args[start + i] = start + i; 1277 if (n <= 1) 1278 return; 1279 for(i = 0; i < n - 1; i++) { 1280 for(j = i + 1; j < n; j++) { 1281 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1282 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1283 if (p1 < p2) { 1284 tmp = def->sorted_args[start + i]; 1285 def->sorted_args[start + i] = def->sorted_args[start + j]; 1286 def->sorted_args[start + j] = tmp; 1287 } 1288 } 1289 } 1290 } 1291 1292 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs) 1293 { 1294 TCGOpcode op; 1295 TCGOpDef *def; 1296 const char *ct_str; 1297 int i, nb_args; 1298 1299 for(;;) { 1300 if (tdefs->op == (TCGOpcode)-1) 1301 break; 1302 op = tdefs->op; 1303 assert((unsigned)op < NB_OPS); 1304 def = &tcg_op_defs[op]; 1305 #if defined(CONFIG_DEBUG_TCG) 1306 /* Duplicate entry in op definitions? */ 1307 assert(!def->used); 1308 def->used = 1; 1309 #endif 1310 nb_args = def->nb_iargs + def->nb_oargs; 1311 for(i = 0; i < nb_args; i++) { 1312 ct_str = tdefs->args_ct_str[i]; 1313 /* Incomplete TCGTargetOpDef entry? */ 1314 assert(ct_str != NULL); 1315 tcg_regset_clear(def->args_ct[i].u.regs); 1316 def->args_ct[i].ct = 0; 1317 if (ct_str[0] >= '0' && ct_str[0] <= '9') { 1318 int oarg; 1319 oarg = ct_str[0] - '0'; 1320 assert(oarg < def->nb_oargs); 1321 assert(def->args_ct[oarg].ct & TCG_CT_REG); 1322 /* TCG_CT_ALIAS is for the output arguments. The input 1323 argument is tagged with TCG_CT_IALIAS. */ 1324 def->args_ct[i] = def->args_ct[oarg]; 1325 def->args_ct[oarg].ct = TCG_CT_ALIAS; 1326 def->args_ct[oarg].alias_index = i; 1327 def->args_ct[i].ct |= TCG_CT_IALIAS; 1328 def->args_ct[i].alias_index = oarg; 1329 } else { 1330 for(;;) { 1331 if (*ct_str == '\0') 1332 break; 1333 switch(*ct_str) { 1334 case 'i': 1335 def->args_ct[i].ct |= TCG_CT_CONST; 1336 ct_str++; 1337 break; 1338 default: 1339 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) { 1340 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n", 1341 ct_str, i, def->name); 1342 exit(1); 1343 } 1344 } 1345 } 1346 } 1347 } 1348 1349 /* TCGTargetOpDef entry with too much information? */ 1350 assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1351 1352 /* sort the constraints (XXX: this is just an heuristic) */ 1353 sort_constraints(def, 0, def->nb_oargs); 1354 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1355 1356 #if 0 1357 { 1358 int i; 1359 1360 printf("%s: sorted=", def->name); 1361 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++) 1362 printf(" %d", def->sorted_args[i]); 1363 printf("\n"); 1364 } 1365 #endif 1366 tdefs++; 1367 } 1368 1369 #if defined(CONFIG_DEBUG_TCG) 1370 i = 0; 1371 for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) { 1372 const TCGOpDef *def = &tcg_op_defs[op]; 1373 if (def->flags & TCG_OPF_NOT_PRESENT) { 1374 /* Wrong entry in op definitions? */ 1375 if (def->used) { 1376 fprintf(stderr, "Invalid op definition for %s\n", def->name); 1377 i = 1; 1378 } 1379 } else { 1380 /* Missing entry in op definitions? */ 1381 if (!def->used) { 1382 fprintf(stderr, "Missing op definition for %s\n", def->name); 1383 i = 1; 1384 } 1385 } 1386 } 1387 if (i == 1) { 1388 tcg_abort(); 1389 } 1390 #endif 1391 } 1392 1393 #ifdef USE_LIVENESS_ANALYSIS 1394 1395 /* set a nop for an operation using 'nb_args' */ 1396 static inline void tcg_set_nop(TCGContext *s, uint16_t *opc_ptr, 1397 TCGArg *args, int nb_args) 1398 { 1399 if (nb_args == 0) { 1400 *opc_ptr = INDEX_op_nop; 1401 } else { 1402 *opc_ptr = INDEX_op_nopn; 1403 args[0] = nb_args; 1404 args[nb_args - 1] = nb_args; 1405 } 1406 } 1407 1408 /* liveness analysis: end of function: all temps are dead, and globals 1409 should be in memory. */ 1410 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps, 1411 uint8_t *mem_temps) 1412 { 1413 memset(dead_temps, 1, s->nb_temps); 1414 memset(mem_temps, 1, s->nb_globals); 1415 memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals); 1416 } 1417 1418 /* liveness analysis: end of basic block: all temps are dead, globals 1419 and local temps should be in memory. */ 1420 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps, 1421 uint8_t *mem_temps) 1422 { 1423 int i; 1424 1425 memset(dead_temps, 1, s->nb_temps); 1426 memset(mem_temps, 1, s->nb_globals); 1427 for(i = s->nb_globals; i < s->nb_temps; i++) { 1428 mem_temps[i] = s->temps[i].temp_local; 1429 } 1430 } 1431 1432 /* Liveness analysis : update the opc_dead_args array to tell if a 1433 given input arguments is dead. Instructions updating dead 1434 temporaries are removed. */ 1435 static void tcg_liveness_analysis(TCGContext *s) 1436 { 1437 int i, op_index, nb_args, nb_iargs, nb_oargs, arg, nb_ops; 1438 TCGOpcode op, op_new, op_new2; 1439 TCGArg *args; 1440 const TCGOpDef *def; 1441 uint8_t *dead_temps, *mem_temps; 1442 uint16_t dead_args; 1443 uint8_t sync_args; 1444 bool have_op_new2; 1445 1446 s->gen_opc_ptr++; /* skip end */ 1447 1448 nb_ops = s->gen_opc_ptr - s->gen_opc_buf; 1449 1450 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1451 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1452 1453 dead_temps = tcg_malloc(s->nb_temps); 1454 mem_temps = tcg_malloc(s->nb_temps); 1455 tcg_la_func_end(s, dead_temps, mem_temps); 1456 1457 args = s->gen_opparam_ptr; 1458 op_index = nb_ops - 1; 1459 while (op_index >= 0) { 1460 op = s->gen_opc_buf[op_index]; 1461 def = &tcg_op_defs[op]; 1462 switch(op) { 1463 case INDEX_op_call: 1464 { 1465 int call_flags; 1466 1467 nb_args = args[-1]; 1468 args -= nb_args; 1469 nb_iargs = args[0] & 0xffff; 1470 nb_oargs = args[0] >> 16; 1471 args++; 1472 call_flags = args[nb_oargs + nb_iargs]; 1473 1474 /* pure functions can be removed if their result is not 1475 used */ 1476 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 1477 for(i = 0; i < nb_oargs; i++) { 1478 arg = args[i]; 1479 if (!dead_temps[arg] || mem_temps[arg]) { 1480 goto do_not_remove_call; 1481 } 1482 } 1483 tcg_set_nop(s, s->gen_opc_buf + op_index, 1484 args - 1, nb_args); 1485 } else { 1486 do_not_remove_call: 1487 1488 /* output args are dead */ 1489 dead_args = 0; 1490 sync_args = 0; 1491 for(i = 0; i < nb_oargs; i++) { 1492 arg = args[i]; 1493 if (dead_temps[arg]) { 1494 dead_args |= (1 << i); 1495 } 1496 if (mem_temps[arg]) { 1497 sync_args |= (1 << i); 1498 } 1499 dead_temps[arg] = 1; 1500 mem_temps[arg] = 0; 1501 } 1502 1503 if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 1504 /* globals should be synced to memory */ 1505 memset(mem_temps, 1, s->nb_globals); 1506 } 1507 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 1508 TCG_CALL_NO_READ_GLOBALS))) { 1509 /* globals should go back to memory */ 1510 memset(dead_temps, 1, s->nb_globals); 1511 } 1512 1513 /* input args are live */ 1514 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1515 arg = args[i]; 1516 if (arg != TCG_CALL_DUMMY_ARG) { 1517 if (dead_temps[arg]) { 1518 dead_args |= (1 << i); 1519 } 1520 dead_temps[arg] = 0; 1521 } 1522 } 1523 s->op_dead_args[op_index] = dead_args; 1524 s->op_sync_args[op_index] = sync_args; 1525 } 1526 args--; 1527 } 1528 break; 1529 case INDEX_op_debug_insn_start: 1530 args -= def->nb_args; 1531 break; 1532 case INDEX_op_nopn: 1533 nb_args = args[-1]; 1534 args -= nb_args; 1535 break; 1536 case INDEX_op_discard: 1537 args--; 1538 /* mark the temporary as dead */ 1539 dead_temps[args[0]] = 1; 1540 mem_temps[args[0]] = 0; 1541 break; 1542 case INDEX_op_end: 1543 break; 1544 1545 case INDEX_op_add2_i32: 1546 op_new = INDEX_op_add_i32; 1547 goto do_addsub2; 1548 case INDEX_op_sub2_i32: 1549 op_new = INDEX_op_sub_i32; 1550 goto do_addsub2; 1551 case INDEX_op_add2_i64: 1552 op_new = INDEX_op_add_i64; 1553 goto do_addsub2; 1554 case INDEX_op_sub2_i64: 1555 op_new = INDEX_op_sub_i64; 1556 do_addsub2: 1557 args -= 6; 1558 nb_iargs = 4; 1559 nb_oargs = 2; 1560 /* Test if the high part of the operation is dead, but not 1561 the low part. The result can be optimized to a simple 1562 add or sub. This happens often for x86_64 guest when the 1563 cpu mode is set to 32 bit. */ 1564 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1565 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1566 goto do_remove; 1567 } 1568 /* Create the single operation plus nop. */ 1569 s->gen_opc_buf[op_index] = op = op_new; 1570 args[1] = args[2]; 1571 args[2] = args[4]; 1572 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop); 1573 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 3); 1574 /* Fall through and mark the single-word operation live. */ 1575 nb_iargs = 2; 1576 nb_oargs = 1; 1577 } 1578 goto do_not_remove; 1579 1580 case INDEX_op_mulu2_i32: 1581 op_new = INDEX_op_mul_i32; 1582 op_new2 = INDEX_op_muluh_i32; 1583 have_op_new2 = TCG_TARGET_HAS_muluh_i32; 1584 goto do_mul2; 1585 case INDEX_op_muls2_i32: 1586 op_new = INDEX_op_mul_i32; 1587 op_new2 = INDEX_op_mulsh_i32; 1588 have_op_new2 = TCG_TARGET_HAS_mulsh_i32; 1589 goto do_mul2; 1590 case INDEX_op_mulu2_i64: 1591 op_new = INDEX_op_mul_i64; 1592 op_new2 = INDEX_op_muluh_i64; 1593 have_op_new2 = TCG_TARGET_HAS_muluh_i64; 1594 goto do_mul2; 1595 case INDEX_op_muls2_i64: 1596 op_new = INDEX_op_mul_i64; 1597 op_new2 = INDEX_op_mulsh_i64; 1598 have_op_new2 = TCG_TARGET_HAS_mulsh_i64; 1599 goto do_mul2; 1600 do_mul2: 1601 args -= 4; 1602 nb_iargs = 2; 1603 nb_oargs = 2; 1604 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1605 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1606 /* Both parts of the operation are dead. */ 1607 goto do_remove; 1608 } 1609 /* The high part of the operation is dead; generate the low. */ 1610 s->gen_opc_buf[op_index] = op = op_new; 1611 args[1] = args[2]; 1612 args[2] = args[3]; 1613 } else if (have_op_new2 && dead_temps[args[0]] 1614 && !mem_temps[args[0]]) { 1615 /* The low part of the operation is dead; generate the high. */ 1616 s->gen_opc_buf[op_index] = op = op_new2; 1617 args[0] = args[1]; 1618 args[1] = args[2]; 1619 args[2] = args[3]; 1620 } else { 1621 goto do_not_remove; 1622 } 1623 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop); 1624 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1); 1625 /* Mark the single-word operation live. */ 1626 nb_oargs = 1; 1627 goto do_not_remove; 1628 1629 default: 1630 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 1631 args -= def->nb_args; 1632 nb_iargs = def->nb_iargs; 1633 nb_oargs = def->nb_oargs; 1634 1635 /* Test if the operation can be removed because all 1636 its outputs are dead. We assume that nb_oargs == 0 1637 implies side effects */ 1638 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 1639 for(i = 0; i < nb_oargs; i++) { 1640 arg = args[i]; 1641 if (!dead_temps[arg] || mem_temps[arg]) { 1642 goto do_not_remove; 1643 } 1644 } 1645 do_remove: 1646 tcg_set_nop(s, s->gen_opc_buf + op_index, args, def->nb_args); 1647 #ifdef CONFIG_PROFILER 1648 s->del_op_count++; 1649 #endif 1650 } else { 1651 do_not_remove: 1652 1653 /* output args are dead */ 1654 dead_args = 0; 1655 sync_args = 0; 1656 for(i = 0; i < nb_oargs; i++) { 1657 arg = args[i]; 1658 if (dead_temps[arg]) { 1659 dead_args |= (1 << i); 1660 } 1661 if (mem_temps[arg]) { 1662 sync_args |= (1 << i); 1663 } 1664 dead_temps[arg] = 1; 1665 mem_temps[arg] = 0; 1666 } 1667 1668 /* if end of basic block, update */ 1669 if (def->flags & TCG_OPF_BB_END) { 1670 tcg_la_bb_end(s, dead_temps, mem_temps); 1671 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 1672 /* globals should be synced to memory */ 1673 memset(mem_temps, 1, s->nb_globals); 1674 } 1675 1676 /* input args are live */ 1677 for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1678 arg = args[i]; 1679 if (dead_temps[arg]) { 1680 dead_args |= (1 << i); 1681 } 1682 dead_temps[arg] = 0; 1683 } 1684 s->op_dead_args[op_index] = dead_args; 1685 s->op_sync_args[op_index] = sync_args; 1686 } 1687 break; 1688 } 1689 op_index--; 1690 } 1691 1692 if (args != s->gen_opparam_buf) { 1693 tcg_abort(); 1694 } 1695 } 1696 #else 1697 /* dummy liveness analysis */ 1698 static void tcg_liveness_analysis(TCGContext *s) 1699 { 1700 int nb_ops; 1701 nb_ops = s->gen_opc_ptr - s->gen_opc_buf; 1702 1703 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1704 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t)); 1705 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1706 memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t)); 1707 } 1708 #endif 1709 1710 #ifndef NDEBUG 1711 static void dump_regs(TCGContext *s) 1712 { 1713 TCGTemp *ts; 1714 int i; 1715 char buf[64]; 1716 1717 for(i = 0; i < s->nb_temps; i++) { 1718 ts = &s->temps[i]; 1719 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i)); 1720 switch(ts->val_type) { 1721 case TEMP_VAL_REG: 1722 printf("%s", tcg_target_reg_names[ts->reg]); 1723 break; 1724 case TEMP_VAL_MEM: 1725 printf("%d(%s)", (int)ts->mem_offset, tcg_target_reg_names[ts->mem_reg]); 1726 break; 1727 case TEMP_VAL_CONST: 1728 printf("$0x%" TCG_PRIlx, ts->val); 1729 break; 1730 case TEMP_VAL_DEAD: 1731 printf("D"); 1732 break; 1733 default: 1734 printf("???"); 1735 break; 1736 } 1737 printf("\n"); 1738 } 1739 1740 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1741 if (s->reg_to_temp[i] >= 0) { 1742 printf("%s: %s\n", 1743 tcg_target_reg_names[i], 1744 tcg_get_arg_str_idx(s, buf, sizeof(buf), s->reg_to_temp[i])); 1745 } 1746 } 1747 } 1748 1749 static void check_regs(TCGContext *s) 1750 { 1751 int reg, k; 1752 TCGTemp *ts; 1753 char buf[64]; 1754 1755 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 1756 k = s->reg_to_temp[reg]; 1757 if (k >= 0) { 1758 ts = &s->temps[k]; 1759 if (ts->val_type != TEMP_VAL_REG || 1760 ts->reg != reg) { 1761 printf("Inconsistency for register %s:\n", 1762 tcg_target_reg_names[reg]); 1763 goto fail; 1764 } 1765 } 1766 } 1767 for(k = 0; k < s->nb_temps; k++) { 1768 ts = &s->temps[k]; 1769 if (ts->val_type == TEMP_VAL_REG && 1770 !ts->fixed_reg && 1771 s->reg_to_temp[ts->reg] != k) { 1772 printf("Inconsistency for temp %s:\n", 1773 tcg_get_arg_str_idx(s, buf, sizeof(buf), k)); 1774 fail: 1775 printf("reg state:\n"); 1776 dump_regs(s); 1777 tcg_abort(); 1778 } 1779 } 1780 } 1781 #endif 1782 1783 static void temp_allocate_frame(TCGContext *s, int temp) 1784 { 1785 TCGTemp *ts; 1786 ts = &s->temps[temp]; 1787 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 1788 /* Sparc64 stack is accessed with offset of 2047 */ 1789 s->current_frame_offset = (s->current_frame_offset + 1790 (tcg_target_long)sizeof(tcg_target_long) - 1) & 1791 ~(sizeof(tcg_target_long) - 1); 1792 #endif 1793 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 1794 s->frame_end) { 1795 tcg_abort(); 1796 } 1797 ts->mem_offset = s->current_frame_offset; 1798 ts->mem_reg = s->frame_reg; 1799 ts->mem_allocated = 1; 1800 s->current_frame_offset += sizeof(tcg_target_long); 1801 } 1802 1803 /* sync register 'reg' by saving it to the corresponding temporary */ 1804 static inline void tcg_reg_sync(TCGContext *s, int reg) 1805 { 1806 TCGTemp *ts; 1807 int temp; 1808 1809 temp = s->reg_to_temp[reg]; 1810 ts = &s->temps[temp]; 1811 assert(ts->val_type == TEMP_VAL_REG); 1812 if (!ts->mem_coherent && !ts->fixed_reg) { 1813 if (!ts->mem_allocated) { 1814 temp_allocate_frame(s, temp); 1815 } 1816 tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1817 } 1818 ts->mem_coherent = 1; 1819 } 1820 1821 /* free register 'reg' by spilling the corresponding temporary if necessary */ 1822 static void tcg_reg_free(TCGContext *s, int reg) 1823 { 1824 int temp; 1825 1826 temp = s->reg_to_temp[reg]; 1827 if (temp != -1) { 1828 tcg_reg_sync(s, reg); 1829 s->temps[temp].val_type = TEMP_VAL_MEM; 1830 s->reg_to_temp[reg] = -1; 1831 } 1832 } 1833 1834 /* Allocate a register belonging to reg1 & ~reg2 */ 1835 static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2) 1836 { 1837 int i, reg; 1838 TCGRegSet reg_ct; 1839 1840 tcg_regset_andnot(reg_ct, reg1, reg2); 1841 1842 /* first try free registers */ 1843 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1844 reg = tcg_target_reg_alloc_order[i]; 1845 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == -1) 1846 return reg; 1847 } 1848 1849 /* XXX: do better spill choice */ 1850 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1851 reg = tcg_target_reg_alloc_order[i]; 1852 if (tcg_regset_test_reg(reg_ct, reg)) { 1853 tcg_reg_free(s, reg); 1854 return reg; 1855 } 1856 } 1857 1858 tcg_abort(); 1859 } 1860 1861 /* mark a temporary as dead. */ 1862 static inline void temp_dead(TCGContext *s, int temp) 1863 { 1864 TCGTemp *ts; 1865 1866 ts = &s->temps[temp]; 1867 if (!ts->fixed_reg) { 1868 if (ts->val_type == TEMP_VAL_REG) { 1869 s->reg_to_temp[ts->reg] = -1; 1870 } 1871 if (temp < s->nb_globals || ts->temp_local) { 1872 ts->val_type = TEMP_VAL_MEM; 1873 } else { 1874 ts->val_type = TEMP_VAL_DEAD; 1875 } 1876 } 1877 } 1878 1879 /* sync a temporary to memory. 'allocated_regs' is used in case a 1880 temporary registers needs to be allocated to store a constant. */ 1881 static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs) 1882 { 1883 TCGTemp *ts; 1884 1885 ts = &s->temps[temp]; 1886 if (!ts->fixed_reg) { 1887 switch(ts->val_type) { 1888 case TEMP_VAL_CONST: 1889 ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 1890 allocated_regs); 1891 ts->val_type = TEMP_VAL_REG; 1892 s->reg_to_temp[ts->reg] = temp; 1893 ts->mem_coherent = 0; 1894 tcg_out_movi(s, ts->type, ts->reg, ts->val); 1895 /* fallthrough*/ 1896 case TEMP_VAL_REG: 1897 tcg_reg_sync(s, ts->reg); 1898 break; 1899 case TEMP_VAL_DEAD: 1900 case TEMP_VAL_MEM: 1901 break; 1902 default: 1903 tcg_abort(); 1904 } 1905 } 1906 } 1907 1908 /* save a temporary to memory. 'allocated_regs' is used in case a 1909 temporary registers needs to be allocated to store a constant. */ 1910 static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs) 1911 { 1912 #ifdef USE_LIVENESS_ANALYSIS 1913 /* The liveness analysis already ensures that globals are back 1914 in memory. Keep an assert for safety. */ 1915 assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg); 1916 #else 1917 temp_sync(s, temp, allocated_regs); 1918 temp_dead(s, temp); 1919 #endif 1920 } 1921 1922 /* save globals to their canonical location and assume they can be 1923 modified be the following code. 'allocated_regs' is used in case a 1924 temporary registers needs to be allocated to store a constant. */ 1925 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 1926 { 1927 int i; 1928 1929 for(i = 0; i < s->nb_globals; i++) { 1930 temp_save(s, i, allocated_regs); 1931 } 1932 } 1933 1934 /* sync globals to their canonical location and assume they can be 1935 read by the following code. 'allocated_regs' is used in case a 1936 temporary registers needs to be allocated to store a constant. */ 1937 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 1938 { 1939 int i; 1940 1941 for (i = 0; i < s->nb_globals; i++) { 1942 #ifdef USE_LIVENESS_ANALYSIS 1943 assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg || 1944 s->temps[i].mem_coherent); 1945 #else 1946 temp_sync(s, i, allocated_regs); 1947 #endif 1948 } 1949 } 1950 1951 /* at the end of a basic block, we assume all temporaries are dead and 1952 all globals are stored at their canonical location. */ 1953 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 1954 { 1955 TCGTemp *ts; 1956 int i; 1957 1958 for(i = s->nb_globals; i < s->nb_temps; i++) { 1959 ts = &s->temps[i]; 1960 if (ts->temp_local) { 1961 temp_save(s, i, allocated_regs); 1962 } else { 1963 #ifdef USE_LIVENESS_ANALYSIS 1964 /* The liveness analysis already ensures that temps are dead. 1965 Keep an assert for safety. */ 1966 assert(ts->val_type == TEMP_VAL_DEAD); 1967 #else 1968 temp_dead(s, i); 1969 #endif 1970 } 1971 } 1972 1973 save_globals(s, allocated_regs); 1974 } 1975 1976 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1) 1977 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1) 1978 1979 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args, 1980 uint16_t dead_args, uint8_t sync_args) 1981 { 1982 TCGTemp *ots; 1983 tcg_target_ulong val; 1984 1985 ots = &s->temps[args[0]]; 1986 val = args[1]; 1987 1988 if (ots->fixed_reg) { 1989 /* for fixed registers, we do not do any constant 1990 propagation */ 1991 tcg_out_movi(s, ots->type, ots->reg, val); 1992 } else { 1993 /* The movi is not explicitly generated here */ 1994 if (ots->val_type == TEMP_VAL_REG) 1995 s->reg_to_temp[ots->reg] = -1; 1996 ots->val_type = TEMP_VAL_CONST; 1997 ots->val = val; 1998 } 1999 if (NEED_SYNC_ARG(0)) { 2000 temp_sync(s, args[0], s->reserved_regs); 2001 } 2002 if (IS_DEAD_ARG(0)) { 2003 temp_dead(s, args[0]); 2004 } 2005 } 2006 2007 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, 2008 const TCGArg *args, uint16_t dead_args, 2009 uint8_t sync_args) 2010 { 2011 TCGRegSet allocated_regs; 2012 TCGTemp *ts, *ots; 2013 const TCGArgConstraint *arg_ct, *oarg_ct; 2014 2015 tcg_regset_set(allocated_regs, s->reserved_regs); 2016 ots = &s->temps[args[0]]; 2017 ts = &s->temps[args[1]]; 2018 oarg_ct = &def->args_ct[0]; 2019 arg_ct = &def->args_ct[1]; 2020 2021 /* If the source value is not in a register, and we're going to be 2022 forced to have it in a register in order to perform the copy, 2023 then copy the SOURCE value into its own register first. That way 2024 we don't have to reload SOURCE the next time it is used. */ 2025 if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG) 2026 || ts->val_type == TEMP_VAL_MEM) { 2027 ts->reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2028 if (ts->val_type == TEMP_VAL_MEM) { 2029 tcg_out_ld(s, ts->type, ts->reg, ts->mem_reg, ts->mem_offset); 2030 ts->mem_coherent = 1; 2031 } else if (ts->val_type == TEMP_VAL_CONST) { 2032 tcg_out_movi(s, ts->type, ts->reg, ts->val); 2033 } 2034 s->reg_to_temp[ts->reg] = args[1]; 2035 ts->val_type = TEMP_VAL_REG; 2036 } 2037 2038 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 2039 /* mov to a non-saved dead register makes no sense (even with 2040 liveness analysis disabled). */ 2041 assert(NEED_SYNC_ARG(0)); 2042 /* The code above should have moved the temp to a register. */ 2043 assert(ts->val_type == TEMP_VAL_REG); 2044 if (!ots->mem_allocated) { 2045 temp_allocate_frame(s, args[0]); 2046 } 2047 tcg_out_st(s, ots->type, ts->reg, ots->mem_reg, ots->mem_offset); 2048 if (IS_DEAD_ARG(1)) { 2049 temp_dead(s, args[1]); 2050 } 2051 temp_dead(s, args[0]); 2052 } else if (ts->val_type == TEMP_VAL_CONST) { 2053 /* propagate constant */ 2054 if (ots->val_type == TEMP_VAL_REG) { 2055 s->reg_to_temp[ots->reg] = -1; 2056 } 2057 ots->val_type = TEMP_VAL_CONST; 2058 ots->val = ts->val; 2059 } else { 2060 /* The code in the first if block should have moved the 2061 temp to a register. */ 2062 assert(ts->val_type == TEMP_VAL_REG); 2063 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 2064 /* the mov can be suppressed */ 2065 if (ots->val_type == TEMP_VAL_REG) { 2066 s->reg_to_temp[ots->reg] = -1; 2067 } 2068 ots->reg = ts->reg; 2069 temp_dead(s, args[1]); 2070 } else { 2071 if (ots->val_type != TEMP_VAL_REG) { 2072 /* When allocating a new register, make sure to not spill the 2073 input one. */ 2074 tcg_regset_set_reg(allocated_regs, ts->reg); 2075 ots->reg = tcg_reg_alloc(s, oarg_ct->u.regs, allocated_regs); 2076 } 2077 tcg_out_mov(s, ots->type, ots->reg, ts->reg); 2078 } 2079 ots->val_type = TEMP_VAL_REG; 2080 ots->mem_coherent = 0; 2081 s->reg_to_temp[ots->reg] = args[0]; 2082 if (NEED_SYNC_ARG(0)) { 2083 tcg_reg_sync(s, ots->reg); 2084 } 2085 } 2086 } 2087 2088 static void tcg_reg_alloc_op(TCGContext *s, 2089 const TCGOpDef *def, TCGOpcode opc, 2090 const TCGArg *args, uint16_t dead_args, 2091 uint8_t sync_args) 2092 { 2093 TCGRegSet allocated_regs; 2094 int i, k, nb_iargs, nb_oargs, reg; 2095 TCGArg arg; 2096 const TCGArgConstraint *arg_ct; 2097 TCGTemp *ts; 2098 TCGArg new_args[TCG_MAX_OP_ARGS]; 2099 int const_args[TCG_MAX_OP_ARGS]; 2100 2101 nb_oargs = def->nb_oargs; 2102 nb_iargs = def->nb_iargs; 2103 2104 /* copy constants */ 2105 memcpy(new_args + nb_oargs + nb_iargs, 2106 args + nb_oargs + nb_iargs, 2107 sizeof(TCGArg) * def->nb_cargs); 2108 2109 /* satisfy input constraints */ 2110 tcg_regset_set(allocated_regs, s->reserved_regs); 2111 for(k = 0; k < nb_iargs; k++) { 2112 i = def->sorted_args[nb_oargs + k]; 2113 arg = args[i]; 2114 arg_ct = &def->args_ct[i]; 2115 ts = &s->temps[arg]; 2116 if (ts->val_type == TEMP_VAL_MEM) { 2117 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2118 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2119 ts->val_type = TEMP_VAL_REG; 2120 ts->reg = reg; 2121 ts->mem_coherent = 1; 2122 s->reg_to_temp[reg] = arg; 2123 } else if (ts->val_type == TEMP_VAL_CONST) { 2124 if (tcg_target_const_match(ts->val, ts->type, arg_ct)) { 2125 /* constant is OK for instruction */ 2126 const_args[i] = 1; 2127 new_args[i] = ts->val; 2128 goto iarg_end; 2129 } else { 2130 /* need to move to a register */ 2131 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2132 tcg_out_movi(s, ts->type, reg, ts->val); 2133 ts->val_type = TEMP_VAL_REG; 2134 ts->reg = reg; 2135 ts->mem_coherent = 0; 2136 s->reg_to_temp[reg] = arg; 2137 } 2138 } 2139 assert(ts->val_type == TEMP_VAL_REG); 2140 if (arg_ct->ct & TCG_CT_IALIAS) { 2141 if (ts->fixed_reg) { 2142 /* if fixed register, we must allocate a new register 2143 if the alias is not the same register */ 2144 if (arg != args[arg_ct->alias_index]) 2145 goto allocate_in_reg; 2146 } else { 2147 /* if the input is aliased to an output and if it is 2148 not dead after the instruction, we must allocate 2149 a new register and move it */ 2150 if (!IS_DEAD_ARG(i)) { 2151 goto allocate_in_reg; 2152 } 2153 } 2154 } 2155 reg = ts->reg; 2156 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2157 /* nothing to do : the constraint is satisfied */ 2158 } else { 2159 allocate_in_reg: 2160 /* allocate a new register matching the constraint 2161 and move the temporary register into it */ 2162 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2163 tcg_out_mov(s, ts->type, reg, ts->reg); 2164 } 2165 new_args[i] = reg; 2166 const_args[i] = 0; 2167 tcg_regset_set_reg(allocated_regs, reg); 2168 iarg_end: ; 2169 } 2170 2171 /* mark dead temporaries and free the associated registers */ 2172 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2173 if (IS_DEAD_ARG(i)) { 2174 temp_dead(s, args[i]); 2175 } 2176 } 2177 2178 if (def->flags & TCG_OPF_BB_END) { 2179 tcg_reg_alloc_bb_end(s, allocated_regs); 2180 } else { 2181 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2182 /* XXX: permit generic clobber register list ? */ 2183 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2184 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 2185 tcg_reg_free(s, reg); 2186 } 2187 } 2188 } 2189 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2190 /* sync globals if the op has side effects and might trigger 2191 an exception. */ 2192 sync_globals(s, allocated_regs); 2193 } 2194 2195 /* satisfy the output constraints */ 2196 tcg_regset_set(allocated_regs, s->reserved_regs); 2197 for(k = 0; k < nb_oargs; k++) { 2198 i = def->sorted_args[k]; 2199 arg = args[i]; 2200 arg_ct = &def->args_ct[i]; 2201 ts = &s->temps[arg]; 2202 if (arg_ct->ct & TCG_CT_ALIAS) { 2203 reg = new_args[arg_ct->alias_index]; 2204 } else { 2205 /* if fixed register, we try to use it */ 2206 reg = ts->reg; 2207 if (ts->fixed_reg && 2208 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2209 goto oarg_end; 2210 } 2211 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2212 } 2213 tcg_regset_set_reg(allocated_regs, reg); 2214 /* if a fixed register is used, then a move will be done afterwards */ 2215 if (!ts->fixed_reg) { 2216 if (ts->val_type == TEMP_VAL_REG) { 2217 s->reg_to_temp[ts->reg] = -1; 2218 } 2219 ts->val_type = TEMP_VAL_REG; 2220 ts->reg = reg; 2221 /* temp value is modified, so the value kept in memory is 2222 potentially not the same */ 2223 ts->mem_coherent = 0; 2224 s->reg_to_temp[reg] = arg; 2225 } 2226 oarg_end: 2227 new_args[i] = reg; 2228 } 2229 } 2230 2231 /* emit instruction */ 2232 tcg_out_op(s, opc, new_args, const_args); 2233 2234 /* move the outputs in the correct register if needed */ 2235 for(i = 0; i < nb_oargs; i++) { 2236 ts = &s->temps[args[i]]; 2237 reg = new_args[i]; 2238 if (ts->fixed_reg && ts->reg != reg) { 2239 tcg_out_mov(s, ts->type, ts->reg, reg); 2240 } 2241 if (NEED_SYNC_ARG(i)) { 2242 tcg_reg_sync(s, reg); 2243 } 2244 if (IS_DEAD_ARG(i)) { 2245 temp_dead(s, args[i]); 2246 } 2247 } 2248 } 2249 2250 #ifdef TCG_TARGET_STACK_GROWSUP 2251 #define STACK_DIR(x) (-(x)) 2252 #else 2253 #define STACK_DIR(x) (x) 2254 #endif 2255 2256 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, 2257 TCGOpcode opc, const TCGArg *args, 2258 uint16_t dead_args, uint8_t sync_args) 2259 { 2260 int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params; 2261 TCGArg arg, func_arg; 2262 TCGTemp *ts; 2263 intptr_t stack_offset; 2264 size_t call_stack_size; 2265 uintptr_t func_addr; 2266 int const_func_arg, allocate_args; 2267 TCGRegSet allocated_regs; 2268 const TCGArgConstraint *arg_ct; 2269 2270 arg = *args++; 2271 2272 nb_oargs = arg >> 16; 2273 nb_iargs = arg & 0xffff; 2274 nb_params = nb_iargs - 1; 2275 2276 flags = args[nb_oargs + nb_iargs]; 2277 2278 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2279 if (nb_regs > nb_params) 2280 nb_regs = nb_params; 2281 2282 /* assign stack slots first */ 2283 call_stack_size = (nb_params - nb_regs) * sizeof(tcg_target_long); 2284 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 2285 ~(TCG_TARGET_STACK_ALIGN - 1); 2286 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 2287 if (allocate_args) { 2288 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 2289 preallocate call stack */ 2290 tcg_abort(); 2291 } 2292 2293 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 2294 for(i = nb_regs; i < nb_params; i++) { 2295 arg = args[nb_oargs + i]; 2296 #ifdef TCG_TARGET_STACK_GROWSUP 2297 stack_offset -= sizeof(tcg_target_long); 2298 #endif 2299 if (arg != TCG_CALL_DUMMY_ARG) { 2300 ts = &s->temps[arg]; 2301 if (ts->val_type == TEMP_VAL_REG) { 2302 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 2303 } else if (ts->val_type == TEMP_VAL_MEM) { 2304 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 2305 s->reserved_regs); 2306 /* XXX: not correct if reading values from the stack */ 2307 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2308 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 2309 } else if (ts->val_type == TEMP_VAL_CONST) { 2310 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 2311 s->reserved_regs); 2312 /* XXX: sign extend may be needed on some targets */ 2313 tcg_out_movi(s, ts->type, reg, ts->val); 2314 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 2315 } else { 2316 tcg_abort(); 2317 } 2318 } 2319 #ifndef TCG_TARGET_STACK_GROWSUP 2320 stack_offset += sizeof(tcg_target_long); 2321 #endif 2322 } 2323 2324 /* assign input registers */ 2325 tcg_regset_set(allocated_regs, s->reserved_regs); 2326 for(i = 0; i < nb_regs; i++) { 2327 arg = args[nb_oargs + i]; 2328 if (arg != TCG_CALL_DUMMY_ARG) { 2329 ts = &s->temps[arg]; 2330 reg = tcg_target_call_iarg_regs[i]; 2331 tcg_reg_free(s, reg); 2332 if (ts->val_type == TEMP_VAL_REG) { 2333 if (ts->reg != reg) { 2334 tcg_out_mov(s, ts->type, reg, ts->reg); 2335 } 2336 } else if (ts->val_type == TEMP_VAL_MEM) { 2337 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2338 } else if (ts->val_type == TEMP_VAL_CONST) { 2339 /* XXX: sign extend ? */ 2340 tcg_out_movi(s, ts->type, reg, ts->val); 2341 } else { 2342 tcg_abort(); 2343 } 2344 tcg_regset_set_reg(allocated_regs, reg); 2345 } 2346 } 2347 2348 /* assign function address */ 2349 func_arg = args[nb_oargs + nb_iargs - 1]; 2350 arg_ct = &def->args_ct[0]; 2351 ts = &s->temps[func_arg]; 2352 func_addr = ts->val; 2353 const_func_arg = 0; 2354 if (ts->val_type == TEMP_VAL_MEM) { 2355 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2356 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2357 func_arg = reg; 2358 tcg_regset_set_reg(allocated_regs, reg); 2359 } else if (ts->val_type == TEMP_VAL_REG) { 2360 reg = ts->reg; 2361 if (!tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2362 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2363 tcg_out_mov(s, ts->type, reg, ts->reg); 2364 } 2365 func_arg = reg; 2366 tcg_regset_set_reg(allocated_regs, reg); 2367 } else if (ts->val_type == TEMP_VAL_CONST) { 2368 if (tcg_target_const_match(func_addr, ts->type, arg_ct)) { 2369 const_func_arg = 1; 2370 func_arg = func_addr; 2371 } else { 2372 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2373 tcg_out_movi(s, ts->type, reg, func_addr); 2374 func_arg = reg; 2375 tcg_regset_set_reg(allocated_regs, reg); 2376 } 2377 } else { 2378 tcg_abort(); 2379 } 2380 2381 2382 /* mark dead temporaries and free the associated registers */ 2383 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2384 if (IS_DEAD_ARG(i)) { 2385 temp_dead(s, args[i]); 2386 } 2387 } 2388 2389 /* clobber call registers */ 2390 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2391 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 2392 tcg_reg_free(s, reg); 2393 } 2394 } 2395 2396 /* Save globals if they might be written by the helper, sync them if 2397 they might be read. */ 2398 if (flags & TCG_CALL_NO_READ_GLOBALS) { 2399 /* Nothing to do */ 2400 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 2401 sync_globals(s, allocated_regs); 2402 } else { 2403 save_globals(s, allocated_regs); 2404 } 2405 2406 tcg_out_op(s, opc, &func_arg, &const_func_arg); 2407 2408 /* assign output registers and emit moves if needed */ 2409 for(i = 0; i < nb_oargs; i++) { 2410 arg = args[i]; 2411 ts = &s->temps[arg]; 2412 reg = tcg_target_call_oarg_regs[i]; 2413 assert(s->reg_to_temp[reg] == -1); 2414 if (ts->fixed_reg) { 2415 if (ts->reg != reg) { 2416 tcg_out_mov(s, ts->type, ts->reg, reg); 2417 } 2418 } else { 2419 if (ts->val_type == TEMP_VAL_REG) { 2420 s->reg_to_temp[ts->reg] = -1; 2421 } 2422 ts->val_type = TEMP_VAL_REG; 2423 ts->reg = reg; 2424 ts->mem_coherent = 0; 2425 s->reg_to_temp[reg] = arg; 2426 if (NEED_SYNC_ARG(i)) { 2427 tcg_reg_sync(s, reg); 2428 } 2429 if (IS_DEAD_ARG(i)) { 2430 temp_dead(s, args[i]); 2431 } 2432 } 2433 } 2434 2435 return nb_iargs + nb_oargs + def->nb_cargs + 1; 2436 } 2437 2438 #ifdef CONFIG_PROFILER 2439 2440 static int64_t tcg_table_op_count[NB_OPS]; 2441 2442 static void dump_op_count(void) 2443 { 2444 int i; 2445 FILE *f; 2446 f = fopen("/tmp/op.log", "w"); 2447 for(i = INDEX_op_end; i < NB_OPS; i++) { 2448 fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]); 2449 } 2450 fclose(f); 2451 } 2452 #endif 2453 2454 2455 static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf, 2456 long search_pc) 2457 { 2458 TCGOpcode opc; 2459 int op_index; 2460 const TCGOpDef *def; 2461 const TCGArg *args; 2462 2463 #ifdef DEBUG_DISAS 2464 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) { 2465 qemu_log("OP:\n"); 2466 tcg_dump_ops(s); 2467 qemu_log("\n"); 2468 } 2469 #endif 2470 2471 #ifdef CONFIG_PROFILER 2472 s->opt_time -= profile_getclock(); 2473 #endif 2474 2475 #ifdef USE_TCG_OPTIMIZATIONS 2476 s->gen_opparam_ptr = 2477 tcg_optimize(s, s->gen_opc_ptr, s->gen_opparam_buf, tcg_op_defs); 2478 #endif 2479 2480 #ifdef CONFIG_PROFILER 2481 s->opt_time += profile_getclock(); 2482 s->la_time -= profile_getclock(); 2483 #endif 2484 2485 tcg_liveness_analysis(s); 2486 2487 #ifdef CONFIG_PROFILER 2488 s->la_time += profile_getclock(); 2489 #endif 2490 2491 #ifdef DEBUG_DISAS 2492 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) { 2493 qemu_log("OP after optimization and liveness analysis:\n"); 2494 tcg_dump_ops(s); 2495 qemu_log("\n"); 2496 } 2497 #endif 2498 2499 tcg_reg_alloc_start(s); 2500 2501 s->code_buf = gen_code_buf; 2502 s->code_ptr = gen_code_buf; 2503 2504 tcg_out_tb_init(s); 2505 2506 args = s->gen_opparam_buf; 2507 op_index = 0; 2508 2509 for(;;) { 2510 opc = s->gen_opc_buf[op_index]; 2511 #ifdef CONFIG_PROFILER 2512 tcg_table_op_count[opc]++; 2513 #endif 2514 def = &tcg_op_defs[opc]; 2515 #if 0 2516 printf("%s: %d %d %d\n", def->name, 2517 def->nb_oargs, def->nb_iargs, def->nb_cargs); 2518 // dump_regs(s); 2519 #endif 2520 switch(opc) { 2521 case INDEX_op_mov_i32: 2522 case INDEX_op_mov_i64: 2523 tcg_reg_alloc_mov(s, def, args, s->op_dead_args[op_index], 2524 s->op_sync_args[op_index]); 2525 break; 2526 case INDEX_op_movi_i32: 2527 case INDEX_op_movi_i64: 2528 tcg_reg_alloc_movi(s, args, s->op_dead_args[op_index], 2529 s->op_sync_args[op_index]); 2530 break; 2531 case INDEX_op_debug_insn_start: 2532 /* debug instruction */ 2533 break; 2534 case INDEX_op_nop: 2535 case INDEX_op_nop1: 2536 case INDEX_op_nop2: 2537 case INDEX_op_nop3: 2538 break; 2539 case INDEX_op_nopn: 2540 args += args[0]; 2541 goto next; 2542 case INDEX_op_discard: 2543 temp_dead(s, args[0]); 2544 break; 2545 case INDEX_op_set_label: 2546 tcg_reg_alloc_bb_end(s, s->reserved_regs); 2547 tcg_out_label(s, args[0], s->code_ptr); 2548 break; 2549 case INDEX_op_call: 2550 args += tcg_reg_alloc_call(s, def, opc, args, 2551 s->op_dead_args[op_index], 2552 s->op_sync_args[op_index]); 2553 goto next; 2554 case INDEX_op_end: 2555 goto the_end; 2556 default: 2557 /* Sanity check that we've not introduced any unhandled opcodes. */ 2558 if (def->flags & TCG_OPF_NOT_PRESENT) { 2559 tcg_abort(); 2560 } 2561 /* Note: in order to speed up the code, it would be much 2562 faster to have specialized register allocator functions for 2563 some common argument patterns */ 2564 tcg_reg_alloc_op(s, def, opc, args, s->op_dead_args[op_index], 2565 s->op_sync_args[op_index]); 2566 break; 2567 } 2568 args += def->nb_args; 2569 next: 2570 if (search_pc >= 0 && search_pc < s->code_ptr - gen_code_buf) { 2571 return op_index; 2572 } 2573 op_index++; 2574 #ifndef NDEBUG 2575 check_regs(s); 2576 #endif 2577 } 2578 the_end: 2579 /* Generate TB finalization at the end of block */ 2580 tcg_out_tb_finalize(s); 2581 return -1; 2582 } 2583 2584 int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf) 2585 { 2586 #ifdef CONFIG_PROFILER 2587 { 2588 int n; 2589 n = (s->gen_opc_ptr - s->gen_opc_buf); 2590 s->op_count += n; 2591 if (n > s->op_count_max) 2592 s->op_count_max = n; 2593 2594 s->temp_count += s->nb_temps; 2595 if (s->nb_temps > s->temp_count_max) 2596 s->temp_count_max = s->nb_temps; 2597 } 2598 #endif 2599 2600 tcg_gen_code_common(s, gen_code_buf, -1); 2601 2602 /* flush instruction cache */ 2603 flush_icache_range((uintptr_t)gen_code_buf, (uintptr_t)s->code_ptr); 2604 2605 return s->code_ptr - gen_code_buf; 2606 } 2607 2608 /* Return the index of the micro operation such as the pc after is < 2609 offset bytes from the start of the TB. The contents of gen_code_buf must 2610 not be changed, though writing the same values is ok. 2611 Return -1 if not found. */ 2612 int tcg_gen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset) 2613 { 2614 return tcg_gen_code_common(s, gen_code_buf, offset); 2615 } 2616 2617 #ifdef CONFIG_PROFILER 2618 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2619 { 2620 TCGContext *s = &tcg_ctx; 2621 int64_t tot; 2622 2623 tot = s->interm_time + s->code_time; 2624 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 2625 tot, tot / 2.4e9); 2626 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 2627 s->tb_count, 2628 s->tb_count1 - s->tb_count, 2629 s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0); 2630 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 2631 s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max); 2632 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 2633 s->tb_count ? 2634 (double)s->del_op_count / s->tb_count : 0); 2635 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 2636 s->tb_count ? 2637 (double)s->temp_count / s->tb_count : 0, 2638 s->temp_count_max); 2639 2640 cpu_fprintf(f, "cycles/op %0.1f\n", 2641 s->op_count ? (double)tot / s->op_count : 0); 2642 cpu_fprintf(f, "cycles/in byte %0.1f\n", 2643 s->code_in_len ? (double)tot / s->code_in_len : 0); 2644 cpu_fprintf(f, "cycles/out byte %0.1f\n", 2645 s->code_out_len ? (double)tot / s->code_out_len : 0); 2646 if (tot == 0) 2647 tot = 1; 2648 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 2649 (double)s->interm_time / tot * 100.0); 2650 cpu_fprintf(f, " gen_code time %0.1f%%\n", 2651 (double)s->code_time / tot * 100.0); 2652 cpu_fprintf(f, "optim./code time %0.1f%%\n", 2653 (double)s->opt_time / (s->code_time ? s->code_time : 1) 2654 * 100.0); 2655 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 2656 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 2657 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 2658 s->restore_count); 2659 cpu_fprintf(f, " avg cycles %0.1f\n", 2660 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 2661 2662 dump_op_count(); 2663 } 2664 #else 2665 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2666 { 2667 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2668 } 2669 #endif 2670 2671 #ifdef ELF_HOST_MACHINE 2672 /* In order to use this feature, the backend needs to do three things: 2673 2674 (1) Define ELF_HOST_MACHINE to indicate both what value to 2675 put into the ELF image and to indicate support for the feature. 2676 2677 (2) Define tcg_register_jit. This should create a buffer containing 2678 the contents of a .debug_frame section that describes the post- 2679 prologue unwind info for the tcg machine. 2680 2681 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 2682 */ 2683 2684 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 2685 typedef enum { 2686 JIT_NOACTION = 0, 2687 JIT_REGISTER_FN, 2688 JIT_UNREGISTER_FN 2689 } jit_actions_t; 2690 2691 struct jit_code_entry { 2692 struct jit_code_entry *next_entry; 2693 struct jit_code_entry *prev_entry; 2694 const void *symfile_addr; 2695 uint64_t symfile_size; 2696 }; 2697 2698 struct jit_descriptor { 2699 uint32_t version; 2700 uint32_t action_flag; 2701 struct jit_code_entry *relevant_entry; 2702 struct jit_code_entry *first_entry; 2703 }; 2704 2705 void __jit_debug_register_code(void) __attribute__((noinline)); 2706 void __jit_debug_register_code(void) 2707 { 2708 asm(""); 2709 } 2710 2711 /* Must statically initialize the version, because GDB may check 2712 the version before we can set it. */ 2713 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 2714 2715 /* End GDB interface. */ 2716 2717 static int find_string(const char *strtab, const char *str) 2718 { 2719 const char *p = strtab + 1; 2720 2721 while (1) { 2722 if (strcmp(p, str) == 0) { 2723 return p - strtab; 2724 } 2725 p += strlen(p) + 1; 2726 } 2727 } 2728 2729 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 2730 void *debug_frame, size_t debug_frame_size) 2731 { 2732 struct __attribute__((packed)) DebugInfo { 2733 uint32_t len; 2734 uint16_t version; 2735 uint32_t abbrev; 2736 uint8_t ptr_size; 2737 uint8_t cu_die; 2738 uint16_t cu_lang; 2739 uintptr_t cu_low_pc; 2740 uintptr_t cu_high_pc; 2741 uint8_t fn_die; 2742 char fn_name[16]; 2743 uintptr_t fn_low_pc; 2744 uintptr_t fn_high_pc; 2745 uint8_t cu_eoc; 2746 }; 2747 2748 struct ElfImage { 2749 ElfW(Ehdr) ehdr; 2750 ElfW(Phdr) phdr; 2751 ElfW(Shdr) shdr[7]; 2752 ElfW(Sym) sym[2]; 2753 struct DebugInfo di; 2754 uint8_t da[24]; 2755 char str[80]; 2756 }; 2757 2758 struct ElfImage *img; 2759 2760 static const struct ElfImage img_template = { 2761 .ehdr = { 2762 .e_ident[EI_MAG0] = ELFMAG0, 2763 .e_ident[EI_MAG1] = ELFMAG1, 2764 .e_ident[EI_MAG2] = ELFMAG2, 2765 .e_ident[EI_MAG3] = ELFMAG3, 2766 .e_ident[EI_CLASS] = ELF_CLASS, 2767 .e_ident[EI_DATA] = ELF_DATA, 2768 .e_ident[EI_VERSION] = EV_CURRENT, 2769 .e_type = ET_EXEC, 2770 .e_machine = ELF_HOST_MACHINE, 2771 .e_version = EV_CURRENT, 2772 .e_phoff = offsetof(struct ElfImage, phdr), 2773 .e_shoff = offsetof(struct ElfImage, shdr), 2774 .e_ehsize = sizeof(ElfW(Shdr)), 2775 .e_phentsize = sizeof(ElfW(Phdr)), 2776 .e_phnum = 1, 2777 .e_shentsize = sizeof(ElfW(Shdr)), 2778 .e_shnum = ARRAY_SIZE(img->shdr), 2779 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 2780 #ifdef ELF_HOST_FLAGS 2781 .e_flags = ELF_HOST_FLAGS, 2782 #endif 2783 #ifdef ELF_OSABI 2784 .e_ident[EI_OSABI] = ELF_OSABI, 2785 #endif 2786 }, 2787 .phdr = { 2788 .p_type = PT_LOAD, 2789 .p_flags = PF_X, 2790 }, 2791 .shdr = { 2792 [0] = { .sh_type = SHT_NULL }, 2793 /* Trick: The contents of code_gen_buffer are not present in 2794 this fake ELF file; that got allocated elsewhere. Therefore 2795 we mark .text as SHT_NOBITS (similar to .bss) so that readers 2796 will not look for contents. We can record any address. */ 2797 [1] = { /* .text */ 2798 .sh_type = SHT_NOBITS, 2799 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 2800 }, 2801 [2] = { /* .debug_info */ 2802 .sh_type = SHT_PROGBITS, 2803 .sh_offset = offsetof(struct ElfImage, di), 2804 .sh_size = sizeof(struct DebugInfo), 2805 }, 2806 [3] = { /* .debug_abbrev */ 2807 .sh_type = SHT_PROGBITS, 2808 .sh_offset = offsetof(struct ElfImage, da), 2809 .sh_size = sizeof(img->da), 2810 }, 2811 [4] = { /* .debug_frame */ 2812 .sh_type = SHT_PROGBITS, 2813 .sh_offset = sizeof(struct ElfImage), 2814 }, 2815 [5] = { /* .symtab */ 2816 .sh_type = SHT_SYMTAB, 2817 .sh_offset = offsetof(struct ElfImage, sym), 2818 .sh_size = sizeof(img->sym), 2819 .sh_info = 1, 2820 .sh_link = ARRAY_SIZE(img->shdr) - 1, 2821 .sh_entsize = sizeof(ElfW(Sym)), 2822 }, 2823 [6] = { /* .strtab */ 2824 .sh_type = SHT_STRTAB, 2825 .sh_offset = offsetof(struct ElfImage, str), 2826 .sh_size = sizeof(img->str), 2827 } 2828 }, 2829 .sym = { 2830 [1] = { /* code_gen_buffer */ 2831 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 2832 .st_shndx = 1, 2833 } 2834 }, 2835 .di = { 2836 .len = sizeof(struct DebugInfo) - 4, 2837 .version = 2, 2838 .ptr_size = sizeof(void *), 2839 .cu_die = 1, 2840 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 2841 .fn_die = 2, 2842 .fn_name = "code_gen_buffer" 2843 }, 2844 .da = { 2845 1, /* abbrev number (the cu) */ 2846 0x11, 1, /* DW_TAG_compile_unit, has children */ 2847 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 2848 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2849 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2850 0, 0, /* end of abbrev */ 2851 2, /* abbrev number (the fn) */ 2852 0x2e, 0, /* DW_TAG_subprogram, no children */ 2853 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 2854 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2855 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2856 0, 0, /* end of abbrev */ 2857 0 /* no more abbrev */ 2858 }, 2859 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 2860 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 2861 }; 2862 2863 /* We only need a single jit entry; statically allocate it. */ 2864 static struct jit_code_entry one_entry; 2865 2866 uintptr_t buf = (uintptr_t)buf_ptr; 2867 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 2868 2869 img = g_malloc(img_size); 2870 *img = img_template; 2871 memcpy(img + 1, debug_frame, debug_frame_size); 2872 2873 img->phdr.p_vaddr = buf; 2874 img->phdr.p_paddr = buf; 2875 img->phdr.p_memsz = buf_size; 2876 2877 img->shdr[1].sh_name = find_string(img->str, ".text"); 2878 img->shdr[1].sh_addr = buf; 2879 img->shdr[1].sh_size = buf_size; 2880 2881 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 2882 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 2883 2884 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 2885 img->shdr[4].sh_size = debug_frame_size; 2886 2887 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 2888 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 2889 2890 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 2891 img->sym[1].st_value = buf; 2892 img->sym[1].st_size = buf_size; 2893 2894 img->di.cu_low_pc = buf; 2895 img->di.cu_high_pc = buf + buf_size; 2896 img->di.fn_low_pc = buf; 2897 img->di.fn_high_pc = buf + buf_size; 2898 2899 #ifdef DEBUG_JIT 2900 /* Enable this block to be able to debug the ELF image file creation. 2901 One can use readelf, objdump, or other inspection utilities. */ 2902 { 2903 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 2904 if (f) { 2905 if (fwrite(img, img_size, 1, f) != img_size) { 2906 /* Avoid stupid unused return value warning for fwrite. */ 2907 } 2908 fclose(f); 2909 } 2910 } 2911 #endif 2912 2913 one_entry.symfile_addr = img; 2914 one_entry.symfile_size = img_size; 2915 2916 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 2917 __jit_debug_descriptor.relevant_entry = &one_entry; 2918 __jit_debug_descriptor.first_entry = &one_entry; 2919 __jit_debug_register_code(); 2920 } 2921 #else 2922 /* No support for the feature. Provide the entry point expected by exec.c, 2923 and implement the internal function we declared earlier. */ 2924 2925 static void tcg_register_jit_int(void *buf, size_t size, 2926 void *debug_frame, size_t debug_frame_size) 2927 { 2928 } 2929 2930 void tcg_register_jit(void *buf, size_t buf_size) 2931 { 2932 } 2933 #endif /* ELF_HOST_MACHINE */ 2934