1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_LIVENESS_ANALYSIS 27 #define USE_TCG_OPTIMIZATIONS 28 29 #include "config.h" 30 31 /* Define to jump the ELF file used to communicate with GDB. */ 32 #undef DEBUG_JIT 33 34 #if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG) 35 /* define it to suppress various consistency checks (faster) */ 36 #define NDEBUG 37 #endif 38 39 #include "qemu-common.h" 40 #include "cache-utils.h" 41 #include "host-utils.h" 42 #include "qemu-timer.h" 43 44 /* Note: the long term plan is to reduce the dependancies on the QEMU 45 CPU definitions. Currently they are used for qemu_ld/st 46 instructions */ 47 #define NO_CPU_IO_DEFS 48 #include "cpu.h" 49 50 #include "tcg-op.h" 51 52 #if TCG_TARGET_REG_BITS == 64 53 # define ELF_CLASS ELFCLASS64 54 #else 55 # define ELF_CLASS ELFCLASS32 56 #endif 57 #ifdef HOST_WORDS_BIGENDIAN 58 # define ELF_DATA ELFDATA2MSB 59 #else 60 # define ELF_DATA ELFDATA2LSB 61 #endif 62 63 #include "elf.h" 64 65 /* Forward declarations for functions declared in tcg-target.c and used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static void patch_reloc(uint8_t *code_ptr, int type, 69 tcg_target_long value, tcg_target_long addend); 70 71 static void tcg_register_jit_int(void *buf, size_t size, 72 void *debug_frame, size_t debug_frame_size) 73 __attribute__((unused)); 74 75 /* Forward declarations for functions declared and used in tcg-target.c. */ 76 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str); 77 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 78 tcg_target_long arg2); 79 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 80 static void tcg_out_movi(TCGContext *s, TCGType type, 81 TCGReg ret, tcg_target_long arg); 82 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 83 const int *const_args); 84 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 85 tcg_target_long arg2); 86 static int tcg_target_const_match(tcg_target_long val, 87 const TCGArgConstraint *arg_ct); 88 89 TCGOpDef tcg_op_defs[] = { 90 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags }, 91 #include "tcg-opc.h" 92 #undef DEF 93 }; 94 const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs); 95 96 static TCGRegSet tcg_target_available_regs[2]; 97 static TCGRegSet tcg_target_call_clobber_regs; 98 99 /* XXX: move that inside the context */ 100 uint16_t *gen_opc_ptr; 101 TCGArg *gen_opparam_ptr; 102 103 static inline void tcg_out8(TCGContext *s, uint8_t v) 104 { 105 *s->code_ptr++ = v; 106 } 107 108 static inline void tcg_out16(TCGContext *s, uint16_t v) 109 { 110 *(uint16_t *)s->code_ptr = v; 111 s->code_ptr += 2; 112 } 113 114 static inline void tcg_out32(TCGContext *s, uint32_t v) 115 { 116 *(uint32_t *)s->code_ptr = v; 117 s->code_ptr += 4; 118 } 119 120 /* label relocation processing */ 121 122 static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type, 123 int label_index, long addend) 124 { 125 TCGLabel *l; 126 TCGRelocation *r; 127 128 l = &s->labels[label_index]; 129 if (l->has_value) { 130 /* FIXME: This may break relocations on RISC targets that 131 modify instruction fields in place. The caller may not have 132 written the initial value. */ 133 patch_reloc(code_ptr, type, l->u.value, addend); 134 } else { 135 /* add a new relocation entry */ 136 r = tcg_malloc(sizeof(TCGRelocation)); 137 r->type = type; 138 r->ptr = code_ptr; 139 r->addend = addend; 140 r->next = l->u.first_reloc; 141 l->u.first_reloc = r; 142 } 143 } 144 145 static void tcg_out_label(TCGContext *s, int label_index, void *ptr) 146 { 147 TCGLabel *l; 148 TCGRelocation *r; 149 tcg_target_long value = (tcg_target_long)ptr; 150 151 l = &s->labels[label_index]; 152 if (l->has_value) 153 tcg_abort(); 154 r = l->u.first_reloc; 155 while (r != NULL) { 156 patch_reloc(r->ptr, r->type, value, r->addend); 157 r = r->next; 158 } 159 l->has_value = 1; 160 l->u.value = value; 161 } 162 163 int gen_new_label(void) 164 { 165 TCGContext *s = &tcg_ctx; 166 int idx; 167 TCGLabel *l; 168 169 if (s->nb_labels >= TCG_MAX_LABELS) 170 tcg_abort(); 171 idx = s->nb_labels++; 172 l = &s->labels[idx]; 173 l->has_value = 0; 174 l->u.first_reloc = NULL; 175 return idx; 176 } 177 178 #include "tcg-target.c" 179 180 /* pool based memory allocation */ 181 void *tcg_malloc_internal(TCGContext *s, int size) 182 { 183 TCGPool *p; 184 int pool_size; 185 186 if (size > TCG_POOL_CHUNK_SIZE) { 187 /* big malloc: insert a new pool (XXX: could optimize) */ 188 p = g_malloc(sizeof(TCGPool) + size); 189 p->size = size; 190 p->next = s->pool_first_large; 191 s->pool_first_large = p; 192 return p->data; 193 } else { 194 p = s->pool_current; 195 if (!p) { 196 p = s->pool_first; 197 if (!p) 198 goto new_pool; 199 } else { 200 if (!p->next) { 201 new_pool: 202 pool_size = TCG_POOL_CHUNK_SIZE; 203 p = g_malloc(sizeof(TCGPool) + pool_size); 204 p->size = pool_size; 205 p->next = NULL; 206 if (s->pool_current) 207 s->pool_current->next = p; 208 else 209 s->pool_first = p; 210 } else { 211 p = p->next; 212 } 213 } 214 } 215 s->pool_current = p; 216 s->pool_cur = p->data + size; 217 s->pool_end = p->data + p->size; 218 return p->data; 219 } 220 221 void tcg_pool_reset(TCGContext *s) 222 { 223 TCGPool *p, *t; 224 for (p = s->pool_first_large; p; p = t) { 225 t = p->next; 226 g_free(p); 227 } 228 s->pool_first_large = NULL; 229 s->pool_cur = s->pool_end = NULL; 230 s->pool_current = NULL; 231 } 232 233 void tcg_context_init(TCGContext *s) 234 { 235 int op, total_args, n; 236 TCGOpDef *def; 237 TCGArgConstraint *args_ct; 238 int *sorted_args; 239 240 memset(s, 0, sizeof(*s)); 241 s->nb_globals = 0; 242 243 /* Count total number of arguments and allocate the corresponding 244 space */ 245 total_args = 0; 246 for(op = 0; op < NB_OPS; op++) { 247 def = &tcg_op_defs[op]; 248 n = def->nb_iargs + def->nb_oargs; 249 total_args += n; 250 } 251 252 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 253 sorted_args = g_malloc(sizeof(int) * total_args); 254 255 for(op = 0; op < NB_OPS; op++) { 256 def = &tcg_op_defs[op]; 257 def->args_ct = args_ct; 258 def->sorted_args = sorted_args; 259 n = def->nb_iargs + def->nb_oargs; 260 sorted_args += n; 261 args_ct += n; 262 } 263 264 tcg_target_init(s); 265 } 266 267 void tcg_prologue_init(TCGContext *s) 268 { 269 /* init global prologue and epilogue */ 270 s->code_buf = code_gen_prologue; 271 s->code_ptr = s->code_buf; 272 tcg_target_qemu_prologue(s); 273 flush_icache_range((tcg_target_ulong)s->code_buf, 274 (tcg_target_ulong)s->code_ptr); 275 } 276 277 void tcg_set_frame(TCGContext *s, int reg, 278 tcg_target_long start, tcg_target_long size) 279 { 280 s->frame_start = start; 281 s->frame_end = start + size; 282 s->frame_reg = reg; 283 } 284 285 void tcg_func_start(TCGContext *s) 286 { 287 int i; 288 tcg_pool_reset(s); 289 s->nb_temps = s->nb_globals; 290 for(i = 0; i < (TCG_TYPE_COUNT * 2); i++) 291 s->first_free_temp[i] = -1; 292 s->labels = tcg_malloc(sizeof(TCGLabel) * TCG_MAX_LABELS); 293 s->nb_labels = 0; 294 s->current_frame_offset = s->frame_start; 295 296 #ifdef CONFIG_DEBUG_TCG 297 s->goto_tb_issue_mask = 0; 298 #endif 299 300 gen_opc_ptr = gen_opc_buf; 301 gen_opparam_ptr = gen_opparam_buf; 302 } 303 304 static inline void tcg_temp_alloc(TCGContext *s, int n) 305 { 306 if (n > TCG_MAX_TEMPS) 307 tcg_abort(); 308 } 309 310 static inline int tcg_global_reg_new_internal(TCGType type, int reg, 311 const char *name) 312 { 313 TCGContext *s = &tcg_ctx; 314 TCGTemp *ts; 315 int idx; 316 317 #if TCG_TARGET_REG_BITS == 32 318 if (type != TCG_TYPE_I32) 319 tcg_abort(); 320 #endif 321 if (tcg_regset_test_reg(s->reserved_regs, reg)) 322 tcg_abort(); 323 idx = s->nb_globals; 324 tcg_temp_alloc(s, s->nb_globals + 1); 325 ts = &s->temps[s->nb_globals]; 326 ts->base_type = type; 327 ts->type = type; 328 ts->fixed_reg = 1; 329 ts->reg = reg; 330 ts->name = name; 331 s->nb_globals++; 332 tcg_regset_set_reg(s->reserved_regs, reg); 333 return idx; 334 } 335 336 TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name) 337 { 338 int idx; 339 340 idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name); 341 return MAKE_TCGV_I32(idx); 342 } 343 344 TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name) 345 { 346 int idx; 347 348 idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name); 349 return MAKE_TCGV_I64(idx); 350 } 351 352 static inline int tcg_global_mem_new_internal(TCGType type, int reg, 353 tcg_target_long offset, 354 const char *name) 355 { 356 TCGContext *s = &tcg_ctx; 357 TCGTemp *ts; 358 int idx; 359 360 idx = s->nb_globals; 361 #if TCG_TARGET_REG_BITS == 32 362 if (type == TCG_TYPE_I64) { 363 char buf[64]; 364 tcg_temp_alloc(s, s->nb_globals + 2); 365 ts = &s->temps[s->nb_globals]; 366 ts->base_type = type; 367 ts->type = TCG_TYPE_I32; 368 ts->fixed_reg = 0; 369 ts->mem_allocated = 1; 370 ts->mem_reg = reg; 371 #ifdef TCG_TARGET_WORDS_BIGENDIAN 372 ts->mem_offset = offset + 4; 373 #else 374 ts->mem_offset = offset; 375 #endif 376 pstrcpy(buf, sizeof(buf), name); 377 pstrcat(buf, sizeof(buf), "_0"); 378 ts->name = strdup(buf); 379 ts++; 380 381 ts->base_type = type; 382 ts->type = TCG_TYPE_I32; 383 ts->fixed_reg = 0; 384 ts->mem_allocated = 1; 385 ts->mem_reg = reg; 386 #ifdef TCG_TARGET_WORDS_BIGENDIAN 387 ts->mem_offset = offset; 388 #else 389 ts->mem_offset = offset + 4; 390 #endif 391 pstrcpy(buf, sizeof(buf), name); 392 pstrcat(buf, sizeof(buf), "_1"); 393 ts->name = strdup(buf); 394 395 s->nb_globals += 2; 396 } else 397 #endif 398 { 399 tcg_temp_alloc(s, s->nb_globals + 1); 400 ts = &s->temps[s->nb_globals]; 401 ts->base_type = type; 402 ts->type = type; 403 ts->fixed_reg = 0; 404 ts->mem_allocated = 1; 405 ts->mem_reg = reg; 406 ts->mem_offset = offset; 407 ts->name = name; 408 s->nb_globals++; 409 } 410 return idx; 411 } 412 413 TCGv_i32 tcg_global_mem_new_i32(int reg, tcg_target_long offset, 414 const char *name) 415 { 416 int idx; 417 418 idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name); 419 return MAKE_TCGV_I32(idx); 420 } 421 422 TCGv_i64 tcg_global_mem_new_i64(int reg, tcg_target_long offset, 423 const char *name) 424 { 425 int idx; 426 427 idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name); 428 return MAKE_TCGV_I64(idx); 429 } 430 431 static inline int tcg_temp_new_internal(TCGType type, int temp_local) 432 { 433 TCGContext *s = &tcg_ctx; 434 TCGTemp *ts; 435 int idx, k; 436 437 k = type; 438 if (temp_local) 439 k += TCG_TYPE_COUNT; 440 idx = s->first_free_temp[k]; 441 if (idx != -1) { 442 /* There is already an available temp with the 443 right type */ 444 ts = &s->temps[idx]; 445 s->first_free_temp[k] = ts->next_free_temp; 446 ts->temp_allocated = 1; 447 assert(ts->temp_local == temp_local); 448 } else { 449 idx = s->nb_temps; 450 #if TCG_TARGET_REG_BITS == 32 451 if (type == TCG_TYPE_I64) { 452 tcg_temp_alloc(s, s->nb_temps + 2); 453 ts = &s->temps[s->nb_temps]; 454 ts->base_type = type; 455 ts->type = TCG_TYPE_I32; 456 ts->temp_allocated = 1; 457 ts->temp_local = temp_local; 458 ts->name = NULL; 459 ts++; 460 ts->base_type = TCG_TYPE_I32; 461 ts->type = TCG_TYPE_I32; 462 ts->temp_allocated = 1; 463 ts->temp_local = temp_local; 464 ts->name = NULL; 465 s->nb_temps += 2; 466 } else 467 #endif 468 { 469 tcg_temp_alloc(s, s->nb_temps + 1); 470 ts = &s->temps[s->nb_temps]; 471 ts->base_type = type; 472 ts->type = type; 473 ts->temp_allocated = 1; 474 ts->temp_local = temp_local; 475 ts->name = NULL; 476 s->nb_temps++; 477 } 478 } 479 480 #if defined(CONFIG_DEBUG_TCG) 481 s->temps_in_use++; 482 #endif 483 return idx; 484 } 485 486 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 487 { 488 int idx; 489 490 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 491 return MAKE_TCGV_I32(idx); 492 } 493 494 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 495 { 496 int idx; 497 498 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 499 return MAKE_TCGV_I64(idx); 500 } 501 502 static inline void tcg_temp_free_internal(int idx) 503 { 504 TCGContext *s = &tcg_ctx; 505 TCGTemp *ts; 506 int k; 507 508 #if defined(CONFIG_DEBUG_TCG) 509 s->temps_in_use--; 510 if (s->temps_in_use < 0) { 511 fprintf(stderr, "More temporaries freed than allocated!\n"); 512 } 513 #endif 514 515 assert(idx >= s->nb_globals && idx < s->nb_temps); 516 ts = &s->temps[idx]; 517 assert(ts->temp_allocated != 0); 518 ts->temp_allocated = 0; 519 k = ts->base_type; 520 if (ts->temp_local) 521 k += TCG_TYPE_COUNT; 522 ts->next_free_temp = s->first_free_temp[k]; 523 s->first_free_temp[k] = idx; 524 } 525 526 void tcg_temp_free_i32(TCGv_i32 arg) 527 { 528 tcg_temp_free_internal(GET_TCGV_I32(arg)); 529 } 530 531 void tcg_temp_free_i64(TCGv_i64 arg) 532 { 533 tcg_temp_free_internal(GET_TCGV_I64(arg)); 534 } 535 536 TCGv_i32 tcg_const_i32(int32_t val) 537 { 538 TCGv_i32 t0; 539 t0 = tcg_temp_new_i32(); 540 tcg_gen_movi_i32(t0, val); 541 return t0; 542 } 543 544 TCGv_i64 tcg_const_i64(int64_t val) 545 { 546 TCGv_i64 t0; 547 t0 = tcg_temp_new_i64(); 548 tcg_gen_movi_i64(t0, val); 549 return t0; 550 } 551 552 TCGv_i32 tcg_const_local_i32(int32_t val) 553 { 554 TCGv_i32 t0; 555 t0 = tcg_temp_local_new_i32(); 556 tcg_gen_movi_i32(t0, val); 557 return t0; 558 } 559 560 TCGv_i64 tcg_const_local_i64(int64_t val) 561 { 562 TCGv_i64 t0; 563 t0 = tcg_temp_local_new_i64(); 564 tcg_gen_movi_i64(t0, val); 565 return t0; 566 } 567 568 #if defined(CONFIG_DEBUG_TCG) 569 void tcg_clear_temp_count(void) 570 { 571 TCGContext *s = &tcg_ctx; 572 s->temps_in_use = 0; 573 } 574 575 int tcg_check_temp_count(void) 576 { 577 TCGContext *s = &tcg_ctx; 578 if (s->temps_in_use) { 579 /* Clear the count so that we don't give another 580 * warning immediately next time around. 581 */ 582 s->temps_in_use = 0; 583 return 1; 584 } 585 return 0; 586 } 587 #endif 588 589 void tcg_register_helper(void *func, const char *name) 590 { 591 TCGContext *s = &tcg_ctx; 592 int n; 593 if ((s->nb_helpers + 1) > s->allocated_helpers) { 594 n = s->allocated_helpers; 595 if (n == 0) { 596 n = 4; 597 } else { 598 n *= 2; 599 } 600 s->helpers = realloc(s->helpers, n * sizeof(TCGHelperInfo)); 601 s->allocated_helpers = n; 602 } 603 s->helpers[s->nb_helpers].func = (tcg_target_ulong)func; 604 s->helpers[s->nb_helpers].name = name; 605 s->nb_helpers++; 606 } 607 608 /* Note: we convert the 64 bit args to 32 bit and do some alignment 609 and endian swap. Maybe it would be better to do the alignment 610 and endian swap in tcg_reg_alloc_call(). */ 611 void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags, 612 int sizemask, TCGArg ret, int nargs, TCGArg *args) 613 { 614 int i; 615 int real_args; 616 int nb_rets; 617 TCGArg *nparam; 618 619 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 620 for (i = 0; i < nargs; ++i) { 621 int is_64bit = sizemask & (1 << (i+1)*2); 622 int is_signed = sizemask & (2 << (i+1)*2); 623 if (!is_64bit) { 624 TCGv_i64 temp = tcg_temp_new_i64(); 625 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 626 if (is_signed) { 627 tcg_gen_ext32s_i64(temp, orig); 628 } else { 629 tcg_gen_ext32u_i64(temp, orig); 630 } 631 args[i] = GET_TCGV_I64(temp); 632 } 633 } 634 #endif /* TCG_TARGET_EXTEND_ARGS */ 635 636 *gen_opc_ptr++ = INDEX_op_call; 637 nparam = gen_opparam_ptr++; 638 if (ret != TCG_CALL_DUMMY_ARG) { 639 #if TCG_TARGET_REG_BITS < 64 640 if (sizemask & 1) { 641 #ifdef TCG_TARGET_WORDS_BIGENDIAN 642 *gen_opparam_ptr++ = ret + 1; 643 *gen_opparam_ptr++ = ret; 644 #else 645 *gen_opparam_ptr++ = ret; 646 *gen_opparam_ptr++ = ret + 1; 647 #endif 648 nb_rets = 2; 649 } else 650 #endif 651 { 652 *gen_opparam_ptr++ = ret; 653 nb_rets = 1; 654 } 655 } else { 656 nb_rets = 0; 657 } 658 real_args = 0; 659 for (i = 0; i < nargs; i++) { 660 #if TCG_TARGET_REG_BITS < 64 661 int is_64bit = sizemask & (1 << (i+1)*2); 662 if (is_64bit) { 663 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 664 /* some targets want aligned 64 bit args */ 665 if (real_args & 1) { 666 *gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG; 667 real_args++; 668 } 669 #endif 670 /* If stack grows up, then we will be placing successive 671 arguments at lower addresses, which means we need to 672 reverse the order compared to how we would normally 673 treat either big or little-endian. For those arguments 674 that will wind up in registers, this still works for 675 HPPA (the only current STACK_GROWSUP target) since the 676 argument registers are *also* allocated in decreasing 677 order. If another such target is added, this logic may 678 have to get more complicated to differentiate between 679 stack arguments and register arguments. */ 680 #if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 681 *gen_opparam_ptr++ = args[i] + 1; 682 *gen_opparam_ptr++ = args[i]; 683 #else 684 *gen_opparam_ptr++ = args[i]; 685 *gen_opparam_ptr++ = args[i] + 1; 686 #endif 687 real_args += 2; 688 continue; 689 } 690 #endif /* TCG_TARGET_REG_BITS < 64 */ 691 692 *gen_opparam_ptr++ = args[i]; 693 real_args++; 694 } 695 *gen_opparam_ptr++ = GET_TCGV_PTR(func); 696 697 *gen_opparam_ptr++ = flags; 698 699 *nparam = (nb_rets << 16) | (real_args + 1); 700 701 /* total parameters, needed to go backward in the instruction stream */ 702 *gen_opparam_ptr++ = 1 + nb_rets + real_args + 3; 703 704 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 705 for (i = 0; i < nargs; ++i) { 706 int is_64bit = sizemask & (1 << (i+1)*2); 707 if (!is_64bit) { 708 TCGv_i64 temp = MAKE_TCGV_I64(args[i]); 709 tcg_temp_free_i64(temp); 710 } 711 } 712 #endif /* TCG_TARGET_EXTEND_ARGS */ 713 } 714 715 #if TCG_TARGET_REG_BITS == 32 716 void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 717 int c, int right, int arith) 718 { 719 if (c == 0) { 720 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 721 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 722 } else if (c >= 32) { 723 c -= 32; 724 if (right) { 725 if (arith) { 726 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 727 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 728 } else { 729 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 730 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 731 } 732 } else { 733 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 734 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 735 } 736 } else { 737 TCGv_i32 t0, t1; 738 739 t0 = tcg_temp_new_i32(); 740 t1 = tcg_temp_new_i32(); 741 if (right) { 742 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 743 if (arith) 744 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 745 else 746 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 747 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 748 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 749 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 750 } else { 751 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 752 /* Note: ret can be the same as arg1, so we use t1 */ 753 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 754 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 755 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 756 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 757 } 758 tcg_temp_free_i32(t0); 759 tcg_temp_free_i32(t1); 760 } 761 } 762 #endif 763 764 765 static void tcg_reg_alloc_start(TCGContext *s) 766 { 767 int i; 768 TCGTemp *ts; 769 for(i = 0; i < s->nb_globals; i++) { 770 ts = &s->temps[i]; 771 if (ts->fixed_reg) { 772 ts->val_type = TEMP_VAL_REG; 773 } else { 774 ts->val_type = TEMP_VAL_MEM; 775 } 776 } 777 for(i = s->nb_globals; i < s->nb_temps; i++) { 778 ts = &s->temps[i]; 779 ts->val_type = TEMP_VAL_DEAD; 780 ts->mem_allocated = 0; 781 ts->fixed_reg = 0; 782 } 783 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 784 s->reg_to_temp[i] = -1; 785 } 786 } 787 788 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size, 789 int idx) 790 { 791 TCGTemp *ts; 792 793 assert(idx >= 0 && idx < s->nb_temps); 794 ts = &s->temps[idx]; 795 assert(ts); 796 if (idx < s->nb_globals) { 797 pstrcpy(buf, buf_size, ts->name); 798 } else { 799 if (ts->temp_local) 800 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 801 else 802 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 803 } 804 return buf; 805 } 806 807 char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg) 808 { 809 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg)); 810 } 811 812 char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg) 813 { 814 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg)); 815 } 816 817 static int helper_cmp(const void *p1, const void *p2) 818 { 819 const TCGHelperInfo *th1 = p1; 820 const TCGHelperInfo *th2 = p2; 821 if (th1->func < th2->func) 822 return -1; 823 else if (th1->func == th2->func) 824 return 0; 825 else 826 return 1; 827 } 828 829 /* find helper definition (Note: A hash table would be better) */ 830 static TCGHelperInfo *tcg_find_helper(TCGContext *s, tcg_target_ulong val) 831 { 832 int m, m_min, m_max; 833 TCGHelperInfo *th; 834 tcg_target_ulong v; 835 836 if (unlikely(!s->helpers_sorted)) { 837 qsort(s->helpers, s->nb_helpers, sizeof(TCGHelperInfo), 838 helper_cmp); 839 s->helpers_sorted = 1; 840 } 841 842 /* binary search */ 843 m_min = 0; 844 m_max = s->nb_helpers - 1; 845 while (m_min <= m_max) { 846 m = (m_min + m_max) >> 1; 847 th = &s->helpers[m]; 848 v = th->func; 849 if (v == val) 850 return th; 851 else if (val < v) { 852 m_max = m - 1; 853 } else { 854 m_min = m + 1; 855 } 856 } 857 return NULL; 858 } 859 860 static const char * const cond_name[] = 861 { 862 [TCG_COND_NEVER] = "never", 863 [TCG_COND_ALWAYS] = "always", 864 [TCG_COND_EQ] = "eq", 865 [TCG_COND_NE] = "ne", 866 [TCG_COND_LT] = "lt", 867 [TCG_COND_GE] = "ge", 868 [TCG_COND_LE] = "le", 869 [TCG_COND_GT] = "gt", 870 [TCG_COND_LTU] = "ltu", 871 [TCG_COND_GEU] = "geu", 872 [TCG_COND_LEU] = "leu", 873 [TCG_COND_GTU] = "gtu" 874 }; 875 876 void tcg_dump_ops(TCGContext *s) 877 { 878 const uint16_t *opc_ptr; 879 const TCGArg *args; 880 TCGArg arg; 881 TCGOpcode c; 882 int i, k, nb_oargs, nb_iargs, nb_cargs, first_insn; 883 const TCGOpDef *def; 884 char buf[128]; 885 886 first_insn = 1; 887 opc_ptr = gen_opc_buf; 888 args = gen_opparam_buf; 889 while (opc_ptr < gen_opc_ptr) { 890 c = *opc_ptr++; 891 def = &tcg_op_defs[c]; 892 if (c == INDEX_op_debug_insn_start) { 893 uint64_t pc; 894 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 895 pc = ((uint64_t)args[1] << 32) | args[0]; 896 #else 897 pc = args[0]; 898 #endif 899 if (!first_insn) { 900 qemu_log("\n"); 901 } 902 qemu_log(" ---- 0x%" PRIx64, pc); 903 first_insn = 0; 904 nb_oargs = def->nb_oargs; 905 nb_iargs = def->nb_iargs; 906 nb_cargs = def->nb_cargs; 907 } else if (c == INDEX_op_call) { 908 TCGArg arg; 909 910 /* variable number of arguments */ 911 arg = *args++; 912 nb_oargs = arg >> 16; 913 nb_iargs = arg & 0xffff; 914 nb_cargs = def->nb_cargs; 915 916 qemu_log(" %s ", def->name); 917 918 /* function name */ 919 qemu_log("%s", 920 tcg_get_arg_str_idx(s, buf, sizeof(buf), 921 args[nb_oargs + nb_iargs - 1])); 922 /* flags */ 923 qemu_log(",$0x%" TCG_PRIlx, args[nb_oargs + nb_iargs]); 924 /* nb out args */ 925 qemu_log(",$%d", nb_oargs); 926 for(i = 0; i < nb_oargs; i++) { 927 qemu_log(","); 928 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 929 args[i])); 930 } 931 for(i = 0; i < (nb_iargs - 1); i++) { 932 qemu_log(","); 933 if (args[nb_oargs + i] == TCG_CALL_DUMMY_ARG) { 934 qemu_log("<dummy>"); 935 } else { 936 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 937 args[nb_oargs + i])); 938 } 939 } 940 } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) { 941 tcg_target_ulong val; 942 TCGHelperInfo *th; 943 944 nb_oargs = def->nb_oargs; 945 nb_iargs = def->nb_iargs; 946 nb_cargs = def->nb_cargs; 947 qemu_log(" %s %s,$", def->name, 948 tcg_get_arg_str_idx(s, buf, sizeof(buf), args[0])); 949 val = args[1]; 950 th = tcg_find_helper(s, val); 951 if (th) { 952 qemu_log("%s", th->name); 953 } else { 954 if (c == INDEX_op_movi_i32) { 955 qemu_log("0x%x", (uint32_t)val); 956 } else { 957 qemu_log("0x%" PRIx64 , (uint64_t)val); 958 } 959 } 960 } else { 961 qemu_log(" %s ", def->name); 962 if (c == INDEX_op_nopn) { 963 /* variable number of arguments */ 964 nb_cargs = *args; 965 nb_oargs = 0; 966 nb_iargs = 0; 967 } else { 968 nb_oargs = def->nb_oargs; 969 nb_iargs = def->nb_iargs; 970 nb_cargs = def->nb_cargs; 971 } 972 973 k = 0; 974 for(i = 0; i < nb_oargs; i++) { 975 if (k != 0) { 976 qemu_log(","); 977 } 978 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 979 args[k++])); 980 } 981 for(i = 0; i < nb_iargs; i++) { 982 if (k != 0) { 983 qemu_log(","); 984 } 985 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 986 args[k++])); 987 } 988 switch (c) { 989 case INDEX_op_brcond_i32: 990 case INDEX_op_setcond_i32: 991 case INDEX_op_movcond_i32: 992 case INDEX_op_brcond2_i32: 993 case INDEX_op_setcond2_i32: 994 case INDEX_op_brcond_i64: 995 case INDEX_op_setcond_i64: 996 case INDEX_op_movcond_i64: 997 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) { 998 qemu_log(",%s", cond_name[args[k++]]); 999 } else { 1000 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1001 } 1002 i = 1; 1003 break; 1004 default: 1005 i = 0; 1006 break; 1007 } 1008 for(; i < nb_cargs; i++) { 1009 if (k != 0) { 1010 qemu_log(","); 1011 } 1012 arg = args[k++]; 1013 qemu_log("$0x%" TCG_PRIlx, arg); 1014 } 1015 } 1016 qemu_log("\n"); 1017 args += nb_iargs + nb_oargs + nb_cargs; 1018 } 1019 } 1020 1021 /* we give more priority to constraints with less registers */ 1022 static int get_constraint_priority(const TCGOpDef *def, int k) 1023 { 1024 const TCGArgConstraint *arg_ct; 1025 1026 int i, n; 1027 arg_ct = &def->args_ct[k]; 1028 if (arg_ct->ct & TCG_CT_ALIAS) { 1029 /* an alias is equivalent to a single register */ 1030 n = 1; 1031 } else { 1032 if (!(arg_ct->ct & TCG_CT_REG)) 1033 return 0; 1034 n = 0; 1035 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1036 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1037 n++; 1038 } 1039 } 1040 return TCG_TARGET_NB_REGS - n + 1; 1041 } 1042 1043 /* sort from highest priority to lowest */ 1044 static void sort_constraints(TCGOpDef *def, int start, int n) 1045 { 1046 int i, j, p1, p2, tmp; 1047 1048 for(i = 0; i < n; i++) 1049 def->sorted_args[start + i] = start + i; 1050 if (n <= 1) 1051 return; 1052 for(i = 0; i < n - 1; i++) { 1053 for(j = i + 1; j < n; j++) { 1054 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1055 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1056 if (p1 < p2) { 1057 tmp = def->sorted_args[start + i]; 1058 def->sorted_args[start + i] = def->sorted_args[start + j]; 1059 def->sorted_args[start + j] = tmp; 1060 } 1061 } 1062 } 1063 } 1064 1065 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs) 1066 { 1067 TCGOpcode op; 1068 TCGOpDef *def; 1069 const char *ct_str; 1070 int i, nb_args; 1071 1072 for(;;) { 1073 if (tdefs->op == (TCGOpcode)-1) 1074 break; 1075 op = tdefs->op; 1076 assert((unsigned)op < NB_OPS); 1077 def = &tcg_op_defs[op]; 1078 #if defined(CONFIG_DEBUG_TCG) 1079 /* Duplicate entry in op definitions? */ 1080 assert(!def->used); 1081 def->used = 1; 1082 #endif 1083 nb_args = def->nb_iargs + def->nb_oargs; 1084 for(i = 0; i < nb_args; i++) { 1085 ct_str = tdefs->args_ct_str[i]; 1086 /* Incomplete TCGTargetOpDef entry? */ 1087 assert(ct_str != NULL); 1088 tcg_regset_clear(def->args_ct[i].u.regs); 1089 def->args_ct[i].ct = 0; 1090 if (ct_str[0] >= '0' && ct_str[0] <= '9') { 1091 int oarg; 1092 oarg = ct_str[0] - '0'; 1093 assert(oarg < def->nb_oargs); 1094 assert(def->args_ct[oarg].ct & TCG_CT_REG); 1095 /* TCG_CT_ALIAS is for the output arguments. The input 1096 argument is tagged with TCG_CT_IALIAS. */ 1097 def->args_ct[i] = def->args_ct[oarg]; 1098 def->args_ct[oarg].ct = TCG_CT_ALIAS; 1099 def->args_ct[oarg].alias_index = i; 1100 def->args_ct[i].ct |= TCG_CT_IALIAS; 1101 def->args_ct[i].alias_index = oarg; 1102 } else { 1103 for(;;) { 1104 if (*ct_str == '\0') 1105 break; 1106 switch(*ct_str) { 1107 case 'i': 1108 def->args_ct[i].ct |= TCG_CT_CONST; 1109 ct_str++; 1110 break; 1111 default: 1112 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) { 1113 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n", 1114 ct_str, i, def->name); 1115 exit(1); 1116 } 1117 } 1118 } 1119 } 1120 } 1121 1122 /* TCGTargetOpDef entry with too much information? */ 1123 assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1124 1125 /* sort the constraints (XXX: this is just an heuristic) */ 1126 sort_constraints(def, 0, def->nb_oargs); 1127 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1128 1129 #if 0 1130 { 1131 int i; 1132 1133 printf("%s: sorted=", def->name); 1134 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++) 1135 printf(" %d", def->sorted_args[i]); 1136 printf("\n"); 1137 } 1138 #endif 1139 tdefs++; 1140 } 1141 1142 #if defined(CONFIG_DEBUG_TCG) 1143 i = 0; 1144 for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) { 1145 const TCGOpDef *def = &tcg_op_defs[op]; 1146 if (op < INDEX_op_call 1147 || op == INDEX_op_debug_insn_start 1148 || (def->flags & TCG_OPF_NOT_PRESENT)) { 1149 /* Wrong entry in op definitions? */ 1150 if (def->used) { 1151 fprintf(stderr, "Invalid op definition for %s\n", def->name); 1152 i = 1; 1153 } 1154 } else { 1155 /* Missing entry in op definitions? */ 1156 if (!def->used) { 1157 fprintf(stderr, "Missing op definition for %s\n", def->name); 1158 i = 1; 1159 } 1160 } 1161 } 1162 if (i == 1) { 1163 tcg_abort(); 1164 } 1165 #endif 1166 } 1167 1168 #ifdef USE_LIVENESS_ANALYSIS 1169 1170 /* set a nop for an operation using 'nb_args' */ 1171 static inline void tcg_set_nop(TCGContext *s, uint16_t *opc_ptr, 1172 TCGArg *args, int nb_args) 1173 { 1174 if (nb_args == 0) { 1175 *opc_ptr = INDEX_op_nop; 1176 } else { 1177 *opc_ptr = INDEX_op_nopn; 1178 args[0] = nb_args; 1179 args[nb_args - 1] = nb_args; 1180 } 1181 } 1182 1183 /* liveness analysis: end of function: globals are live, temps are 1184 dead. */ 1185 /* XXX: at this stage, not used as there would be little gains because 1186 most TBs end with a conditional jump. */ 1187 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps) 1188 { 1189 memset(dead_temps, 0, s->nb_globals); 1190 memset(dead_temps + s->nb_globals, 1, s->nb_temps - s->nb_globals); 1191 } 1192 1193 /* liveness analysis: end of basic block: globals are live, temps are 1194 dead, local temps are live. */ 1195 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps) 1196 { 1197 int i; 1198 TCGTemp *ts; 1199 1200 memset(dead_temps, 0, s->nb_globals); 1201 ts = &s->temps[s->nb_globals]; 1202 for(i = s->nb_globals; i < s->nb_temps; i++) { 1203 if (ts->temp_local) 1204 dead_temps[i] = 0; 1205 else 1206 dead_temps[i] = 1; 1207 ts++; 1208 } 1209 } 1210 1211 /* Liveness analysis : update the opc_dead_args array to tell if a 1212 given input arguments is dead. Instructions updating dead 1213 temporaries are removed. */ 1214 static void tcg_liveness_analysis(TCGContext *s) 1215 { 1216 int i, op_index, nb_args, nb_iargs, nb_oargs, arg, nb_ops; 1217 TCGOpcode op; 1218 TCGArg *args; 1219 const TCGOpDef *def; 1220 uint8_t *dead_temps; 1221 unsigned int dead_args; 1222 1223 gen_opc_ptr++; /* skip end */ 1224 1225 nb_ops = gen_opc_ptr - gen_opc_buf; 1226 1227 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1228 1229 dead_temps = tcg_malloc(s->nb_temps); 1230 memset(dead_temps, 1, s->nb_temps); 1231 1232 args = gen_opparam_ptr; 1233 op_index = nb_ops - 1; 1234 while (op_index >= 0) { 1235 op = gen_opc_buf[op_index]; 1236 def = &tcg_op_defs[op]; 1237 switch(op) { 1238 case INDEX_op_call: 1239 { 1240 int call_flags; 1241 1242 nb_args = args[-1]; 1243 args -= nb_args; 1244 nb_iargs = args[0] & 0xffff; 1245 nb_oargs = args[0] >> 16; 1246 args++; 1247 call_flags = args[nb_oargs + nb_iargs]; 1248 1249 /* pure functions can be removed if their result is not 1250 used */ 1251 if (call_flags & TCG_CALL_PURE) { 1252 for(i = 0; i < nb_oargs; i++) { 1253 arg = args[i]; 1254 if (!dead_temps[arg]) 1255 goto do_not_remove_call; 1256 } 1257 tcg_set_nop(s, gen_opc_buf + op_index, 1258 args - 1, nb_args); 1259 } else { 1260 do_not_remove_call: 1261 1262 /* output args are dead */ 1263 dead_args = 0; 1264 for(i = 0; i < nb_oargs; i++) { 1265 arg = args[i]; 1266 if (dead_temps[arg]) { 1267 dead_args |= (1 << i); 1268 } 1269 dead_temps[arg] = 1; 1270 } 1271 1272 if (!(call_flags & TCG_CALL_CONST)) { 1273 /* globals are live (they may be used by the call) */ 1274 memset(dead_temps, 0, s->nb_globals); 1275 } 1276 1277 /* input args are live */ 1278 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1279 arg = args[i]; 1280 if (arg != TCG_CALL_DUMMY_ARG) { 1281 if (dead_temps[arg]) { 1282 dead_args |= (1 << i); 1283 } 1284 dead_temps[arg] = 0; 1285 } 1286 } 1287 s->op_dead_args[op_index] = dead_args; 1288 } 1289 args--; 1290 } 1291 break; 1292 case INDEX_op_debug_insn_start: 1293 args -= def->nb_args; 1294 break; 1295 case INDEX_op_nopn: 1296 nb_args = args[-1]; 1297 args -= nb_args; 1298 break; 1299 case INDEX_op_discard: 1300 args--; 1301 /* mark the temporary as dead */ 1302 dead_temps[args[0]] = 1; 1303 break; 1304 case INDEX_op_end: 1305 break; 1306 1307 case INDEX_op_add2_i32: 1308 case INDEX_op_sub2_i32: 1309 args -= 6; 1310 nb_iargs = 4; 1311 nb_oargs = 2; 1312 /* Test if the high part of the operation is dead, but not 1313 the low part. The result can be optimized to a simple 1314 add or sub. This happens often for x86_64 guest when the 1315 cpu mode is set to 32 bit. */ 1316 if (dead_temps[args[1]]) { 1317 if (dead_temps[args[0]]) { 1318 goto do_remove; 1319 } 1320 /* Create the single operation plus nop. */ 1321 if (op == INDEX_op_add2_i32) { 1322 op = INDEX_op_add_i32; 1323 } else { 1324 op = INDEX_op_sub_i32; 1325 } 1326 gen_opc_buf[op_index] = op; 1327 args[1] = args[2]; 1328 args[2] = args[4]; 1329 assert(gen_opc_buf[op_index + 1] == INDEX_op_nop); 1330 tcg_set_nop(s, gen_opc_buf + op_index + 1, args + 3, 3); 1331 /* Fall through and mark the single-word operation live. */ 1332 nb_iargs = 2; 1333 nb_oargs = 1; 1334 } 1335 goto do_not_remove; 1336 1337 case INDEX_op_mulu2_i32: 1338 args -= 4; 1339 nb_iargs = 2; 1340 nb_oargs = 2; 1341 /* Likewise, test for the high part of the operation dead. */ 1342 if (dead_temps[args[1]]) { 1343 if (dead_temps[args[0]]) { 1344 goto do_remove; 1345 } 1346 gen_opc_buf[op_index] = op = INDEX_op_mul_i32; 1347 args[1] = args[2]; 1348 args[2] = args[3]; 1349 assert(gen_opc_buf[op_index + 1] == INDEX_op_nop); 1350 tcg_set_nop(s, gen_opc_buf + op_index + 1, args + 3, 1); 1351 /* Fall through and mark the single-word operation live. */ 1352 nb_oargs = 1; 1353 } 1354 goto do_not_remove; 1355 1356 default: 1357 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 1358 args -= def->nb_args; 1359 nb_iargs = def->nb_iargs; 1360 nb_oargs = def->nb_oargs; 1361 1362 /* Test if the operation can be removed because all 1363 its outputs are dead. We assume that nb_oargs == 0 1364 implies side effects */ 1365 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 1366 for(i = 0; i < nb_oargs; i++) { 1367 arg = args[i]; 1368 if (!dead_temps[arg]) 1369 goto do_not_remove; 1370 } 1371 do_remove: 1372 tcg_set_nop(s, gen_opc_buf + op_index, args, def->nb_args); 1373 #ifdef CONFIG_PROFILER 1374 s->del_op_count++; 1375 #endif 1376 } else { 1377 do_not_remove: 1378 1379 /* output args are dead */ 1380 dead_args = 0; 1381 for(i = 0; i < nb_oargs; i++) { 1382 arg = args[i]; 1383 if (dead_temps[arg]) { 1384 dead_args |= (1 << i); 1385 } 1386 dead_temps[arg] = 1; 1387 } 1388 1389 /* if end of basic block, update */ 1390 if (def->flags & TCG_OPF_BB_END) { 1391 tcg_la_bb_end(s, dead_temps); 1392 } else if (def->flags & TCG_OPF_CALL_CLOBBER) { 1393 /* globals are live */ 1394 memset(dead_temps, 0, s->nb_globals); 1395 } 1396 1397 /* input args are live */ 1398 for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1399 arg = args[i]; 1400 if (dead_temps[arg]) { 1401 dead_args |= (1 << i); 1402 } 1403 dead_temps[arg] = 0; 1404 } 1405 s->op_dead_args[op_index] = dead_args; 1406 } 1407 break; 1408 } 1409 op_index--; 1410 } 1411 1412 if (args != gen_opparam_buf) 1413 tcg_abort(); 1414 } 1415 #else 1416 /* dummy liveness analysis */ 1417 static void tcg_liveness_analysis(TCGContext *s) 1418 { 1419 int nb_ops; 1420 nb_ops = gen_opc_ptr - gen_opc_buf; 1421 1422 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1423 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t)); 1424 } 1425 #endif 1426 1427 #ifndef NDEBUG 1428 static void dump_regs(TCGContext *s) 1429 { 1430 TCGTemp *ts; 1431 int i; 1432 char buf[64]; 1433 1434 for(i = 0; i < s->nb_temps; i++) { 1435 ts = &s->temps[i]; 1436 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i)); 1437 switch(ts->val_type) { 1438 case TEMP_VAL_REG: 1439 printf("%s", tcg_target_reg_names[ts->reg]); 1440 break; 1441 case TEMP_VAL_MEM: 1442 printf("%d(%s)", (int)ts->mem_offset, tcg_target_reg_names[ts->mem_reg]); 1443 break; 1444 case TEMP_VAL_CONST: 1445 printf("$0x%" TCG_PRIlx, ts->val); 1446 break; 1447 case TEMP_VAL_DEAD: 1448 printf("D"); 1449 break; 1450 default: 1451 printf("???"); 1452 break; 1453 } 1454 printf("\n"); 1455 } 1456 1457 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1458 if (s->reg_to_temp[i] >= 0) { 1459 printf("%s: %s\n", 1460 tcg_target_reg_names[i], 1461 tcg_get_arg_str_idx(s, buf, sizeof(buf), s->reg_to_temp[i])); 1462 } 1463 } 1464 } 1465 1466 static void check_regs(TCGContext *s) 1467 { 1468 int reg, k; 1469 TCGTemp *ts; 1470 char buf[64]; 1471 1472 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 1473 k = s->reg_to_temp[reg]; 1474 if (k >= 0) { 1475 ts = &s->temps[k]; 1476 if (ts->val_type != TEMP_VAL_REG || 1477 ts->reg != reg) { 1478 printf("Inconsistency for register %s:\n", 1479 tcg_target_reg_names[reg]); 1480 goto fail; 1481 } 1482 } 1483 } 1484 for(k = 0; k < s->nb_temps; k++) { 1485 ts = &s->temps[k]; 1486 if (ts->val_type == TEMP_VAL_REG && 1487 !ts->fixed_reg && 1488 s->reg_to_temp[ts->reg] != k) { 1489 printf("Inconsistency for temp %s:\n", 1490 tcg_get_arg_str_idx(s, buf, sizeof(buf), k)); 1491 fail: 1492 printf("reg state:\n"); 1493 dump_regs(s); 1494 tcg_abort(); 1495 } 1496 } 1497 } 1498 #endif 1499 1500 static void temp_allocate_frame(TCGContext *s, int temp) 1501 { 1502 TCGTemp *ts; 1503 ts = &s->temps[temp]; 1504 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 1505 /* Sparc64 stack is accessed with offset of 2047 */ 1506 s->current_frame_offset = (s->current_frame_offset + 1507 (tcg_target_long)sizeof(tcg_target_long) - 1) & 1508 ~(sizeof(tcg_target_long) - 1); 1509 #endif 1510 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 1511 s->frame_end) { 1512 tcg_abort(); 1513 } 1514 ts->mem_offset = s->current_frame_offset; 1515 ts->mem_reg = s->frame_reg; 1516 ts->mem_allocated = 1; 1517 s->current_frame_offset += (tcg_target_long)sizeof(tcg_target_long); 1518 } 1519 1520 /* free register 'reg' by spilling the corresponding temporary if necessary */ 1521 static void tcg_reg_free(TCGContext *s, int reg) 1522 { 1523 TCGTemp *ts; 1524 int temp; 1525 1526 temp = s->reg_to_temp[reg]; 1527 if (temp != -1) { 1528 ts = &s->temps[temp]; 1529 assert(ts->val_type == TEMP_VAL_REG); 1530 if (!ts->mem_coherent) { 1531 if (!ts->mem_allocated) 1532 temp_allocate_frame(s, temp); 1533 tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1534 } 1535 ts->val_type = TEMP_VAL_MEM; 1536 s->reg_to_temp[reg] = -1; 1537 } 1538 } 1539 1540 /* Allocate a register belonging to reg1 & ~reg2 */ 1541 static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2) 1542 { 1543 int i, reg; 1544 TCGRegSet reg_ct; 1545 1546 tcg_regset_andnot(reg_ct, reg1, reg2); 1547 1548 /* first try free registers */ 1549 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1550 reg = tcg_target_reg_alloc_order[i]; 1551 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == -1) 1552 return reg; 1553 } 1554 1555 /* XXX: do better spill choice */ 1556 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1557 reg = tcg_target_reg_alloc_order[i]; 1558 if (tcg_regset_test_reg(reg_ct, reg)) { 1559 tcg_reg_free(s, reg); 1560 return reg; 1561 } 1562 } 1563 1564 tcg_abort(); 1565 } 1566 1567 /* save a temporary to memory. 'allocated_regs' is used in case a 1568 temporary registers needs to be allocated to store a constant. */ 1569 static void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs) 1570 { 1571 TCGTemp *ts; 1572 int reg; 1573 1574 ts = &s->temps[temp]; 1575 if (!ts->fixed_reg) { 1576 switch(ts->val_type) { 1577 case TEMP_VAL_REG: 1578 tcg_reg_free(s, ts->reg); 1579 break; 1580 case TEMP_VAL_DEAD: 1581 ts->val_type = TEMP_VAL_MEM; 1582 break; 1583 case TEMP_VAL_CONST: 1584 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 1585 allocated_regs); 1586 if (!ts->mem_allocated) 1587 temp_allocate_frame(s, temp); 1588 tcg_out_movi(s, ts->type, reg, ts->val); 1589 tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1590 ts->val_type = TEMP_VAL_MEM; 1591 break; 1592 case TEMP_VAL_MEM: 1593 break; 1594 default: 1595 tcg_abort(); 1596 } 1597 } 1598 } 1599 1600 /* save globals to their canonical location and assume they can be 1601 modified be the following code. 'allocated_regs' is used in case a 1602 temporary registers needs to be allocated to store a constant. */ 1603 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 1604 { 1605 int i; 1606 1607 for(i = 0; i < s->nb_globals; i++) { 1608 temp_save(s, i, allocated_regs); 1609 } 1610 } 1611 1612 /* at the end of a basic block, we assume all temporaries are dead and 1613 all globals are stored at their canonical location. */ 1614 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 1615 { 1616 TCGTemp *ts; 1617 int i; 1618 1619 for(i = s->nb_globals; i < s->nb_temps; i++) { 1620 ts = &s->temps[i]; 1621 if (ts->temp_local) { 1622 temp_save(s, i, allocated_regs); 1623 } else { 1624 if (ts->val_type == TEMP_VAL_REG) { 1625 s->reg_to_temp[ts->reg] = -1; 1626 } 1627 ts->val_type = TEMP_VAL_DEAD; 1628 } 1629 } 1630 1631 save_globals(s, allocated_regs); 1632 } 1633 1634 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1) 1635 1636 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args) 1637 { 1638 TCGTemp *ots; 1639 tcg_target_ulong val; 1640 1641 ots = &s->temps[args[0]]; 1642 val = args[1]; 1643 1644 if (ots->fixed_reg) { 1645 /* for fixed registers, we do not do any constant 1646 propagation */ 1647 tcg_out_movi(s, ots->type, ots->reg, val); 1648 } else { 1649 /* The movi is not explicitly generated here */ 1650 if (ots->val_type == TEMP_VAL_REG) 1651 s->reg_to_temp[ots->reg] = -1; 1652 ots->val_type = TEMP_VAL_CONST; 1653 ots->val = val; 1654 } 1655 } 1656 1657 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, 1658 const TCGArg *args, 1659 unsigned int dead_args) 1660 { 1661 TCGTemp *ts, *ots; 1662 int reg; 1663 const TCGArgConstraint *arg_ct; 1664 1665 ots = &s->temps[args[0]]; 1666 ts = &s->temps[args[1]]; 1667 arg_ct = &def->args_ct[0]; 1668 1669 /* XXX: always mark arg dead if IS_DEAD_ARG(1) */ 1670 if (ts->val_type == TEMP_VAL_REG) { 1671 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 1672 /* the mov can be suppressed */ 1673 if (ots->val_type == TEMP_VAL_REG) 1674 s->reg_to_temp[ots->reg] = -1; 1675 reg = ts->reg; 1676 s->reg_to_temp[reg] = -1; 1677 ts->val_type = TEMP_VAL_DEAD; 1678 } else { 1679 if (ots->val_type == TEMP_VAL_REG) { 1680 reg = ots->reg; 1681 } else { 1682 reg = tcg_reg_alloc(s, arg_ct->u.regs, s->reserved_regs); 1683 } 1684 if (ts->reg != reg) { 1685 tcg_out_mov(s, ots->type, reg, ts->reg); 1686 } 1687 } 1688 } else if (ts->val_type == TEMP_VAL_MEM) { 1689 if (ots->val_type == TEMP_VAL_REG) { 1690 reg = ots->reg; 1691 } else { 1692 reg = tcg_reg_alloc(s, arg_ct->u.regs, s->reserved_regs); 1693 } 1694 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1695 } else if (ts->val_type == TEMP_VAL_CONST) { 1696 if (ots->fixed_reg) { 1697 reg = ots->reg; 1698 tcg_out_movi(s, ots->type, reg, ts->val); 1699 } else { 1700 /* propagate constant */ 1701 if (ots->val_type == TEMP_VAL_REG) 1702 s->reg_to_temp[ots->reg] = -1; 1703 ots->val_type = TEMP_VAL_CONST; 1704 ots->val = ts->val; 1705 return; 1706 } 1707 } else { 1708 tcg_abort(); 1709 } 1710 s->reg_to_temp[reg] = args[0]; 1711 ots->reg = reg; 1712 ots->val_type = TEMP_VAL_REG; 1713 ots->mem_coherent = 0; 1714 } 1715 1716 static void tcg_reg_alloc_op(TCGContext *s, 1717 const TCGOpDef *def, TCGOpcode opc, 1718 const TCGArg *args, 1719 unsigned int dead_args) 1720 { 1721 TCGRegSet allocated_regs; 1722 int i, k, nb_iargs, nb_oargs, reg; 1723 TCGArg arg; 1724 const TCGArgConstraint *arg_ct; 1725 TCGTemp *ts; 1726 TCGArg new_args[TCG_MAX_OP_ARGS]; 1727 int const_args[TCG_MAX_OP_ARGS]; 1728 1729 nb_oargs = def->nb_oargs; 1730 nb_iargs = def->nb_iargs; 1731 1732 /* copy constants */ 1733 memcpy(new_args + nb_oargs + nb_iargs, 1734 args + nb_oargs + nb_iargs, 1735 sizeof(TCGArg) * def->nb_cargs); 1736 1737 /* satisfy input constraints */ 1738 tcg_regset_set(allocated_regs, s->reserved_regs); 1739 for(k = 0; k < nb_iargs; k++) { 1740 i = def->sorted_args[nb_oargs + k]; 1741 arg = args[i]; 1742 arg_ct = &def->args_ct[i]; 1743 ts = &s->temps[arg]; 1744 if (ts->val_type == TEMP_VAL_MEM) { 1745 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 1746 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1747 ts->val_type = TEMP_VAL_REG; 1748 ts->reg = reg; 1749 ts->mem_coherent = 1; 1750 s->reg_to_temp[reg] = arg; 1751 } else if (ts->val_type == TEMP_VAL_CONST) { 1752 if (tcg_target_const_match(ts->val, arg_ct)) { 1753 /* constant is OK for instruction */ 1754 const_args[i] = 1; 1755 new_args[i] = ts->val; 1756 goto iarg_end; 1757 } else { 1758 /* need to move to a register */ 1759 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 1760 tcg_out_movi(s, ts->type, reg, ts->val); 1761 ts->val_type = TEMP_VAL_REG; 1762 ts->reg = reg; 1763 ts->mem_coherent = 0; 1764 s->reg_to_temp[reg] = arg; 1765 } 1766 } 1767 assert(ts->val_type == TEMP_VAL_REG); 1768 if (arg_ct->ct & TCG_CT_IALIAS) { 1769 if (ts->fixed_reg) { 1770 /* if fixed register, we must allocate a new register 1771 if the alias is not the same register */ 1772 if (arg != args[arg_ct->alias_index]) 1773 goto allocate_in_reg; 1774 } else { 1775 /* if the input is aliased to an output and if it is 1776 not dead after the instruction, we must allocate 1777 a new register and move it */ 1778 if (!IS_DEAD_ARG(i)) { 1779 goto allocate_in_reg; 1780 } 1781 } 1782 } 1783 reg = ts->reg; 1784 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 1785 /* nothing to do : the constraint is satisfied */ 1786 } else { 1787 allocate_in_reg: 1788 /* allocate a new register matching the constraint 1789 and move the temporary register into it */ 1790 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 1791 tcg_out_mov(s, ts->type, reg, ts->reg); 1792 } 1793 new_args[i] = reg; 1794 const_args[i] = 0; 1795 tcg_regset_set_reg(allocated_regs, reg); 1796 iarg_end: ; 1797 } 1798 1799 if (def->flags & TCG_OPF_BB_END) { 1800 tcg_reg_alloc_bb_end(s, allocated_regs); 1801 } else { 1802 /* mark dead temporaries and free the associated registers */ 1803 for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1804 arg = args[i]; 1805 if (IS_DEAD_ARG(i)) { 1806 ts = &s->temps[arg]; 1807 if (!ts->fixed_reg) { 1808 if (ts->val_type == TEMP_VAL_REG) 1809 s->reg_to_temp[ts->reg] = -1; 1810 ts->val_type = TEMP_VAL_DEAD; 1811 } 1812 } 1813 } 1814 1815 if (def->flags & TCG_OPF_CALL_CLOBBER) { 1816 /* XXX: permit generic clobber register list ? */ 1817 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 1818 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 1819 tcg_reg_free(s, reg); 1820 } 1821 } 1822 /* XXX: for load/store we could do that only for the slow path 1823 (i.e. when a memory callback is called) */ 1824 1825 /* store globals and free associated registers (we assume the insn 1826 can modify any global. */ 1827 save_globals(s, allocated_regs); 1828 } 1829 1830 /* satisfy the output constraints */ 1831 tcg_regset_set(allocated_regs, s->reserved_regs); 1832 for(k = 0; k < nb_oargs; k++) { 1833 i = def->sorted_args[k]; 1834 arg = args[i]; 1835 arg_ct = &def->args_ct[i]; 1836 ts = &s->temps[arg]; 1837 if (arg_ct->ct & TCG_CT_ALIAS) { 1838 reg = new_args[arg_ct->alias_index]; 1839 } else { 1840 /* if fixed register, we try to use it */ 1841 reg = ts->reg; 1842 if (ts->fixed_reg && 1843 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 1844 goto oarg_end; 1845 } 1846 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 1847 } 1848 tcg_regset_set_reg(allocated_regs, reg); 1849 /* if a fixed register is used, then a move will be done afterwards */ 1850 if (!ts->fixed_reg) { 1851 if (ts->val_type == TEMP_VAL_REG) 1852 s->reg_to_temp[ts->reg] = -1; 1853 if (IS_DEAD_ARG(i)) { 1854 ts->val_type = TEMP_VAL_DEAD; 1855 } else { 1856 ts->val_type = TEMP_VAL_REG; 1857 ts->reg = reg; 1858 /* temp value is modified, so the value kept in memory is 1859 potentially not the same */ 1860 ts->mem_coherent = 0; 1861 s->reg_to_temp[reg] = arg; 1862 } 1863 } 1864 oarg_end: 1865 new_args[i] = reg; 1866 } 1867 } 1868 1869 /* emit instruction */ 1870 tcg_out_op(s, opc, new_args, const_args); 1871 1872 /* move the outputs in the correct register if needed */ 1873 for(i = 0; i < nb_oargs; i++) { 1874 ts = &s->temps[args[i]]; 1875 reg = new_args[i]; 1876 if (ts->fixed_reg && ts->reg != reg) { 1877 tcg_out_mov(s, ts->type, ts->reg, reg); 1878 } 1879 } 1880 } 1881 1882 #ifdef TCG_TARGET_STACK_GROWSUP 1883 #define STACK_DIR(x) (-(x)) 1884 #else 1885 #define STACK_DIR(x) (x) 1886 #endif 1887 1888 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, 1889 TCGOpcode opc, const TCGArg *args, 1890 unsigned int dead_args) 1891 { 1892 int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params; 1893 TCGArg arg, func_arg; 1894 TCGTemp *ts; 1895 tcg_target_long stack_offset, call_stack_size, func_addr; 1896 int const_func_arg, allocate_args; 1897 TCGRegSet allocated_regs; 1898 const TCGArgConstraint *arg_ct; 1899 1900 arg = *args++; 1901 1902 nb_oargs = arg >> 16; 1903 nb_iargs = arg & 0xffff; 1904 nb_params = nb_iargs - 1; 1905 1906 flags = args[nb_oargs + nb_iargs]; 1907 1908 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 1909 if (nb_regs > nb_params) 1910 nb_regs = nb_params; 1911 1912 /* assign stack slots first */ 1913 call_stack_size = (nb_params - nb_regs) * sizeof(tcg_target_long); 1914 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 1915 ~(TCG_TARGET_STACK_ALIGN - 1); 1916 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 1917 if (allocate_args) { 1918 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 1919 preallocate call stack */ 1920 tcg_abort(); 1921 } 1922 1923 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 1924 for(i = nb_regs; i < nb_params; i++) { 1925 arg = args[nb_oargs + i]; 1926 #ifdef TCG_TARGET_STACK_GROWSUP 1927 stack_offset -= sizeof(tcg_target_long); 1928 #endif 1929 if (arg != TCG_CALL_DUMMY_ARG) { 1930 ts = &s->temps[arg]; 1931 if (ts->val_type == TEMP_VAL_REG) { 1932 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 1933 } else if (ts->val_type == TEMP_VAL_MEM) { 1934 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 1935 s->reserved_regs); 1936 /* XXX: not correct if reading values from the stack */ 1937 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1938 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 1939 } else if (ts->val_type == TEMP_VAL_CONST) { 1940 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 1941 s->reserved_regs); 1942 /* XXX: sign extend may be needed on some targets */ 1943 tcg_out_movi(s, ts->type, reg, ts->val); 1944 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 1945 } else { 1946 tcg_abort(); 1947 } 1948 } 1949 #ifndef TCG_TARGET_STACK_GROWSUP 1950 stack_offset += sizeof(tcg_target_long); 1951 #endif 1952 } 1953 1954 /* assign input registers */ 1955 tcg_regset_set(allocated_regs, s->reserved_regs); 1956 for(i = 0; i < nb_regs; i++) { 1957 arg = args[nb_oargs + i]; 1958 if (arg != TCG_CALL_DUMMY_ARG) { 1959 ts = &s->temps[arg]; 1960 reg = tcg_target_call_iarg_regs[i]; 1961 tcg_reg_free(s, reg); 1962 if (ts->val_type == TEMP_VAL_REG) { 1963 if (ts->reg != reg) { 1964 tcg_out_mov(s, ts->type, reg, ts->reg); 1965 } 1966 } else if (ts->val_type == TEMP_VAL_MEM) { 1967 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1968 } else if (ts->val_type == TEMP_VAL_CONST) { 1969 /* XXX: sign extend ? */ 1970 tcg_out_movi(s, ts->type, reg, ts->val); 1971 } else { 1972 tcg_abort(); 1973 } 1974 tcg_regset_set_reg(allocated_regs, reg); 1975 } 1976 } 1977 1978 /* assign function address */ 1979 func_arg = args[nb_oargs + nb_iargs - 1]; 1980 arg_ct = &def->args_ct[0]; 1981 ts = &s->temps[func_arg]; 1982 func_addr = ts->val; 1983 const_func_arg = 0; 1984 if (ts->val_type == TEMP_VAL_MEM) { 1985 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 1986 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1987 func_arg = reg; 1988 tcg_regset_set_reg(allocated_regs, reg); 1989 } else if (ts->val_type == TEMP_VAL_REG) { 1990 reg = ts->reg; 1991 if (!tcg_regset_test_reg(arg_ct->u.regs, reg)) { 1992 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 1993 tcg_out_mov(s, ts->type, reg, ts->reg); 1994 } 1995 func_arg = reg; 1996 tcg_regset_set_reg(allocated_regs, reg); 1997 } else if (ts->val_type == TEMP_VAL_CONST) { 1998 if (tcg_target_const_match(func_addr, arg_ct)) { 1999 const_func_arg = 1; 2000 func_arg = func_addr; 2001 } else { 2002 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2003 tcg_out_movi(s, ts->type, reg, func_addr); 2004 func_arg = reg; 2005 tcg_regset_set_reg(allocated_regs, reg); 2006 } 2007 } else { 2008 tcg_abort(); 2009 } 2010 2011 2012 /* mark dead temporaries and free the associated registers */ 2013 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2014 arg = args[i]; 2015 if (IS_DEAD_ARG(i)) { 2016 ts = &s->temps[arg]; 2017 if (!ts->fixed_reg) { 2018 if (ts->val_type == TEMP_VAL_REG) 2019 s->reg_to_temp[ts->reg] = -1; 2020 ts->val_type = TEMP_VAL_DEAD; 2021 } 2022 } 2023 } 2024 2025 /* clobber call registers */ 2026 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2027 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 2028 tcg_reg_free(s, reg); 2029 } 2030 } 2031 2032 /* store globals and free associated registers (we assume the call 2033 can modify any global. */ 2034 if (!(flags & TCG_CALL_CONST)) { 2035 save_globals(s, allocated_regs); 2036 } 2037 2038 tcg_out_op(s, opc, &func_arg, &const_func_arg); 2039 2040 /* assign output registers and emit moves if needed */ 2041 for(i = 0; i < nb_oargs; i++) { 2042 arg = args[i]; 2043 ts = &s->temps[arg]; 2044 reg = tcg_target_call_oarg_regs[i]; 2045 assert(s->reg_to_temp[reg] == -1); 2046 if (ts->fixed_reg) { 2047 if (ts->reg != reg) { 2048 tcg_out_mov(s, ts->type, ts->reg, reg); 2049 } 2050 } else { 2051 if (ts->val_type == TEMP_VAL_REG) 2052 s->reg_to_temp[ts->reg] = -1; 2053 if (IS_DEAD_ARG(i)) { 2054 ts->val_type = TEMP_VAL_DEAD; 2055 } else { 2056 ts->val_type = TEMP_VAL_REG; 2057 ts->reg = reg; 2058 ts->mem_coherent = 0; 2059 s->reg_to_temp[reg] = arg; 2060 } 2061 } 2062 } 2063 2064 return nb_iargs + nb_oargs + def->nb_cargs + 1; 2065 } 2066 2067 #ifdef CONFIG_PROFILER 2068 2069 static int64_t tcg_table_op_count[NB_OPS]; 2070 2071 static void dump_op_count(void) 2072 { 2073 int i; 2074 FILE *f; 2075 f = fopen("/tmp/op.log", "w"); 2076 for(i = INDEX_op_end; i < NB_OPS; i++) { 2077 fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]); 2078 } 2079 fclose(f); 2080 } 2081 #endif 2082 2083 2084 static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf, 2085 long search_pc) 2086 { 2087 TCGOpcode opc; 2088 int op_index; 2089 const TCGOpDef *def; 2090 unsigned int dead_args; 2091 const TCGArg *args; 2092 2093 #ifdef DEBUG_DISAS 2094 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) { 2095 qemu_log("OP:\n"); 2096 tcg_dump_ops(s); 2097 qemu_log("\n"); 2098 } 2099 #endif 2100 2101 #ifdef CONFIG_PROFILER 2102 s->opt_time -= profile_getclock(); 2103 #endif 2104 2105 #ifdef USE_TCG_OPTIMIZATIONS 2106 gen_opparam_ptr = 2107 tcg_optimize(s, gen_opc_ptr, gen_opparam_buf, tcg_op_defs); 2108 #endif 2109 2110 #ifdef CONFIG_PROFILER 2111 s->opt_time += profile_getclock(); 2112 s->la_time -= profile_getclock(); 2113 #endif 2114 2115 tcg_liveness_analysis(s); 2116 2117 #ifdef CONFIG_PROFILER 2118 s->la_time += profile_getclock(); 2119 #endif 2120 2121 #ifdef DEBUG_DISAS 2122 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) { 2123 qemu_log("OP after optimization and liveness analysis:\n"); 2124 tcg_dump_ops(s); 2125 qemu_log("\n"); 2126 } 2127 #endif 2128 2129 tcg_reg_alloc_start(s); 2130 2131 s->code_buf = gen_code_buf; 2132 s->code_ptr = gen_code_buf; 2133 2134 args = gen_opparam_buf; 2135 op_index = 0; 2136 2137 for(;;) { 2138 opc = gen_opc_buf[op_index]; 2139 #ifdef CONFIG_PROFILER 2140 tcg_table_op_count[opc]++; 2141 #endif 2142 def = &tcg_op_defs[opc]; 2143 #if 0 2144 printf("%s: %d %d %d\n", def->name, 2145 def->nb_oargs, def->nb_iargs, def->nb_cargs); 2146 // dump_regs(s); 2147 #endif 2148 switch(opc) { 2149 case INDEX_op_mov_i32: 2150 case INDEX_op_mov_i64: 2151 dead_args = s->op_dead_args[op_index]; 2152 tcg_reg_alloc_mov(s, def, args, dead_args); 2153 break; 2154 case INDEX_op_movi_i32: 2155 case INDEX_op_movi_i64: 2156 tcg_reg_alloc_movi(s, args); 2157 break; 2158 case INDEX_op_debug_insn_start: 2159 /* debug instruction */ 2160 break; 2161 case INDEX_op_nop: 2162 case INDEX_op_nop1: 2163 case INDEX_op_nop2: 2164 case INDEX_op_nop3: 2165 break; 2166 case INDEX_op_nopn: 2167 args += args[0]; 2168 goto next; 2169 case INDEX_op_discard: 2170 { 2171 TCGTemp *ts; 2172 ts = &s->temps[args[0]]; 2173 /* mark the temporary as dead */ 2174 if (!ts->fixed_reg) { 2175 if (ts->val_type == TEMP_VAL_REG) 2176 s->reg_to_temp[ts->reg] = -1; 2177 ts->val_type = TEMP_VAL_DEAD; 2178 } 2179 } 2180 break; 2181 case INDEX_op_set_label: 2182 tcg_reg_alloc_bb_end(s, s->reserved_regs); 2183 tcg_out_label(s, args[0], s->code_ptr); 2184 break; 2185 case INDEX_op_call: 2186 dead_args = s->op_dead_args[op_index]; 2187 args += tcg_reg_alloc_call(s, def, opc, args, dead_args); 2188 goto next; 2189 case INDEX_op_end: 2190 goto the_end; 2191 default: 2192 /* Sanity check that we've not introduced any unhandled opcodes. */ 2193 if (def->flags & TCG_OPF_NOT_PRESENT) { 2194 tcg_abort(); 2195 } 2196 /* Note: in order to speed up the code, it would be much 2197 faster to have specialized register allocator functions for 2198 some common argument patterns */ 2199 dead_args = s->op_dead_args[op_index]; 2200 tcg_reg_alloc_op(s, def, opc, args, dead_args); 2201 break; 2202 } 2203 args += def->nb_args; 2204 next: 2205 if (search_pc >= 0 && search_pc < s->code_ptr - gen_code_buf) { 2206 return op_index; 2207 } 2208 op_index++; 2209 #ifndef NDEBUG 2210 check_regs(s); 2211 #endif 2212 } 2213 the_end: 2214 return -1; 2215 } 2216 2217 int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf) 2218 { 2219 #ifdef CONFIG_PROFILER 2220 { 2221 int n; 2222 n = (gen_opc_ptr - gen_opc_buf); 2223 s->op_count += n; 2224 if (n > s->op_count_max) 2225 s->op_count_max = n; 2226 2227 s->temp_count += s->nb_temps; 2228 if (s->nb_temps > s->temp_count_max) 2229 s->temp_count_max = s->nb_temps; 2230 } 2231 #endif 2232 2233 tcg_gen_code_common(s, gen_code_buf, -1); 2234 2235 /* flush instruction cache */ 2236 flush_icache_range((tcg_target_ulong)gen_code_buf, 2237 (tcg_target_ulong)s->code_ptr); 2238 2239 return s->code_ptr - gen_code_buf; 2240 } 2241 2242 /* Return the index of the micro operation such as the pc after is < 2243 offset bytes from the start of the TB. The contents of gen_code_buf must 2244 not be changed, though writing the same values is ok. 2245 Return -1 if not found. */ 2246 int tcg_gen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset) 2247 { 2248 return tcg_gen_code_common(s, gen_code_buf, offset); 2249 } 2250 2251 #ifdef CONFIG_PROFILER 2252 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2253 { 2254 TCGContext *s = &tcg_ctx; 2255 int64_t tot; 2256 2257 tot = s->interm_time + s->code_time; 2258 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 2259 tot, tot / 2.4e9); 2260 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 2261 s->tb_count, 2262 s->tb_count1 - s->tb_count, 2263 s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0); 2264 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 2265 s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max); 2266 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 2267 s->tb_count ? 2268 (double)s->del_op_count / s->tb_count : 0); 2269 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 2270 s->tb_count ? 2271 (double)s->temp_count / s->tb_count : 0, 2272 s->temp_count_max); 2273 2274 cpu_fprintf(f, "cycles/op %0.1f\n", 2275 s->op_count ? (double)tot / s->op_count : 0); 2276 cpu_fprintf(f, "cycles/in byte %0.1f\n", 2277 s->code_in_len ? (double)tot / s->code_in_len : 0); 2278 cpu_fprintf(f, "cycles/out byte %0.1f\n", 2279 s->code_out_len ? (double)tot / s->code_out_len : 0); 2280 if (tot == 0) 2281 tot = 1; 2282 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 2283 (double)s->interm_time / tot * 100.0); 2284 cpu_fprintf(f, " gen_code time %0.1f%%\n", 2285 (double)s->code_time / tot * 100.0); 2286 cpu_fprintf(f, "optim./code time %0.1f%%\n", 2287 (double)s->opt_time / (s->code_time ? s->code_time : 1) 2288 * 100.0); 2289 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 2290 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 2291 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 2292 s->restore_count); 2293 cpu_fprintf(f, " avg cycles %0.1f\n", 2294 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 2295 2296 dump_op_count(); 2297 } 2298 #else 2299 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2300 { 2301 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2302 } 2303 #endif 2304 2305 #ifdef ELF_HOST_MACHINE 2306 /* In order to use this feature, the backend needs to do three things: 2307 2308 (1) Define ELF_HOST_MACHINE to indicate both what value to 2309 put into the ELF image and to indicate support for the feature. 2310 2311 (2) Define tcg_register_jit. This should create a buffer containing 2312 the contents of a .debug_frame section that describes the post- 2313 prologue unwind info for the tcg machine. 2314 2315 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 2316 */ 2317 2318 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 2319 typedef enum { 2320 JIT_NOACTION = 0, 2321 JIT_REGISTER_FN, 2322 JIT_UNREGISTER_FN 2323 } jit_actions_t; 2324 2325 struct jit_code_entry { 2326 struct jit_code_entry *next_entry; 2327 struct jit_code_entry *prev_entry; 2328 const void *symfile_addr; 2329 uint64_t symfile_size; 2330 }; 2331 2332 struct jit_descriptor { 2333 uint32_t version; 2334 uint32_t action_flag; 2335 struct jit_code_entry *relevant_entry; 2336 struct jit_code_entry *first_entry; 2337 }; 2338 2339 void __jit_debug_register_code(void) __attribute__((noinline)); 2340 void __jit_debug_register_code(void) 2341 { 2342 asm(""); 2343 } 2344 2345 /* Must statically initialize the version, because GDB may check 2346 the version before we can set it. */ 2347 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 2348 2349 /* End GDB interface. */ 2350 2351 static int find_string(const char *strtab, const char *str) 2352 { 2353 const char *p = strtab + 1; 2354 2355 while (1) { 2356 if (strcmp(p, str) == 0) { 2357 return p - strtab; 2358 } 2359 p += strlen(p) + 1; 2360 } 2361 } 2362 2363 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 2364 void *debug_frame, size_t debug_frame_size) 2365 { 2366 struct __attribute__((packed)) DebugInfo { 2367 uint32_t len; 2368 uint16_t version; 2369 uint32_t abbrev; 2370 uint8_t ptr_size; 2371 uint8_t cu_die; 2372 uint16_t cu_lang; 2373 uintptr_t cu_low_pc; 2374 uintptr_t cu_high_pc; 2375 uint8_t fn_die; 2376 char fn_name[16]; 2377 uintptr_t fn_low_pc; 2378 uintptr_t fn_high_pc; 2379 uint8_t cu_eoc; 2380 }; 2381 2382 struct ElfImage { 2383 ElfW(Ehdr) ehdr; 2384 ElfW(Phdr) phdr; 2385 ElfW(Shdr) shdr[7]; 2386 ElfW(Sym) sym[2]; 2387 struct DebugInfo di; 2388 uint8_t da[24]; 2389 char str[80]; 2390 }; 2391 2392 struct ElfImage *img; 2393 2394 static const struct ElfImage img_template = { 2395 .ehdr = { 2396 .e_ident[EI_MAG0] = ELFMAG0, 2397 .e_ident[EI_MAG1] = ELFMAG1, 2398 .e_ident[EI_MAG2] = ELFMAG2, 2399 .e_ident[EI_MAG3] = ELFMAG3, 2400 .e_ident[EI_CLASS] = ELF_CLASS, 2401 .e_ident[EI_DATA] = ELF_DATA, 2402 .e_ident[EI_VERSION] = EV_CURRENT, 2403 .e_type = ET_EXEC, 2404 .e_machine = ELF_HOST_MACHINE, 2405 .e_version = EV_CURRENT, 2406 .e_phoff = offsetof(struct ElfImage, phdr), 2407 .e_shoff = offsetof(struct ElfImage, shdr), 2408 .e_ehsize = sizeof(ElfW(Shdr)), 2409 .e_phentsize = sizeof(ElfW(Phdr)), 2410 .e_phnum = 1, 2411 .e_shentsize = sizeof(ElfW(Shdr)), 2412 .e_shnum = ARRAY_SIZE(img->shdr), 2413 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 2414 #ifdef ELF_HOST_FLAGS 2415 .e_flags = ELF_HOST_FLAGS, 2416 #endif 2417 #ifdef ELF_OSABI 2418 .e_ident[EI_OSABI] = ELF_OSABI, 2419 #endif 2420 }, 2421 .phdr = { 2422 .p_type = PT_LOAD, 2423 .p_flags = PF_X, 2424 }, 2425 .shdr = { 2426 [0] = { .sh_type = SHT_NULL }, 2427 /* Trick: The contents of code_gen_buffer are not present in 2428 this fake ELF file; that got allocated elsewhere. Therefore 2429 we mark .text as SHT_NOBITS (similar to .bss) so that readers 2430 will not look for contents. We can record any address. */ 2431 [1] = { /* .text */ 2432 .sh_type = SHT_NOBITS, 2433 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 2434 }, 2435 [2] = { /* .debug_info */ 2436 .sh_type = SHT_PROGBITS, 2437 .sh_offset = offsetof(struct ElfImage, di), 2438 .sh_size = sizeof(struct DebugInfo), 2439 }, 2440 [3] = { /* .debug_abbrev */ 2441 .sh_type = SHT_PROGBITS, 2442 .sh_offset = offsetof(struct ElfImage, da), 2443 .sh_size = sizeof(img->da), 2444 }, 2445 [4] = { /* .debug_frame */ 2446 .sh_type = SHT_PROGBITS, 2447 .sh_offset = sizeof(struct ElfImage), 2448 }, 2449 [5] = { /* .symtab */ 2450 .sh_type = SHT_SYMTAB, 2451 .sh_offset = offsetof(struct ElfImage, sym), 2452 .sh_size = sizeof(img->sym), 2453 .sh_info = 1, 2454 .sh_link = ARRAY_SIZE(img->shdr) - 1, 2455 .sh_entsize = sizeof(ElfW(Sym)), 2456 }, 2457 [6] = { /* .strtab */ 2458 .sh_type = SHT_STRTAB, 2459 .sh_offset = offsetof(struct ElfImage, str), 2460 .sh_size = sizeof(img->str), 2461 } 2462 }, 2463 .sym = { 2464 [1] = { /* code_gen_buffer */ 2465 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 2466 .st_shndx = 1, 2467 } 2468 }, 2469 .di = { 2470 .len = sizeof(struct DebugInfo) - 4, 2471 .version = 2, 2472 .ptr_size = sizeof(void *), 2473 .cu_die = 1, 2474 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 2475 .fn_die = 2, 2476 .fn_name = "code_gen_buffer" 2477 }, 2478 .da = { 2479 1, /* abbrev number (the cu) */ 2480 0x11, 1, /* DW_TAG_compile_unit, has children */ 2481 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 2482 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2483 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2484 0, 0, /* end of abbrev */ 2485 2, /* abbrev number (the fn) */ 2486 0x2e, 0, /* DW_TAG_subprogram, no children */ 2487 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 2488 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2489 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2490 0, 0, /* end of abbrev */ 2491 0 /* no more abbrev */ 2492 }, 2493 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 2494 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 2495 }; 2496 2497 /* We only need a single jit entry; statically allocate it. */ 2498 static struct jit_code_entry one_entry; 2499 2500 uintptr_t buf = (uintptr_t)buf_ptr; 2501 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 2502 2503 img = g_malloc(img_size); 2504 *img = img_template; 2505 memcpy(img + 1, debug_frame, debug_frame_size); 2506 2507 img->phdr.p_vaddr = buf; 2508 img->phdr.p_paddr = buf; 2509 img->phdr.p_memsz = buf_size; 2510 2511 img->shdr[1].sh_name = find_string(img->str, ".text"); 2512 img->shdr[1].sh_addr = buf; 2513 img->shdr[1].sh_size = buf_size; 2514 2515 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 2516 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 2517 2518 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 2519 img->shdr[4].sh_size = debug_frame_size; 2520 2521 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 2522 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 2523 2524 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 2525 img->sym[1].st_value = buf; 2526 img->sym[1].st_size = buf_size; 2527 2528 img->di.cu_low_pc = buf; 2529 img->di.cu_high_pc = buf_size; 2530 img->di.fn_low_pc = buf; 2531 img->di.fn_high_pc = buf_size; 2532 2533 #ifdef DEBUG_JIT 2534 /* Enable this block to be able to debug the ELF image file creation. 2535 One can use readelf, objdump, or other inspection utilities. */ 2536 { 2537 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 2538 if (f) { 2539 if (fwrite(img, img_size, 1, f) != img_size) { 2540 /* Avoid stupid unused return value warning for fwrite. */ 2541 } 2542 fclose(f); 2543 } 2544 } 2545 #endif 2546 2547 one_entry.symfile_addr = img; 2548 one_entry.symfile_size = img_size; 2549 2550 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 2551 __jit_debug_descriptor.relevant_entry = &one_entry; 2552 __jit_debug_descriptor.first_entry = &one_entry; 2553 __jit_debug_register_code(); 2554 } 2555 #else 2556 /* No support for the feature. Provide the entry point expected by exec.c, 2557 and implement the internal function we declared earlier. */ 2558 2559 static void tcg_register_jit_int(void *buf, size_t size, 2560 void *debug_frame, size_t debug_frame_size) 2561 { 2562 } 2563 2564 void tcg_register_jit(void *buf, size_t buf_size) 2565 { 2566 } 2567 #endif /* ELF_HOST_MACHINE */ 2568