1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/error-report.h" 34 #include "qemu/cutils.h" 35 #include "qemu/host-utils.h" 36 #include "qemu/timer.h" 37 38 /* Note: the long term plan is to reduce the dependencies on the QEMU 39 CPU definitions. Currently they are used for qemu_ld/st 40 instructions */ 41 #define NO_CPU_IO_DEFS 42 #include "cpu.h" 43 44 #include "exec/cpu-common.h" 45 #include "exec/exec-all.h" 46 47 #include "tcg-op.h" 48 49 #if UINTPTR_MAX == UINT32_MAX 50 # define ELF_CLASS ELFCLASS32 51 #else 52 # define ELF_CLASS ELFCLASS64 53 #endif 54 #ifdef HOST_WORDS_BIGENDIAN 55 # define ELF_DATA ELFDATA2MSB 56 #else 57 # define ELF_DATA ELFDATA2LSB 58 #endif 59 60 #include "elf.h" 61 #include "exec/log.h" 62 #include "sysemu/sysemu.h" 63 64 /* Forward declarations for functions declared in tcg-target.inc.c and 65 used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode); 68 static void tcg_target_qemu_prologue(TCGContext *s); 69 static void patch_reloc(tcg_insn_unit *code_ptr, int type, 70 intptr_t value, intptr_t addend); 71 72 /* The CIE and FDE header definitions will be common to all hosts. */ 73 typedef struct { 74 uint32_t len __attribute__((aligned((sizeof(void *))))); 75 uint32_t id; 76 uint8_t version; 77 char augmentation[1]; 78 uint8_t code_align; 79 uint8_t data_align; 80 uint8_t return_column; 81 } DebugFrameCIE; 82 83 typedef struct QEMU_PACKED { 84 uint32_t len __attribute__((aligned((sizeof(void *))))); 85 uint32_t cie_offset; 86 uintptr_t func_start; 87 uintptr_t func_len; 88 } DebugFrameFDEHeader; 89 90 typedef struct QEMU_PACKED { 91 DebugFrameCIE cie; 92 DebugFrameFDEHeader fde; 93 } DebugFrameHeader; 94 95 static void tcg_register_jit_int(void *buf, size_t size, 96 const void *debug_frame, 97 size_t debug_frame_size) 98 __attribute__((unused)); 99 100 /* Forward declarations for functions declared and used in tcg-target.inc.c. */ 101 static const char *target_parse_constraint(TCGArgConstraint *ct, 102 const char *ct_str, TCGType type); 103 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 104 intptr_t arg2); 105 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 106 static void tcg_out_movi(TCGContext *s, TCGType type, 107 TCGReg ret, tcg_target_long arg); 108 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 109 const int *const_args); 110 #if TCG_TARGET_MAYBE_vec 111 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, 112 unsigned vece, const TCGArg *args, 113 const int *const_args); 114 #else 115 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, 116 unsigned vece, const TCGArg *args, 117 const int *const_args) 118 { 119 g_assert_not_reached(); 120 } 121 #endif 122 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 123 intptr_t arg2); 124 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 125 TCGReg base, intptr_t ofs); 126 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); 127 static int tcg_target_const_match(tcg_target_long val, TCGType type, 128 const TCGArgConstraint *arg_ct); 129 #ifdef TCG_TARGET_NEED_LDST_LABELS 130 static bool tcg_out_ldst_finalize(TCGContext *s); 131 #endif 132 133 #define TCG_HIGHWATER 1024 134 135 static TCGContext **tcg_ctxs; 136 static unsigned int n_tcg_ctxs; 137 TCGv_env cpu_env = 0; 138 139 struct tcg_region_tree { 140 QemuMutex lock; 141 GTree *tree; 142 /* padding to avoid false sharing is computed at run-time */ 143 }; 144 145 /* 146 * We divide code_gen_buffer into equally-sized "regions" that TCG threads 147 * dynamically allocate from as demand dictates. Given appropriate region 148 * sizing, this minimizes flushes even when some TCG threads generate a lot 149 * more code than others. 150 */ 151 struct tcg_region_state { 152 QemuMutex lock; 153 154 /* fields set at init time */ 155 void *start; 156 void *start_aligned; 157 void *end; 158 size_t n; 159 size_t size; /* size of one region */ 160 size_t stride; /* .size + guard size */ 161 162 /* fields protected by the lock */ 163 size_t current; /* current region index */ 164 size_t agg_size_full; /* aggregate size of full regions */ 165 }; 166 167 static struct tcg_region_state region; 168 /* 169 * This is an array of struct tcg_region_tree's, with padding. 170 * We use void * to simplify the computation of region_trees[i]; each 171 * struct is found every tree_size bytes. 172 */ 173 static void *region_trees; 174 static size_t tree_size; 175 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 176 static TCGRegSet tcg_target_call_clobber_regs; 177 178 #if TCG_TARGET_INSN_UNIT_SIZE == 1 179 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 180 { 181 *s->code_ptr++ = v; 182 } 183 184 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 185 uint8_t v) 186 { 187 *p = v; 188 } 189 #endif 190 191 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 192 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 193 { 194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 195 *s->code_ptr++ = v; 196 } else { 197 tcg_insn_unit *p = s->code_ptr; 198 memcpy(p, &v, sizeof(v)); 199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 200 } 201 } 202 203 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 204 uint16_t v) 205 { 206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 207 *p = v; 208 } else { 209 memcpy(p, &v, sizeof(v)); 210 } 211 } 212 #endif 213 214 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 215 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 216 { 217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 218 *s->code_ptr++ = v; 219 } else { 220 tcg_insn_unit *p = s->code_ptr; 221 memcpy(p, &v, sizeof(v)); 222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 223 } 224 } 225 226 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 227 uint32_t v) 228 { 229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 230 *p = v; 231 } else { 232 memcpy(p, &v, sizeof(v)); 233 } 234 } 235 #endif 236 237 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 238 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 239 { 240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 241 *s->code_ptr++ = v; 242 } else { 243 tcg_insn_unit *p = s->code_ptr; 244 memcpy(p, &v, sizeof(v)); 245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 246 } 247 } 248 249 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 250 uint64_t v) 251 { 252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 253 *p = v; 254 } else { 255 memcpy(p, &v, sizeof(v)); 256 } 257 } 258 #endif 259 260 /* label relocation processing */ 261 262 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 263 TCGLabel *l, intptr_t addend) 264 { 265 TCGRelocation *r; 266 267 if (l->has_value) { 268 /* FIXME: This may break relocations on RISC targets that 269 modify instruction fields in place. The caller may not have 270 written the initial value. */ 271 patch_reloc(code_ptr, type, l->u.value, addend); 272 } else { 273 /* add a new relocation entry */ 274 r = tcg_malloc(sizeof(TCGRelocation)); 275 r->type = type; 276 r->ptr = code_ptr; 277 r->addend = addend; 278 r->next = l->u.first_reloc; 279 l->u.first_reloc = r; 280 } 281 } 282 283 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr) 284 { 285 intptr_t value = (intptr_t)ptr; 286 TCGRelocation *r; 287 288 tcg_debug_assert(!l->has_value); 289 290 for (r = l->u.first_reloc; r != NULL; r = r->next) { 291 patch_reloc(r->ptr, r->type, value, r->addend); 292 } 293 294 l->has_value = 1; 295 l->u.value_ptr = ptr; 296 } 297 298 TCGLabel *gen_new_label(void) 299 { 300 TCGContext *s = tcg_ctx; 301 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 302 303 *l = (TCGLabel){ 304 .id = s->nb_labels++ 305 }; 306 307 return l; 308 } 309 310 static void set_jmp_reset_offset(TCGContext *s, int which) 311 { 312 size_t off = tcg_current_code_size(s); 313 s->tb_jmp_reset_offset[which] = off; 314 /* Make sure that we didn't overflow the stored offset. */ 315 assert(s->tb_jmp_reset_offset[which] == off); 316 } 317 318 #include "tcg-target.inc.c" 319 320 /* compare a pointer @ptr and a tb_tc @s */ 321 static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s) 322 { 323 if (ptr >= s->ptr + s->size) { 324 return 1; 325 } else if (ptr < s->ptr) { 326 return -1; 327 } 328 return 0; 329 } 330 331 static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp) 332 { 333 const struct tb_tc *a = ap; 334 const struct tb_tc *b = bp; 335 336 /* 337 * When both sizes are set, we know this isn't a lookup. 338 * This is the most likely case: every TB must be inserted; lookups 339 * are a lot less frequent. 340 */ 341 if (likely(a->size && b->size)) { 342 if (a->ptr > b->ptr) { 343 return 1; 344 } else if (a->ptr < b->ptr) { 345 return -1; 346 } 347 /* a->ptr == b->ptr should happen only on deletions */ 348 g_assert(a->size == b->size); 349 return 0; 350 } 351 /* 352 * All lookups have either .size field set to 0. 353 * From the glib sources we see that @ap is always the lookup key. However 354 * the docs provide no guarantee, so we just mark this case as likely. 355 */ 356 if (likely(a->size == 0)) { 357 return ptr_cmp_tb_tc(a->ptr, b); 358 } 359 return ptr_cmp_tb_tc(b->ptr, a); 360 } 361 362 static void tcg_region_trees_init(void) 363 { 364 size_t i; 365 366 tree_size = ROUND_UP(sizeof(struct tcg_region_tree), qemu_dcache_linesize); 367 region_trees = qemu_memalign(qemu_dcache_linesize, region.n * tree_size); 368 for (i = 0; i < region.n; i++) { 369 struct tcg_region_tree *rt = region_trees + i * tree_size; 370 371 qemu_mutex_init(&rt->lock); 372 rt->tree = g_tree_new(tb_tc_cmp); 373 } 374 } 375 376 static struct tcg_region_tree *tc_ptr_to_region_tree(void *p) 377 { 378 size_t region_idx; 379 380 if (p < region.start_aligned) { 381 region_idx = 0; 382 } else { 383 ptrdiff_t offset = p - region.start_aligned; 384 385 if (offset > region.stride * (region.n - 1)) { 386 region_idx = region.n - 1; 387 } else { 388 region_idx = offset / region.stride; 389 } 390 } 391 return region_trees + region_idx * tree_size; 392 } 393 394 void tcg_tb_insert(TranslationBlock *tb) 395 { 396 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr); 397 398 qemu_mutex_lock(&rt->lock); 399 g_tree_insert(rt->tree, &tb->tc, tb); 400 qemu_mutex_unlock(&rt->lock); 401 } 402 403 void tcg_tb_remove(TranslationBlock *tb) 404 { 405 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr); 406 407 qemu_mutex_lock(&rt->lock); 408 g_tree_remove(rt->tree, &tb->tc); 409 qemu_mutex_unlock(&rt->lock); 410 } 411 412 /* 413 * Find the TB 'tb' such that 414 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size 415 * Return NULL if not found. 416 */ 417 TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr) 418 { 419 struct tcg_region_tree *rt = tc_ptr_to_region_tree((void *)tc_ptr); 420 TranslationBlock *tb; 421 struct tb_tc s = { .ptr = (void *)tc_ptr }; 422 423 qemu_mutex_lock(&rt->lock); 424 tb = g_tree_lookup(rt->tree, &s); 425 qemu_mutex_unlock(&rt->lock); 426 return tb; 427 } 428 429 static void tcg_region_tree_lock_all(void) 430 { 431 size_t i; 432 433 for (i = 0; i < region.n; i++) { 434 struct tcg_region_tree *rt = region_trees + i * tree_size; 435 436 qemu_mutex_lock(&rt->lock); 437 } 438 } 439 440 static void tcg_region_tree_unlock_all(void) 441 { 442 size_t i; 443 444 for (i = 0; i < region.n; i++) { 445 struct tcg_region_tree *rt = region_trees + i * tree_size; 446 447 qemu_mutex_unlock(&rt->lock); 448 } 449 } 450 451 void tcg_tb_foreach(GTraverseFunc func, gpointer user_data) 452 { 453 size_t i; 454 455 tcg_region_tree_lock_all(); 456 for (i = 0; i < region.n; i++) { 457 struct tcg_region_tree *rt = region_trees + i * tree_size; 458 459 g_tree_foreach(rt->tree, func, user_data); 460 } 461 tcg_region_tree_unlock_all(); 462 } 463 464 size_t tcg_nb_tbs(void) 465 { 466 size_t nb_tbs = 0; 467 size_t i; 468 469 tcg_region_tree_lock_all(); 470 for (i = 0; i < region.n; i++) { 471 struct tcg_region_tree *rt = region_trees + i * tree_size; 472 473 nb_tbs += g_tree_nnodes(rt->tree); 474 } 475 tcg_region_tree_unlock_all(); 476 return nb_tbs; 477 } 478 479 static void tcg_region_tree_reset_all(void) 480 { 481 size_t i; 482 483 tcg_region_tree_lock_all(); 484 for (i = 0; i < region.n; i++) { 485 struct tcg_region_tree *rt = region_trees + i * tree_size; 486 487 /* Increment the refcount first so that destroy acts as a reset */ 488 g_tree_ref(rt->tree); 489 g_tree_destroy(rt->tree); 490 } 491 tcg_region_tree_unlock_all(); 492 } 493 494 static void tcg_region_bounds(size_t curr_region, void **pstart, void **pend) 495 { 496 void *start, *end; 497 498 start = region.start_aligned + curr_region * region.stride; 499 end = start + region.size; 500 501 if (curr_region == 0) { 502 start = region.start; 503 } 504 if (curr_region == region.n - 1) { 505 end = region.end; 506 } 507 508 *pstart = start; 509 *pend = end; 510 } 511 512 static void tcg_region_assign(TCGContext *s, size_t curr_region) 513 { 514 void *start, *end; 515 516 tcg_region_bounds(curr_region, &start, &end); 517 518 s->code_gen_buffer = start; 519 s->code_gen_ptr = start; 520 s->code_gen_buffer_size = end - start; 521 s->code_gen_highwater = end - TCG_HIGHWATER; 522 } 523 524 static bool tcg_region_alloc__locked(TCGContext *s) 525 { 526 if (region.current == region.n) { 527 return true; 528 } 529 tcg_region_assign(s, region.current); 530 region.current++; 531 return false; 532 } 533 534 /* 535 * Request a new region once the one in use has filled up. 536 * Returns true on error. 537 */ 538 static bool tcg_region_alloc(TCGContext *s) 539 { 540 bool err; 541 /* read the region size now; alloc__locked will overwrite it on success */ 542 size_t size_full = s->code_gen_buffer_size; 543 544 qemu_mutex_lock(®ion.lock); 545 err = tcg_region_alloc__locked(s); 546 if (!err) { 547 region.agg_size_full += size_full - TCG_HIGHWATER; 548 } 549 qemu_mutex_unlock(®ion.lock); 550 return err; 551 } 552 553 /* 554 * Perform a context's first region allocation. 555 * This function does _not_ increment region.agg_size_full. 556 */ 557 static inline bool tcg_region_initial_alloc__locked(TCGContext *s) 558 { 559 return tcg_region_alloc__locked(s); 560 } 561 562 /* Call from a safe-work context */ 563 void tcg_region_reset_all(void) 564 { 565 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 566 unsigned int i; 567 568 qemu_mutex_lock(®ion.lock); 569 region.current = 0; 570 region.agg_size_full = 0; 571 572 for (i = 0; i < n_ctxs; i++) { 573 TCGContext *s = atomic_read(&tcg_ctxs[i]); 574 bool err = tcg_region_initial_alloc__locked(s); 575 576 g_assert(!err); 577 } 578 qemu_mutex_unlock(®ion.lock); 579 580 tcg_region_tree_reset_all(); 581 } 582 583 #ifdef CONFIG_USER_ONLY 584 static size_t tcg_n_regions(void) 585 { 586 return 1; 587 } 588 #else 589 /* 590 * It is likely that some vCPUs will translate more code than others, so we 591 * first try to set more regions than max_cpus, with those regions being of 592 * reasonable size. If that's not possible we make do by evenly dividing 593 * the code_gen_buffer among the vCPUs. 594 */ 595 static size_t tcg_n_regions(void) 596 { 597 size_t i; 598 599 /* Use a single region if all we have is one vCPU thread */ 600 if (max_cpus == 1 || !qemu_tcg_mttcg_enabled()) { 601 return 1; 602 } 603 604 /* Try to have more regions than max_cpus, with each region being >= 2 MB */ 605 for (i = 8; i > 0; i--) { 606 size_t regions_per_thread = i; 607 size_t region_size; 608 609 region_size = tcg_init_ctx.code_gen_buffer_size; 610 region_size /= max_cpus * regions_per_thread; 611 612 if (region_size >= 2 * 1024u * 1024) { 613 return max_cpus * regions_per_thread; 614 } 615 } 616 /* If we can't, then just allocate one region per vCPU thread */ 617 return max_cpus; 618 } 619 #endif 620 621 /* 622 * Initializes region partitioning. 623 * 624 * Called at init time from the parent thread (i.e. the one calling 625 * tcg_context_init), after the target's TCG globals have been set. 626 * 627 * Region partitioning works by splitting code_gen_buffer into separate regions, 628 * and then assigning regions to TCG threads so that the threads can translate 629 * code in parallel without synchronization. 630 * 631 * In softmmu the number of TCG threads is bounded by max_cpus, so we use at 632 * least max_cpus regions in MTTCG. In !MTTCG we use a single region. 633 * Note that the TCG options from the command-line (i.e. -accel accel=tcg,[...]) 634 * must have been parsed before calling this function, since it calls 635 * qemu_tcg_mttcg_enabled(). 636 * 637 * In user-mode we use a single region. Having multiple regions in user-mode 638 * is not supported, because the number of vCPU threads (recall that each thread 639 * spawned by the guest corresponds to a vCPU thread) is only bounded by the 640 * OS, and usually this number is huge (tens of thousands is not uncommon). 641 * Thus, given this large bound on the number of vCPU threads and the fact 642 * that code_gen_buffer is allocated at compile-time, we cannot guarantee 643 * that the availability of at least one region per vCPU thread. 644 * 645 * However, this user-mode limitation is unlikely to be a significant problem 646 * in practice. Multi-threaded guests share most if not all of their translated 647 * code, which makes parallel code generation less appealing than in softmmu. 648 */ 649 void tcg_region_init(void) 650 { 651 void *buf = tcg_init_ctx.code_gen_buffer; 652 void *aligned; 653 size_t size = tcg_init_ctx.code_gen_buffer_size; 654 size_t page_size = qemu_real_host_page_size; 655 size_t region_size; 656 size_t n_regions; 657 size_t i; 658 659 n_regions = tcg_n_regions(); 660 661 /* The first region will be 'aligned - buf' bytes larger than the others */ 662 aligned = QEMU_ALIGN_PTR_UP(buf, page_size); 663 g_assert(aligned < tcg_init_ctx.code_gen_buffer + size); 664 /* 665 * Make region_size a multiple of page_size, using aligned as the start. 666 * As a result of this we might end up with a few extra pages at the end of 667 * the buffer; we will assign those to the last region. 668 */ 669 region_size = (size - (aligned - buf)) / n_regions; 670 region_size = QEMU_ALIGN_DOWN(region_size, page_size); 671 672 /* A region must have at least 2 pages; one code, one guard */ 673 g_assert(region_size >= 2 * page_size); 674 675 /* init the region struct */ 676 qemu_mutex_init(®ion.lock); 677 region.n = n_regions; 678 region.size = region_size - page_size; 679 region.stride = region_size; 680 region.start = buf; 681 region.start_aligned = aligned; 682 /* page-align the end, since its last page will be a guard page */ 683 region.end = QEMU_ALIGN_PTR_DOWN(buf + size, page_size); 684 /* account for that last guard page */ 685 region.end -= page_size; 686 687 /* set guard pages */ 688 for (i = 0; i < region.n; i++) { 689 void *start, *end; 690 int rc; 691 692 tcg_region_bounds(i, &start, &end); 693 rc = qemu_mprotect_none(end, page_size); 694 g_assert(!rc); 695 } 696 697 tcg_region_trees_init(); 698 699 /* In user-mode we support only one ctx, so do the initial allocation now */ 700 #ifdef CONFIG_USER_ONLY 701 { 702 bool err = tcg_region_initial_alloc__locked(tcg_ctx); 703 704 g_assert(!err); 705 } 706 #endif 707 } 708 709 /* 710 * All TCG threads except the parent (i.e. the one that called tcg_context_init 711 * and registered the target's TCG globals) must register with this function 712 * before initiating translation. 713 * 714 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 715 * of tcg_region_init() for the reasoning behind this. 716 * 717 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 718 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 719 * is not used anymore for translation once this function is called. 720 * 721 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 722 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 723 */ 724 #ifdef CONFIG_USER_ONLY 725 void tcg_register_thread(void) 726 { 727 tcg_ctx = &tcg_init_ctx; 728 } 729 #else 730 void tcg_register_thread(void) 731 { 732 TCGContext *s = g_malloc(sizeof(*s)); 733 unsigned int i, n; 734 bool err; 735 736 *s = tcg_init_ctx; 737 738 /* Relink mem_base. */ 739 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 740 if (tcg_init_ctx.temps[i].mem_base) { 741 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 742 tcg_debug_assert(b >= 0 && b < n); 743 s->temps[i].mem_base = &s->temps[b]; 744 } 745 } 746 747 /* Claim an entry in tcg_ctxs */ 748 n = atomic_fetch_inc(&n_tcg_ctxs); 749 g_assert(n < max_cpus); 750 atomic_set(&tcg_ctxs[n], s); 751 752 tcg_ctx = s; 753 qemu_mutex_lock(®ion.lock); 754 err = tcg_region_initial_alloc__locked(tcg_ctx); 755 g_assert(!err); 756 qemu_mutex_unlock(®ion.lock); 757 } 758 #endif /* !CONFIG_USER_ONLY */ 759 760 /* 761 * Returns the size (in bytes) of all translated code (i.e. from all regions) 762 * currently in the cache. 763 * See also: tcg_code_capacity() 764 * Do not confuse with tcg_current_code_size(); that one applies to a single 765 * TCG context. 766 */ 767 size_t tcg_code_size(void) 768 { 769 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 770 unsigned int i; 771 size_t total; 772 773 qemu_mutex_lock(®ion.lock); 774 total = region.agg_size_full; 775 for (i = 0; i < n_ctxs; i++) { 776 const TCGContext *s = atomic_read(&tcg_ctxs[i]); 777 size_t size; 778 779 size = atomic_read(&s->code_gen_ptr) - s->code_gen_buffer; 780 g_assert(size <= s->code_gen_buffer_size); 781 total += size; 782 } 783 qemu_mutex_unlock(®ion.lock); 784 return total; 785 } 786 787 /* 788 * Returns the code capacity (in bytes) of the entire cache, i.e. including all 789 * regions. 790 * See also: tcg_code_size() 791 */ 792 size_t tcg_code_capacity(void) 793 { 794 size_t guard_size, capacity; 795 796 /* no need for synchronization; these variables are set at init time */ 797 guard_size = region.stride - region.size; 798 capacity = region.end + guard_size - region.start; 799 capacity -= region.n * (guard_size + TCG_HIGHWATER); 800 return capacity; 801 } 802 803 size_t tcg_tb_phys_invalidate_count(void) 804 { 805 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 806 unsigned int i; 807 size_t total = 0; 808 809 for (i = 0; i < n_ctxs; i++) { 810 const TCGContext *s = atomic_read(&tcg_ctxs[i]); 811 812 total += atomic_read(&s->tb_phys_invalidate_count); 813 } 814 return total; 815 } 816 817 /* pool based memory allocation */ 818 void *tcg_malloc_internal(TCGContext *s, int size) 819 { 820 TCGPool *p; 821 int pool_size; 822 823 if (size > TCG_POOL_CHUNK_SIZE) { 824 /* big malloc: insert a new pool (XXX: could optimize) */ 825 p = g_malloc(sizeof(TCGPool) + size); 826 p->size = size; 827 p->next = s->pool_first_large; 828 s->pool_first_large = p; 829 return p->data; 830 } else { 831 p = s->pool_current; 832 if (!p) { 833 p = s->pool_first; 834 if (!p) 835 goto new_pool; 836 } else { 837 if (!p->next) { 838 new_pool: 839 pool_size = TCG_POOL_CHUNK_SIZE; 840 p = g_malloc(sizeof(TCGPool) + pool_size); 841 p->size = pool_size; 842 p->next = NULL; 843 if (s->pool_current) 844 s->pool_current->next = p; 845 else 846 s->pool_first = p; 847 } else { 848 p = p->next; 849 } 850 } 851 } 852 s->pool_current = p; 853 s->pool_cur = p->data + size; 854 s->pool_end = p->data + p->size; 855 return p->data; 856 } 857 858 void tcg_pool_reset(TCGContext *s) 859 { 860 TCGPool *p, *t; 861 for (p = s->pool_first_large; p; p = t) { 862 t = p->next; 863 g_free(p); 864 } 865 s->pool_first_large = NULL; 866 s->pool_cur = s->pool_end = NULL; 867 s->pool_current = NULL; 868 } 869 870 typedef struct TCGHelperInfo { 871 void *func; 872 const char *name; 873 unsigned flags; 874 unsigned sizemask; 875 } TCGHelperInfo; 876 877 #include "exec/helper-proto.h" 878 879 static const TCGHelperInfo all_helpers[] = { 880 #include "exec/helper-tcg.h" 881 }; 882 static GHashTable *helper_table; 883 884 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 885 static void process_op_defs(TCGContext *s); 886 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 887 TCGReg reg, const char *name); 888 889 void tcg_context_init(TCGContext *s) 890 { 891 int op, total_args, n, i; 892 TCGOpDef *def; 893 TCGArgConstraint *args_ct; 894 int *sorted_args; 895 TCGTemp *ts; 896 897 memset(s, 0, sizeof(*s)); 898 s->nb_globals = 0; 899 900 /* Count total number of arguments and allocate the corresponding 901 space */ 902 total_args = 0; 903 for(op = 0; op < NB_OPS; op++) { 904 def = &tcg_op_defs[op]; 905 n = def->nb_iargs + def->nb_oargs; 906 total_args += n; 907 } 908 909 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 910 sorted_args = g_malloc(sizeof(int) * total_args); 911 912 for(op = 0; op < NB_OPS; op++) { 913 def = &tcg_op_defs[op]; 914 def->args_ct = args_ct; 915 def->sorted_args = sorted_args; 916 n = def->nb_iargs + def->nb_oargs; 917 sorted_args += n; 918 args_ct += n; 919 } 920 921 /* Register helpers. */ 922 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 923 helper_table = g_hash_table_new(NULL, NULL); 924 925 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 926 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 927 (gpointer)&all_helpers[i]); 928 } 929 930 tcg_target_init(s); 931 process_op_defs(s); 932 933 /* Reverse the order of the saved registers, assuming they're all at 934 the start of tcg_target_reg_alloc_order. */ 935 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 936 int r = tcg_target_reg_alloc_order[n]; 937 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 938 break; 939 } 940 } 941 for (i = 0; i < n; ++i) { 942 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 943 } 944 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 945 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 946 } 947 948 tcg_ctx = s; 949 /* 950 * In user-mode we simply share the init context among threads, since we 951 * use a single region. See the documentation tcg_region_init() for the 952 * reasoning behind this. 953 * In softmmu we will have at most max_cpus TCG threads. 954 */ 955 #ifdef CONFIG_USER_ONLY 956 tcg_ctxs = &tcg_ctx; 957 n_tcg_ctxs = 1; 958 #else 959 tcg_ctxs = g_new(TCGContext *, max_cpus); 960 #endif 961 962 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 963 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 964 cpu_env = temp_tcgv_ptr(ts); 965 } 966 967 /* 968 * Allocate TBs right before their corresponding translated code, making 969 * sure that TBs and code are on different cache lines. 970 */ 971 TranslationBlock *tcg_tb_alloc(TCGContext *s) 972 { 973 uintptr_t align = qemu_icache_linesize; 974 TranslationBlock *tb; 975 void *next; 976 977 retry: 978 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 979 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 980 981 if (unlikely(next > s->code_gen_highwater)) { 982 if (tcg_region_alloc(s)) { 983 return NULL; 984 } 985 goto retry; 986 } 987 atomic_set(&s->code_gen_ptr, next); 988 s->data_gen_ptr = NULL; 989 return tb; 990 } 991 992 void tcg_prologue_init(TCGContext *s) 993 { 994 size_t prologue_size, total_size; 995 void *buf0, *buf1; 996 997 /* Put the prologue at the beginning of code_gen_buffer. */ 998 buf0 = s->code_gen_buffer; 999 total_size = s->code_gen_buffer_size; 1000 s->code_ptr = buf0; 1001 s->code_buf = buf0; 1002 s->data_gen_ptr = NULL; 1003 s->code_gen_prologue = buf0; 1004 1005 /* Compute a high-water mark, at which we voluntarily flush the buffer 1006 and start over. The size here is arbitrary, significantly larger 1007 than we expect the code generation for any one opcode to require. */ 1008 s->code_gen_highwater = s->code_gen_buffer + (total_size - TCG_HIGHWATER); 1009 1010 #ifdef TCG_TARGET_NEED_POOL_LABELS 1011 s->pool_labels = NULL; 1012 #endif 1013 1014 /* Generate the prologue. */ 1015 tcg_target_qemu_prologue(s); 1016 1017 #ifdef TCG_TARGET_NEED_POOL_LABELS 1018 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 1019 { 1020 bool ok = tcg_out_pool_finalize(s); 1021 tcg_debug_assert(ok); 1022 } 1023 #endif 1024 1025 buf1 = s->code_ptr; 1026 flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1); 1027 1028 /* Deduct the prologue from the buffer. */ 1029 prologue_size = tcg_current_code_size(s); 1030 s->code_gen_ptr = buf1; 1031 s->code_gen_buffer = buf1; 1032 s->code_buf = buf1; 1033 total_size -= prologue_size; 1034 s->code_gen_buffer_size = total_size; 1035 1036 tcg_register_jit(s->code_gen_buffer, total_size); 1037 1038 #ifdef DEBUG_DISAS 1039 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 1040 qemu_log_lock(); 1041 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 1042 if (s->data_gen_ptr) { 1043 size_t code_size = s->data_gen_ptr - buf0; 1044 size_t data_size = prologue_size - code_size; 1045 size_t i; 1046 1047 log_disas(buf0, code_size); 1048 1049 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1050 if (sizeof(tcg_target_ulong) == 8) { 1051 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1052 (uintptr_t)s->data_gen_ptr + i, 1053 *(uint64_t *)(s->data_gen_ptr + i)); 1054 } else { 1055 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1056 (uintptr_t)s->data_gen_ptr + i, 1057 *(uint32_t *)(s->data_gen_ptr + i)); 1058 } 1059 } 1060 } else { 1061 log_disas(buf0, prologue_size); 1062 } 1063 qemu_log("\n"); 1064 qemu_log_flush(); 1065 qemu_log_unlock(); 1066 } 1067 #endif 1068 1069 /* Assert that goto_ptr is implemented completely. */ 1070 if (TCG_TARGET_HAS_goto_ptr) { 1071 tcg_debug_assert(s->code_gen_epilogue != NULL); 1072 } 1073 } 1074 1075 void tcg_func_start(TCGContext *s) 1076 { 1077 tcg_pool_reset(s); 1078 s->nb_temps = s->nb_globals; 1079 1080 /* No temps have been previously allocated for size or locality. */ 1081 memset(s->free_temps, 0, sizeof(s->free_temps)); 1082 1083 s->nb_ops = 0; 1084 s->nb_labels = 0; 1085 s->current_frame_offset = s->frame_start; 1086 1087 #ifdef CONFIG_DEBUG_TCG 1088 s->goto_tb_issue_mask = 0; 1089 #endif 1090 1091 QTAILQ_INIT(&s->ops); 1092 QTAILQ_INIT(&s->free_ops); 1093 } 1094 1095 static inline TCGTemp *tcg_temp_alloc(TCGContext *s) 1096 { 1097 int n = s->nb_temps++; 1098 tcg_debug_assert(n < TCG_MAX_TEMPS); 1099 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 1100 } 1101 1102 static inline TCGTemp *tcg_global_alloc(TCGContext *s) 1103 { 1104 TCGTemp *ts; 1105 1106 tcg_debug_assert(s->nb_globals == s->nb_temps); 1107 s->nb_globals++; 1108 ts = tcg_temp_alloc(s); 1109 ts->temp_global = 1; 1110 1111 return ts; 1112 } 1113 1114 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1115 TCGReg reg, const char *name) 1116 { 1117 TCGTemp *ts; 1118 1119 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 1120 tcg_abort(); 1121 } 1122 1123 ts = tcg_global_alloc(s); 1124 ts->base_type = type; 1125 ts->type = type; 1126 ts->fixed_reg = 1; 1127 ts->reg = reg; 1128 ts->name = name; 1129 tcg_regset_set_reg(s->reserved_regs, reg); 1130 1131 return ts; 1132 } 1133 1134 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 1135 { 1136 s->frame_start = start; 1137 s->frame_end = start + size; 1138 s->frame_temp 1139 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 1140 } 1141 1142 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 1143 intptr_t offset, const char *name) 1144 { 1145 TCGContext *s = tcg_ctx; 1146 TCGTemp *base_ts = tcgv_ptr_temp(base); 1147 TCGTemp *ts = tcg_global_alloc(s); 1148 int indirect_reg = 0, bigendian = 0; 1149 #ifdef HOST_WORDS_BIGENDIAN 1150 bigendian = 1; 1151 #endif 1152 1153 if (!base_ts->fixed_reg) { 1154 /* We do not support double-indirect registers. */ 1155 tcg_debug_assert(!base_ts->indirect_reg); 1156 base_ts->indirect_base = 1; 1157 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 1158 ? 2 : 1); 1159 indirect_reg = 1; 1160 } 1161 1162 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1163 TCGTemp *ts2 = tcg_global_alloc(s); 1164 char buf[64]; 1165 1166 ts->base_type = TCG_TYPE_I64; 1167 ts->type = TCG_TYPE_I32; 1168 ts->indirect_reg = indirect_reg; 1169 ts->mem_allocated = 1; 1170 ts->mem_base = base_ts; 1171 ts->mem_offset = offset + bigendian * 4; 1172 pstrcpy(buf, sizeof(buf), name); 1173 pstrcat(buf, sizeof(buf), "_0"); 1174 ts->name = strdup(buf); 1175 1176 tcg_debug_assert(ts2 == ts + 1); 1177 ts2->base_type = TCG_TYPE_I64; 1178 ts2->type = TCG_TYPE_I32; 1179 ts2->indirect_reg = indirect_reg; 1180 ts2->mem_allocated = 1; 1181 ts2->mem_base = base_ts; 1182 ts2->mem_offset = offset + (1 - bigendian) * 4; 1183 pstrcpy(buf, sizeof(buf), name); 1184 pstrcat(buf, sizeof(buf), "_1"); 1185 ts2->name = strdup(buf); 1186 } else { 1187 ts->base_type = type; 1188 ts->type = type; 1189 ts->indirect_reg = indirect_reg; 1190 ts->mem_allocated = 1; 1191 ts->mem_base = base_ts; 1192 ts->mem_offset = offset; 1193 ts->name = name; 1194 } 1195 return ts; 1196 } 1197 1198 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) 1199 { 1200 TCGContext *s = tcg_ctx; 1201 TCGTemp *ts; 1202 int idx, k; 1203 1204 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 1205 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 1206 if (idx < TCG_MAX_TEMPS) { 1207 /* There is already an available temp with the right type. */ 1208 clear_bit(idx, s->free_temps[k].l); 1209 1210 ts = &s->temps[idx]; 1211 ts->temp_allocated = 1; 1212 tcg_debug_assert(ts->base_type == type); 1213 tcg_debug_assert(ts->temp_local == temp_local); 1214 } else { 1215 ts = tcg_temp_alloc(s); 1216 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1217 TCGTemp *ts2 = tcg_temp_alloc(s); 1218 1219 ts->base_type = type; 1220 ts->type = TCG_TYPE_I32; 1221 ts->temp_allocated = 1; 1222 ts->temp_local = temp_local; 1223 1224 tcg_debug_assert(ts2 == ts + 1); 1225 ts2->base_type = TCG_TYPE_I64; 1226 ts2->type = TCG_TYPE_I32; 1227 ts2->temp_allocated = 1; 1228 ts2->temp_local = temp_local; 1229 } else { 1230 ts->base_type = type; 1231 ts->type = type; 1232 ts->temp_allocated = 1; 1233 ts->temp_local = temp_local; 1234 } 1235 } 1236 1237 #if defined(CONFIG_DEBUG_TCG) 1238 s->temps_in_use++; 1239 #endif 1240 return ts; 1241 } 1242 1243 TCGv_vec tcg_temp_new_vec(TCGType type) 1244 { 1245 TCGTemp *t; 1246 1247 #ifdef CONFIG_DEBUG_TCG 1248 switch (type) { 1249 case TCG_TYPE_V64: 1250 assert(TCG_TARGET_HAS_v64); 1251 break; 1252 case TCG_TYPE_V128: 1253 assert(TCG_TARGET_HAS_v128); 1254 break; 1255 case TCG_TYPE_V256: 1256 assert(TCG_TARGET_HAS_v256); 1257 break; 1258 default: 1259 g_assert_not_reached(); 1260 } 1261 #endif 1262 1263 t = tcg_temp_new_internal(type, 0); 1264 return temp_tcgv_vec(t); 1265 } 1266 1267 /* Create a new temp of the same type as an existing temp. */ 1268 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1269 { 1270 TCGTemp *t = tcgv_vec_temp(match); 1271 1272 tcg_debug_assert(t->temp_allocated != 0); 1273 1274 t = tcg_temp_new_internal(t->base_type, 0); 1275 return temp_tcgv_vec(t); 1276 } 1277 1278 void tcg_temp_free_internal(TCGTemp *ts) 1279 { 1280 TCGContext *s = tcg_ctx; 1281 int k, idx; 1282 1283 #if defined(CONFIG_DEBUG_TCG) 1284 s->temps_in_use--; 1285 if (s->temps_in_use < 0) { 1286 fprintf(stderr, "More temporaries freed than allocated!\n"); 1287 } 1288 #endif 1289 1290 tcg_debug_assert(ts->temp_global == 0); 1291 tcg_debug_assert(ts->temp_allocated != 0); 1292 ts->temp_allocated = 0; 1293 1294 idx = temp_idx(ts); 1295 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 1296 set_bit(idx, s->free_temps[k].l); 1297 } 1298 1299 TCGv_i32 tcg_const_i32(int32_t val) 1300 { 1301 TCGv_i32 t0; 1302 t0 = tcg_temp_new_i32(); 1303 tcg_gen_movi_i32(t0, val); 1304 return t0; 1305 } 1306 1307 TCGv_i64 tcg_const_i64(int64_t val) 1308 { 1309 TCGv_i64 t0; 1310 t0 = tcg_temp_new_i64(); 1311 tcg_gen_movi_i64(t0, val); 1312 return t0; 1313 } 1314 1315 TCGv_i32 tcg_const_local_i32(int32_t val) 1316 { 1317 TCGv_i32 t0; 1318 t0 = tcg_temp_local_new_i32(); 1319 tcg_gen_movi_i32(t0, val); 1320 return t0; 1321 } 1322 1323 TCGv_i64 tcg_const_local_i64(int64_t val) 1324 { 1325 TCGv_i64 t0; 1326 t0 = tcg_temp_local_new_i64(); 1327 tcg_gen_movi_i64(t0, val); 1328 return t0; 1329 } 1330 1331 #if defined(CONFIG_DEBUG_TCG) 1332 void tcg_clear_temp_count(void) 1333 { 1334 TCGContext *s = tcg_ctx; 1335 s->temps_in_use = 0; 1336 } 1337 1338 int tcg_check_temp_count(void) 1339 { 1340 TCGContext *s = tcg_ctx; 1341 if (s->temps_in_use) { 1342 /* Clear the count so that we don't give another 1343 * warning immediately next time around. 1344 */ 1345 s->temps_in_use = 0; 1346 return 1; 1347 } 1348 return 0; 1349 } 1350 #endif 1351 1352 /* Return true if OP may appear in the opcode stream. 1353 Test the runtime variable that controls each opcode. */ 1354 bool tcg_op_supported(TCGOpcode op) 1355 { 1356 const bool have_vec 1357 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1358 1359 switch (op) { 1360 case INDEX_op_discard: 1361 case INDEX_op_set_label: 1362 case INDEX_op_call: 1363 case INDEX_op_br: 1364 case INDEX_op_mb: 1365 case INDEX_op_insn_start: 1366 case INDEX_op_exit_tb: 1367 case INDEX_op_goto_tb: 1368 case INDEX_op_qemu_ld_i32: 1369 case INDEX_op_qemu_st_i32: 1370 case INDEX_op_qemu_ld_i64: 1371 case INDEX_op_qemu_st_i64: 1372 return true; 1373 1374 case INDEX_op_goto_ptr: 1375 return TCG_TARGET_HAS_goto_ptr; 1376 1377 case INDEX_op_mov_i32: 1378 case INDEX_op_movi_i32: 1379 case INDEX_op_setcond_i32: 1380 case INDEX_op_brcond_i32: 1381 case INDEX_op_ld8u_i32: 1382 case INDEX_op_ld8s_i32: 1383 case INDEX_op_ld16u_i32: 1384 case INDEX_op_ld16s_i32: 1385 case INDEX_op_ld_i32: 1386 case INDEX_op_st8_i32: 1387 case INDEX_op_st16_i32: 1388 case INDEX_op_st_i32: 1389 case INDEX_op_add_i32: 1390 case INDEX_op_sub_i32: 1391 case INDEX_op_mul_i32: 1392 case INDEX_op_and_i32: 1393 case INDEX_op_or_i32: 1394 case INDEX_op_xor_i32: 1395 case INDEX_op_shl_i32: 1396 case INDEX_op_shr_i32: 1397 case INDEX_op_sar_i32: 1398 return true; 1399 1400 case INDEX_op_movcond_i32: 1401 return TCG_TARGET_HAS_movcond_i32; 1402 case INDEX_op_div_i32: 1403 case INDEX_op_divu_i32: 1404 return TCG_TARGET_HAS_div_i32; 1405 case INDEX_op_rem_i32: 1406 case INDEX_op_remu_i32: 1407 return TCG_TARGET_HAS_rem_i32; 1408 case INDEX_op_div2_i32: 1409 case INDEX_op_divu2_i32: 1410 return TCG_TARGET_HAS_div2_i32; 1411 case INDEX_op_rotl_i32: 1412 case INDEX_op_rotr_i32: 1413 return TCG_TARGET_HAS_rot_i32; 1414 case INDEX_op_deposit_i32: 1415 return TCG_TARGET_HAS_deposit_i32; 1416 case INDEX_op_extract_i32: 1417 return TCG_TARGET_HAS_extract_i32; 1418 case INDEX_op_sextract_i32: 1419 return TCG_TARGET_HAS_sextract_i32; 1420 case INDEX_op_add2_i32: 1421 return TCG_TARGET_HAS_add2_i32; 1422 case INDEX_op_sub2_i32: 1423 return TCG_TARGET_HAS_sub2_i32; 1424 case INDEX_op_mulu2_i32: 1425 return TCG_TARGET_HAS_mulu2_i32; 1426 case INDEX_op_muls2_i32: 1427 return TCG_TARGET_HAS_muls2_i32; 1428 case INDEX_op_muluh_i32: 1429 return TCG_TARGET_HAS_muluh_i32; 1430 case INDEX_op_mulsh_i32: 1431 return TCG_TARGET_HAS_mulsh_i32; 1432 case INDEX_op_ext8s_i32: 1433 return TCG_TARGET_HAS_ext8s_i32; 1434 case INDEX_op_ext16s_i32: 1435 return TCG_TARGET_HAS_ext16s_i32; 1436 case INDEX_op_ext8u_i32: 1437 return TCG_TARGET_HAS_ext8u_i32; 1438 case INDEX_op_ext16u_i32: 1439 return TCG_TARGET_HAS_ext16u_i32; 1440 case INDEX_op_bswap16_i32: 1441 return TCG_TARGET_HAS_bswap16_i32; 1442 case INDEX_op_bswap32_i32: 1443 return TCG_TARGET_HAS_bswap32_i32; 1444 case INDEX_op_not_i32: 1445 return TCG_TARGET_HAS_not_i32; 1446 case INDEX_op_neg_i32: 1447 return TCG_TARGET_HAS_neg_i32; 1448 case INDEX_op_andc_i32: 1449 return TCG_TARGET_HAS_andc_i32; 1450 case INDEX_op_orc_i32: 1451 return TCG_TARGET_HAS_orc_i32; 1452 case INDEX_op_eqv_i32: 1453 return TCG_TARGET_HAS_eqv_i32; 1454 case INDEX_op_nand_i32: 1455 return TCG_TARGET_HAS_nand_i32; 1456 case INDEX_op_nor_i32: 1457 return TCG_TARGET_HAS_nor_i32; 1458 case INDEX_op_clz_i32: 1459 return TCG_TARGET_HAS_clz_i32; 1460 case INDEX_op_ctz_i32: 1461 return TCG_TARGET_HAS_ctz_i32; 1462 case INDEX_op_ctpop_i32: 1463 return TCG_TARGET_HAS_ctpop_i32; 1464 1465 case INDEX_op_brcond2_i32: 1466 case INDEX_op_setcond2_i32: 1467 return TCG_TARGET_REG_BITS == 32; 1468 1469 case INDEX_op_mov_i64: 1470 case INDEX_op_movi_i64: 1471 case INDEX_op_setcond_i64: 1472 case INDEX_op_brcond_i64: 1473 case INDEX_op_ld8u_i64: 1474 case INDEX_op_ld8s_i64: 1475 case INDEX_op_ld16u_i64: 1476 case INDEX_op_ld16s_i64: 1477 case INDEX_op_ld32u_i64: 1478 case INDEX_op_ld32s_i64: 1479 case INDEX_op_ld_i64: 1480 case INDEX_op_st8_i64: 1481 case INDEX_op_st16_i64: 1482 case INDEX_op_st32_i64: 1483 case INDEX_op_st_i64: 1484 case INDEX_op_add_i64: 1485 case INDEX_op_sub_i64: 1486 case INDEX_op_mul_i64: 1487 case INDEX_op_and_i64: 1488 case INDEX_op_or_i64: 1489 case INDEX_op_xor_i64: 1490 case INDEX_op_shl_i64: 1491 case INDEX_op_shr_i64: 1492 case INDEX_op_sar_i64: 1493 case INDEX_op_ext_i32_i64: 1494 case INDEX_op_extu_i32_i64: 1495 return TCG_TARGET_REG_BITS == 64; 1496 1497 case INDEX_op_movcond_i64: 1498 return TCG_TARGET_HAS_movcond_i64; 1499 case INDEX_op_div_i64: 1500 case INDEX_op_divu_i64: 1501 return TCG_TARGET_HAS_div_i64; 1502 case INDEX_op_rem_i64: 1503 case INDEX_op_remu_i64: 1504 return TCG_TARGET_HAS_rem_i64; 1505 case INDEX_op_div2_i64: 1506 case INDEX_op_divu2_i64: 1507 return TCG_TARGET_HAS_div2_i64; 1508 case INDEX_op_rotl_i64: 1509 case INDEX_op_rotr_i64: 1510 return TCG_TARGET_HAS_rot_i64; 1511 case INDEX_op_deposit_i64: 1512 return TCG_TARGET_HAS_deposit_i64; 1513 case INDEX_op_extract_i64: 1514 return TCG_TARGET_HAS_extract_i64; 1515 case INDEX_op_sextract_i64: 1516 return TCG_TARGET_HAS_sextract_i64; 1517 case INDEX_op_extrl_i64_i32: 1518 return TCG_TARGET_HAS_extrl_i64_i32; 1519 case INDEX_op_extrh_i64_i32: 1520 return TCG_TARGET_HAS_extrh_i64_i32; 1521 case INDEX_op_ext8s_i64: 1522 return TCG_TARGET_HAS_ext8s_i64; 1523 case INDEX_op_ext16s_i64: 1524 return TCG_TARGET_HAS_ext16s_i64; 1525 case INDEX_op_ext32s_i64: 1526 return TCG_TARGET_HAS_ext32s_i64; 1527 case INDEX_op_ext8u_i64: 1528 return TCG_TARGET_HAS_ext8u_i64; 1529 case INDEX_op_ext16u_i64: 1530 return TCG_TARGET_HAS_ext16u_i64; 1531 case INDEX_op_ext32u_i64: 1532 return TCG_TARGET_HAS_ext32u_i64; 1533 case INDEX_op_bswap16_i64: 1534 return TCG_TARGET_HAS_bswap16_i64; 1535 case INDEX_op_bswap32_i64: 1536 return TCG_TARGET_HAS_bswap32_i64; 1537 case INDEX_op_bswap64_i64: 1538 return TCG_TARGET_HAS_bswap64_i64; 1539 case INDEX_op_not_i64: 1540 return TCG_TARGET_HAS_not_i64; 1541 case INDEX_op_neg_i64: 1542 return TCG_TARGET_HAS_neg_i64; 1543 case INDEX_op_andc_i64: 1544 return TCG_TARGET_HAS_andc_i64; 1545 case INDEX_op_orc_i64: 1546 return TCG_TARGET_HAS_orc_i64; 1547 case INDEX_op_eqv_i64: 1548 return TCG_TARGET_HAS_eqv_i64; 1549 case INDEX_op_nand_i64: 1550 return TCG_TARGET_HAS_nand_i64; 1551 case INDEX_op_nor_i64: 1552 return TCG_TARGET_HAS_nor_i64; 1553 case INDEX_op_clz_i64: 1554 return TCG_TARGET_HAS_clz_i64; 1555 case INDEX_op_ctz_i64: 1556 return TCG_TARGET_HAS_ctz_i64; 1557 case INDEX_op_ctpop_i64: 1558 return TCG_TARGET_HAS_ctpop_i64; 1559 case INDEX_op_add2_i64: 1560 return TCG_TARGET_HAS_add2_i64; 1561 case INDEX_op_sub2_i64: 1562 return TCG_TARGET_HAS_sub2_i64; 1563 case INDEX_op_mulu2_i64: 1564 return TCG_TARGET_HAS_mulu2_i64; 1565 case INDEX_op_muls2_i64: 1566 return TCG_TARGET_HAS_muls2_i64; 1567 case INDEX_op_muluh_i64: 1568 return TCG_TARGET_HAS_muluh_i64; 1569 case INDEX_op_mulsh_i64: 1570 return TCG_TARGET_HAS_mulsh_i64; 1571 1572 case INDEX_op_mov_vec: 1573 case INDEX_op_dup_vec: 1574 case INDEX_op_dupi_vec: 1575 case INDEX_op_ld_vec: 1576 case INDEX_op_st_vec: 1577 case INDEX_op_add_vec: 1578 case INDEX_op_sub_vec: 1579 case INDEX_op_and_vec: 1580 case INDEX_op_or_vec: 1581 case INDEX_op_xor_vec: 1582 case INDEX_op_cmp_vec: 1583 return have_vec; 1584 case INDEX_op_dup2_vec: 1585 return have_vec && TCG_TARGET_REG_BITS == 32; 1586 case INDEX_op_not_vec: 1587 return have_vec && TCG_TARGET_HAS_not_vec; 1588 case INDEX_op_neg_vec: 1589 return have_vec && TCG_TARGET_HAS_neg_vec; 1590 case INDEX_op_andc_vec: 1591 return have_vec && TCG_TARGET_HAS_andc_vec; 1592 case INDEX_op_orc_vec: 1593 return have_vec && TCG_TARGET_HAS_orc_vec; 1594 case INDEX_op_mul_vec: 1595 return have_vec && TCG_TARGET_HAS_mul_vec; 1596 case INDEX_op_shli_vec: 1597 case INDEX_op_shri_vec: 1598 case INDEX_op_sari_vec: 1599 return have_vec && TCG_TARGET_HAS_shi_vec; 1600 case INDEX_op_shls_vec: 1601 case INDEX_op_shrs_vec: 1602 case INDEX_op_sars_vec: 1603 return have_vec && TCG_TARGET_HAS_shs_vec; 1604 case INDEX_op_shlv_vec: 1605 case INDEX_op_shrv_vec: 1606 case INDEX_op_sarv_vec: 1607 return have_vec && TCG_TARGET_HAS_shv_vec; 1608 1609 default: 1610 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1611 return true; 1612 } 1613 } 1614 1615 /* Note: we convert the 64 bit args to 32 bit and do some alignment 1616 and endian swap. Maybe it would be better to do the alignment 1617 and endian swap in tcg_reg_alloc_call(). */ 1618 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1619 { 1620 int i, real_args, nb_rets, pi; 1621 unsigned sizemask, flags; 1622 TCGHelperInfo *info; 1623 TCGOp *op; 1624 1625 info = g_hash_table_lookup(helper_table, (gpointer)func); 1626 flags = info->flags; 1627 sizemask = info->sizemask; 1628 1629 #if defined(__sparc__) && !defined(__arch64__) \ 1630 && !defined(CONFIG_TCG_INTERPRETER) 1631 /* We have 64-bit values in one register, but need to pass as two 1632 separate parameters. Split them. */ 1633 int orig_sizemask = sizemask; 1634 int orig_nargs = nargs; 1635 TCGv_i64 retl, reth; 1636 TCGTemp *split_args[MAX_OPC_PARAM]; 1637 1638 retl = NULL; 1639 reth = NULL; 1640 if (sizemask != 0) { 1641 for (i = real_args = 0; i < nargs; ++i) { 1642 int is_64bit = sizemask & (1 << (i+1)*2); 1643 if (is_64bit) { 1644 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1645 TCGv_i32 h = tcg_temp_new_i32(); 1646 TCGv_i32 l = tcg_temp_new_i32(); 1647 tcg_gen_extr_i64_i32(l, h, orig); 1648 split_args[real_args++] = tcgv_i32_temp(h); 1649 split_args[real_args++] = tcgv_i32_temp(l); 1650 } else { 1651 split_args[real_args++] = args[i]; 1652 } 1653 } 1654 nargs = real_args; 1655 args = split_args; 1656 sizemask = 0; 1657 } 1658 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1659 for (i = 0; i < nargs; ++i) { 1660 int is_64bit = sizemask & (1 << (i+1)*2); 1661 int is_signed = sizemask & (2 << (i+1)*2); 1662 if (!is_64bit) { 1663 TCGv_i64 temp = tcg_temp_new_i64(); 1664 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1665 if (is_signed) { 1666 tcg_gen_ext32s_i64(temp, orig); 1667 } else { 1668 tcg_gen_ext32u_i64(temp, orig); 1669 } 1670 args[i] = tcgv_i64_temp(temp); 1671 } 1672 } 1673 #endif /* TCG_TARGET_EXTEND_ARGS */ 1674 1675 op = tcg_emit_op(INDEX_op_call); 1676 1677 pi = 0; 1678 if (ret != NULL) { 1679 #if defined(__sparc__) && !defined(__arch64__) \ 1680 && !defined(CONFIG_TCG_INTERPRETER) 1681 if (orig_sizemask & 1) { 1682 /* The 32-bit ABI is going to return the 64-bit value in 1683 the %o0/%o1 register pair. Prepare for this by using 1684 two return temporaries, and reassemble below. */ 1685 retl = tcg_temp_new_i64(); 1686 reth = tcg_temp_new_i64(); 1687 op->args[pi++] = tcgv_i64_arg(reth); 1688 op->args[pi++] = tcgv_i64_arg(retl); 1689 nb_rets = 2; 1690 } else { 1691 op->args[pi++] = temp_arg(ret); 1692 nb_rets = 1; 1693 } 1694 #else 1695 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { 1696 #ifdef HOST_WORDS_BIGENDIAN 1697 op->args[pi++] = temp_arg(ret + 1); 1698 op->args[pi++] = temp_arg(ret); 1699 #else 1700 op->args[pi++] = temp_arg(ret); 1701 op->args[pi++] = temp_arg(ret + 1); 1702 #endif 1703 nb_rets = 2; 1704 } else { 1705 op->args[pi++] = temp_arg(ret); 1706 nb_rets = 1; 1707 } 1708 #endif 1709 } else { 1710 nb_rets = 0; 1711 } 1712 TCGOP_CALLO(op) = nb_rets; 1713 1714 real_args = 0; 1715 for (i = 0; i < nargs; i++) { 1716 int is_64bit = sizemask & (1 << (i+1)*2); 1717 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 1718 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 1719 /* some targets want aligned 64 bit args */ 1720 if (real_args & 1) { 1721 op->args[pi++] = TCG_CALL_DUMMY_ARG; 1722 real_args++; 1723 } 1724 #endif 1725 /* If stack grows up, then we will be placing successive 1726 arguments at lower addresses, which means we need to 1727 reverse the order compared to how we would normally 1728 treat either big or little-endian. For those arguments 1729 that will wind up in registers, this still works for 1730 HPPA (the only current STACK_GROWSUP target) since the 1731 argument registers are *also* allocated in decreasing 1732 order. If another such target is added, this logic may 1733 have to get more complicated to differentiate between 1734 stack arguments and register arguments. */ 1735 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 1736 op->args[pi++] = temp_arg(args[i] + 1); 1737 op->args[pi++] = temp_arg(args[i]); 1738 #else 1739 op->args[pi++] = temp_arg(args[i]); 1740 op->args[pi++] = temp_arg(args[i] + 1); 1741 #endif 1742 real_args += 2; 1743 continue; 1744 } 1745 1746 op->args[pi++] = temp_arg(args[i]); 1747 real_args++; 1748 } 1749 op->args[pi++] = (uintptr_t)func; 1750 op->args[pi++] = flags; 1751 TCGOP_CALLI(op) = real_args; 1752 1753 /* Make sure the fields didn't overflow. */ 1754 tcg_debug_assert(TCGOP_CALLI(op) == real_args); 1755 tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); 1756 1757 #if defined(__sparc__) && !defined(__arch64__) \ 1758 && !defined(CONFIG_TCG_INTERPRETER) 1759 /* Free all of the parts we allocated above. */ 1760 for (i = real_args = 0; i < orig_nargs; ++i) { 1761 int is_64bit = orig_sizemask & (1 << (i+1)*2); 1762 if (is_64bit) { 1763 tcg_temp_free_internal(args[real_args++]); 1764 tcg_temp_free_internal(args[real_args++]); 1765 } else { 1766 real_args++; 1767 } 1768 } 1769 if (orig_sizemask & 1) { 1770 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 1771 Note that describing these as TCGv_i64 eliminates an unnecessary 1772 zero-extension that tcg_gen_concat_i32_i64 would create. */ 1773 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth); 1774 tcg_temp_free_i64(retl); 1775 tcg_temp_free_i64(reth); 1776 } 1777 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1778 for (i = 0; i < nargs; ++i) { 1779 int is_64bit = sizemask & (1 << (i+1)*2); 1780 if (!is_64bit) { 1781 tcg_temp_free_internal(args[i]); 1782 } 1783 } 1784 #endif /* TCG_TARGET_EXTEND_ARGS */ 1785 } 1786 1787 static void tcg_reg_alloc_start(TCGContext *s) 1788 { 1789 int i, n; 1790 TCGTemp *ts; 1791 1792 for (i = 0, n = s->nb_globals; i < n; i++) { 1793 ts = &s->temps[i]; 1794 ts->val_type = (ts->fixed_reg ? TEMP_VAL_REG : TEMP_VAL_MEM); 1795 } 1796 for (n = s->nb_temps; i < n; i++) { 1797 ts = &s->temps[i]; 1798 ts->val_type = (ts->temp_local ? TEMP_VAL_MEM : TEMP_VAL_DEAD); 1799 ts->mem_allocated = 0; 1800 ts->fixed_reg = 0; 1801 } 1802 1803 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1804 } 1805 1806 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1807 TCGTemp *ts) 1808 { 1809 int idx = temp_idx(ts); 1810 1811 if (ts->temp_global) { 1812 pstrcpy(buf, buf_size, ts->name); 1813 } else if (ts->temp_local) { 1814 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1815 } else { 1816 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1817 } 1818 return buf; 1819 } 1820 1821 static char *tcg_get_arg_str(TCGContext *s, char *buf, 1822 int buf_size, TCGArg arg) 1823 { 1824 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1825 } 1826 1827 /* Find helper name. */ 1828 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 1829 { 1830 const char *ret = NULL; 1831 if (helper_table) { 1832 TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val); 1833 if (info) { 1834 ret = info->name; 1835 } 1836 } 1837 return ret; 1838 } 1839 1840 static const char * const cond_name[] = 1841 { 1842 [TCG_COND_NEVER] = "never", 1843 [TCG_COND_ALWAYS] = "always", 1844 [TCG_COND_EQ] = "eq", 1845 [TCG_COND_NE] = "ne", 1846 [TCG_COND_LT] = "lt", 1847 [TCG_COND_GE] = "ge", 1848 [TCG_COND_LE] = "le", 1849 [TCG_COND_GT] = "gt", 1850 [TCG_COND_LTU] = "ltu", 1851 [TCG_COND_GEU] = "geu", 1852 [TCG_COND_LEU] = "leu", 1853 [TCG_COND_GTU] = "gtu" 1854 }; 1855 1856 static const char * const ldst_name[] = 1857 { 1858 [MO_UB] = "ub", 1859 [MO_SB] = "sb", 1860 [MO_LEUW] = "leuw", 1861 [MO_LESW] = "lesw", 1862 [MO_LEUL] = "leul", 1863 [MO_LESL] = "lesl", 1864 [MO_LEQ] = "leq", 1865 [MO_BEUW] = "beuw", 1866 [MO_BESW] = "besw", 1867 [MO_BEUL] = "beul", 1868 [MO_BESL] = "besl", 1869 [MO_BEQ] = "beq", 1870 }; 1871 1872 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1873 #ifdef ALIGNED_ONLY 1874 [MO_UNALN >> MO_ASHIFT] = "un+", 1875 [MO_ALIGN >> MO_ASHIFT] = "", 1876 #else 1877 [MO_UNALN >> MO_ASHIFT] = "", 1878 [MO_ALIGN >> MO_ASHIFT] = "al+", 1879 #endif 1880 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1881 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1882 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1883 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1884 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1885 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1886 }; 1887 1888 void tcg_dump_ops(TCGContext *s) 1889 { 1890 char buf[128]; 1891 TCGOp *op; 1892 1893 QTAILQ_FOREACH(op, &s->ops, link) { 1894 int i, k, nb_oargs, nb_iargs, nb_cargs; 1895 const TCGOpDef *def; 1896 TCGOpcode c; 1897 int col = 0; 1898 1899 c = op->opc; 1900 def = &tcg_op_defs[c]; 1901 1902 if (c == INDEX_op_insn_start) { 1903 col += qemu_log("\n ----"); 1904 1905 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1906 target_ulong a; 1907 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1908 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 1909 #else 1910 a = op->args[i]; 1911 #endif 1912 col += qemu_log(" " TARGET_FMT_lx, a); 1913 } 1914 } else if (c == INDEX_op_call) { 1915 /* variable number of arguments */ 1916 nb_oargs = TCGOP_CALLO(op); 1917 nb_iargs = TCGOP_CALLI(op); 1918 nb_cargs = def->nb_cargs; 1919 1920 /* function name, flags, out args */ 1921 col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, 1922 tcg_find_helper(s, op->args[nb_oargs + nb_iargs]), 1923 op->args[nb_oargs + nb_iargs + 1], nb_oargs); 1924 for (i = 0; i < nb_oargs; i++) { 1925 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf), 1926 op->args[i])); 1927 } 1928 for (i = 0; i < nb_iargs; i++) { 1929 TCGArg arg = op->args[nb_oargs + i]; 1930 const char *t = "<dummy>"; 1931 if (arg != TCG_CALL_DUMMY_ARG) { 1932 t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 1933 } 1934 col += qemu_log(",%s", t); 1935 } 1936 } else { 1937 col += qemu_log(" %s ", def->name); 1938 1939 nb_oargs = def->nb_oargs; 1940 nb_iargs = def->nb_iargs; 1941 nb_cargs = def->nb_cargs; 1942 1943 if (def->flags & TCG_OPF_VECTOR) { 1944 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op), 1945 8 << TCGOP_VECE(op)); 1946 } 1947 1948 k = 0; 1949 for (i = 0; i < nb_oargs; i++) { 1950 if (k != 0) { 1951 col += qemu_log(","); 1952 } 1953 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1954 op->args[k++])); 1955 } 1956 for (i = 0; i < nb_iargs; i++) { 1957 if (k != 0) { 1958 col += qemu_log(","); 1959 } 1960 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1961 op->args[k++])); 1962 } 1963 switch (c) { 1964 case INDEX_op_brcond_i32: 1965 case INDEX_op_setcond_i32: 1966 case INDEX_op_movcond_i32: 1967 case INDEX_op_brcond2_i32: 1968 case INDEX_op_setcond2_i32: 1969 case INDEX_op_brcond_i64: 1970 case INDEX_op_setcond_i64: 1971 case INDEX_op_movcond_i64: 1972 case INDEX_op_cmp_vec: 1973 if (op->args[k] < ARRAY_SIZE(cond_name) 1974 && cond_name[op->args[k]]) { 1975 col += qemu_log(",%s", cond_name[op->args[k++]]); 1976 } else { 1977 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]); 1978 } 1979 i = 1; 1980 break; 1981 case INDEX_op_qemu_ld_i32: 1982 case INDEX_op_qemu_st_i32: 1983 case INDEX_op_qemu_ld_i64: 1984 case INDEX_op_qemu_st_i64: 1985 { 1986 TCGMemOpIdx oi = op->args[k++]; 1987 TCGMemOp op = get_memop(oi); 1988 unsigned ix = get_mmuidx(oi); 1989 1990 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1991 col += qemu_log(",$0x%x,%u", op, ix); 1992 } else { 1993 const char *s_al, *s_op; 1994 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1995 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1996 col += qemu_log(",%s%s,%u", s_al, s_op, ix); 1997 } 1998 i = 1; 1999 } 2000 break; 2001 default: 2002 i = 0; 2003 break; 2004 } 2005 switch (c) { 2006 case INDEX_op_set_label: 2007 case INDEX_op_br: 2008 case INDEX_op_brcond_i32: 2009 case INDEX_op_brcond_i64: 2010 case INDEX_op_brcond2_i32: 2011 col += qemu_log("%s$L%d", k ? "," : "", 2012 arg_label(op->args[k])->id); 2013 i++, k++; 2014 break; 2015 default: 2016 break; 2017 } 2018 for (; i < nb_cargs; i++, k++) { 2019 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]); 2020 } 2021 } 2022 if (op->life) { 2023 unsigned life = op->life; 2024 2025 for (; col < 48; ++col) { 2026 putc(' ', qemu_logfile); 2027 } 2028 2029 if (life & (SYNC_ARG * 3)) { 2030 qemu_log(" sync:"); 2031 for (i = 0; i < 2; ++i) { 2032 if (life & (SYNC_ARG << i)) { 2033 qemu_log(" %d", i); 2034 } 2035 } 2036 } 2037 life /= DEAD_ARG; 2038 if (life) { 2039 qemu_log(" dead:"); 2040 for (i = 0; life; ++i, life >>= 1) { 2041 if (life & 1) { 2042 qemu_log(" %d", i); 2043 } 2044 } 2045 } 2046 } 2047 qemu_log("\n"); 2048 } 2049 } 2050 2051 /* we give more priority to constraints with less registers */ 2052 static int get_constraint_priority(const TCGOpDef *def, int k) 2053 { 2054 const TCGArgConstraint *arg_ct; 2055 2056 int i, n; 2057 arg_ct = &def->args_ct[k]; 2058 if (arg_ct->ct & TCG_CT_ALIAS) { 2059 /* an alias is equivalent to a single register */ 2060 n = 1; 2061 } else { 2062 if (!(arg_ct->ct & TCG_CT_REG)) 2063 return 0; 2064 n = 0; 2065 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 2066 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 2067 n++; 2068 } 2069 } 2070 return TCG_TARGET_NB_REGS - n + 1; 2071 } 2072 2073 /* sort from highest priority to lowest */ 2074 static void sort_constraints(TCGOpDef *def, int start, int n) 2075 { 2076 int i, j, p1, p2, tmp; 2077 2078 for(i = 0; i < n; i++) 2079 def->sorted_args[start + i] = start + i; 2080 if (n <= 1) 2081 return; 2082 for(i = 0; i < n - 1; i++) { 2083 for(j = i + 1; j < n; j++) { 2084 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 2085 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 2086 if (p1 < p2) { 2087 tmp = def->sorted_args[start + i]; 2088 def->sorted_args[start + i] = def->sorted_args[start + j]; 2089 def->sorted_args[start + j] = tmp; 2090 } 2091 } 2092 } 2093 } 2094 2095 static void process_op_defs(TCGContext *s) 2096 { 2097 TCGOpcode op; 2098 2099 for (op = 0; op < NB_OPS; op++) { 2100 TCGOpDef *def = &tcg_op_defs[op]; 2101 const TCGTargetOpDef *tdefs; 2102 TCGType type; 2103 int i, nb_args; 2104 2105 if (def->flags & TCG_OPF_NOT_PRESENT) { 2106 continue; 2107 } 2108 2109 nb_args = def->nb_iargs + def->nb_oargs; 2110 if (nb_args == 0) { 2111 continue; 2112 } 2113 2114 tdefs = tcg_target_op_def(op); 2115 /* Missing TCGTargetOpDef entry. */ 2116 tcg_debug_assert(tdefs != NULL); 2117 2118 type = (def->flags & TCG_OPF_64BIT ? TCG_TYPE_I64 : TCG_TYPE_I32); 2119 for (i = 0; i < nb_args; i++) { 2120 const char *ct_str = tdefs->args_ct_str[i]; 2121 /* Incomplete TCGTargetOpDef entry. */ 2122 tcg_debug_assert(ct_str != NULL); 2123 2124 def->args_ct[i].u.regs = 0; 2125 def->args_ct[i].ct = 0; 2126 while (*ct_str != '\0') { 2127 switch(*ct_str) { 2128 case '0' ... '9': 2129 { 2130 int oarg = *ct_str - '0'; 2131 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]); 2132 tcg_debug_assert(oarg < def->nb_oargs); 2133 tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG); 2134 /* TCG_CT_ALIAS is for the output arguments. 2135 The input is tagged with TCG_CT_IALIAS. */ 2136 def->args_ct[i] = def->args_ct[oarg]; 2137 def->args_ct[oarg].ct |= TCG_CT_ALIAS; 2138 def->args_ct[oarg].alias_index = i; 2139 def->args_ct[i].ct |= TCG_CT_IALIAS; 2140 def->args_ct[i].alias_index = oarg; 2141 } 2142 ct_str++; 2143 break; 2144 case '&': 2145 def->args_ct[i].ct |= TCG_CT_NEWREG; 2146 ct_str++; 2147 break; 2148 case 'i': 2149 def->args_ct[i].ct |= TCG_CT_CONST; 2150 ct_str++; 2151 break; 2152 default: 2153 ct_str = target_parse_constraint(&def->args_ct[i], 2154 ct_str, type); 2155 /* Typo in TCGTargetOpDef constraint. */ 2156 tcg_debug_assert(ct_str != NULL); 2157 } 2158 } 2159 } 2160 2161 /* TCGTargetOpDef entry with too much information? */ 2162 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2163 2164 /* sort the constraints (XXX: this is just an heuristic) */ 2165 sort_constraints(def, 0, def->nb_oargs); 2166 sort_constraints(def, def->nb_oargs, def->nb_iargs); 2167 } 2168 } 2169 2170 void tcg_op_remove(TCGContext *s, TCGOp *op) 2171 { 2172 QTAILQ_REMOVE(&s->ops, op, link); 2173 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2174 s->nb_ops--; 2175 2176 #ifdef CONFIG_PROFILER 2177 atomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 2178 #endif 2179 } 2180 2181 static TCGOp *tcg_op_alloc(TCGOpcode opc) 2182 { 2183 TCGContext *s = tcg_ctx; 2184 TCGOp *op; 2185 2186 if (likely(QTAILQ_EMPTY(&s->free_ops))) { 2187 op = tcg_malloc(sizeof(TCGOp)); 2188 } else { 2189 op = QTAILQ_FIRST(&s->free_ops); 2190 QTAILQ_REMOVE(&s->free_ops, op, link); 2191 } 2192 memset(op, 0, offsetof(TCGOp, link)); 2193 op->opc = opc; 2194 s->nb_ops++; 2195 2196 return op; 2197 } 2198 2199 TCGOp *tcg_emit_op(TCGOpcode opc) 2200 { 2201 TCGOp *op = tcg_op_alloc(opc); 2202 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2203 return op; 2204 } 2205 2206 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 2207 TCGOpcode opc, int nargs) 2208 { 2209 TCGOp *new_op = tcg_op_alloc(opc); 2210 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 2211 return new_op; 2212 } 2213 2214 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 2215 TCGOpcode opc, int nargs) 2216 { 2217 TCGOp *new_op = tcg_op_alloc(opc); 2218 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 2219 return new_op; 2220 } 2221 2222 #define TS_DEAD 1 2223 #define TS_MEM 2 2224 2225 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 2226 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 2227 2228 /* liveness analysis: end of function: all temps are dead, and globals 2229 should be in memory. */ 2230 static void tcg_la_func_end(TCGContext *s) 2231 { 2232 int ng = s->nb_globals; 2233 int nt = s->nb_temps; 2234 int i; 2235 2236 for (i = 0; i < ng; ++i) { 2237 s->temps[i].state = TS_DEAD | TS_MEM; 2238 } 2239 for (i = ng; i < nt; ++i) { 2240 s->temps[i].state = TS_DEAD; 2241 } 2242 } 2243 2244 /* liveness analysis: end of basic block: all temps are dead, globals 2245 and local temps should be in memory. */ 2246 static void tcg_la_bb_end(TCGContext *s) 2247 { 2248 int ng = s->nb_globals; 2249 int nt = s->nb_temps; 2250 int i; 2251 2252 for (i = 0; i < ng; ++i) { 2253 s->temps[i].state = TS_DEAD | TS_MEM; 2254 } 2255 for (i = ng; i < nt; ++i) { 2256 s->temps[i].state = (s->temps[i].temp_local 2257 ? TS_DEAD | TS_MEM 2258 : TS_DEAD); 2259 } 2260 } 2261 2262 /* Liveness analysis : update the opc_arg_life array to tell if a 2263 given input arguments is dead. Instructions updating dead 2264 temporaries are removed. */ 2265 static void liveness_pass_1(TCGContext *s) 2266 { 2267 int nb_globals = s->nb_globals; 2268 TCGOp *op, *op_prev; 2269 2270 tcg_la_func_end(s); 2271 2272 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, TCGOpHead, link, op_prev) { 2273 int i, nb_iargs, nb_oargs; 2274 TCGOpcode opc_new, opc_new2; 2275 bool have_opc_new2; 2276 TCGLifeData arg_life = 0; 2277 TCGTemp *arg_ts; 2278 TCGOpcode opc = op->opc; 2279 const TCGOpDef *def = &tcg_op_defs[opc]; 2280 2281 switch (opc) { 2282 case INDEX_op_call: 2283 { 2284 int call_flags; 2285 2286 nb_oargs = TCGOP_CALLO(op); 2287 nb_iargs = TCGOP_CALLI(op); 2288 call_flags = op->args[nb_oargs + nb_iargs + 1]; 2289 2290 /* pure functions can be removed if their result is unused */ 2291 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 2292 for (i = 0; i < nb_oargs; i++) { 2293 arg_ts = arg_temp(op->args[i]); 2294 if (arg_ts->state != TS_DEAD) { 2295 goto do_not_remove_call; 2296 } 2297 } 2298 goto do_remove; 2299 } else { 2300 do_not_remove_call: 2301 2302 /* output args are dead */ 2303 for (i = 0; i < nb_oargs; i++) { 2304 arg_ts = arg_temp(op->args[i]); 2305 if (arg_ts->state & TS_DEAD) { 2306 arg_life |= DEAD_ARG << i; 2307 } 2308 if (arg_ts->state & TS_MEM) { 2309 arg_life |= SYNC_ARG << i; 2310 } 2311 arg_ts->state = TS_DEAD; 2312 } 2313 2314 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 2315 TCG_CALL_NO_READ_GLOBALS))) { 2316 /* globals should go back to memory */ 2317 for (i = 0; i < nb_globals; i++) { 2318 s->temps[i].state = TS_DEAD | TS_MEM; 2319 } 2320 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 2321 /* globals should be synced to memory */ 2322 for (i = 0; i < nb_globals; i++) { 2323 s->temps[i].state |= TS_MEM; 2324 } 2325 } 2326 2327 /* record arguments that die in this helper */ 2328 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2329 arg_ts = arg_temp(op->args[i]); 2330 if (arg_ts && arg_ts->state & TS_DEAD) { 2331 arg_life |= DEAD_ARG << i; 2332 } 2333 } 2334 /* input arguments are live for preceding opcodes */ 2335 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2336 arg_ts = arg_temp(op->args[i]); 2337 if (arg_ts) { 2338 arg_ts->state &= ~TS_DEAD; 2339 } 2340 } 2341 } 2342 } 2343 break; 2344 case INDEX_op_insn_start: 2345 break; 2346 case INDEX_op_discard: 2347 /* mark the temporary as dead */ 2348 arg_temp(op->args[0])->state = TS_DEAD; 2349 break; 2350 2351 case INDEX_op_add2_i32: 2352 opc_new = INDEX_op_add_i32; 2353 goto do_addsub2; 2354 case INDEX_op_sub2_i32: 2355 opc_new = INDEX_op_sub_i32; 2356 goto do_addsub2; 2357 case INDEX_op_add2_i64: 2358 opc_new = INDEX_op_add_i64; 2359 goto do_addsub2; 2360 case INDEX_op_sub2_i64: 2361 opc_new = INDEX_op_sub_i64; 2362 do_addsub2: 2363 nb_iargs = 4; 2364 nb_oargs = 2; 2365 /* Test if the high part of the operation is dead, but not 2366 the low part. The result can be optimized to a simple 2367 add or sub. This happens often for x86_64 guest when the 2368 cpu mode is set to 32 bit. */ 2369 if (arg_temp(op->args[1])->state == TS_DEAD) { 2370 if (arg_temp(op->args[0])->state == TS_DEAD) { 2371 goto do_remove; 2372 } 2373 /* Replace the opcode and adjust the args in place, 2374 leaving 3 unused args at the end. */ 2375 op->opc = opc = opc_new; 2376 op->args[1] = op->args[2]; 2377 op->args[2] = op->args[4]; 2378 /* Fall through and mark the single-word operation live. */ 2379 nb_iargs = 2; 2380 nb_oargs = 1; 2381 } 2382 goto do_not_remove; 2383 2384 case INDEX_op_mulu2_i32: 2385 opc_new = INDEX_op_mul_i32; 2386 opc_new2 = INDEX_op_muluh_i32; 2387 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 2388 goto do_mul2; 2389 case INDEX_op_muls2_i32: 2390 opc_new = INDEX_op_mul_i32; 2391 opc_new2 = INDEX_op_mulsh_i32; 2392 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 2393 goto do_mul2; 2394 case INDEX_op_mulu2_i64: 2395 opc_new = INDEX_op_mul_i64; 2396 opc_new2 = INDEX_op_muluh_i64; 2397 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 2398 goto do_mul2; 2399 case INDEX_op_muls2_i64: 2400 opc_new = INDEX_op_mul_i64; 2401 opc_new2 = INDEX_op_mulsh_i64; 2402 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 2403 goto do_mul2; 2404 do_mul2: 2405 nb_iargs = 2; 2406 nb_oargs = 2; 2407 if (arg_temp(op->args[1])->state == TS_DEAD) { 2408 if (arg_temp(op->args[0])->state == TS_DEAD) { 2409 /* Both parts of the operation are dead. */ 2410 goto do_remove; 2411 } 2412 /* The high part of the operation is dead; generate the low. */ 2413 op->opc = opc = opc_new; 2414 op->args[1] = op->args[2]; 2415 op->args[2] = op->args[3]; 2416 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 2417 /* The low part of the operation is dead; generate the high. */ 2418 op->opc = opc = opc_new2; 2419 op->args[0] = op->args[1]; 2420 op->args[1] = op->args[2]; 2421 op->args[2] = op->args[3]; 2422 } else { 2423 goto do_not_remove; 2424 } 2425 /* Mark the single-word operation live. */ 2426 nb_oargs = 1; 2427 goto do_not_remove; 2428 2429 default: 2430 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 2431 nb_iargs = def->nb_iargs; 2432 nb_oargs = def->nb_oargs; 2433 2434 /* Test if the operation can be removed because all 2435 its outputs are dead. We assume that nb_oargs == 0 2436 implies side effects */ 2437 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 2438 for (i = 0; i < nb_oargs; i++) { 2439 if (arg_temp(op->args[i])->state != TS_DEAD) { 2440 goto do_not_remove; 2441 } 2442 } 2443 do_remove: 2444 tcg_op_remove(s, op); 2445 } else { 2446 do_not_remove: 2447 /* output args are dead */ 2448 for (i = 0; i < nb_oargs; i++) { 2449 arg_ts = arg_temp(op->args[i]); 2450 if (arg_ts->state & TS_DEAD) { 2451 arg_life |= DEAD_ARG << i; 2452 } 2453 if (arg_ts->state & TS_MEM) { 2454 arg_life |= SYNC_ARG << i; 2455 } 2456 arg_ts->state = TS_DEAD; 2457 } 2458 2459 /* if end of basic block, update */ 2460 if (def->flags & TCG_OPF_BB_END) { 2461 tcg_la_bb_end(s); 2462 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2463 /* globals should be synced to memory */ 2464 for (i = 0; i < nb_globals; i++) { 2465 s->temps[i].state |= TS_MEM; 2466 } 2467 } 2468 2469 /* record arguments that die in this opcode */ 2470 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2471 arg_ts = arg_temp(op->args[i]); 2472 if (arg_ts->state & TS_DEAD) { 2473 arg_life |= DEAD_ARG << i; 2474 } 2475 } 2476 /* input arguments are live for preceding opcodes */ 2477 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2478 arg_temp(op->args[i])->state &= ~TS_DEAD; 2479 } 2480 } 2481 break; 2482 } 2483 op->life = arg_life; 2484 } 2485 } 2486 2487 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 2488 static bool liveness_pass_2(TCGContext *s) 2489 { 2490 int nb_globals = s->nb_globals; 2491 int nb_temps, i; 2492 bool changes = false; 2493 TCGOp *op, *op_next; 2494 2495 /* Create a temporary for each indirect global. */ 2496 for (i = 0; i < nb_globals; ++i) { 2497 TCGTemp *its = &s->temps[i]; 2498 if (its->indirect_reg) { 2499 TCGTemp *dts = tcg_temp_alloc(s); 2500 dts->type = its->type; 2501 dts->base_type = its->base_type; 2502 its->state_ptr = dts; 2503 } else { 2504 its->state_ptr = NULL; 2505 } 2506 /* All globals begin dead. */ 2507 its->state = TS_DEAD; 2508 } 2509 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 2510 TCGTemp *its = &s->temps[i]; 2511 its->state_ptr = NULL; 2512 its->state = TS_DEAD; 2513 } 2514 2515 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2516 TCGOpcode opc = op->opc; 2517 const TCGOpDef *def = &tcg_op_defs[opc]; 2518 TCGLifeData arg_life = op->life; 2519 int nb_iargs, nb_oargs, call_flags; 2520 TCGTemp *arg_ts, *dir_ts; 2521 2522 if (opc == INDEX_op_call) { 2523 nb_oargs = TCGOP_CALLO(op); 2524 nb_iargs = TCGOP_CALLI(op); 2525 call_flags = op->args[nb_oargs + nb_iargs + 1]; 2526 } else { 2527 nb_iargs = def->nb_iargs; 2528 nb_oargs = def->nb_oargs; 2529 2530 /* Set flags similar to how calls require. */ 2531 if (def->flags & TCG_OPF_BB_END) { 2532 /* Like writing globals: save_globals */ 2533 call_flags = 0; 2534 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2535 /* Like reading globals: sync_globals */ 2536 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2537 } else { 2538 /* No effect on globals. */ 2539 call_flags = (TCG_CALL_NO_READ_GLOBALS | 2540 TCG_CALL_NO_WRITE_GLOBALS); 2541 } 2542 } 2543 2544 /* Make sure that input arguments are available. */ 2545 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2546 arg_ts = arg_temp(op->args[i]); 2547 if (arg_ts) { 2548 dir_ts = arg_ts->state_ptr; 2549 if (dir_ts && arg_ts->state == TS_DEAD) { 2550 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 2551 ? INDEX_op_ld_i32 2552 : INDEX_op_ld_i64); 2553 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 2554 2555 lop->args[0] = temp_arg(dir_ts); 2556 lop->args[1] = temp_arg(arg_ts->mem_base); 2557 lop->args[2] = arg_ts->mem_offset; 2558 2559 /* Loaded, but synced with memory. */ 2560 arg_ts->state = TS_MEM; 2561 } 2562 } 2563 } 2564 2565 /* Perform input replacement, and mark inputs that became dead. 2566 No action is required except keeping temp_state up to date 2567 so that we reload when needed. */ 2568 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2569 arg_ts = arg_temp(op->args[i]); 2570 if (arg_ts) { 2571 dir_ts = arg_ts->state_ptr; 2572 if (dir_ts) { 2573 op->args[i] = temp_arg(dir_ts); 2574 changes = true; 2575 if (IS_DEAD_ARG(i)) { 2576 arg_ts->state = TS_DEAD; 2577 } 2578 } 2579 } 2580 } 2581 2582 /* Liveness analysis should ensure that the following are 2583 all correct, for call sites and basic block end points. */ 2584 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 2585 /* Nothing to do */ 2586 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 2587 for (i = 0; i < nb_globals; ++i) { 2588 /* Liveness should see that globals are synced back, 2589 that is, either TS_DEAD or TS_MEM. */ 2590 arg_ts = &s->temps[i]; 2591 tcg_debug_assert(arg_ts->state_ptr == 0 2592 || arg_ts->state != 0); 2593 } 2594 } else { 2595 for (i = 0; i < nb_globals; ++i) { 2596 /* Liveness should see that globals are saved back, 2597 that is, TS_DEAD, waiting to be reloaded. */ 2598 arg_ts = &s->temps[i]; 2599 tcg_debug_assert(arg_ts->state_ptr == 0 2600 || arg_ts->state == TS_DEAD); 2601 } 2602 } 2603 2604 /* Outputs become available. */ 2605 for (i = 0; i < nb_oargs; i++) { 2606 arg_ts = arg_temp(op->args[i]); 2607 dir_ts = arg_ts->state_ptr; 2608 if (!dir_ts) { 2609 continue; 2610 } 2611 op->args[i] = temp_arg(dir_ts); 2612 changes = true; 2613 2614 /* The output is now live and modified. */ 2615 arg_ts->state = 0; 2616 2617 /* Sync outputs upon their last write. */ 2618 if (NEED_SYNC_ARG(i)) { 2619 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 2620 ? INDEX_op_st_i32 2621 : INDEX_op_st_i64); 2622 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 2623 2624 sop->args[0] = temp_arg(dir_ts); 2625 sop->args[1] = temp_arg(arg_ts->mem_base); 2626 sop->args[2] = arg_ts->mem_offset; 2627 2628 arg_ts->state = TS_MEM; 2629 } 2630 /* Drop outputs that are dead. */ 2631 if (IS_DEAD_ARG(i)) { 2632 arg_ts->state = TS_DEAD; 2633 } 2634 } 2635 } 2636 2637 return changes; 2638 } 2639 2640 #ifdef CONFIG_DEBUG_TCG 2641 static void dump_regs(TCGContext *s) 2642 { 2643 TCGTemp *ts; 2644 int i; 2645 char buf[64]; 2646 2647 for(i = 0; i < s->nb_temps; i++) { 2648 ts = &s->temps[i]; 2649 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2650 switch(ts->val_type) { 2651 case TEMP_VAL_REG: 2652 printf("%s", tcg_target_reg_names[ts->reg]); 2653 break; 2654 case TEMP_VAL_MEM: 2655 printf("%d(%s)", (int)ts->mem_offset, 2656 tcg_target_reg_names[ts->mem_base->reg]); 2657 break; 2658 case TEMP_VAL_CONST: 2659 printf("$0x%" TCG_PRIlx, ts->val); 2660 break; 2661 case TEMP_VAL_DEAD: 2662 printf("D"); 2663 break; 2664 default: 2665 printf("???"); 2666 break; 2667 } 2668 printf("\n"); 2669 } 2670 2671 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 2672 if (s->reg_to_temp[i] != NULL) { 2673 printf("%s: %s\n", 2674 tcg_target_reg_names[i], 2675 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 2676 } 2677 } 2678 } 2679 2680 static void check_regs(TCGContext *s) 2681 { 2682 int reg; 2683 int k; 2684 TCGTemp *ts; 2685 char buf[64]; 2686 2687 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2688 ts = s->reg_to_temp[reg]; 2689 if (ts != NULL) { 2690 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 2691 printf("Inconsistency for register %s:\n", 2692 tcg_target_reg_names[reg]); 2693 goto fail; 2694 } 2695 } 2696 } 2697 for (k = 0; k < s->nb_temps; k++) { 2698 ts = &s->temps[k]; 2699 if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg 2700 && s->reg_to_temp[ts->reg] != ts) { 2701 printf("Inconsistency for temp %s:\n", 2702 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2703 fail: 2704 printf("reg state:\n"); 2705 dump_regs(s); 2706 tcg_abort(); 2707 } 2708 } 2709 } 2710 #endif 2711 2712 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 2713 { 2714 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 2715 /* Sparc64 stack is accessed with offset of 2047 */ 2716 s->current_frame_offset = (s->current_frame_offset + 2717 (tcg_target_long)sizeof(tcg_target_long) - 1) & 2718 ~(sizeof(tcg_target_long) - 1); 2719 #endif 2720 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 2721 s->frame_end) { 2722 tcg_abort(); 2723 } 2724 ts->mem_offset = s->current_frame_offset; 2725 ts->mem_base = s->frame_temp; 2726 ts->mem_allocated = 1; 2727 s->current_frame_offset += sizeof(tcg_target_long); 2728 } 2729 2730 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet); 2731 2732 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 2733 mark it free; otherwise mark it dead. */ 2734 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 2735 { 2736 if (ts->fixed_reg) { 2737 return; 2738 } 2739 if (ts->val_type == TEMP_VAL_REG) { 2740 s->reg_to_temp[ts->reg] = NULL; 2741 } 2742 ts->val_type = (free_or_dead < 0 2743 || ts->temp_local 2744 || ts->temp_global 2745 ? TEMP_VAL_MEM : TEMP_VAL_DEAD); 2746 } 2747 2748 /* Mark a temporary as dead. */ 2749 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 2750 { 2751 temp_free_or_dead(s, ts, 1); 2752 } 2753 2754 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 2755 registers needs to be allocated to store a constant. If 'free_or_dead' 2756 is non-zero, subsequently release the temporary; if it is positive, the 2757 temp is dead; if it is negative, the temp is free. */ 2758 static void temp_sync(TCGContext *s, TCGTemp *ts, 2759 TCGRegSet allocated_regs, int free_or_dead) 2760 { 2761 if (ts->fixed_reg) { 2762 return; 2763 } 2764 if (!ts->mem_coherent) { 2765 if (!ts->mem_allocated) { 2766 temp_allocate_frame(s, ts); 2767 } 2768 switch (ts->val_type) { 2769 case TEMP_VAL_CONST: 2770 /* If we're going to free the temp immediately, then we won't 2771 require it later in a register, so attempt to store the 2772 constant to memory directly. */ 2773 if (free_or_dead 2774 && tcg_out_sti(s, ts->type, ts->val, 2775 ts->mem_base->reg, ts->mem_offset)) { 2776 break; 2777 } 2778 temp_load(s, ts, tcg_target_available_regs[ts->type], 2779 allocated_regs); 2780 /* fallthrough */ 2781 2782 case TEMP_VAL_REG: 2783 tcg_out_st(s, ts->type, ts->reg, 2784 ts->mem_base->reg, ts->mem_offset); 2785 break; 2786 2787 case TEMP_VAL_MEM: 2788 break; 2789 2790 case TEMP_VAL_DEAD: 2791 default: 2792 tcg_abort(); 2793 } 2794 ts->mem_coherent = 1; 2795 } 2796 if (free_or_dead) { 2797 temp_free_or_dead(s, ts, free_or_dead); 2798 } 2799 } 2800 2801 /* free register 'reg' by spilling the corresponding temporary if necessary */ 2802 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 2803 { 2804 TCGTemp *ts = s->reg_to_temp[reg]; 2805 if (ts != NULL) { 2806 temp_sync(s, ts, allocated_regs, -1); 2807 } 2808 } 2809 2810 /* Allocate a register belonging to reg1 & ~reg2 */ 2811 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs, 2812 TCGRegSet allocated_regs, bool rev) 2813 { 2814 int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 2815 const int *order; 2816 TCGReg reg; 2817 TCGRegSet reg_ct; 2818 2819 reg_ct = desired_regs & ~allocated_regs; 2820 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 2821 2822 /* first try free registers */ 2823 for(i = 0; i < n; i++) { 2824 reg = order[i]; 2825 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL) 2826 return reg; 2827 } 2828 2829 /* XXX: do better spill choice */ 2830 for(i = 0; i < n; i++) { 2831 reg = order[i]; 2832 if (tcg_regset_test_reg(reg_ct, reg)) { 2833 tcg_reg_free(s, reg, allocated_regs); 2834 return reg; 2835 } 2836 } 2837 2838 tcg_abort(); 2839 } 2840 2841 /* Make sure the temporary is in a register. If needed, allocate the register 2842 from DESIRED while avoiding ALLOCATED. */ 2843 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 2844 TCGRegSet allocated_regs) 2845 { 2846 TCGReg reg; 2847 2848 switch (ts->val_type) { 2849 case TEMP_VAL_REG: 2850 return; 2851 case TEMP_VAL_CONST: 2852 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 2853 tcg_out_movi(s, ts->type, reg, ts->val); 2854 ts->mem_coherent = 0; 2855 break; 2856 case TEMP_VAL_MEM: 2857 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 2858 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 2859 ts->mem_coherent = 1; 2860 break; 2861 case TEMP_VAL_DEAD: 2862 default: 2863 tcg_abort(); 2864 } 2865 ts->reg = reg; 2866 ts->val_type = TEMP_VAL_REG; 2867 s->reg_to_temp[reg] = ts; 2868 } 2869 2870 /* Save a temporary to memory. 'allocated_regs' is used in case a 2871 temporary registers needs to be allocated to store a constant. */ 2872 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 2873 { 2874 /* The liveness analysis already ensures that globals are back 2875 in memory. Keep an tcg_debug_assert for safety. */ 2876 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg); 2877 } 2878 2879 /* save globals to their canonical location and assume they can be 2880 modified be the following code. 'allocated_regs' is used in case a 2881 temporary registers needs to be allocated to store a constant. */ 2882 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 2883 { 2884 int i, n; 2885 2886 for (i = 0, n = s->nb_globals; i < n; i++) { 2887 temp_save(s, &s->temps[i], allocated_regs); 2888 } 2889 } 2890 2891 /* sync globals to their canonical location and assume they can be 2892 read by the following code. 'allocated_regs' is used in case a 2893 temporary registers needs to be allocated to store a constant. */ 2894 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 2895 { 2896 int i, n; 2897 2898 for (i = 0, n = s->nb_globals; i < n; i++) { 2899 TCGTemp *ts = &s->temps[i]; 2900 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 2901 || ts->fixed_reg 2902 || ts->mem_coherent); 2903 } 2904 } 2905 2906 /* at the end of a basic block, we assume all temporaries are dead and 2907 all globals are stored at their canonical location. */ 2908 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 2909 { 2910 int i; 2911 2912 for (i = s->nb_globals; i < s->nb_temps; i++) { 2913 TCGTemp *ts = &s->temps[i]; 2914 if (ts->temp_local) { 2915 temp_save(s, ts, allocated_regs); 2916 } else { 2917 /* The liveness analysis already ensures that temps are dead. 2918 Keep an tcg_debug_assert for safety. */ 2919 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 2920 } 2921 } 2922 2923 save_globals(s, allocated_regs); 2924 } 2925 2926 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 2927 tcg_target_ulong val, TCGLifeData arg_life) 2928 { 2929 if (ots->fixed_reg) { 2930 /* For fixed registers, we do not do any constant propagation. */ 2931 tcg_out_movi(s, ots->type, ots->reg, val); 2932 return; 2933 } 2934 2935 /* The movi is not explicitly generated here. */ 2936 if (ots->val_type == TEMP_VAL_REG) { 2937 s->reg_to_temp[ots->reg] = NULL; 2938 } 2939 ots->val_type = TEMP_VAL_CONST; 2940 ots->val = val; 2941 ots->mem_coherent = 0; 2942 if (NEED_SYNC_ARG(0)) { 2943 temp_sync(s, ots, s->reserved_regs, IS_DEAD_ARG(0)); 2944 } else if (IS_DEAD_ARG(0)) { 2945 temp_dead(s, ots); 2946 } 2947 } 2948 2949 static void tcg_reg_alloc_movi(TCGContext *s, const TCGOp *op) 2950 { 2951 TCGTemp *ots = arg_temp(op->args[0]); 2952 tcg_target_ulong val = op->args[1]; 2953 2954 tcg_reg_alloc_do_movi(s, ots, val, op->life); 2955 } 2956 2957 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 2958 { 2959 const TCGLifeData arg_life = op->life; 2960 TCGRegSet allocated_regs; 2961 TCGTemp *ts, *ots; 2962 TCGType otype, itype; 2963 2964 allocated_regs = s->reserved_regs; 2965 ots = arg_temp(op->args[0]); 2966 ts = arg_temp(op->args[1]); 2967 2968 /* Note that otype != itype for no-op truncation. */ 2969 otype = ots->type; 2970 itype = ts->type; 2971 2972 if (ts->val_type == TEMP_VAL_CONST) { 2973 /* propagate constant or generate sti */ 2974 tcg_target_ulong val = ts->val; 2975 if (IS_DEAD_ARG(1)) { 2976 temp_dead(s, ts); 2977 } 2978 tcg_reg_alloc_do_movi(s, ots, val, arg_life); 2979 return; 2980 } 2981 2982 /* If the source value is in memory we're going to be forced 2983 to have it in a register in order to perform the copy. Copy 2984 the SOURCE value into its own register first, that way we 2985 don't have to reload SOURCE the next time it is used. */ 2986 if (ts->val_type == TEMP_VAL_MEM) { 2987 temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs); 2988 } 2989 2990 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 2991 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 2992 /* mov to a non-saved dead register makes no sense (even with 2993 liveness analysis disabled). */ 2994 tcg_debug_assert(NEED_SYNC_ARG(0)); 2995 if (!ots->mem_allocated) { 2996 temp_allocate_frame(s, ots); 2997 } 2998 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 2999 if (IS_DEAD_ARG(1)) { 3000 temp_dead(s, ts); 3001 } 3002 temp_dead(s, ots); 3003 } else { 3004 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 3005 /* the mov can be suppressed */ 3006 if (ots->val_type == TEMP_VAL_REG) { 3007 s->reg_to_temp[ots->reg] = NULL; 3008 } 3009 ots->reg = ts->reg; 3010 temp_dead(s, ts); 3011 } else { 3012 if (ots->val_type != TEMP_VAL_REG) { 3013 /* When allocating a new register, make sure to not spill the 3014 input one. */ 3015 tcg_regset_set_reg(allocated_regs, ts->reg); 3016 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 3017 allocated_regs, ots->indirect_base); 3018 } 3019 tcg_out_mov(s, otype, ots->reg, ts->reg); 3020 } 3021 ots->val_type = TEMP_VAL_REG; 3022 ots->mem_coherent = 0; 3023 s->reg_to_temp[ots->reg] = ots; 3024 if (NEED_SYNC_ARG(0)) { 3025 temp_sync(s, ots, allocated_regs, 0); 3026 } 3027 } 3028 } 3029 3030 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 3031 { 3032 const TCGLifeData arg_life = op->life; 3033 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 3034 TCGRegSet i_allocated_regs; 3035 TCGRegSet o_allocated_regs; 3036 int i, k, nb_iargs, nb_oargs; 3037 TCGReg reg; 3038 TCGArg arg; 3039 const TCGArgConstraint *arg_ct; 3040 TCGTemp *ts; 3041 TCGArg new_args[TCG_MAX_OP_ARGS]; 3042 int const_args[TCG_MAX_OP_ARGS]; 3043 3044 nb_oargs = def->nb_oargs; 3045 nb_iargs = def->nb_iargs; 3046 3047 /* copy constants */ 3048 memcpy(new_args + nb_oargs + nb_iargs, 3049 op->args + nb_oargs + nb_iargs, 3050 sizeof(TCGArg) * def->nb_cargs); 3051 3052 i_allocated_regs = s->reserved_regs; 3053 o_allocated_regs = s->reserved_regs; 3054 3055 /* satisfy input constraints */ 3056 for (k = 0; k < nb_iargs; k++) { 3057 i = def->sorted_args[nb_oargs + k]; 3058 arg = op->args[i]; 3059 arg_ct = &def->args_ct[i]; 3060 ts = arg_temp(arg); 3061 3062 if (ts->val_type == TEMP_VAL_CONST 3063 && tcg_target_const_match(ts->val, ts->type, arg_ct)) { 3064 /* constant is OK for instruction */ 3065 const_args[i] = 1; 3066 new_args[i] = ts->val; 3067 goto iarg_end; 3068 } 3069 3070 temp_load(s, ts, arg_ct->u.regs, i_allocated_regs); 3071 3072 if (arg_ct->ct & TCG_CT_IALIAS) { 3073 if (ts->fixed_reg) { 3074 /* if fixed register, we must allocate a new register 3075 if the alias is not the same register */ 3076 if (arg != op->args[arg_ct->alias_index]) 3077 goto allocate_in_reg; 3078 } else { 3079 /* if the input is aliased to an output and if it is 3080 not dead after the instruction, we must allocate 3081 a new register and move it */ 3082 if (!IS_DEAD_ARG(i)) { 3083 goto allocate_in_reg; 3084 } 3085 /* check if the current register has already been allocated 3086 for another input aliased to an output */ 3087 int k2, i2; 3088 for (k2 = 0 ; k2 < k ; k2++) { 3089 i2 = def->sorted_args[nb_oargs + k2]; 3090 if ((def->args_ct[i2].ct & TCG_CT_IALIAS) && 3091 (new_args[i2] == ts->reg)) { 3092 goto allocate_in_reg; 3093 } 3094 } 3095 } 3096 } 3097 reg = ts->reg; 3098 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 3099 /* nothing to do : the constraint is satisfied */ 3100 } else { 3101 allocate_in_reg: 3102 /* allocate a new register matching the constraint 3103 and move the temporary register into it */ 3104 reg = tcg_reg_alloc(s, arg_ct->u.regs, i_allocated_regs, 3105 ts->indirect_base); 3106 tcg_out_mov(s, ts->type, reg, ts->reg); 3107 } 3108 new_args[i] = reg; 3109 const_args[i] = 0; 3110 tcg_regset_set_reg(i_allocated_regs, reg); 3111 iarg_end: ; 3112 } 3113 3114 /* mark dead temporaries and free the associated registers */ 3115 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3116 if (IS_DEAD_ARG(i)) { 3117 temp_dead(s, arg_temp(op->args[i])); 3118 } 3119 } 3120 3121 if (def->flags & TCG_OPF_BB_END) { 3122 tcg_reg_alloc_bb_end(s, i_allocated_regs); 3123 } else { 3124 if (def->flags & TCG_OPF_CALL_CLOBBER) { 3125 /* XXX: permit generic clobber register list ? */ 3126 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3127 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3128 tcg_reg_free(s, i, i_allocated_regs); 3129 } 3130 } 3131 } 3132 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3133 /* sync globals if the op has side effects and might trigger 3134 an exception. */ 3135 sync_globals(s, i_allocated_regs); 3136 } 3137 3138 /* satisfy the output constraints */ 3139 for(k = 0; k < nb_oargs; k++) { 3140 i = def->sorted_args[k]; 3141 arg = op->args[i]; 3142 arg_ct = &def->args_ct[i]; 3143 ts = arg_temp(arg); 3144 if ((arg_ct->ct & TCG_CT_ALIAS) 3145 && !const_args[arg_ct->alias_index]) { 3146 reg = new_args[arg_ct->alias_index]; 3147 } else if (arg_ct->ct & TCG_CT_NEWREG) { 3148 reg = tcg_reg_alloc(s, arg_ct->u.regs, 3149 i_allocated_regs | o_allocated_regs, 3150 ts->indirect_base); 3151 } else { 3152 /* if fixed register, we try to use it */ 3153 reg = ts->reg; 3154 if (ts->fixed_reg && 3155 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 3156 goto oarg_end; 3157 } 3158 reg = tcg_reg_alloc(s, arg_ct->u.regs, o_allocated_regs, 3159 ts->indirect_base); 3160 } 3161 tcg_regset_set_reg(o_allocated_regs, reg); 3162 /* if a fixed register is used, then a move will be done afterwards */ 3163 if (!ts->fixed_reg) { 3164 if (ts->val_type == TEMP_VAL_REG) { 3165 s->reg_to_temp[ts->reg] = NULL; 3166 } 3167 ts->val_type = TEMP_VAL_REG; 3168 ts->reg = reg; 3169 /* temp value is modified, so the value kept in memory is 3170 potentially not the same */ 3171 ts->mem_coherent = 0; 3172 s->reg_to_temp[reg] = ts; 3173 } 3174 oarg_end: 3175 new_args[i] = reg; 3176 } 3177 } 3178 3179 /* emit instruction */ 3180 if (def->flags & TCG_OPF_VECTOR) { 3181 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 3182 new_args, const_args); 3183 } else { 3184 tcg_out_op(s, op->opc, new_args, const_args); 3185 } 3186 3187 /* move the outputs in the correct register if needed */ 3188 for(i = 0; i < nb_oargs; i++) { 3189 ts = arg_temp(op->args[i]); 3190 reg = new_args[i]; 3191 if (ts->fixed_reg && ts->reg != reg) { 3192 tcg_out_mov(s, ts->type, ts->reg, reg); 3193 } 3194 if (NEED_SYNC_ARG(i)) { 3195 temp_sync(s, ts, o_allocated_regs, IS_DEAD_ARG(i)); 3196 } else if (IS_DEAD_ARG(i)) { 3197 temp_dead(s, ts); 3198 } 3199 } 3200 } 3201 3202 #ifdef TCG_TARGET_STACK_GROWSUP 3203 #define STACK_DIR(x) (-(x)) 3204 #else 3205 #define STACK_DIR(x) (x) 3206 #endif 3207 3208 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 3209 { 3210 const int nb_oargs = TCGOP_CALLO(op); 3211 const int nb_iargs = TCGOP_CALLI(op); 3212 const TCGLifeData arg_life = op->life; 3213 int flags, nb_regs, i; 3214 TCGReg reg; 3215 TCGArg arg; 3216 TCGTemp *ts; 3217 intptr_t stack_offset; 3218 size_t call_stack_size; 3219 tcg_insn_unit *func_addr; 3220 int allocate_args; 3221 TCGRegSet allocated_regs; 3222 3223 func_addr = (tcg_insn_unit *)(intptr_t)op->args[nb_oargs + nb_iargs]; 3224 flags = op->args[nb_oargs + nb_iargs + 1]; 3225 3226 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 3227 if (nb_regs > nb_iargs) { 3228 nb_regs = nb_iargs; 3229 } 3230 3231 /* assign stack slots first */ 3232 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 3233 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 3234 ~(TCG_TARGET_STACK_ALIGN - 1); 3235 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 3236 if (allocate_args) { 3237 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 3238 preallocate call stack */ 3239 tcg_abort(); 3240 } 3241 3242 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 3243 for (i = nb_regs; i < nb_iargs; i++) { 3244 arg = op->args[nb_oargs + i]; 3245 #ifdef TCG_TARGET_STACK_GROWSUP 3246 stack_offset -= sizeof(tcg_target_long); 3247 #endif 3248 if (arg != TCG_CALL_DUMMY_ARG) { 3249 ts = arg_temp(arg); 3250 temp_load(s, ts, tcg_target_available_regs[ts->type], 3251 s->reserved_regs); 3252 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 3253 } 3254 #ifndef TCG_TARGET_STACK_GROWSUP 3255 stack_offset += sizeof(tcg_target_long); 3256 #endif 3257 } 3258 3259 /* assign input registers */ 3260 allocated_regs = s->reserved_regs; 3261 for (i = 0; i < nb_regs; i++) { 3262 arg = op->args[nb_oargs + i]; 3263 if (arg != TCG_CALL_DUMMY_ARG) { 3264 ts = arg_temp(arg); 3265 reg = tcg_target_call_iarg_regs[i]; 3266 tcg_reg_free(s, reg, allocated_regs); 3267 3268 if (ts->val_type == TEMP_VAL_REG) { 3269 if (ts->reg != reg) { 3270 tcg_out_mov(s, ts->type, reg, ts->reg); 3271 } 3272 } else { 3273 TCGRegSet arg_set = 0; 3274 3275 tcg_regset_set_reg(arg_set, reg); 3276 temp_load(s, ts, arg_set, allocated_regs); 3277 } 3278 3279 tcg_regset_set_reg(allocated_regs, reg); 3280 } 3281 } 3282 3283 /* mark dead temporaries and free the associated registers */ 3284 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3285 if (IS_DEAD_ARG(i)) { 3286 temp_dead(s, arg_temp(op->args[i])); 3287 } 3288 } 3289 3290 /* clobber call registers */ 3291 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3292 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3293 tcg_reg_free(s, i, allocated_regs); 3294 } 3295 } 3296 3297 /* Save globals if they might be written by the helper, sync them if 3298 they might be read. */ 3299 if (flags & TCG_CALL_NO_READ_GLOBALS) { 3300 /* Nothing to do */ 3301 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 3302 sync_globals(s, allocated_regs); 3303 } else { 3304 save_globals(s, allocated_regs); 3305 } 3306 3307 tcg_out_call(s, func_addr); 3308 3309 /* assign output registers and emit moves if needed */ 3310 for(i = 0; i < nb_oargs; i++) { 3311 arg = op->args[i]; 3312 ts = arg_temp(arg); 3313 reg = tcg_target_call_oarg_regs[i]; 3314 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3315 3316 if (ts->fixed_reg) { 3317 if (ts->reg != reg) { 3318 tcg_out_mov(s, ts->type, ts->reg, reg); 3319 } 3320 } else { 3321 if (ts->val_type == TEMP_VAL_REG) { 3322 s->reg_to_temp[ts->reg] = NULL; 3323 } 3324 ts->val_type = TEMP_VAL_REG; 3325 ts->reg = reg; 3326 ts->mem_coherent = 0; 3327 s->reg_to_temp[reg] = ts; 3328 if (NEED_SYNC_ARG(i)) { 3329 temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i)); 3330 } else if (IS_DEAD_ARG(i)) { 3331 temp_dead(s, ts); 3332 } 3333 } 3334 } 3335 } 3336 3337 #ifdef CONFIG_PROFILER 3338 3339 /* avoid copy/paste errors */ 3340 #define PROF_ADD(to, from, field) \ 3341 do { \ 3342 (to)->field += atomic_read(&((from)->field)); \ 3343 } while (0) 3344 3345 #define PROF_MAX(to, from, field) \ 3346 do { \ 3347 typeof((from)->field) val__ = atomic_read(&((from)->field)); \ 3348 if (val__ > (to)->field) { \ 3349 (to)->field = val__; \ 3350 } \ 3351 } while (0) 3352 3353 /* Pass in a zero'ed @prof */ 3354 static inline 3355 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 3356 { 3357 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 3358 unsigned int i; 3359 3360 for (i = 0; i < n_ctxs; i++) { 3361 TCGContext *s = atomic_read(&tcg_ctxs[i]); 3362 const TCGProfile *orig = &s->prof; 3363 3364 if (counters) { 3365 PROF_ADD(prof, orig, cpu_exec_time); 3366 PROF_ADD(prof, orig, tb_count1); 3367 PROF_ADD(prof, orig, tb_count); 3368 PROF_ADD(prof, orig, op_count); 3369 PROF_MAX(prof, orig, op_count_max); 3370 PROF_ADD(prof, orig, temp_count); 3371 PROF_MAX(prof, orig, temp_count_max); 3372 PROF_ADD(prof, orig, del_op_count); 3373 PROF_ADD(prof, orig, code_in_len); 3374 PROF_ADD(prof, orig, code_out_len); 3375 PROF_ADD(prof, orig, search_out_len); 3376 PROF_ADD(prof, orig, interm_time); 3377 PROF_ADD(prof, orig, code_time); 3378 PROF_ADD(prof, orig, la_time); 3379 PROF_ADD(prof, orig, opt_time); 3380 PROF_ADD(prof, orig, restore_count); 3381 PROF_ADD(prof, orig, restore_time); 3382 } 3383 if (table) { 3384 int i; 3385 3386 for (i = 0; i < NB_OPS; i++) { 3387 PROF_ADD(prof, orig, table_op_count[i]); 3388 } 3389 } 3390 } 3391 } 3392 3393 #undef PROF_ADD 3394 #undef PROF_MAX 3395 3396 static void tcg_profile_snapshot_counters(TCGProfile *prof) 3397 { 3398 tcg_profile_snapshot(prof, true, false); 3399 } 3400 3401 static void tcg_profile_snapshot_table(TCGProfile *prof) 3402 { 3403 tcg_profile_snapshot(prof, false, true); 3404 } 3405 3406 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 3407 { 3408 TCGProfile prof = {}; 3409 int i; 3410 3411 tcg_profile_snapshot_table(&prof); 3412 for (i = 0; i < NB_OPS; i++) { 3413 cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, 3414 prof.table_op_count[i]); 3415 } 3416 } 3417 3418 int64_t tcg_cpu_exec_time(void) 3419 { 3420 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 3421 unsigned int i; 3422 int64_t ret = 0; 3423 3424 for (i = 0; i < n_ctxs; i++) { 3425 const TCGContext *s = atomic_read(&tcg_ctxs[i]); 3426 const TCGProfile *prof = &s->prof; 3427 3428 ret += atomic_read(&prof->cpu_exec_time); 3429 } 3430 return ret; 3431 } 3432 #else 3433 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 3434 { 3435 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 3436 } 3437 3438 int64_t tcg_cpu_exec_time(void) 3439 { 3440 error_report("%s: TCG profiler not compiled", __func__); 3441 exit(EXIT_FAILURE); 3442 } 3443 #endif 3444 3445 3446 int tcg_gen_code(TCGContext *s, TranslationBlock *tb) 3447 { 3448 #ifdef CONFIG_PROFILER 3449 TCGProfile *prof = &s->prof; 3450 #endif 3451 int i, num_insns; 3452 TCGOp *op; 3453 3454 #ifdef CONFIG_PROFILER 3455 { 3456 int n = 0; 3457 3458 QTAILQ_FOREACH(op, &s->ops, link) { 3459 n++; 3460 } 3461 atomic_set(&prof->op_count, prof->op_count + n); 3462 if (n > prof->op_count_max) { 3463 atomic_set(&prof->op_count_max, n); 3464 } 3465 3466 n = s->nb_temps; 3467 atomic_set(&prof->temp_count, prof->temp_count + n); 3468 if (n > prof->temp_count_max) { 3469 atomic_set(&prof->temp_count_max, n); 3470 } 3471 } 3472 #endif 3473 3474 #ifdef DEBUG_DISAS 3475 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 3476 && qemu_log_in_addr_range(tb->pc))) { 3477 qemu_log_lock(); 3478 qemu_log("OP:\n"); 3479 tcg_dump_ops(s); 3480 qemu_log("\n"); 3481 qemu_log_unlock(); 3482 } 3483 #endif 3484 3485 #ifdef CONFIG_PROFILER 3486 atomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 3487 #endif 3488 3489 #ifdef USE_TCG_OPTIMIZATIONS 3490 tcg_optimize(s); 3491 #endif 3492 3493 #ifdef CONFIG_PROFILER 3494 atomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 3495 atomic_set(&prof->la_time, prof->la_time - profile_getclock()); 3496 #endif 3497 3498 liveness_pass_1(s); 3499 3500 if (s->nb_indirects > 0) { 3501 #ifdef DEBUG_DISAS 3502 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 3503 && qemu_log_in_addr_range(tb->pc))) { 3504 qemu_log_lock(); 3505 qemu_log("OP before indirect lowering:\n"); 3506 tcg_dump_ops(s); 3507 qemu_log("\n"); 3508 qemu_log_unlock(); 3509 } 3510 #endif 3511 /* Replace indirect temps with direct temps. */ 3512 if (liveness_pass_2(s)) { 3513 /* If changes were made, re-run liveness. */ 3514 liveness_pass_1(s); 3515 } 3516 } 3517 3518 #ifdef CONFIG_PROFILER 3519 atomic_set(&prof->la_time, prof->la_time + profile_getclock()); 3520 #endif 3521 3522 #ifdef DEBUG_DISAS 3523 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 3524 && qemu_log_in_addr_range(tb->pc))) { 3525 qemu_log_lock(); 3526 qemu_log("OP after optimization and liveness analysis:\n"); 3527 tcg_dump_ops(s); 3528 qemu_log("\n"); 3529 qemu_log_unlock(); 3530 } 3531 #endif 3532 3533 tcg_reg_alloc_start(s); 3534 3535 s->code_buf = tb->tc.ptr; 3536 s->code_ptr = tb->tc.ptr; 3537 3538 #ifdef TCG_TARGET_NEED_LDST_LABELS 3539 QSIMPLEQ_INIT(&s->ldst_labels); 3540 #endif 3541 #ifdef TCG_TARGET_NEED_POOL_LABELS 3542 s->pool_labels = NULL; 3543 #endif 3544 3545 num_insns = -1; 3546 QTAILQ_FOREACH(op, &s->ops, link) { 3547 TCGOpcode opc = op->opc; 3548 3549 #ifdef CONFIG_PROFILER 3550 atomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 3551 #endif 3552 3553 switch (opc) { 3554 case INDEX_op_mov_i32: 3555 case INDEX_op_mov_i64: 3556 case INDEX_op_mov_vec: 3557 tcg_reg_alloc_mov(s, op); 3558 break; 3559 case INDEX_op_movi_i32: 3560 case INDEX_op_movi_i64: 3561 case INDEX_op_dupi_vec: 3562 tcg_reg_alloc_movi(s, op); 3563 break; 3564 case INDEX_op_insn_start: 3565 if (num_insns >= 0) { 3566 size_t off = tcg_current_code_size(s); 3567 s->gen_insn_end_off[num_insns] = off; 3568 /* Assert that we do not overflow our stored offset. */ 3569 assert(s->gen_insn_end_off[num_insns] == off); 3570 } 3571 num_insns++; 3572 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 3573 target_ulong a; 3574 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 3575 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 3576 #else 3577 a = op->args[i]; 3578 #endif 3579 s->gen_insn_data[num_insns][i] = a; 3580 } 3581 break; 3582 case INDEX_op_discard: 3583 temp_dead(s, arg_temp(op->args[0])); 3584 break; 3585 case INDEX_op_set_label: 3586 tcg_reg_alloc_bb_end(s, s->reserved_regs); 3587 tcg_out_label(s, arg_label(op->args[0]), s->code_ptr); 3588 break; 3589 case INDEX_op_call: 3590 tcg_reg_alloc_call(s, op); 3591 break; 3592 default: 3593 /* Sanity check that we've not introduced any unhandled opcodes. */ 3594 tcg_debug_assert(tcg_op_supported(opc)); 3595 /* Note: in order to speed up the code, it would be much 3596 faster to have specialized register allocator functions for 3597 some common argument patterns */ 3598 tcg_reg_alloc_op(s, op); 3599 break; 3600 } 3601 #ifdef CONFIG_DEBUG_TCG 3602 check_regs(s); 3603 #endif 3604 /* Test for (pending) buffer overflow. The assumption is that any 3605 one operation beginning below the high water mark cannot overrun 3606 the buffer completely. Thus we can test for overflow after 3607 generating code without having to check during generation. */ 3608 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 3609 return -1; 3610 } 3611 } 3612 tcg_debug_assert(num_insns >= 0); 3613 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 3614 3615 /* Generate TB finalization at the end of block */ 3616 #ifdef TCG_TARGET_NEED_LDST_LABELS 3617 if (!tcg_out_ldst_finalize(s)) { 3618 return -1; 3619 } 3620 #endif 3621 #ifdef TCG_TARGET_NEED_POOL_LABELS 3622 if (!tcg_out_pool_finalize(s)) { 3623 return -1; 3624 } 3625 #endif 3626 3627 /* flush instruction cache */ 3628 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 3629 3630 return tcg_current_code_size(s); 3631 } 3632 3633 #ifdef CONFIG_PROFILER 3634 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 3635 { 3636 TCGProfile prof = {}; 3637 const TCGProfile *s; 3638 int64_t tb_count; 3639 int64_t tb_div_count; 3640 int64_t tot; 3641 3642 tcg_profile_snapshot_counters(&prof); 3643 s = &prof; 3644 tb_count = s->tb_count; 3645 tb_div_count = tb_count ? tb_count : 1; 3646 tot = s->interm_time + s->code_time; 3647 3648 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 3649 tot, tot / 2.4e9); 3650 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 3651 tb_count, s->tb_count1 - tb_count, 3652 (double)(s->tb_count1 - s->tb_count) 3653 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 3654 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 3655 (double)s->op_count / tb_div_count, s->op_count_max); 3656 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 3657 (double)s->del_op_count / tb_div_count); 3658 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 3659 (double)s->temp_count / tb_div_count, s->temp_count_max); 3660 cpu_fprintf(f, "avg host code/TB %0.1f\n", 3661 (double)s->code_out_len / tb_div_count); 3662 cpu_fprintf(f, "avg search data/TB %0.1f\n", 3663 (double)s->search_out_len / tb_div_count); 3664 3665 cpu_fprintf(f, "cycles/op %0.1f\n", 3666 s->op_count ? (double)tot / s->op_count : 0); 3667 cpu_fprintf(f, "cycles/in byte %0.1f\n", 3668 s->code_in_len ? (double)tot / s->code_in_len : 0); 3669 cpu_fprintf(f, "cycles/out byte %0.1f\n", 3670 s->code_out_len ? (double)tot / s->code_out_len : 0); 3671 cpu_fprintf(f, "cycles/search byte %0.1f\n", 3672 s->search_out_len ? (double)tot / s->search_out_len : 0); 3673 if (tot == 0) { 3674 tot = 1; 3675 } 3676 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 3677 (double)s->interm_time / tot * 100.0); 3678 cpu_fprintf(f, " gen_code time %0.1f%%\n", 3679 (double)s->code_time / tot * 100.0); 3680 cpu_fprintf(f, "optim./code time %0.1f%%\n", 3681 (double)s->opt_time / (s->code_time ? s->code_time : 1) 3682 * 100.0); 3683 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 3684 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 3685 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 3686 s->restore_count); 3687 cpu_fprintf(f, " avg cycles %0.1f\n", 3688 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 3689 } 3690 #else 3691 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 3692 { 3693 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 3694 } 3695 #endif 3696 3697 #ifdef ELF_HOST_MACHINE 3698 /* In order to use this feature, the backend needs to do three things: 3699 3700 (1) Define ELF_HOST_MACHINE to indicate both what value to 3701 put into the ELF image and to indicate support for the feature. 3702 3703 (2) Define tcg_register_jit. This should create a buffer containing 3704 the contents of a .debug_frame section that describes the post- 3705 prologue unwind info for the tcg machine. 3706 3707 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 3708 */ 3709 3710 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 3711 typedef enum { 3712 JIT_NOACTION = 0, 3713 JIT_REGISTER_FN, 3714 JIT_UNREGISTER_FN 3715 } jit_actions_t; 3716 3717 struct jit_code_entry { 3718 struct jit_code_entry *next_entry; 3719 struct jit_code_entry *prev_entry; 3720 const void *symfile_addr; 3721 uint64_t symfile_size; 3722 }; 3723 3724 struct jit_descriptor { 3725 uint32_t version; 3726 uint32_t action_flag; 3727 struct jit_code_entry *relevant_entry; 3728 struct jit_code_entry *first_entry; 3729 }; 3730 3731 void __jit_debug_register_code(void) __attribute__((noinline)); 3732 void __jit_debug_register_code(void) 3733 { 3734 asm(""); 3735 } 3736 3737 /* Must statically initialize the version, because GDB may check 3738 the version before we can set it. */ 3739 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 3740 3741 /* End GDB interface. */ 3742 3743 static int find_string(const char *strtab, const char *str) 3744 { 3745 const char *p = strtab + 1; 3746 3747 while (1) { 3748 if (strcmp(p, str) == 0) { 3749 return p - strtab; 3750 } 3751 p += strlen(p) + 1; 3752 } 3753 } 3754 3755 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 3756 const void *debug_frame, 3757 size_t debug_frame_size) 3758 { 3759 struct __attribute__((packed)) DebugInfo { 3760 uint32_t len; 3761 uint16_t version; 3762 uint32_t abbrev; 3763 uint8_t ptr_size; 3764 uint8_t cu_die; 3765 uint16_t cu_lang; 3766 uintptr_t cu_low_pc; 3767 uintptr_t cu_high_pc; 3768 uint8_t fn_die; 3769 char fn_name[16]; 3770 uintptr_t fn_low_pc; 3771 uintptr_t fn_high_pc; 3772 uint8_t cu_eoc; 3773 }; 3774 3775 struct ElfImage { 3776 ElfW(Ehdr) ehdr; 3777 ElfW(Phdr) phdr; 3778 ElfW(Shdr) shdr[7]; 3779 ElfW(Sym) sym[2]; 3780 struct DebugInfo di; 3781 uint8_t da[24]; 3782 char str[80]; 3783 }; 3784 3785 struct ElfImage *img; 3786 3787 static const struct ElfImage img_template = { 3788 .ehdr = { 3789 .e_ident[EI_MAG0] = ELFMAG0, 3790 .e_ident[EI_MAG1] = ELFMAG1, 3791 .e_ident[EI_MAG2] = ELFMAG2, 3792 .e_ident[EI_MAG3] = ELFMAG3, 3793 .e_ident[EI_CLASS] = ELF_CLASS, 3794 .e_ident[EI_DATA] = ELF_DATA, 3795 .e_ident[EI_VERSION] = EV_CURRENT, 3796 .e_type = ET_EXEC, 3797 .e_machine = ELF_HOST_MACHINE, 3798 .e_version = EV_CURRENT, 3799 .e_phoff = offsetof(struct ElfImage, phdr), 3800 .e_shoff = offsetof(struct ElfImage, shdr), 3801 .e_ehsize = sizeof(ElfW(Shdr)), 3802 .e_phentsize = sizeof(ElfW(Phdr)), 3803 .e_phnum = 1, 3804 .e_shentsize = sizeof(ElfW(Shdr)), 3805 .e_shnum = ARRAY_SIZE(img->shdr), 3806 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 3807 #ifdef ELF_HOST_FLAGS 3808 .e_flags = ELF_HOST_FLAGS, 3809 #endif 3810 #ifdef ELF_OSABI 3811 .e_ident[EI_OSABI] = ELF_OSABI, 3812 #endif 3813 }, 3814 .phdr = { 3815 .p_type = PT_LOAD, 3816 .p_flags = PF_X, 3817 }, 3818 .shdr = { 3819 [0] = { .sh_type = SHT_NULL }, 3820 /* Trick: The contents of code_gen_buffer are not present in 3821 this fake ELF file; that got allocated elsewhere. Therefore 3822 we mark .text as SHT_NOBITS (similar to .bss) so that readers 3823 will not look for contents. We can record any address. */ 3824 [1] = { /* .text */ 3825 .sh_type = SHT_NOBITS, 3826 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 3827 }, 3828 [2] = { /* .debug_info */ 3829 .sh_type = SHT_PROGBITS, 3830 .sh_offset = offsetof(struct ElfImage, di), 3831 .sh_size = sizeof(struct DebugInfo), 3832 }, 3833 [3] = { /* .debug_abbrev */ 3834 .sh_type = SHT_PROGBITS, 3835 .sh_offset = offsetof(struct ElfImage, da), 3836 .sh_size = sizeof(img->da), 3837 }, 3838 [4] = { /* .debug_frame */ 3839 .sh_type = SHT_PROGBITS, 3840 .sh_offset = sizeof(struct ElfImage), 3841 }, 3842 [5] = { /* .symtab */ 3843 .sh_type = SHT_SYMTAB, 3844 .sh_offset = offsetof(struct ElfImage, sym), 3845 .sh_size = sizeof(img->sym), 3846 .sh_info = 1, 3847 .sh_link = ARRAY_SIZE(img->shdr) - 1, 3848 .sh_entsize = sizeof(ElfW(Sym)), 3849 }, 3850 [6] = { /* .strtab */ 3851 .sh_type = SHT_STRTAB, 3852 .sh_offset = offsetof(struct ElfImage, str), 3853 .sh_size = sizeof(img->str), 3854 } 3855 }, 3856 .sym = { 3857 [1] = { /* code_gen_buffer */ 3858 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 3859 .st_shndx = 1, 3860 } 3861 }, 3862 .di = { 3863 .len = sizeof(struct DebugInfo) - 4, 3864 .version = 2, 3865 .ptr_size = sizeof(void *), 3866 .cu_die = 1, 3867 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 3868 .fn_die = 2, 3869 .fn_name = "code_gen_buffer" 3870 }, 3871 .da = { 3872 1, /* abbrev number (the cu) */ 3873 0x11, 1, /* DW_TAG_compile_unit, has children */ 3874 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 3875 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 3876 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 3877 0, 0, /* end of abbrev */ 3878 2, /* abbrev number (the fn) */ 3879 0x2e, 0, /* DW_TAG_subprogram, no children */ 3880 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 3881 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 3882 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 3883 0, 0, /* end of abbrev */ 3884 0 /* no more abbrev */ 3885 }, 3886 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 3887 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 3888 }; 3889 3890 /* We only need a single jit entry; statically allocate it. */ 3891 static struct jit_code_entry one_entry; 3892 3893 uintptr_t buf = (uintptr_t)buf_ptr; 3894 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 3895 DebugFrameHeader *dfh; 3896 3897 img = g_malloc(img_size); 3898 *img = img_template; 3899 3900 img->phdr.p_vaddr = buf; 3901 img->phdr.p_paddr = buf; 3902 img->phdr.p_memsz = buf_size; 3903 3904 img->shdr[1].sh_name = find_string(img->str, ".text"); 3905 img->shdr[1].sh_addr = buf; 3906 img->shdr[1].sh_size = buf_size; 3907 3908 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 3909 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 3910 3911 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 3912 img->shdr[4].sh_size = debug_frame_size; 3913 3914 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 3915 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 3916 3917 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 3918 img->sym[1].st_value = buf; 3919 img->sym[1].st_size = buf_size; 3920 3921 img->di.cu_low_pc = buf; 3922 img->di.cu_high_pc = buf + buf_size; 3923 img->di.fn_low_pc = buf; 3924 img->di.fn_high_pc = buf + buf_size; 3925 3926 dfh = (DebugFrameHeader *)(img + 1); 3927 memcpy(dfh, debug_frame, debug_frame_size); 3928 dfh->fde.func_start = buf; 3929 dfh->fde.func_len = buf_size; 3930 3931 #ifdef DEBUG_JIT 3932 /* Enable this block to be able to debug the ELF image file creation. 3933 One can use readelf, objdump, or other inspection utilities. */ 3934 { 3935 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 3936 if (f) { 3937 if (fwrite(img, img_size, 1, f) != img_size) { 3938 /* Avoid stupid unused return value warning for fwrite. */ 3939 } 3940 fclose(f); 3941 } 3942 } 3943 #endif 3944 3945 one_entry.symfile_addr = img; 3946 one_entry.symfile_size = img_size; 3947 3948 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 3949 __jit_debug_descriptor.relevant_entry = &one_entry; 3950 __jit_debug_descriptor.first_entry = &one_entry; 3951 __jit_debug_register_code(); 3952 } 3953 #else 3954 /* No support for the feature. Provide the entry point expected by exec.c, 3955 and implement the internal function we declared earlier. */ 3956 3957 static void tcg_register_jit_int(void *buf, size_t size, 3958 const void *debug_frame, 3959 size_t debug_frame_size) 3960 { 3961 } 3962 3963 void tcg_register_jit(void *buf, size_t buf_size) 3964 { 3965 } 3966 #endif /* ELF_HOST_MACHINE */ 3967 3968 #if !TCG_TARGET_MAYBE_vec 3969 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 3970 { 3971 g_assert_not_reached(); 3972 } 3973 #endif 3974