1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/cutils.h" 34 #include "qemu/host-utils.h" 35 #include "qemu/timer.h" 36 37 /* Note: the long term plan is to reduce the dependencies on the QEMU 38 CPU definitions. Currently they are used for qemu_ld/st 39 instructions */ 40 #define NO_CPU_IO_DEFS 41 #include "cpu.h" 42 43 #include "exec/cpu-common.h" 44 #include "exec/exec-all.h" 45 46 #include "tcg-op.h" 47 48 #if UINTPTR_MAX == UINT32_MAX 49 # define ELF_CLASS ELFCLASS32 50 #else 51 # define ELF_CLASS ELFCLASS64 52 #endif 53 #ifdef HOST_WORDS_BIGENDIAN 54 # define ELF_DATA ELFDATA2MSB 55 #else 56 # define ELF_DATA ELFDATA2LSB 57 #endif 58 59 #include "elf.h" 60 #include "exec/log.h" 61 #include "sysemu/sysemu.h" 62 63 /* Forward declarations for functions declared in tcg-target.inc.c and 64 used here. */ 65 static void tcg_target_init(TCGContext *s); 66 static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static void patch_reloc(tcg_insn_unit *code_ptr, int type, 69 intptr_t value, intptr_t addend); 70 71 /* The CIE and FDE header definitions will be common to all hosts. */ 72 typedef struct { 73 uint32_t len __attribute__((aligned((sizeof(void *))))); 74 uint32_t id; 75 uint8_t version; 76 char augmentation[1]; 77 uint8_t code_align; 78 uint8_t data_align; 79 uint8_t return_column; 80 } DebugFrameCIE; 81 82 typedef struct QEMU_PACKED { 83 uint32_t len __attribute__((aligned((sizeof(void *))))); 84 uint32_t cie_offset; 85 uintptr_t func_start; 86 uintptr_t func_len; 87 } DebugFrameFDEHeader; 88 89 typedef struct QEMU_PACKED { 90 DebugFrameCIE cie; 91 DebugFrameFDEHeader fde; 92 } DebugFrameHeader; 93 94 static void tcg_register_jit_int(void *buf, size_t size, 95 const void *debug_frame, 96 size_t debug_frame_size) 97 __attribute__((unused)); 98 99 /* Forward declarations for functions declared and used in tcg-target.inc.c. */ 100 static const char *target_parse_constraint(TCGArgConstraint *ct, 101 const char *ct_str, TCGType type); 102 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 103 intptr_t arg2); 104 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 105 static void tcg_out_movi(TCGContext *s, TCGType type, 106 TCGReg ret, tcg_target_long arg); 107 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 108 const int *const_args); 109 #if TCG_TARGET_MAYBE_vec 110 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, 111 unsigned vece, const TCGArg *args, 112 const int *const_args); 113 #else 114 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, 115 unsigned vece, const TCGArg *args, 116 const int *const_args) 117 { 118 g_assert_not_reached(); 119 } 120 #endif 121 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 122 intptr_t arg2); 123 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 124 TCGReg base, intptr_t ofs); 125 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); 126 static int tcg_target_const_match(tcg_target_long val, TCGType type, 127 const TCGArgConstraint *arg_ct); 128 #ifdef TCG_TARGET_NEED_LDST_LABELS 129 static bool tcg_out_ldst_finalize(TCGContext *s); 130 #endif 131 132 #define TCG_HIGHWATER 1024 133 134 static TCGContext **tcg_ctxs; 135 static unsigned int n_tcg_ctxs; 136 TCGv_env cpu_env = 0; 137 138 /* 139 * We divide code_gen_buffer into equally-sized "regions" that TCG threads 140 * dynamically allocate from as demand dictates. Given appropriate region 141 * sizing, this minimizes flushes even when some TCG threads generate a lot 142 * more code than others. 143 */ 144 struct tcg_region_state { 145 QemuMutex lock; 146 147 /* fields set at init time */ 148 void *start; 149 void *start_aligned; 150 void *end; 151 size_t n; 152 size_t size; /* size of one region */ 153 size_t stride; /* .size + guard size */ 154 155 /* fields protected by the lock */ 156 size_t current; /* current region index */ 157 size_t agg_size_full; /* aggregate size of full regions */ 158 }; 159 160 static struct tcg_region_state region; 161 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 162 static TCGRegSet tcg_target_call_clobber_regs; 163 164 #if TCG_TARGET_INSN_UNIT_SIZE == 1 165 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 166 { 167 *s->code_ptr++ = v; 168 } 169 170 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 171 uint8_t v) 172 { 173 *p = v; 174 } 175 #endif 176 177 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 178 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 179 { 180 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 181 *s->code_ptr++ = v; 182 } else { 183 tcg_insn_unit *p = s->code_ptr; 184 memcpy(p, &v, sizeof(v)); 185 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 186 } 187 } 188 189 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 190 uint16_t v) 191 { 192 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 193 *p = v; 194 } else { 195 memcpy(p, &v, sizeof(v)); 196 } 197 } 198 #endif 199 200 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 201 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 202 { 203 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 204 *s->code_ptr++ = v; 205 } else { 206 tcg_insn_unit *p = s->code_ptr; 207 memcpy(p, &v, sizeof(v)); 208 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 209 } 210 } 211 212 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 213 uint32_t v) 214 { 215 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 216 *p = v; 217 } else { 218 memcpy(p, &v, sizeof(v)); 219 } 220 } 221 #endif 222 223 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 224 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 225 { 226 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 227 *s->code_ptr++ = v; 228 } else { 229 tcg_insn_unit *p = s->code_ptr; 230 memcpy(p, &v, sizeof(v)); 231 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 232 } 233 } 234 235 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 236 uint64_t v) 237 { 238 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 239 *p = v; 240 } else { 241 memcpy(p, &v, sizeof(v)); 242 } 243 } 244 #endif 245 246 /* label relocation processing */ 247 248 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 249 TCGLabel *l, intptr_t addend) 250 { 251 TCGRelocation *r; 252 253 if (l->has_value) { 254 /* FIXME: This may break relocations on RISC targets that 255 modify instruction fields in place. The caller may not have 256 written the initial value. */ 257 patch_reloc(code_ptr, type, l->u.value, addend); 258 } else { 259 /* add a new relocation entry */ 260 r = tcg_malloc(sizeof(TCGRelocation)); 261 r->type = type; 262 r->ptr = code_ptr; 263 r->addend = addend; 264 r->next = l->u.first_reloc; 265 l->u.first_reloc = r; 266 } 267 } 268 269 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr) 270 { 271 intptr_t value = (intptr_t)ptr; 272 TCGRelocation *r; 273 274 tcg_debug_assert(!l->has_value); 275 276 for (r = l->u.first_reloc; r != NULL; r = r->next) { 277 patch_reloc(r->ptr, r->type, value, r->addend); 278 } 279 280 l->has_value = 1; 281 l->u.value_ptr = ptr; 282 } 283 284 TCGLabel *gen_new_label(void) 285 { 286 TCGContext *s = tcg_ctx; 287 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 288 289 *l = (TCGLabel){ 290 .id = s->nb_labels++ 291 }; 292 293 return l; 294 } 295 296 #include "tcg-target.inc.c" 297 298 static void tcg_region_bounds(size_t curr_region, void **pstart, void **pend) 299 { 300 void *start, *end; 301 302 start = region.start_aligned + curr_region * region.stride; 303 end = start + region.size; 304 305 if (curr_region == 0) { 306 start = region.start; 307 } 308 if (curr_region == region.n - 1) { 309 end = region.end; 310 } 311 312 *pstart = start; 313 *pend = end; 314 } 315 316 static void tcg_region_assign(TCGContext *s, size_t curr_region) 317 { 318 void *start, *end; 319 320 tcg_region_bounds(curr_region, &start, &end); 321 322 s->code_gen_buffer = start; 323 s->code_gen_ptr = start; 324 s->code_gen_buffer_size = end - start; 325 s->code_gen_highwater = end - TCG_HIGHWATER; 326 } 327 328 static bool tcg_region_alloc__locked(TCGContext *s) 329 { 330 if (region.current == region.n) { 331 return true; 332 } 333 tcg_region_assign(s, region.current); 334 region.current++; 335 return false; 336 } 337 338 /* 339 * Request a new region once the one in use has filled up. 340 * Returns true on error. 341 */ 342 static bool tcg_region_alloc(TCGContext *s) 343 { 344 bool err; 345 /* read the region size now; alloc__locked will overwrite it on success */ 346 size_t size_full = s->code_gen_buffer_size; 347 348 qemu_mutex_lock(®ion.lock); 349 err = tcg_region_alloc__locked(s); 350 if (!err) { 351 region.agg_size_full += size_full - TCG_HIGHWATER; 352 } 353 qemu_mutex_unlock(®ion.lock); 354 return err; 355 } 356 357 /* 358 * Perform a context's first region allocation. 359 * This function does _not_ increment region.agg_size_full. 360 */ 361 static inline bool tcg_region_initial_alloc__locked(TCGContext *s) 362 { 363 return tcg_region_alloc__locked(s); 364 } 365 366 /* Call from a safe-work context */ 367 void tcg_region_reset_all(void) 368 { 369 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 370 unsigned int i; 371 372 qemu_mutex_lock(®ion.lock); 373 region.current = 0; 374 region.agg_size_full = 0; 375 376 for (i = 0; i < n_ctxs; i++) { 377 TCGContext *s = atomic_read(&tcg_ctxs[i]); 378 bool err = tcg_region_initial_alloc__locked(s); 379 380 g_assert(!err); 381 } 382 qemu_mutex_unlock(®ion.lock); 383 } 384 385 #ifdef CONFIG_USER_ONLY 386 static size_t tcg_n_regions(void) 387 { 388 return 1; 389 } 390 #else 391 /* 392 * It is likely that some vCPUs will translate more code than others, so we 393 * first try to set more regions than max_cpus, with those regions being of 394 * reasonable size. If that's not possible we make do by evenly dividing 395 * the code_gen_buffer among the vCPUs. 396 */ 397 static size_t tcg_n_regions(void) 398 { 399 size_t i; 400 401 /* Use a single region if all we have is one vCPU thread */ 402 if (max_cpus == 1 || !qemu_tcg_mttcg_enabled()) { 403 return 1; 404 } 405 406 /* Try to have more regions than max_cpus, with each region being >= 2 MB */ 407 for (i = 8; i > 0; i--) { 408 size_t regions_per_thread = i; 409 size_t region_size; 410 411 region_size = tcg_init_ctx.code_gen_buffer_size; 412 region_size /= max_cpus * regions_per_thread; 413 414 if (region_size >= 2 * 1024u * 1024) { 415 return max_cpus * regions_per_thread; 416 } 417 } 418 /* If we can't, then just allocate one region per vCPU thread */ 419 return max_cpus; 420 } 421 #endif 422 423 /* 424 * Initializes region partitioning. 425 * 426 * Called at init time from the parent thread (i.e. the one calling 427 * tcg_context_init), after the target's TCG globals have been set. 428 * 429 * Region partitioning works by splitting code_gen_buffer into separate regions, 430 * and then assigning regions to TCG threads so that the threads can translate 431 * code in parallel without synchronization. 432 * 433 * In softmmu the number of TCG threads is bounded by max_cpus, so we use at 434 * least max_cpus regions in MTTCG. In !MTTCG we use a single region. 435 * Note that the TCG options from the command-line (i.e. -accel accel=tcg,[...]) 436 * must have been parsed before calling this function, since it calls 437 * qemu_tcg_mttcg_enabled(). 438 * 439 * In user-mode we use a single region. Having multiple regions in user-mode 440 * is not supported, because the number of vCPU threads (recall that each thread 441 * spawned by the guest corresponds to a vCPU thread) is only bounded by the 442 * OS, and usually this number is huge (tens of thousands is not uncommon). 443 * Thus, given this large bound on the number of vCPU threads and the fact 444 * that code_gen_buffer is allocated at compile-time, we cannot guarantee 445 * that the availability of at least one region per vCPU thread. 446 * 447 * However, this user-mode limitation is unlikely to be a significant problem 448 * in practice. Multi-threaded guests share most if not all of their translated 449 * code, which makes parallel code generation less appealing than in softmmu. 450 */ 451 void tcg_region_init(void) 452 { 453 void *buf = tcg_init_ctx.code_gen_buffer; 454 void *aligned; 455 size_t size = tcg_init_ctx.code_gen_buffer_size; 456 size_t page_size = qemu_real_host_page_size; 457 size_t region_size; 458 size_t n_regions; 459 size_t i; 460 461 n_regions = tcg_n_regions(); 462 463 /* The first region will be 'aligned - buf' bytes larger than the others */ 464 aligned = QEMU_ALIGN_PTR_UP(buf, page_size); 465 g_assert(aligned < tcg_init_ctx.code_gen_buffer + size); 466 /* 467 * Make region_size a multiple of page_size, using aligned as the start. 468 * As a result of this we might end up with a few extra pages at the end of 469 * the buffer; we will assign those to the last region. 470 */ 471 region_size = (size - (aligned - buf)) / n_regions; 472 region_size = QEMU_ALIGN_DOWN(region_size, page_size); 473 474 /* A region must have at least 2 pages; one code, one guard */ 475 g_assert(region_size >= 2 * page_size); 476 477 /* init the region struct */ 478 qemu_mutex_init(®ion.lock); 479 region.n = n_regions; 480 region.size = region_size - page_size; 481 region.stride = region_size; 482 region.start = buf; 483 region.start_aligned = aligned; 484 /* page-align the end, since its last page will be a guard page */ 485 region.end = QEMU_ALIGN_PTR_DOWN(buf + size, page_size); 486 /* account for that last guard page */ 487 region.end -= page_size; 488 489 /* set guard pages */ 490 for (i = 0; i < region.n; i++) { 491 void *start, *end; 492 int rc; 493 494 tcg_region_bounds(i, &start, &end); 495 rc = qemu_mprotect_none(end, page_size); 496 g_assert(!rc); 497 } 498 499 /* In user-mode we support only one ctx, so do the initial allocation now */ 500 #ifdef CONFIG_USER_ONLY 501 { 502 bool err = tcg_region_initial_alloc__locked(tcg_ctx); 503 504 g_assert(!err); 505 } 506 #endif 507 } 508 509 /* 510 * All TCG threads except the parent (i.e. the one that called tcg_context_init 511 * and registered the target's TCG globals) must register with this function 512 * before initiating translation. 513 * 514 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 515 * of tcg_region_init() for the reasoning behind this. 516 * 517 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 518 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 519 * is not used anymore for translation once this function is called. 520 * 521 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 522 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 523 */ 524 #ifdef CONFIG_USER_ONLY 525 void tcg_register_thread(void) 526 { 527 tcg_ctx = &tcg_init_ctx; 528 } 529 #else 530 void tcg_register_thread(void) 531 { 532 TCGContext *s = g_malloc(sizeof(*s)); 533 unsigned int i, n; 534 bool err; 535 536 *s = tcg_init_ctx; 537 538 /* Relink mem_base. */ 539 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 540 if (tcg_init_ctx.temps[i].mem_base) { 541 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 542 tcg_debug_assert(b >= 0 && b < n); 543 s->temps[i].mem_base = &s->temps[b]; 544 } 545 } 546 547 /* Claim an entry in tcg_ctxs */ 548 n = atomic_fetch_inc(&n_tcg_ctxs); 549 g_assert(n < max_cpus); 550 atomic_set(&tcg_ctxs[n], s); 551 552 tcg_ctx = s; 553 qemu_mutex_lock(®ion.lock); 554 err = tcg_region_initial_alloc__locked(tcg_ctx); 555 g_assert(!err); 556 qemu_mutex_unlock(®ion.lock); 557 } 558 #endif /* !CONFIG_USER_ONLY */ 559 560 /* 561 * Returns the size (in bytes) of all translated code (i.e. from all regions) 562 * currently in the cache. 563 * See also: tcg_code_capacity() 564 * Do not confuse with tcg_current_code_size(); that one applies to a single 565 * TCG context. 566 */ 567 size_t tcg_code_size(void) 568 { 569 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 570 unsigned int i; 571 size_t total; 572 573 qemu_mutex_lock(®ion.lock); 574 total = region.agg_size_full; 575 for (i = 0; i < n_ctxs; i++) { 576 const TCGContext *s = atomic_read(&tcg_ctxs[i]); 577 size_t size; 578 579 size = atomic_read(&s->code_gen_ptr) - s->code_gen_buffer; 580 g_assert(size <= s->code_gen_buffer_size); 581 total += size; 582 } 583 qemu_mutex_unlock(®ion.lock); 584 return total; 585 } 586 587 /* 588 * Returns the code capacity (in bytes) of the entire cache, i.e. including all 589 * regions. 590 * See also: tcg_code_size() 591 */ 592 size_t tcg_code_capacity(void) 593 { 594 size_t guard_size, capacity; 595 596 /* no need for synchronization; these variables are set at init time */ 597 guard_size = region.stride - region.size; 598 capacity = region.end + guard_size - region.start; 599 capacity -= region.n * (guard_size + TCG_HIGHWATER); 600 return capacity; 601 } 602 603 /* pool based memory allocation */ 604 void *tcg_malloc_internal(TCGContext *s, int size) 605 { 606 TCGPool *p; 607 int pool_size; 608 609 if (size > TCG_POOL_CHUNK_SIZE) { 610 /* big malloc: insert a new pool (XXX: could optimize) */ 611 p = g_malloc(sizeof(TCGPool) + size); 612 p->size = size; 613 p->next = s->pool_first_large; 614 s->pool_first_large = p; 615 return p->data; 616 } else { 617 p = s->pool_current; 618 if (!p) { 619 p = s->pool_first; 620 if (!p) 621 goto new_pool; 622 } else { 623 if (!p->next) { 624 new_pool: 625 pool_size = TCG_POOL_CHUNK_SIZE; 626 p = g_malloc(sizeof(TCGPool) + pool_size); 627 p->size = pool_size; 628 p->next = NULL; 629 if (s->pool_current) 630 s->pool_current->next = p; 631 else 632 s->pool_first = p; 633 } else { 634 p = p->next; 635 } 636 } 637 } 638 s->pool_current = p; 639 s->pool_cur = p->data + size; 640 s->pool_end = p->data + p->size; 641 return p->data; 642 } 643 644 void tcg_pool_reset(TCGContext *s) 645 { 646 TCGPool *p, *t; 647 for (p = s->pool_first_large; p; p = t) { 648 t = p->next; 649 g_free(p); 650 } 651 s->pool_first_large = NULL; 652 s->pool_cur = s->pool_end = NULL; 653 s->pool_current = NULL; 654 } 655 656 typedef struct TCGHelperInfo { 657 void *func; 658 const char *name; 659 unsigned flags; 660 unsigned sizemask; 661 } TCGHelperInfo; 662 663 #include "exec/helper-proto.h" 664 665 static const TCGHelperInfo all_helpers[] = { 666 #include "exec/helper-tcg.h" 667 }; 668 static GHashTable *helper_table; 669 670 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 671 static void process_op_defs(TCGContext *s); 672 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 673 TCGReg reg, const char *name); 674 675 void tcg_context_init(TCGContext *s) 676 { 677 int op, total_args, n, i; 678 TCGOpDef *def; 679 TCGArgConstraint *args_ct; 680 int *sorted_args; 681 TCGTemp *ts; 682 683 memset(s, 0, sizeof(*s)); 684 s->nb_globals = 0; 685 686 /* Count total number of arguments and allocate the corresponding 687 space */ 688 total_args = 0; 689 for(op = 0; op < NB_OPS; op++) { 690 def = &tcg_op_defs[op]; 691 n = def->nb_iargs + def->nb_oargs; 692 total_args += n; 693 } 694 695 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 696 sorted_args = g_malloc(sizeof(int) * total_args); 697 698 for(op = 0; op < NB_OPS; op++) { 699 def = &tcg_op_defs[op]; 700 def->args_ct = args_ct; 701 def->sorted_args = sorted_args; 702 n = def->nb_iargs + def->nb_oargs; 703 sorted_args += n; 704 args_ct += n; 705 } 706 707 /* Register helpers. */ 708 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 709 helper_table = g_hash_table_new(NULL, NULL); 710 711 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 712 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 713 (gpointer)&all_helpers[i]); 714 } 715 716 tcg_target_init(s); 717 process_op_defs(s); 718 719 /* Reverse the order of the saved registers, assuming they're all at 720 the start of tcg_target_reg_alloc_order. */ 721 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 722 int r = tcg_target_reg_alloc_order[n]; 723 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 724 break; 725 } 726 } 727 for (i = 0; i < n; ++i) { 728 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 729 } 730 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 731 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 732 } 733 734 tcg_ctx = s; 735 /* 736 * In user-mode we simply share the init context among threads, since we 737 * use a single region. See the documentation tcg_region_init() for the 738 * reasoning behind this. 739 * In softmmu we will have at most max_cpus TCG threads. 740 */ 741 #ifdef CONFIG_USER_ONLY 742 tcg_ctxs = &tcg_ctx; 743 n_tcg_ctxs = 1; 744 #else 745 tcg_ctxs = g_new(TCGContext *, max_cpus); 746 #endif 747 748 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 749 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 750 cpu_env = temp_tcgv_ptr(ts); 751 } 752 753 /* 754 * Allocate TBs right before their corresponding translated code, making 755 * sure that TBs and code are on different cache lines. 756 */ 757 TranslationBlock *tcg_tb_alloc(TCGContext *s) 758 { 759 uintptr_t align = qemu_icache_linesize; 760 TranslationBlock *tb; 761 void *next; 762 763 retry: 764 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 765 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 766 767 if (unlikely(next > s->code_gen_highwater)) { 768 if (tcg_region_alloc(s)) { 769 return NULL; 770 } 771 goto retry; 772 } 773 atomic_set(&s->code_gen_ptr, next); 774 s->data_gen_ptr = NULL; 775 return tb; 776 } 777 778 void tcg_prologue_init(TCGContext *s) 779 { 780 size_t prologue_size, total_size; 781 void *buf0, *buf1; 782 783 /* Put the prologue at the beginning of code_gen_buffer. */ 784 buf0 = s->code_gen_buffer; 785 total_size = s->code_gen_buffer_size; 786 s->code_ptr = buf0; 787 s->code_buf = buf0; 788 s->data_gen_ptr = NULL; 789 s->code_gen_prologue = buf0; 790 791 /* Compute a high-water mark, at which we voluntarily flush the buffer 792 and start over. The size here is arbitrary, significantly larger 793 than we expect the code generation for any one opcode to require. */ 794 s->code_gen_highwater = s->code_gen_buffer + (total_size - TCG_HIGHWATER); 795 796 #ifdef TCG_TARGET_NEED_POOL_LABELS 797 s->pool_labels = NULL; 798 #endif 799 800 /* Generate the prologue. */ 801 tcg_target_qemu_prologue(s); 802 803 #ifdef TCG_TARGET_NEED_POOL_LABELS 804 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 805 { 806 bool ok = tcg_out_pool_finalize(s); 807 tcg_debug_assert(ok); 808 } 809 #endif 810 811 buf1 = s->code_ptr; 812 flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1); 813 814 /* Deduct the prologue from the buffer. */ 815 prologue_size = tcg_current_code_size(s); 816 s->code_gen_ptr = buf1; 817 s->code_gen_buffer = buf1; 818 s->code_buf = buf1; 819 total_size -= prologue_size; 820 s->code_gen_buffer_size = total_size; 821 822 tcg_register_jit(s->code_gen_buffer, total_size); 823 824 #ifdef DEBUG_DISAS 825 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 826 qemu_log_lock(); 827 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 828 if (s->data_gen_ptr) { 829 size_t code_size = s->data_gen_ptr - buf0; 830 size_t data_size = prologue_size - code_size; 831 size_t i; 832 833 log_disas(buf0, code_size); 834 835 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 836 if (sizeof(tcg_target_ulong) == 8) { 837 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 838 (uintptr_t)s->data_gen_ptr + i, 839 *(uint64_t *)(s->data_gen_ptr + i)); 840 } else { 841 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 842 (uintptr_t)s->data_gen_ptr + i, 843 *(uint32_t *)(s->data_gen_ptr + i)); 844 } 845 } 846 } else { 847 log_disas(buf0, prologue_size); 848 } 849 qemu_log("\n"); 850 qemu_log_flush(); 851 qemu_log_unlock(); 852 } 853 #endif 854 855 /* Assert that goto_ptr is implemented completely. */ 856 if (TCG_TARGET_HAS_goto_ptr) { 857 tcg_debug_assert(s->code_gen_epilogue != NULL); 858 } 859 } 860 861 void tcg_func_start(TCGContext *s) 862 { 863 tcg_pool_reset(s); 864 s->nb_temps = s->nb_globals; 865 866 /* No temps have been previously allocated for size or locality. */ 867 memset(s->free_temps, 0, sizeof(s->free_temps)); 868 869 s->nb_labels = 0; 870 s->current_frame_offset = s->frame_start; 871 872 #ifdef CONFIG_DEBUG_TCG 873 s->goto_tb_issue_mask = 0; 874 #endif 875 876 QTAILQ_INIT(&s->ops); 877 QTAILQ_INIT(&s->free_ops); 878 } 879 880 static inline TCGTemp *tcg_temp_alloc(TCGContext *s) 881 { 882 int n = s->nb_temps++; 883 tcg_debug_assert(n < TCG_MAX_TEMPS); 884 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 885 } 886 887 static inline TCGTemp *tcg_global_alloc(TCGContext *s) 888 { 889 TCGTemp *ts; 890 891 tcg_debug_assert(s->nb_globals == s->nb_temps); 892 s->nb_globals++; 893 ts = tcg_temp_alloc(s); 894 ts->temp_global = 1; 895 896 return ts; 897 } 898 899 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 900 TCGReg reg, const char *name) 901 { 902 TCGTemp *ts; 903 904 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 905 tcg_abort(); 906 } 907 908 ts = tcg_global_alloc(s); 909 ts->base_type = type; 910 ts->type = type; 911 ts->fixed_reg = 1; 912 ts->reg = reg; 913 ts->name = name; 914 tcg_regset_set_reg(s->reserved_regs, reg); 915 916 return ts; 917 } 918 919 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 920 { 921 s->frame_start = start; 922 s->frame_end = start + size; 923 s->frame_temp 924 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 925 } 926 927 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 928 intptr_t offset, const char *name) 929 { 930 TCGContext *s = tcg_ctx; 931 TCGTemp *base_ts = tcgv_ptr_temp(base); 932 TCGTemp *ts = tcg_global_alloc(s); 933 int indirect_reg = 0, bigendian = 0; 934 #ifdef HOST_WORDS_BIGENDIAN 935 bigendian = 1; 936 #endif 937 938 if (!base_ts->fixed_reg) { 939 /* We do not support double-indirect registers. */ 940 tcg_debug_assert(!base_ts->indirect_reg); 941 base_ts->indirect_base = 1; 942 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 943 ? 2 : 1); 944 indirect_reg = 1; 945 } 946 947 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 948 TCGTemp *ts2 = tcg_global_alloc(s); 949 char buf[64]; 950 951 ts->base_type = TCG_TYPE_I64; 952 ts->type = TCG_TYPE_I32; 953 ts->indirect_reg = indirect_reg; 954 ts->mem_allocated = 1; 955 ts->mem_base = base_ts; 956 ts->mem_offset = offset + bigendian * 4; 957 pstrcpy(buf, sizeof(buf), name); 958 pstrcat(buf, sizeof(buf), "_0"); 959 ts->name = strdup(buf); 960 961 tcg_debug_assert(ts2 == ts + 1); 962 ts2->base_type = TCG_TYPE_I64; 963 ts2->type = TCG_TYPE_I32; 964 ts2->indirect_reg = indirect_reg; 965 ts2->mem_allocated = 1; 966 ts2->mem_base = base_ts; 967 ts2->mem_offset = offset + (1 - bigendian) * 4; 968 pstrcpy(buf, sizeof(buf), name); 969 pstrcat(buf, sizeof(buf), "_1"); 970 ts2->name = strdup(buf); 971 } else { 972 ts->base_type = type; 973 ts->type = type; 974 ts->indirect_reg = indirect_reg; 975 ts->mem_allocated = 1; 976 ts->mem_base = base_ts; 977 ts->mem_offset = offset; 978 ts->name = name; 979 } 980 return ts; 981 } 982 983 static TCGTemp *tcg_temp_new_internal(TCGType type, int temp_local) 984 { 985 TCGContext *s = tcg_ctx; 986 TCGTemp *ts; 987 int idx, k; 988 989 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 990 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 991 if (idx < TCG_MAX_TEMPS) { 992 /* There is already an available temp with the right type. */ 993 clear_bit(idx, s->free_temps[k].l); 994 995 ts = &s->temps[idx]; 996 ts->temp_allocated = 1; 997 tcg_debug_assert(ts->base_type == type); 998 tcg_debug_assert(ts->temp_local == temp_local); 999 } else { 1000 ts = tcg_temp_alloc(s); 1001 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1002 TCGTemp *ts2 = tcg_temp_alloc(s); 1003 1004 ts->base_type = type; 1005 ts->type = TCG_TYPE_I32; 1006 ts->temp_allocated = 1; 1007 ts->temp_local = temp_local; 1008 1009 tcg_debug_assert(ts2 == ts + 1); 1010 ts2->base_type = TCG_TYPE_I64; 1011 ts2->type = TCG_TYPE_I32; 1012 ts2->temp_allocated = 1; 1013 ts2->temp_local = temp_local; 1014 } else { 1015 ts->base_type = type; 1016 ts->type = type; 1017 ts->temp_allocated = 1; 1018 ts->temp_local = temp_local; 1019 } 1020 } 1021 1022 #if defined(CONFIG_DEBUG_TCG) 1023 s->temps_in_use++; 1024 #endif 1025 return ts; 1026 } 1027 1028 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 1029 { 1030 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 1031 return temp_tcgv_i32(t); 1032 } 1033 1034 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 1035 { 1036 TCGTemp *t = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 1037 return temp_tcgv_i64(t); 1038 } 1039 1040 TCGv_vec tcg_temp_new_vec(TCGType type) 1041 { 1042 TCGTemp *t; 1043 1044 #ifdef CONFIG_DEBUG_TCG 1045 switch (type) { 1046 case TCG_TYPE_V64: 1047 assert(TCG_TARGET_HAS_v64); 1048 break; 1049 case TCG_TYPE_V128: 1050 assert(TCG_TARGET_HAS_v128); 1051 break; 1052 case TCG_TYPE_V256: 1053 assert(TCG_TARGET_HAS_v256); 1054 break; 1055 default: 1056 g_assert_not_reached(); 1057 } 1058 #endif 1059 1060 t = tcg_temp_new_internal(type, 0); 1061 return temp_tcgv_vec(t); 1062 } 1063 1064 /* Create a new temp of the same type as an existing temp. */ 1065 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1066 { 1067 TCGTemp *t = tcgv_vec_temp(match); 1068 1069 tcg_debug_assert(t->temp_allocated != 0); 1070 1071 t = tcg_temp_new_internal(t->base_type, 0); 1072 return temp_tcgv_vec(t); 1073 } 1074 1075 static void tcg_temp_free_internal(TCGTemp *ts) 1076 { 1077 TCGContext *s = tcg_ctx; 1078 int k, idx; 1079 1080 #if defined(CONFIG_DEBUG_TCG) 1081 s->temps_in_use--; 1082 if (s->temps_in_use < 0) { 1083 fprintf(stderr, "More temporaries freed than allocated!\n"); 1084 } 1085 #endif 1086 1087 tcg_debug_assert(ts->temp_global == 0); 1088 tcg_debug_assert(ts->temp_allocated != 0); 1089 ts->temp_allocated = 0; 1090 1091 idx = temp_idx(ts); 1092 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 1093 set_bit(idx, s->free_temps[k].l); 1094 } 1095 1096 void tcg_temp_free_i32(TCGv_i32 arg) 1097 { 1098 tcg_temp_free_internal(tcgv_i32_temp(arg)); 1099 } 1100 1101 void tcg_temp_free_i64(TCGv_i64 arg) 1102 { 1103 tcg_temp_free_internal(tcgv_i64_temp(arg)); 1104 } 1105 1106 void tcg_temp_free_vec(TCGv_vec arg) 1107 { 1108 tcg_temp_free_internal(tcgv_vec_temp(arg)); 1109 } 1110 1111 TCGv_i32 tcg_const_i32(int32_t val) 1112 { 1113 TCGv_i32 t0; 1114 t0 = tcg_temp_new_i32(); 1115 tcg_gen_movi_i32(t0, val); 1116 return t0; 1117 } 1118 1119 TCGv_i64 tcg_const_i64(int64_t val) 1120 { 1121 TCGv_i64 t0; 1122 t0 = tcg_temp_new_i64(); 1123 tcg_gen_movi_i64(t0, val); 1124 return t0; 1125 } 1126 1127 TCGv_i32 tcg_const_local_i32(int32_t val) 1128 { 1129 TCGv_i32 t0; 1130 t0 = tcg_temp_local_new_i32(); 1131 tcg_gen_movi_i32(t0, val); 1132 return t0; 1133 } 1134 1135 TCGv_i64 tcg_const_local_i64(int64_t val) 1136 { 1137 TCGv_i64 t0; 1138 t0 = tcg_temp_local_new_i64(); 1139 tcg_gen_movi_i64(t0, val); 1140 return t0; 1141 } 1142 1143 #if defined(CONFIG_DEBUG_TCG) 1144 void tcg_clear_temp_count(void) 1145 { 1146 TCGContext *s = tcg_ctx; 1147 s->temps_in_use = 0; 1148 } 1149 1150 int tcg_check_temp_count(void) 1151 { 1152 TCGContext *s = tcg_ctx; 1153 if (s->temps_in_use) { 1154 /* Clear the count so that we don't give another 1155 * warning immediately next time around. 1156 */ 1157 s->temps_in_use = 0; 1158 return 1; 1159 } 1160 return 0; 1161 } 1162 #endif 1163 1164 /* Return true if OP may appear in the opcode stream. 1165 Test the runtime variable that controls each opcode. */ 1166 bool tcg_op_supported(TCGOpcode op) 1167 { 1168 const bool have_vec 1169 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1170 1171 switch (op) { 1172 case INDEX_op_discard: 1173 case INDEX_op_set_label: 1174 case INDEX_op_call: 1175 case INDEX_op_br: 1176 case INDEX_op_mb: 1177 case INDEX_op_insn_start: 1178 case INDEX_op_exit_tb: 1179 case INDEX_op_goto_tb: 1180 case INDEX_op_qemu_ld_i32: 1181 case INDEX_op_qemu_st_i32: 1182 case INDEX_op_qemu_ld_i64: 1183 case INDEX_op_qemu_st_i64: 1184 return true; 1185 1186 case INDEX_op_goto_ptr: 1187 return TCG_TARGET_HAS_goto_ptr; 1188 1189 case INDEX_op_mov_i32: 1190 case INDEX_op_movi_i32: 1191 case INDEX_op_setcond_i32: 1192 case INDEX_op_brcond_i32: 1193 case INDEX_op_ld8u_i32: 1194 case INDEX_op_ld8s_i32: 1195 case INDEX_op_ld16u_i32: 1196 case INDEX_op_ld16s_i32: 1197 case INDEX_op_ld_i32: 1198 case INDEX_op_st8_i32: 1199 case INDEX_op_st16_i32: 1200 case INDEX_op_st_i32: 1201 case INDEX_op_add_i32: 1202 case INDEX_op_sub_i32: 1203 case INDEX_op_mul_i32: 1204 case INDEX_op_and_i32: 1205 case INDEX_op_or_i32: 1206 case INDEX_op_xor_i32: 1207 case INDEX_op_shl_i32: 1208 case INDEX_op_shr_i32: 1209 case INDEX_op_sar_i32: 1210 return true; 1211 1212 case INDEX_op_movcond_i32: 1213 return TCG_TARGET_HAS_movcond_i32; 1214 case INDEX_op_div_i32: 1215 case INDEX_op_divu_i32: 1216 return TCG_TARGET_HAS_div_i32; 1217 case INDEX_op_rem_i32: 1218 case INDEX_op_remu_i32: 1219 return TCG_TARGET_HAS_rem_i32; 1220 case INDEX_op_div2_i32: 1221 case INDEX_op_divu2_i32: 1222 return TCG_TARGET_HAS_div2_i32; 1223 case INDEX_op_rotl_i32: 1224 case INDEX_op_rotr_i32: 1225 return TCG_TARGET_HAS_rot_i32; 1226 case INDEX_op_deposit_i32: 1227 return TCG_TARGET_HAS_deposit_i32; 1228 case INDEX_op_extract_i32: 1229 return TCG_TARGET_HAS_extract_i32; 1230 case INDEX_op_sextract_i32: 1231 return TCG_TARGET_HAS_sextract_i32; 1232 case INDEX_op_add2_i32: 1233 return TCG_TARGET_HAS_add2_i32; 1234 case INDEX_op_sub2_i32: 1235 return TCG_TARGET_HAS_sub2_i32; 1236 case INDEX_op_mulu2_i32: 1237 return TCG_TARGET_HAS_mulu2_i32; 1238 case INDEX_op_muls2_i32: 1239 return TCG_TARGET_HAS_muls2_i32; 1240 case INDEX_op_muluh_i32: 1241 return TCG_TARGET_HAS_muluh_i32; 1242 case INDEX_op_mulsh_i32: 1243 return TCG_TARGET_HAS_mulsh_i32; 1244 case INDEX_op_ext8s_i32: 1245 return TCG_TARGET_HAS_ext8s_i32; 1246 case INDEX_op_ext16s_i32: 1247 return TCG_TARGET_HAS_ext16s_i32; 1248 case INDEX_op_ext8u_i32: 1249 return TCG_TARGET_HAS_ext8u_i32; 1250 case INDEX_op_ext16u_i32: 1251 return TCG_TARGET_HAS_ext16u_i32; 1252 case INDEX_op_bswap16_i32: 1253 return TCG_TARGET_HAS_bswap16_i32; 1254 case INDEX_op_bswap32_i32: 1255 return TCG_TARGET_HAS_bswap32_i32; 1256 case INDEX_op_not_i32: 1257 return TCG_TARGET_HAS_not_i32; 1258 case INDEX_op_neg_i32: 1259 return TCG_TARGET_HAS_neg_i32; 1260 case INDEX_op_andc_i32: 1261 return TCG_TARGET_HAS_andc_i32; 1262 case INDEX_op_orc_i32: 1263 return TCG_TARGET_HAS_orc_i32; 1264 case INDEX_op_eqv_i32: 1265 return TCG_TARGET_HAS_eqv_i32; 1266 case INDEX_op_nand_i32: 1267 return TCG_TARGET_HAS_nand_i32; 1268 case INDEX_op_nor_i32: 1269 return TCG_TARGET_HAS_nor_i32; 1270 case INDEX_op_clz_i32: 1271 return TCG_TARGET_HAS_clz_i32; 1272 case INDEX_op_ctz_i32: 1273 return TCG_TARGET_HAS_ctz_i32; 1274 case INDEX_op_ctpop_i32: 1275 return TCG_TARGET_HAS_ctpop_i32; 1276 1277 case INDEX_op_brcond2_i32: 1278 case INDEX_op_setcond2_i32: 1279 return TCG_TARGET_REG_BITS == 32; 1280 1281 case INDEX_op_mov_i64: 1282 case INDEX_op_movi_i64: 1283 case INDEX_op_setcond_i64: 1284 case INDEX_op_brcond_i64: 1285 case INDEX_op_ld8u_i64: 1286 case INDEX_op_ld8s_i64: 1287 case INDEX_op_ld16u_i64: 1288 case INDEX_op_ld16s_i64: 1289 case INDEX_op_ld32u_i64: 1290 case INDEX_op_ld32s_i64: 1291 case INDEX_op_ld_i64: 1292 case INDEX_op_st8_i64: 1293 case INDEX_op_st16_i64: 1294 case INDEX_op_st32_i64: 1295 case INDEX_op_st_i64: 1296 case INDEX_op_add_i64: 1297 case INDEX_op_sub_i64: 1298 case INDEX_op_mul_i64: 1299 case INDEX_op_and_i64: 1300 case INDEX_op_or_i64: 1301 case INDEX_op_xor_i64: 1302 case INDEX_op_shl_i64: 1303 case INDEX_op_shr_i64: 1304 case INDEX_op_sar_i64: 1305 case INDEX_op_ext_i32_i64: 1306 case INDEX_op_extu_i32_i64: 1307 return TCG_TARGET_REG_BITS == 64; 1308 1309 case INDEX_op_movcond_i64: 1310 return TCG_TARGET_HAS_movcond_i64; 1311 case INDEX_op_div_i64: 1312 case INDEX_op_divu_i64: 1313 return TCG_TARGET_HAS_div_i64; 1314 case INDEX_op_rem_i64: 1315 case INDEX_op_remu_i64: 1316 return TCG_TARGET_HAS_rem_i64; 1317 case INDEX_op_div2_i64: 1318 case INDEX_op_divu2_i64: 1319 return TCG_TARGET_HAS_div2_i64; 1320 case INDEX_op_rotl_i64: 1321 case INDEX_op_rotr_i64: 1322 return TCG_TARGET_HAS_rot_i64; 1323 case INDEX_op_deposit_i64: 1324 return TCG_TARGET_HAS_deposit_i64; 1325 case INDEX_op_extract_i64: 1326 return TCG_TARGET_HAS_extract_i64; 1327 case INDEX_op_sextract_i64: 1328 return TCG_TARGET_HAS_sextract_i64; 1329 case INDEX_op_extrl_i64_i32: 1330 return TCG_TARGET_HAS_extrl_i64_i32; 1331 case INDEX_op_extrh_i64_i32: 1332 return TCG_TARGET_HAS_extrh_i64_i32; 1333 case INDEX_op_ext8s_i64: 1334 return TCG_TARGET_HAS_ext8s_i64; 1335 case INDEX_op_ext16s_i64: 1336 return TCG_TARGET_HAS_ext16s_i64; 1337 case INDEX_op_ext32s_i64: 1338 return TCG_TARGET_HAS_ext32s_i64; 1339 case INDEX_op_ext8u_i64: 1340 return TCG_TARGET_HAS_ext8u_i64; 1341 case INDEX_op_ext16u_i64: 1342 return TCG_TARGET_HAS_ext16u_i64; 1343 case INDEX_op_ext32u_i64: 1344 return TCG_TARGET_HAS_ext32u_i64; 1345 case INDEX_op_bswap16_i64: 1346 return TCG_TARGET_HAS_bswap16_i64; 1347 case INDEX_op_bswap32_i64: 1348 return TCG_TARGET_HAS_bswap32_i64; 1349 case INDEX_op_bswap64_i64: 1350 return TCG_TARGET_HAS_bswap64_i64; 1351 case INDEX_op_not_i64: 1352 return TCG_TARGET_HAS_not_i64; 1353 case INDEX_op_neg_i64: 1354 return TCG_TARGET_HAS_neg_i64; 1355 case INDEX_op_andc_i64: 1356 return TCG_TARGET_HAS_andc_i64; 1357 case INDEX_op_orc_i64: 1358 return TCG_TARGET_HAS_orc_i64; 1359 case INDEX_op_eqv_i64: 1360 return TCG_TARGET_HAS_eqv_i64; 1361 case INDEX_op_nand_i64: 1362 return TCG_TARGET_HAS_nand_i64; 1363 case INDEX_op_nor_i64: 1364 return TCG_TARGET_HAS_nor_i64; 1365 case INDEX_op_clz_i64: 1366 return TCG_TARGET_HAS_clz_i64; 1367 case INDEX_op_ctz_i64: 1368 return TCG_TARGET_HAS_ctz_i64; 1369 case INDEX_op_ctpop_i64: 1370 return TCG_TARGET_HAS_ctpop_i64; 1371 case INDEX_op_add2_i64: 1372 return TCG_TARGET_HAS_add2_i64; 1373 case INDEX_op_sub2_i64: 1374 return TCG_TARGET_HAS_sub2_i64; 1375 case INDEX_op_mulu2_i64: 1376 return TCG_TARGET_HAS_mulu2_i64; 1377 case INDEX_op_muls2_i64: 1378 return TCG_TARGET_HAS_muls2_i64; 1379 case INDEX_op_muluh_i64: 1380 return TCG_TARGET_HAS_muluh_i64; 1381 case INDEX_op_mulsh_i64: 1382 return TCG_TARGET_HAS_mulsh_i64; 1383 1384 case INDEX_op_mov_vec: 1385 case INDEX_op_dup_vec: 1386 case INDEX_op_dupi_vec: 1387 case INDEX_op_ld_vec: 1388 case INDEX_op_st_vec: 1389 case INDEX_op_add_vec: 1390 case INDEX_op_sub_vec: 1391 case INDEX_op_and_vec: 1392 case INDEX_op_or_vec: 1393 case INDEX_op_xor_vec: 1394 return have_vec; 1395 case INDEX_op_dup2_vec: 1396 return have_vec && TCG_TARGET_REG_BITS == 32; 1397 case INDEX_op_not_vec: 1398 return have_vec && TCG_TARGET_HAS_not_vec; 1399 case INDEX_op_neg_vec: 1400 return have_vec && TCG_TARGET_HAS_neg_vec; 1401 case INDEX_op_andc_vec: 1402 return have_vec && TCG_TARGET_HAS_andc_vec; 1403 case INDEX_op_orc_vec: 1404 return have_vec && TCG_TARGET_HAS_orc_vec; 1405 1406 default: 1407 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1408 return true; 1409 } 1410 } 1411 1412 /* Note: we convert the 64 bit args to 32 bit and do some alignment 1413 and endian swap. Maybe it would be better to do the alignment 1414 and endian swap in tcg_reg_alloc_call(). */ 1415 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1416 { 1417 int i, real_args, nb_rets, pi; 1418 unsigned sizemask, flags; 1419 TCGHelperInfo *info; 1420 TCGOp *op; 1421 1422 info = g_hash_table_lookup(helper_table, (gpointer)func); 1423 flags = info->flags; 1424 sizemask = info->sizemask; 1425 1426 #if defined(__sparc__) && !defined(__arch64__) \ 1427 && !defined(CONFIG_TCG_INTERPRETER) 1428 /* We have 64-bit values in one register, but need to pass as two 1429 separate parameters. Split them. */ 1430 int orig_sizemask = sizemask; 1431 int orig_nargs = nargs; 1432 TCGv_i64 retl, reth; 1433 TCGTemp *split_args[MAX_OPC_PARAM]; 1434 1435 retl = NULL; 1436 reth = NULL; 1437 if (sizemask != 0) { 1438 for (i = real_args = 0; i < nargs; ++i) { 1439 int is_64bit = sizemask & (1 << (i+1)*2); 1440 if (is_64bit) { 1441 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1442 TCGv_i32 h = tcg_temp_new_i32(); 1443 TCGv_i32 l = tcg_temp_new_i32(); 1444 tcg_gen_extr_i64_i32(l, h, orig); 1445 split_args[real_args++] = tcgv_i32_temp(h); 1446 split_args[real_args++] = tcgv_i32_temp(l); 1447 } else { 1448 split_args[real_args++] = args[i]; 1449 } 1450 } 1451 nargs = real_args; 1452 args = split_args; 1453 sizemask = 0; 1454 } 1455 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1456 for (i = 0; i < nargs; ++i) { 1457 int is_64bit = sizemask & (1 << (i+1)*2); 1458 int is_signed = sizemask & (2 << (i+1)*2); 1459 if (!is_64bit) { 1460 TCGv_i64 temp = tcg_temp_new_i64(); 1461 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1462 if (is_signed) { 1463 tcg_gen_ext32s_i64(temp, orig); 1464 } else { 1465 tcg_gen_ext32u_i64(temp, orig); 1466 } 1467 args[i] = tcgv_i64_temp(temp); 1468 } 1469 } 1470 #endif /* TCG_TARGET_EXTEND_ARGS */ 1471 1472 op = tcg_emit_op(INDEX_op_call); 1473 1474 pi = 0; 1475 if (ret != NULL) { 1476 #if defined(__sparc__) && !defined(__arch64__) \ 1477 && !defined(CONFIG_TCG_INTERPRETER) 1478 if (orig_sizemask & 1) { 1479 /* The 32-bit ABI is going to return the 64-bit value in 1480 the %o0/%o1 register pair. Prepare for this by using 1481 two return temporaries, and reassemble below. */ 1482 retl = tcg_temp_new_i64(); 1483 reth = tcg_temp_new_i64(); 1484 op->args[pi++] = tcgv_i64_arg(reth); 1485 op->args[pi++] = tcgv_i64_arg(retl); 1486 nb_rets = 2; 1487 } else { 1488 op->args[pi++] = temp_arg(ret); 1489 nb_rets = 1; 1490 } 1491 #else 1492 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { 1493 #ifdef HOST_WORDS_BIGENDIAN 1494 op->args[pi++] = temp_arg(ret + 1); 1495 op->args[pi++] = temp_arg(ret); 1496 #else 1497 op->args[pi++] = temp_arg(ret); 1498 op->args[pi++] = temp_arg(ret + 1); 1499 #endif 1500 nb_rets = 2; 1501 } else { 1502 op->args[pi++] = temp_arg(ret); 1503 nb_rets = 1; 1504 } 1505 #endif 1506 } else { 1507 nb_rets = 0; 1508 } 1509 TCGOP_CALLO(op) = nb_rets; 1510 1511 real_args = 0; 1512 for (i = 0; i < nargs; i++) { 1513 int is_64bit = sizemask & (1 << (i+1)*2); 1514 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 1515 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 1516 /* some targets want aligned 64 bit args */ 1517 if (real_args & 1) { 1518 op->args[pi++] = TCG_CALL_DUMMY_ARG; 1519 real_args++; 1520 } 1521 #endif 1522 /* If stack grows up, then we will be placing successive 1523 arguments at lower addresses, which means we need to 1524 reverse the order compared to how we would normally 1525 treat either big or little-endian. For those arguments 1526 that will wind up in registers, this still works for 1527 HPPA (the only current STACK_GROWSUP target) since the 1528 argument registers are *also* allocated in decreasing 1529 order. If another such target is added, this logic may 1530 have to get more complicated to differentiate between 1531 stack arguments and register arguments. */ 1532 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 1533 op->args[pi++] = temp_arg(args[i] + 1); 1534 op->args[pi++] = temp_arg(args[i]); 1535 #else 1536 op->args[pi++] = temp_arg(args[i]); 1537 op->args[pi++] = temp_arg(args[i] + 1); 1538 #endif 1539 real_args += 2; 1540 continue; 1541 } 1542 1543 op->args[pi++] = temp_arg(args[i]); 1544 real_args++; 1545 } 1546 op->args[pi++] = (uintptr_t)func; 1547 op->args[pi++] = flags; 1548 TCGOP_CALLI(op) = real_args; 1549 1550 /* Make sure the fields didn't overflow. */ 1551 tcg_debug_assert(TCGOP_CALLI(op) == real_args); 1552 tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); 1553 1554 #if defined(__sparc__) && !defined(__arch64__) \ 1555 && !defined(CONFIG_TCG_INTERPRETER) 1556 /* Free all of the parts we allocated above. */ 1557 for (i = real_args = 0; i < orig_nargs; ++i) { 1558 int is_64bit = orig_sizemask & (1 << (i+1)*2); 1559 if (is_64bit) { 1560 tcg_temp_free_internal(args[real_args++]); 1561 tcg_temp_free_internal(args[real_args++]); 1562 } else { 1563 real_args++; 1564 } 1565 } 1566 if (orig_sizemask & 1) { 1567 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 1568 Note that describing these as TCGv_i64 eliminates an unnecessary 1569 zero-extension that tcg_gen_concat_i32_i64 would create. */ 1570 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth); 1571 tcg_temp_free_i64(retl); 1572 tcg_temp_free_i64(reth); 1573 } 1574 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1575 for (i = 0; i < nargs; ++i) { 1576 int is_64bit = sizemask & (1 << (i+1)*2); 1577 if (!is_64bit) { 1578 tcg_temp_free_internal(args[i]); 1579 } 1580 } 1581 #endif /* TCG_TARGET_EXTEND_ARGS */ 1582 } 1583 1584 static void tcg_reg_alloc_start(TCGContext *s) 1585 { 1586 int i, n; 1587 TCGTemp *ts; 1588 1589 for (i = 0, n = s->nb_globals; i < n; i++) { 1590 ts = &s->temps[i]; 1591 ts->val_type = (ts->fixed_reg ? TEMP_VAL_REG : TEMP_VAL_MEM); 1592 } 1593 for (n = s->nb_temps; i < n; i++) { 1594 ts = &s->temps[i]; 1595 ts->val_type = (ts->temp_local ? TEMP_VAL_MEM : TEMP_VAL_DEAD); 1596 ts->mem_allocated = 0; 1597 ts->fixed_reg = 0; 1598 } 1599 1600 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1601 } 1602 1603 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1604 TCGTemp *ts) 1605 { 1606 int idx = temp_idx(ts); 1607 1608 if (ts->temp_global) { 1609 pstrcpy(buf, buf_size, ts->name); 1610 } else if (ts->temp_local) { 1611 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1612 } else { 1613 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1614 } 1615 return buf; 1616 } 1617 1618 static char *tcg_get_arg_str(TCGContext *s, char *buf, 1619 int buf_size, TCGArg arg) 1620 { 1621 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1622 } 1623 1624 /* Find helper name. */ 1625 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 1626 { 1627 const char *ret = NULL; 1628 if (helper_table) { 1629 TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val); 1630 if (info) { 1631 ret = info->name; 1632 } 1633 } 1634 return ret; 1635 } 1636 1637 static const char * const cond_name[] = 1638 { 1639 [TCG_COND_NEVER] = "never", 1640 [TCG_COND_ALWAYS] = "always", 1641 [TCG_COND_EQ] = "eq", 1642 [TCG_COND_NE] = "ne", 1643 [TCG_COND_LT] = "lt", 1644 [TCG_COND_GE] = "ge", 1645 [TCG_COND_LE] = "le", 1646 [TCG_COND_GT] = "gt", 1647 [TCG_COND_LTU] = "ltu", 1648 [TCG_COND_GEU] = "geu", 1649 [TCG_COND_LEU] = "leu", 1650 [TCG_COND_GTU] = "gtu" 1651 }; 1652 1653 static const char * const ldst_name[] = 1654 { 1655 [MO_UB] = "ub", 1656 [MO_SB] = "sb", 1657 [MO_LEUW] = "leuw", 1658 [MO_LESW] = "lesw", 1659 [MO_LEUL] = "leul", 1660 [MO_LESL] = "lesl", 1661 [MO_LEQ] = "leq", 1662 [MO_BEUW] = "beuw", 1663 [MO_BESW] = "besw", 1664 [MO_BEUL] = "beul", 1665 [MO_BESL] = "besl", 1666 [MO_BEQ] = "beq", 1667 }; 1668 1669 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1670 #ifdef ALIGNED_ONLY 1671 [MO_UNALN >> MO_ASHIFT] = "un+", 1672 [MO_ALIGN >> MO_ASHIFT] = "", 1673 #else 1674 [MO_UNALN >> MO_ASHIFT] = "", 1675 [MO_ALIGN >> MO_ASHIFT] = "al+", 1676 #endif 1677 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1678 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1679 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1680 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1681 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1682 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1683 }; 1684 1685 void tcg_dump_ops(TCGContext *s) 1686 { 1687 char buf[128]; 1688 TCGOp *op; 1689 1690 QTAILQ_FOREACH(op, &s->ops, link) { 1691 int i, k, nb_oargs, nb_iargs, nb_cargs; 1692 const TCGOpDef *def; 1693 TCGOpcode c; 1694 int col = 0; 1695 1696 c = op->opc; 1697 def = &tcg_op_defs[c]; 1698 1699 if (c == INDEX_op_insn_start) { 1700 col += qemu_log("\n ----"); 1701 1702 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1703 target_ulong a; 1704 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1705 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 1706 #else 1707 a = op->args[i]; 1708 #endif 1709 col += qemu_log(" " TARGET_FMT_lx, a); 1710 } 1711 } else if (c == INDEX_op_call) { 1712 /* variable number of arguments */ 1713 nb_oargs = TCGOP_CALLO(op); 1714 nb_iargs = TCGOP_CALLI(op); 1715 nb_cargs = def->nb_cargs; 1716 1717 /* function name, flags, out args */ 1718 col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, 1719 tcg_find_helper(s, op->args[nb_oargs + nb_iargs]), 1720 op->args[nb_oargs + nb_iargs + 1], nb_oargs); 1721 for (i = 0; i < nb_oargs; i++) { 1722 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf), 1723 op->args[i])); 1724 } 1725 for (i = 0; i < nb_iargs; i++) { 1726 TCGArg arg = op->args[nb_oargs + i]; 1727 const char *t = "<dummy>"; 1728 if (arg != TCG_CALL_DUMMY_ARG) { 1729 t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 1730 } 1731 col += qemu_log(",%s", t); 1732 } 1733 } else { 1734 col += qemu_log(" %s ", def->name); 1735 1736 nb_oargs = def->nb_oargs; 1737 nb_iargs = def->nb_iargs; 1738 nb_cargs = def->nb_cargs; 1739 1740 if (def->flags & TCG_OPF_VECTOR) { 1741 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op), 1742 8 << TCGOP_VECE(op)); 1743 } 1744 1745 k = 0; 1746 for (i = 0; i < nb_oargs; i++) { 1747 if (k != 0) { 1748 col += qemu_log(","); 1749 } 1750 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1751 op->args[k++])); 1752 } 1753 for (i = 0; i < nb_iargs; i++) { 1754 if (k != 0) { 1755 col += qemu_log(","); 1756 } 1757 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1758 op->args[k++])); 1759 } 1760 switch (c) { 1761 case INDEX_op_brcond_i32: 1762 case INDEX_op_setcond_i32: 1763 case INDEX_op_movcond_i32: 1764 case INDEX_op_brcond2_i32: 1765 case INDEX_op_setcond2_i32: 1766 case INDEX_op_brcond_i64: 1767 case INDEX_op_setcond_i64: 1768 case INDEX_op_movcond_i64: 1769 if (op->args[k] < ARRAY_SIZE(cond_name) 1770 && cond_name[op->args[k]]) { 1771 col += qemu_log(",%s", cond_name[op->args[k++]]); 1772 } else { 1773 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]); 1774 } 1775 i = 1; 1776 break; 1777 case INDEX_op_qemu_ld_i32: 1778 case INDEX_op_qemu_st_i32: 1779 case INDEX_op_qemu_ld_i64: 1780 case INDEX_op_qemu_st_i64: 1781 { 1782 TCGMemOpIdx oi = op->args[k++]; 1783 TCGMemOp op = get_memop(oi); 1784 unsigned ix = get_mmuidx(oi); 1785 1786 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1787 col += qemu_log(",$0x%x,%u", op, ix); 1788 } else { 1789 const char *s_al, *s_op; 1790 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1791 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1792 col += qemu_log(",%s%s,%u", s_al, s_op, ix); 1793 } 1794 i = 1; 1795 } 1796 break; 1797 default: 1798 i = 0; 1799 break; 1800 } 1801 switch (c) { 1802 case INDEX_op_set_label: 1803 case INDEX_op_br: 1804 case INDEX_op_brcond_i32: 1805 case INDEX_op_brcond_i64: 1806 case INDEX_op_brcond2_i32: 1807 col += qemu_log("%s$L%d", k ? "," : "", 1808 arg_label(op->args[k])->id); 1809 i++, k++; 1810 break; 1811 default: 1812 break; 1813 } 1814 for (; i < nb_cargs; i++, k++) { 1815 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]); 1816 } 1817 } 1818 if (op->life) { 1819 unsigned life = op->life; 1820 1821 for (; col < 48; ++col) { 1822 putc(' ', qemu_logfile); 1823 } 1824 1825 if (life & (SYNC_ARG * 3)) { 1826 qemu_log(" sync:"); 1827 for (i = 0; i < 2; ++i) { 1828 if (life & (SYNC_ARG << i)) { 1829 qemu_log(" %d", i); 1830 } 1831 } 1832 } 1833 life /= DEAD_ARG; 1834 if (life) { 1835 qemu_log(" dead:"); 1836 for (i = 0; life; ++i, life >>= 1) { 1837 if (life & 1) { 1838 qemu_log(" %d", i); 1839 } 1840 } 1841 } 1842 } 1843 qemu_log("\n"); 1844 } 1845 } 1846 1847 /* we give more priority to constraints with less registers */ 1848 static int get_constraint_priority(const TCGOpDef *def, int k) 1849 { 1850 const TCGArgConstraint *arg_ct; 1851 1852 int i, n; 1853 arg_ct = &def->args_ct[k]; 1854 if (arg_ct->ct & TCG_CT_ALIAS) { 1855 /* an alias is equivalent to a single register */ 1856 n = 1; 1857 } else { 1858 if (!(arg_ct->ct & TCG_CT_REG)) 1859 return 0; 1860 n = 0; 1861 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1862 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1863 n++; 1864 } 1865 } 1866 return TCG_TARGET_NB_REGS - n + 1; 1867 } 1868 1869 /* sort from highest priority to lowest */ 1870 static void sort_constraints(TCGOpDef *def, int start, int n) 1871 { 1872 int i, j, p1, p2, tmp; 1873 1874 for(i = 0; i < n; i++) 1875 def->sorted_args[start + i] = start + i; 1876 if (n <= 1) 1877 return; 1878 for(i = 0; i < n - 1; i++) { 1879 for(j = i + 1; j < n; j++) { 1880 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1881 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1882 if (p1 < p2) { 1883 tmp = def->sorted_args[start + i]; 1884 def->sorted_args[start + i] = def->sorted_args[start + j]; 1885 def->sorted_args[start + j] = tmp; 1886 } 1887 } 1888 } 1889 } 1890 1891 static void process_op_defs(TCGContext *s) 1892 { 1893 TCGOpcode op; 1894 1895 for (op = 0; op < NB_OPS; op++) { 1896 TCGOpDef *def = &tcg_op_defs[op]; 1897 const TCGTargetOpDef *tdefs; 1898 TCGType type; 1899 int i, nb_args; 1900 1901 if (def->flags & TCG_OPF_NOT_PRESENT) { 1902 continue; 1903 } 1904 1905 nb_args = def->nb_iargs + def->nb_oargs; 1906 if (nb_args == 0) { 1907 continue; 1908 } 1909 1910 tdefs = tcg_target_op_def(op); 1911 /* Missing TCGTargetOpDef entry. */ 1912 tcg_debug_assert(tdefs != NULL); 1913 1914 type = (def->flags & TCG_OPF_64BIT ? TCG_TYPE_I64 : TCG_TYPE_I32); 1915 for (i = 0; i < nb_args; i++) { 1916 const char *ct_str = tdefs->args_ct_str[i]; 1917 /* Incomplete TCGTargetOpDef entry. */ 1918 tcg_debug_assert(ct_str != NULL); 1919 1920 def->args_ct[i].u.regs = 0; 1921 def->args_ct[i].ct = 0; 1922 while (*ct_str != '\0') { 1923 switch(*ct_str) { 1924 case '0' ... '9': 1925 { 1926 int oarg = *ct_str - '0'; 1927 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]); 1928 tcg_debug_assert(oarg < def->nb_oargs); 1929 tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG); 1930 /* TCG_CT_ALIAS is for the output arguments. 1931 The input is tagged with TCG_CT_IALIAS. */ 1932 def->args_ct[i] = def->args_ct[oarg]; 1933 def->args_ct[oarg].ct |= TCG_CT_ALIAS; 1934 def->args_ct[oarg].alias_index = i; 1935 def->args_ct[i].ct |= TCG_CT_IALIAS; 1936 def->args_ct[i].alias_index = oarg; 1937 } 1938 ct_str++; 1939 break; 1940 case '&': 1941 def->args_ct[i].ct |= TCG_CT_NEWREG; 1942 ct_str++; 1943 break; 1944 case 'i': 1945 def->args_ct[i].ct |= TCG_CT_CONST; 1946 ct_str++; 1947 break; 1948 default: 1949 ct_str = target_parse_constraint(&def->args_ct[i], 1950 ct_str, type); 1951 /* Typo in TCGTargetOpDef constraint. */ 1952 tcg_debug_assert(ct_str != NULL); 1953 } 1954 } 1955 } 1956 1957 /* TCGTargetOpDef entry with too much information? */ 1958 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1959 1960 /* sort the constraints (XXX: this is just an heuristic) */ 1961 sort_constraints(def, 0, def->nb_oargs); 1962 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1963 } 1964 } 1965 1966 void tcg_op_remove(TCGContext *s, TCGOp *op) 1967 { 1968 QTAILQ_REMOVE(&s->ops, op, link); 1969 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 1970 1971 #ifdef CONFIG_PROFILER 1972 atomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 1973 #endif 1974 } 1975 1976 static TCGOp *tcg_op_alloc(TCGOpcode opc) 1977 { 1978 TCGContext *s = tcg_ctx; 1979 TCGOp *op; 1980 1981 if (likely(QTAILQ_EMPTY(&s->free_ops))) { 1982 op = tcg_malloc(sizeof(TCGOp)); 1983 } else { 1984 op = QTAILQ_FIRST(&s->free_ops); 1985 QTAILQ_REMOVE(&s->free_ops, op, link); 1986 } 1987 memset(op, 0, offsetof(TCGOp, link)); 1988 op->opc = opc; 1989 1990 return op; 1991 } 1992 1993 TCGOp *tcg_emit_op(TCGOpcode opc) 1994 { 1995 TCGOp *op = tcg_op_alloc(opc); 1996 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 1997 return op; 1998 } 1999 2000 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 2001 TCGOpcode opc, int nargs) 2002 { 2003 TCGOp *new_op = tcg_op_alloc(opc); 2004 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 2005 return new_op; 2006 } 2007 2008 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 2009 TCGOpcode opc, int nargs) 2010 { 2011 TCGOp *new_op = tcg_op_alloc(opc); 2012 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 2013 return new_op; 2014 } 2015 2016 #define TS_DEAD 1 2017 #define TS_MEM 2 2018 2019 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 2020 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 2021 2022 /* liveness analysis: end of function: all temps are dead, and globals 2023 should be in memory. */ 2024 static void tcg_la_func_end(TCGContext *s) 2025 { 2026 int ng = s->nb_globals; 2027 int nt = s->nb_temps; 2028 int i; 2029 2030 for (i = 0; i < ng; ++i) { 2031 s->temps[i].state = TS_DEAD | TS_MEM; 2032 } 2033 for (i = ng; i < nt; ++i) { 2034 s->temps[i].state = TS_DEAD; 2035 } 2036 } 2037 2038 /* liveness analysis: end of basic block: all temps are dead, globals 2039 and local temps should be in memory. */ 2040 static void tcg_la_bb_end(TCGContext *s) 2041 { 2042 int ng = s->nb_globals; 2043 int nt = s->nb_temps; 2044 int i; 2045 2046 for (i = 0; i < ng; ++i) { 2047 s->temps[i].state = TS_DEAD | TS_MEM; 2048 } 2049 for (i = ng; i < nt; ++i) { 2050 s->temps[i].state = (s->temps[i].temp_local 2051 ? TS_DEAD | TS_MEM 2052 : TS_DEAD); 2053 } 2054 } 2055 2056 /* Liveness analysis : update the opc_arg_life array to tell if a 2057 given input arguments is dead. Instructions updating dead 2058 temporaries are removed. */ 2059 static void liveness_pass_1(TCGContext *s) 2060 { 2061 int nb_globals = s->nb_globals; 2062 TCGOp *op, *op_prev; 2063 2064 tcg_la_func_end(s); 2065 2066 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, TCGOpHead, link, op_prev) { 2067 int i, nb_iargs, nb_oargs; 2068 TCGOpcode opc_new, opc_new2; 2069 bool have_opc_new2; 2070 TCGLifeData arg_life = 0; 2071 TCGTemp *arg_ts; 2072 TCGOpcode opc = op->opc; 2073 const TCGOpDef *def = &tcg_op_defs[opc]; 2074 2075 switch (opc) { 2076 case INDEX_op_call: 2077 { 2078 int call_flags; 2079 2080 nb_oargs = TCGOP_CALLO(op); 2081 nb_iargs = TCGOP_CALLI(op); 2082 call_flags = op->args[nb_oargs + nb_iargs + 1]; 2083 2084 /* pure functions can be removed if their result is unused */ 2085 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 2086 for (i = 0; i < nb_oargs; i++) { 2087 arg_ts = arg_temp(op->args[i]); 2088 if (arg_ts->state != TS_DEAD) { 2089 goto do_not_remove_call; 2090 } 2091 } 2092 goto do_remove; 2093 } else { 2094 do_not_remove_call: 2095 2096 /* output args are dead */ 2097 for (i = 0; i < nb_oargs; i++) { 2098 arg_ts = arg_temp(op->args[i]); 2099 if (arg_ts->state & TS_DEAD) { 2100 arg_life |= DEAD_ARG << i; 2101 } 2102 if (arg_ts->state & TS_MEM) { 2103 arg_life |= SYNC_ARG << i; 2104 } 2105 arg_ts->state = TS_DEAD; 2106 } 2107 2108 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 2109 TCG_CALL_NO_READ_GLOBALS))) { 2110 /* globals should go back to memory */ 2111 for (i = 0; i < nb_globals; i++) { 2112 s->temps[i].state = TS_DEAD | TS_MEM; 2113 } 2114 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 2115 /* globals should be synced to memory */ 2116 for (i = 0; i < nb_globals; i++) { 2117 s->temps[i].state |= TS_MEM; 2118 } 2119 } 2120 2121 /* record arguments that die in this helper */ 2122 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2123 arg_ts = arg_temp(op->args[i]); 2124 if (arg_ts && arg_ts->state & TS_DEAD) { 2125 arg_life |= DEAD_ARG << i; 2126 } 2127 } 2128 /* input arguments are live for preceding opcodes */ 2129 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2130 arg_ts = arg_temp(op->args[i]); 2131 if (arg_ts) { 2132 arg_ts->state &= ~TS_DEAD; 2133 } 2134 } 2135 } 2136 } 2137 break; 2138 case INDEX_op_insn_start: 2139 break; 2140 case INDEX_op_discard: 2141 /* mark the temporary as dead */ 2142 arg_temp(op->args[0])->state = TS_DEAD; 2143 break; 2144 2145 case INDEX_op_add2_i32: 2146 opc_new = INDEX_op_add_i32; 2147 goto do_addsub2; 2148 case INDEX_op_sub2_i32: 2149 opc_new = INDEX_op_sub_i32; 2150 goto do_addsub2; 2151 case INDEX_op_add2_i64: 2152 opc_new = INDEX_op_add_i64; 2153 goto do_addsub2; 2154 case INDEX_op_sub2_i64: 2155 opc_new = INDEX_op_sub_i64; 2156 do_addsub2: 2157 nb_iargs = 4; 2158 nb_oargs = 2; 2159 /* Test if the high part of the operation is dead, but not 2160 the low part. The result can be optimized to a simple 2161 add or sub. This happens often for x86_64 guest when the 2162 cpu mode is set to 32 bit. */ 2163 if (arg_temp(op->args[1])->state == TS_DEAD) { 2164 if (arg_temp(op->args[0])->state == TS_DEAD) { 2165 goto do_remove; 2166 } 2167 /* Replace the opcode and adjust the args in place, 2168 leaving 3 unused args at the end. */ 2169 op->opc = opc = opc_new; 2170 op->args[1] = op->args[2]; 2171 op->args[2] = op->args[4]; 2172 /* Fall through and mark the single-word operation live. */ 2173 nb_iargs = 2; 2174 nb_oargs = 1; 2175 } 2176 goto do_not_remove; 2177 2178 case INDEX_op_mulu2_i32: 2179 opc_new = INDEX_op_mul_i32; 2180 opc_new2 = INDEX_op_muluh_i32; 2181 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 2182 goto do_mul2; 2183 case INDEX_op_muls2_i32: 2184 opc_new = INDEX_op_mul_i32; 2185 opc_new2 = INDEX_op_mulsh_i32; 2186 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 2187 goto do_mul2; 2188 case INDEX_op_mulu2_i64: 2189 opc_new = INDEX_op_mul_i64; 2190 opc_new2 = INDEX_op_muluh_i64; 2191 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 2192 goto do_mul2; 2193 case INDEX_op_muls2_i64: 2194 opc_new = INDEX_op_mul_i64; 2195 opc_new2 = INDEX_op_mulsh_i64; 2196 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 2197 goto do_mul2; 2198 do_mul2: 2199 nb_iargs = 2; 2200 nb_oargs = 2; 2201 if (arg_temp(op->args[1])->state == TS_DEAD) { 2202 if (arg_temp(op->args[0])->state == TS_DEAD) { 2203 /* Both parts of the operation are dead. */ 2204 goto do_remove; 2205 } 2206 /* The high part of the operation is dead; generate the low. */ 2207 op->opc = opc = opc_new; 2208 op->args[1] = op->args[2]; 2209 op->args[2] = op->args[3]; 2210 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 2211 /* The low part of the operation is dead; generate the high. */ 2212 op->opc = opc = opc_new2; 2213 op->args[0] = op->args[1]; 2214 op->args[1] = op->args[2]; 2215 op->args[2] = op->args[3]; 2216 } else { 2217 goto do_not_remove; 2218 } 2219 /* Mark the single-word operation live. */ 2220 nb_oargs = 1; 2221 goto do_not_remove; 2222 2223 default: 2224 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 2225 nb_iargs = def->nb_iargs; 2226 nb_oargs = def->nb_oargs; 2227 2228 /* Test if the operation can be removed because all 2229 its outputs are dead. We assume that nb_oargs == 0 2230 implies side effects */ 2231 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 2232 for (i = 0; i < nb_oargs; i++) { 2233 if (arg_temp(op->args[i])->state != TS_DEAD) { 2234 goto do_not_remove; 2235 } 2236 } 2237 do_remove: 2238 tcg_op_remove(s, op); 2239 } else { 2240 do_not_remove: 2241 /* output args are dead */ 2242 for (i = 0; i < nb_oargs; i++) { 2243 arg_ts = arg_temp(op->args[i]); 2244 if (arg_ts->state & TS_DEAD) { 2245 arg_life |= DEAD_ARG << i; 2246 } 2247 if (arg_ts->state & TS_MEM) { 2248 arg_life |= SYNC_ARG << i; 2249 } 2250 arg_ts->state = TS_DEAD; 2251 } 2252 2253 /* if end of basic block, update */ 2254 if (def->flags & TCG_OPF_BB_END) { 2255 tcg_la_bb_end(s); 2256 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2257 /* globals should be synced to memory */ 2258 for (i = 0; i < nb_globals; i++) { 2259 s->temps[i].state |= TS_MEM; 2260 } 2261 } 2262 2263 /* record arguments that die in this opcode */ 2264 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2265 arg_ts = arg_temp(op->args[i]); 2266 if (arg_ts->state & TS_DEAD) { 2267 arg_life |= DEAD_ARG << i; 2268 } 2269 } 2270 /* input arguments are live for preceding opcodes */ 2271 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2272 arg_temp(op->args[i])->state &= ~TS_DEAD; 2273 } 2274 } 2275 break; 2276 } 2277 op->life = arg_life; 2278 } 2279 } 2280 2281 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 2282 static bool liveness_pass_2(TCGContext *s) 2283 { 2284 int nb_globals = s->nb_globals; 2285 int nb_temps, i; 2286 bool changes = false; 2287 TCGOp *op, *op_next; 2288 2289 /* Create a temporary for each indirect global. */ 2290 for (i = 0; i < nb_globals; ++i) { 2291 TCGTemp *its = &s->temps[i]; 2292 if (its->indirect_reg) { 2293 TCGTemp *dts = tcg_temp_alloc(s); 2294 dts->type = its->type; 2295 dts->base_type = its->base_type; 2296 its->state_ptr = dts; 2297 } else { 2298 its->state_ptr = NULL; 2299 } 2300 /* All globals begin dead. */ 2301 its->state = TS_DEAD; 2302 } 2303 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 2304 TCGTemp *its = &s->temps[i]; 2305 its->state_ptr = NULL; 2306 its->state = TS_DEAD; 2307 } 2308 2309 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2310 TCGOpcode opc = op->opc; 2311 const TCGOpDef *def = &tcg_op_defs[opc]; 2312 TCGLifeData arg_life = op->life; 2313 int nb_iargs, nb_oargs, call_flags; 2314 TCGTemp *arg_ts, *dir_ts; 2315 2316 if (opc == INDEX_op_call) { 2317 nb_oargs = TCGOP_CALLO(op); 2318 nb_iargs = TCGOP_CALLI(op); 2319 call_flags = op->args[nb_oargs + nb_iargs + 1]; 2320 } else { 2321 nb_iargs = def->nb_iargs; 2322 nb_oargs = def->nb_oargs; 2323 2324 /* Set flags similar to how calls require. */ 2325 if (def->flags & TCG_OPF_BB_END) { 2326 /* Like writing globals: save_globals */ 2327 call_flags = 0; 2328 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2329 /* Like reading globals: sync_globals */ 2330 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2331 } else { 2332 /* No effect on globals. */ 2333 call_flags = (TCG_CALL_NO_READ_GLOBALS | 2334 TCG_CALL_NO_WRITE_GLOBALS); 2335 } 2336 } 2337 2338 /* Make sure that input arguments are available. */ 2339 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2340 arg_ts = arg_temp(op->args[i]); 2341 if (arg_ts) { 2342 dir_ts = arg_ts->state_ptr; 2343 if (dir_ts && arg_ts->state == TS_DEAD) { 2344 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 2345 ? INDEX_op_ld_i32 2346 : INDEX_op_ld_i64); 2347 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 2348 2349 lop->args[0] = temp_arg(dir_ts); 2350 lop->args[1] = temp_arg(arg_ts->mem_base); 2351 lop->args[2] = arg_ts->mem_offset; 2352 2353 /* Loaded, but synced with memory. */ 2354 arg_ts->state = TS_MEM; 2355 } 2356 } 2357 } 2358 2359 /* Perform input replacement, and mark inputs that became dead. 2360 No action is required except keeping temp_state up to date 2361 so that we reload when needed. */ 2362 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2363 arg_ts = arg_temp(op->args[i]); 2364 if (arg_ts) { 2365 dir_ts = arg_ts->state_ptr; 2366 if (dir_ts) { 2367 op->args[i] = temp_arg(dir_ts); 2368 changes = true; 2369 if (IS_DEAD_ARG(i)) { 2370 arg_ts->state = TS_DEAD; 2371 } 2372 } 2373 } 2374 } 2375 2376 /* Liveness analysis should ensure that the following are 2377 all correct, for call sites and basic block end points. */ 2378 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 2379 /* Nothing to do */ 2380 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 2381 for (i = 0; i < nb_globals; ++i) { 2382 /* Liveness should see that globals are synced back, 2383 that is, either TS_DEAD or TS_MEM. */ 2384 arg_ts = &s->temps[i]; 2385 tcg_debug_assert(arg_ts->state_ptr == 0 2386 || arg_ts->state != 0); 2387 } 2388 } else { 2389 for (i = 0; i < nb_globals; ++i) { 2390 /* Liveness should see that globals are saved back, 2391 that is, TS_DEAD, waiting to be reloaded. */ 2392 arg_ts = &s->temps[i]; 2393 tcg_debug_assert(arg_ts->state_ptr == 0 2394 || arg_ts->state == TS_DEAD); 2395 } 2396 } 2397 2398 /* Outputs become available. */ 2399 for (i = 0; i < nb_oargs; i++) { 2400 arg_ts = arg_temp(op->args[i]); 2401 dir_ts = arg_ts->state_ptr; 2402 if (!dir_ts) { 2403 continue; 2404 } 2405 op->args[i] = temp_arg(dir_ts); 2406 changes = true; 2407 2408 /* The output is now live and modified. */ 2409 arg_ts->state = 0; 2410 2411 /* Sync outputs upon their last write. */ 2412 if (NEED_SYNC_ARG(i)) { 2413 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 2414 ? INDEX_op_st_i32 2415 : INDEX_op_st_i64); 2416 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 2417 2418 sop->args[0] = temp_arg(dir_ts); 2419 sop->args[1] = temp_arg(arg_ts->mem_base); 2420 sop->args[2] = arg_ts->mem_offset; 2421 2422 arg_ts->state = TS_MEM; 2423 } 2424 /* Drop outputs that are dead. */ 2425 if (IS_DEAD_ARG(i)) { 2426 arg_ts->state = TS_DEAD; 2427 } 2428 } 2429 } 2430 2431 return changes; 2432 } 2433 2434 #ifdef CONFIG_DEBUG_TCG 2435 static void dump_regs(TCGContext *s) 2436 { 2437 TCGTemp *ts; 2438 int i; 2439 char buf[64]; 2440 2441 for(i = 0; i < s->nb_temps; i++) { 2442 ts = &s->temps[i]; 2443 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2444 switch(ts->val_type) { 2445 case TEMP_VAL_REG: 2446 printf("%s", tcg_target_reg_names[ts->reg]); 2447 break; 2448 case TEMP_VAL_MEM: 2449 printf("%d(%s)", (int)ts->mem_offset, 2450 tcg_target_reg_names[ts->mem_base->reg]); 2451 break; 2452 case TEMP_VAL_CONST: 2453 printf("$0x%" TCG_PRIlx, ts->val); 2454 break; 2455 case TEMP_VAL_DEAD: 2456 printf("D"); 2457 break; 2458 default: 2459 printf("???"); 2460 break; 2461 } 2462 printf("\n"); 2463 } 2464 2465 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 2466 if (s->reg_to_temp[i] != NULL) { 2467 printf("%s: %s\n", 2468 tcg_target_reg_names[i], 2469 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 2470 } 2471 } 2472 } 2473 2474 static void check_regs(TCGContext *s) 2475 { 2476 int reg; 2477 int k; 2478 TCGTemp *ts; 2479 char buf[64]; 2480 2481 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2482 ts = s->reg_to_temp[reg]; 2483 if (ts != NULL) { 2484 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 2485 printf("Inconsistency for register %s:\n", 2486 tcg_target_reg_names[reg]); 2487 goto fail; 2488 } 2489 } 2490 } 2491 for (k = 0; k < s->nb_temps; k++) { 2492 ts = &s->temps[k]; 2493 if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg 2494 && s->reg_to_temp[ts->reg] != ts) { 2495 printf("Inconsistency for temp %s:\n", 2496 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2497 fail: 2498 printf("reg state:\n"); 2499 dump_regs(s); 2500 tcg_abort(); 2501 } 2502 } 2503 } 2504 #endif 2505 2506 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 2507 { 2508 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 2509 /* Sparc64 stack is accessed with offset of 2047 */ 2510 s->current_frame_offset = (s->current_frame_offset + 2511 (tcg_target_long)sizeof(tcg_target_long) - 1) & 2512 ~(sizeof(tcg_target_long) - 1); 2513 #endif 2514 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 2515 s->frame_end) { 2516 tcg_abort(); 2517 } 2518 ts->mem_offset = s->current_frame_offset; 2519 ts->mem_base = s->frame_temp; 2520 ts->mem_allocated = 1; 2521 s->current_frame_offset += sizeof(tcg_target_long); 2522 } 2523 2524 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet); 2525 2526 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 2527 mark it free; otherwise mark it dead. */ 2528 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 2529 { 2530 if (ts->fixed_reg) { 2531 return; 2532 } 2533 if (ts->val_type == TEMP_VAL_REG) { 2534 s->reg_to_temp[ts->reg] = NULL; 2535 } 2536 ts->val_type = (free_or_dead < 0 2537 || ts->temp_local 2538 || ts->temp_global 2539 ? TEMP_VAL_MEM : TEMP_VAL_DEAD); 2540 } 2541 2542 /* Mark a temporary as dead. */ 2543 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 2544 { 2545 temp_free_or_dead(s, ts, 1); 2546 } 2547 2548 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 2549 registers needs to be allocated to store a constant. If 'free_or_dead' 2550 is non-zero, subsequently release the temporary; if it is positive, the 2551 temp is dead; if it is negative, the temp is free. */ 2552 static void temp_sync(TCGContext *s, TCGTemp *ts, 2553 TCGRegSet allocated_regs, int free_or_dead) 2554 { 2555 if (ts->fixed_reg) { 2556 return; 2557 } 2558 if (!ts->mem_coherent) { 2559 if (!ts->mem_allocated) { 2560 temp_allocate_frame(s, ts); 2561 } 2562 switch (ts->val_type) { 2563 case TEMP_VAL_CONST: 2564 /* If we're going to free the temp immediately, then we won't 2565 require it later in a register, so attempt to store the 2566 constant to memory directly. */ 2567 if (free_or_dead 2568 && tcg_out_sti(s, ts->type, ts->val, 2569 ts->mem_base->reg, ts->mem_offset)) { 2570 break; 2571 } 2572 temp_load(s, ts, tcg_target_available_regs[ts->type], 2573 allocated_regs); 2574 /* fallthrough */ 2575 2576 case TEMP_VAL_REG: 2577 tcg_out_st(s, ts->type, ts->reg, 2578 ts->mem_base->reg, ts->mem_offset); 2579 break; 2580 2581 case TEMP_VAL_MEM: 2582 break; 2583 2584 case TEMP_VAL_DEAD: 2585 default: 2586 tcg_abort(); 2587 } 2588 ts->mem_coherent = 1; 2589 } 2590 if (free_or_dead) { 2591 temp_free_or_dead(s, ts, free_or_dead); 2592 } 2593 } 2594 2595 /* free register 'reg' by spilling the corresponding temporary if necessary */ 2596 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 2597 { 2598 TCGTemp *ts = s->reg_to_temp[reg]; 2599 if (ts != NULL) { 2600 temp_sync(s, ts, allocated_regs, -1); 2601 } 2602 } 2603 2604 /* Allocate a register belonging to reg1 & ~reg2 */ 2605 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs, 2606 TCGRegSet allocated_regs, bool rev) 2607 { 2608 int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 2609 const int *order; 2610 TCGReg reg; 2611 TCGRegSet reg_ct; 2612 2613 reg_ct = desired_regs & ~allocated_regs; 2614 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 2615 2616 /* first try free registers */ 2617 for(i = 0; i < n; i++) { 2618 reg = order[i]; 2619 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL) 2620 return reg; 2621 } 2622 2623 /* XXX: do better spill choice */ 2624 for(i = 0; i < n; i++) { 2625 reg = order[i]; 2626 if (tcg_regset_test_reg(reg_ct, reg)) { 2627 tcg_reg_free(s, reg, allocated_regs); 2628 return reg; 2629 } 2630 } 2631 2632 tcg_abort(); 2633 } 2634 2635 /* Make sure the temporary is in a register. If needed, allocate the register 2636 from DESIRED while avoiding ALLOCATED. */ 2637 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 2638 TCGRegSet allocated_regs) 2639 { 2640 TCGReg reg; 2641 2642 switch (ts->val_type) { 2643 case TEMP_VAL_REG: 2644 return; 2645 case TEMP_VAL_CONST: 2646 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 2647 tcg_out_movi(s, ts->type, reg, ts->val); 2648 ts->mem_coherent = 0; 2649 break; 2650 case TEMP_VAL_MEM: 2651 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 2652 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 2653 ts->mem_coherent = 1; 2654 break; 2655 case TEMP_VAL_DEAD: 2656 default: 2657 tcg_abort(); 2658 } 2659 ts->reg = reg; 2660 ts->val_type = TEMP_VAL_REG; 2661 s->reg_to_temp[reg] = ts; 2662 } 2663 2664 /* Save a temporary to memory. 'allocated_regs' is used in case a 2665 temporary registers needs to be allocated to store a constant. */ 2666 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 2667 { 2668 /* The liveness analysis already ensures that globals are back 2669 in memory. Keep an tcg_debug_assert for safety. */ 2670 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg); 2671 } 2672 2673 /* save globals to their canonical location and assume they can be 2674 modified be the following code. 'allocated_regs' is used in case a 2675 temporary registers needs to be allocated to store a constant. */ 2676 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 2677 { 2678 int i, n; 2679 2680 for (i = 0, n = s->nb_globals; i < n; i++) { 2681 temp_save(s, &s->temps[i], allocated_regs); 2682 } 2683 } 2684 2685 /* sync globals to their canonical location and assume they can be 2686 read by the following code. 'allocated_regs' is used in case a 2687 temporary registers needs to be allocated to store a constant. */ 2688 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 2689 { 2690 int i, n; 2691 2692 for (i = 0, n = s->nb_globals; i < n; i++) { 2693 TCGTemp *ts = &s->temps[i]; 2694 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 2695 || ts->fixed_reg 2696 || ts->mem_coherent); 2697 } 2698 } 2699 2700 /* at the end of a basic block, we assume all temporaries are dead and 2701 all globals are stored at their canonical location. */ 2702 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 2703 { 2704 int i; 2705 2706 for (i = s->nb_globals; i < s->nb_temps; i++) { 2707 TCGTemp *ts = &s->temps[i]; 2708 if (ts->temp_local) { 2709 temp_save(s, ts, allocated_regs); 2710 } else { 2711 /* The liveness analysis already ensures that temps are dead. 2712 Keep an tcg_debug_assert for safety. */ 2713 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 2714 } 2715 } 2716 2717 save_globals(s, allocated_regs); 2718 } 2719 2720 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 2721 tcg_target_ulong val, TCGLifeData arg_life) 2722 { 2723 if (ots->fixed_reg) { 2724 /* For fixed registers, we do not do any constant propagation. */ 2725 tcg_out_movi(s, ots->type, ots->reg, val); 2726 return; 2727 } 2728 2729 /* The movi is not explicitly generated here. */ 2730 if (ots->val_type == TEMP_VAL_REG) { 2731 s->reg_to_temp[ots->reg] = NULL; 2732 } 2733 ots->val_type = TEMP_VAL_CONST; 2734 ots->val = val; 2735 ots->mem_coherent = 0; 2736 if (NEED_SYNC_ARG(0)) { 2737 temp_sync(s, ots, s->reserved_regs, IS_DEAD_ARG(0)); 2738 } else if (IS_DEAD_ARG(0)) { 2739 temp_dead(s, ots); 2740 } 2741 } 2742 2743 static void tcg_reg_alloc_movi(TCGContext *s, const TCGOp *op) 2744 { 2745 TCGTemp *ots = arg_temp(op->args[0]); 2746 tcg_target_ulong val = op->args[1]; 2747 2748 tcg_reg_alloc_do_movi(s, ots, val, op->life); 2749 } 2750 2751 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 2752 { 2753 const TCGLifeData arg_life = op->life; 2754 TCGRegSet allocated_regs; 2755 TCGTemp *ts, *ots; 2756 TCGType otype, itype; 2757 2758 allocated_regs = s->reserved_regs; 2759 ots = arg_temp(op->args[0]); 2760 ts = arg_temp(op->args[1]); 2761 2762 /* Note that otype != itype for no-op truncation. */ 2763 otype = ots->type; 2764 itype = ts->type; 2765 2766 if (ts->val_type == TEMP_VAL_CONST) { 2767 /* propagate constant or generate sti */ 2768 tcg_target_ulong val = ts->val; 2769 if (IS_DEAD_ARG(1)) { 2770 temp_dead(s, ts); 2771 } 2772 tcg_reg_alloc_do_movi(s, ots, val, arg_life); 2773 return; 2774 } 2775 2776 /* If the source value is in memory we're going to be forced 2777 to have it in a register in order to perform the copy. Copy 2778 the SOURCE value into its own register first, that way we 2779 don't have to reload SOURCE the next time it is used. */ 2780 if (ts->val_type == TEMP_VAL_MEM) { 2781 temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs); 2782 } 2783 2784 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 2785 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 2786 /* mov to a non-saved dead register makes no sense (even with 2787 liveness analysis disabled). */ 2788 tcg_debug_assert(NEED_SYNC_ARG(0)); 2789 if (!ots->mem_allocated) { 2790 temp_allocate_frame(s, ots); 2791 } 2792 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 2793 if (IS_DEAD_ARG(1)) { 2794 temp_dead(s, ts); 2795 } 2796 temp_dead(s, ots); 2797 } else { 2798 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 2799 /* the mov can be suppressed */ 2800 if (ots->val_type == TEMP_VAL_REG) { 2801 s->reg_to_temp[ots->reg] = NULL; 2802 } 2803 ots->reg = ts->reg; 2804 temp_dead(s, ts); 2805 } else { 2806 if (ots->val_type != TEMP_VAL_REG) { 2807 /* When allocating a new register, make sure to not spill the 2808 input one. */ 2809 tcg_regset_set_reg(allocated_regs, ts->reg); 2810 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 2811 allocated_regs, ots->indirect_base); 2812 } 2813 tcg_out_mov(s, otype, ots->reg, ts->reg); 2814 } 2815 ots->val_type = TEMP_VAL_REG; 2816 ots->mem_coherent = 0; 2817 s->reg_to_temp[ots->reg] = ots; 2818 if (NEED_SYNC_ARG(0)) { 2819 temp_sync(s, ots, allocated_regs, 0); 2820 } 2821 } 2822 } 2823 2824 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 2825 { 2826 const TCGLifeData arg_life = op->life; 2827 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 2828 TCGRegSet i_allocated_regs; 2829 TCGRegSet o_allocated_regs; 2830 int i, k, nb_iargs, nb_oargs; 2831 TCGReg reg; 2832 TCGArg arg; 2833 const TCGArgConstraint *arg_ct; 2834 TCGTemp *ts; 2835 TCGArg new_args[TCG_MAX_OP_ARGS]; 2836 int const_args[TCG_MAX_OP_ARGS]; 2837 2838 nb_oargs = def->nb_oargs; 2839 nb_iargs = def->nb_iargs; 2840 2841 /* copy constants */ 2842 memcpy(new_args + nb_oargs + nb_iargs, 2843 op->args + nb_oargs + nb_iargs, 2844 sizeof(TCGArg) * def->nb_cargs); 2845 2846 i_allocated_regs = s->reserved_regs; 2847 o_allocated_regs = s->reserved_regs; 2848 2849 /* satisfy input constraints */ 2850 for (k = 0; k < nb_iargs; k++) { 2851 i = def->sorted_args[nb_oargs + k]; 2852 arg = op->args[i]; 2853 arg_ct = &def->args_ct[i]; 2854 ts = arg_temp(arg); 2855 2856 if (ts->val_type == TEMP_VAL_CONST 2857 && tcg_target_const_match(ts->val, ts->type, arg_ct)) { 2858 /* constant is OK for instruction */ 2859 const_args[i] = 1; 2860 new_args[i] = ts->val; 2861 goto iarg_end; 2862 } 2863 2864 temp_load(s, ts, arg_ct->u.regs, i_allocated_regs); 2865 2866 if (arg_ct->ct & TCG_CT_IALIAS) { 2867 if (ts->fixed_reg) { 2868 /* if fixed register, we must allocate a new register 2869 if the alias is not the same register */ 2870 if (arg != op->args[arg_ct->alias_index]) 2871 goto allocate_in_reg; 2872 } else { 2873 /* if the input is aliased to an output and if it is 2874 not dead after the instruction, we must allocate 2875 a new register and move it */ 2876 if (!IS_DEAD_ARG(i)) { 2877 goto allocate_in_reg; 2878 } 2879 /* check if the current register has already been allocated 2880 for another input aliased to an output */ 2881 int k2, i2; 2882 for (k2 = 0 ; k2 < k ; k2++) { 2883 i2 = def->sorted_args[nb_oargs + k2]; 2884 if ((def->args_ct[i2].ct & TCG_CT_IALIAS) && 2885 (new_args[i2] == ts->reg)) { 2886 goto allocate_in_reg; 2887 } 2888 } 2889 } 2890 } 2891 reg = ts->reg; 2892 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2893 /* nothing to do : the constraint is satisfied */ 2894 } else { 2895 allocate_in_reg: 2896 /* allocate a new register matching the constraint 2897 and move the temporary register into it */ 2898 reg = tcg_reg_alloc(s, arg_ct->u.regs, i_allocated_regs, 2899 ts->indirect_base); 2900 tcg_out_mov(s, ts->type, reg, ts->reg); 2901 } 2902 new_args[i] = reg; 2903 const_args[i] = 0; 2904 tcg_regset_set_reg(i_allocated_regs, reg); 2905 iarg_end: ; 2906 } 2907 2908 /* mark dead temporaries and free the associated registers */ 2909 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2910 if (IS_DEAD_ARG(i)) { 2911 temp_dead(s, arg_temp(op->args[i])); 2912 } 2913 } 2914 2915 if (def->flags & TCG_OPF_BB_END) { 2916 tcg_reg_alloc_bb_end(s, i_allocated_regs); 2917 } else { 2918 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2919 /* XXX: permit generic clobber register list ? */ 2920 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 2921 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 2922 tcg_reg_free(s, i, i_allocated_regs); 2923 } 2924 } 2925 } 2926 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2927 /* sync globals if the op has side effects and might trigger 2928 an exception. */ 2929 sync_globals(s, i_allocated_regs); 2930 } 2931 2932 /* satisfy the output constraints */ 2933 for(k = 0; k < nb_oargs; k++) { 2934 i = def->sorted_args[k]; 2935 arg = op->args[i]; 2936 arg_ct = &def->args_ct[i]; 2937 ts = arg_temp(arg); 2938 if ((arg_ct->ct & TCG_CT_ALIAS) 2939 && !const_args[arg_ct->alias_index]) { 2940 reg = new_args[arg_ct->alias_index]; 2941 } else if (arg_ct->ct & TCG_CT_NEWREG) { 2942 reg = tcg_reg_alloc(s, arg_ct->u.regs, 2943 i_allocated_regs | o_allocated_regs, 2944 ts->indirect_base); 2945 } else { 2946 /* if fixed register, we try to use it */ 2947 reg = ts->reg; 2948 if (ts->fixed_reg && 2949 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2950 goto oarg_end; 2951 } 2952 reg = tcg_reg_alloc(s, arg_ct->u.regs, o_allocated_regs, 2953 ts->indirect_base); 2954 } 2955 tcg_regset_set_reg(o_allocated_regs, reg); 2956 /* if a fixed register is used, then a move will be done afterwards */ 2957 if (!ts->fixed_reg) { 2958 if (ts->val_type == TEMP_VAL_REG) { 2959 s->reg_to_temp[ts->reg] = NULL; 2960 } 2961 ts->val_type = TEMP_VAL_REG; 2962 ts->reg = reg; 2963 /* temp value is modified, so the value kept in memory is 2964 potentially not the same */ 2965 ts->mem_coherent = 0; 2966 s->reg_to_temp[reg] = ts; 2967 } 2968 oarg_end: 2969 new_args[i] = reg; 2970 } 2971 } 2972 2973 /* emit instruction */ 2974 if (def->flags & TCG_OPF_VECTOR) { 2975 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 2976 new_args, const_args); 2977 } else { 2978 tcg_out_op(s, op->opc, new_args, const_args); 2979 } 2980 2981 /* move the outputs in the correct register if needed */ 2982 for(i = 0; i < nb_oargs; i++) { 2983 ts = arg_temp(op->args[i]); 2984 reg = new_args[i]; 2985 if (ts->fixed_reg && ts->reg != reg) { 2986 tcg_out_mov(s, ts->type, ts->reg, reg); 2987 } 2988 if (NEED_SYNC_ARG(i)) { 2989 temp_sync(s, ts, o_allocated_regs, IS_DEAD_ARG(i)); 2990 } else if (IS_DEAD_ARG(i)) { 2991 temp_dead(s, ts); 2992 } 2993 } 2994 } 2995 2996 #ifdef TCG_TARGET_STACK_GROWSUP 2997 #define STACK_DIR(x) (-(x)) 2998 #else 2999 #define STACK_DIR(x) (x) 3000 #endif 3001 3002 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 3003 { 3004 const int nb_oargs = TCGOP_CALLO(op); 3005 const int nb_iargs = TCGOP_CALLI(op); 3006 const TCGLifeData arg_life = op->life; 3007 int flags, nb_regs, i; 3008 TCGReg reg; 3009 TCGArg arg; 3010 TCGTemp *ts; 3011 intptr_t stack_offset; 3012 size_t call_stack_size; 3013 tcg_insn_unit *func_addr; 3014 int allocate_args; 3015 TCGRegSet allocated_regs; 3016 3017 func_addr = (tcg_insn_unit *)(intptr_t)op->args[nb_oargs + nb_iargs]; 3018 flags = op->args[nb_oargs + nb_iargs + 1]; 3019 3020 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 3021 if (nb_regs > nb_iargs) { 3022 nb_regs = nb_iargs; 3023 } 3024 3025 /* assign stack slots first */ 3026 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 3027 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 3028 ~(TCG_TARGET_STACK_ALIGN - 1); 3029 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 3030 if (allocate_args) { 3031 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 3032 preallocate call stack */ 3033 tcg_abort(); 3034 } 3035 3036 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 3037 for (i = nb_regs; i < nb_iargs; i++) { 3038 arg = op->args[nb_oargs + i]; 3039 #ifdef TCG_TARGET_STACK_GROWSUP 3040 stack_offset -= sizeof(tcg_target_long); 3041 #endif 3042 if (arg != TCG_CALL_DUMMY_ARG) { 3043 ts = arg_temp(arg); 3044 temp_load(s, ts, tcg_target_available_regs[ts->type], 3045 s->reserved_regs); 3046 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 3047 } 3048 #ifndef TCG_TARGET_STACK_GROWSUP 3049 stack_offset += sizeof(tcg_target_long); 3050 #endif 3051 } 3052 3053 /* assign input registers */ 3054 allocated_regs = s->reserved_regs; 3055 for (i = 0; i < nb_regs; i++) { 3056 arg = op->args[nb_oargs + i]; 3057 if (arg != TCG_CALL_DUMMY_ARG) { 3058 ts = arg_temp(arg); 3059 reg = tcg_target_call_iarg_regs[i]; 3060 tcg_reg_free(s, reg, allocated_regs); 3061 3062 if (ts->val_type == TEMP_VAL_REG) { 3063 if (ts->reg != reg) { 3064 tcg_out_mov(s, ts->type, reg, ts->reg); 3065 } 3066 } else { 3067 TCGRegSet arg_set = 0; 3068 3069 tcg_regset_set_reg(arg_set, reg); 3070 temp_load(s, ts, arg_set, allocated_regs); 3071 } 3072 3073 tcg_regset_set_reg(allocated_regs, reg); 3074 } 3075 } 3076 3077 /* mark dead temporaries and free the associated registers */ 3078 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3079 if (IS_DEAD_ARG(i)) { 3080 temp_dead(s, arg_temp(op->args[i])); 3081 } 3082 } 3083 3084 /* clobber call registers */ 3085 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3086 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3087 tcg_reg_free(s, i, allocated_regs); 3088 } 3089 } 3090 3091 /* Save globals if they might be written by the helper, sync them if 3092 they might be read. */ 3093 if (flags & TCG_CALL_NO_READ_GLOBALS) { 3094 /* Nothing to do */ 3095 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 3096 sync_globals(s, allocated_regs); 3097 } else { 3098 save_globals(s, allocated_regs); 3099 } 3100 3101 tcg_out_call(s, func_addr); 3102 3103 /* assign output registers and emit moves if needed */ 3104 for(i = 0; i < nb_oargs; i++) { 3105 arg = op->args[i]; 3106 ts = arg_temp(arg); 3107 reg = tcg_target_call_oarg_regs[i]; 3108 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3109 3110 if (ts->fixed_reg) { 3111 if (ts->reg != reg) { 3112 tcg_out_mov(s, ts->type, ts->reg, reg); 3113 } 3114 } else { 3115 if (ts->val_type == TEMP_VAL_REG) { 3116 s->reg_to_temp[ts->reg] = NULL; 3117 } 3118 ts->val_type = TEMP_VAL_REG; 3119 ts->reg = reg; 3120 ts->mem_coherent = 0; 3121 s->reg_to_temp[reg] = ts; 3122 if (NEED_SYNC_ARG(i)) { 3123 temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i)); 3124 } else if (IS_DEAD_ARG(i)) { 3125 temp_dead(s, ts); 3126 } 3127 } 3128 } 3129 } 3130 3131 #ifdef CONFIG_PROFILER 3132 3133 /* avoid copy/paste errors */ 3134 #define PROF_ADD(to, from, field) \ 3135 do { \ 3136 (to)->field += atomic_read(&((from)->field)); \ 3137 } while (0) 3138 3139 #define PROF_MAX(to, from, field) \ 3140 do { \ 3141 typeof((from)->field) val__ = atomic_read(&((from)->field)); \ 3142 if (val__ > (to)->field) { \ 3143 (to)->field = val__; \ 3144 } \ 3145 } while (0) 3146 3147 /* Pass in a zero'ed @prof */ 3148 static inline 3149 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 3150 { 3151 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs); 3152 unsigned int i; 3153 3154 for (i = 0; i < n_ctxs; i++) { 3155 TCGContext *s = atomic_read(&tcg_ctxs[i]); 3156 const TCGProfile *orig = &s->prof; 3157 3158 if (counters) { 3159 PROF_ADD(prof, orig, tb_count1); 3160 PROF_ADD(prof, orig, tb_count); 3161 PROF_ADD(prof, orig, op_count); 3162 PROF_MAX(prof, orig, op_count_max); 3163 PROF_ADD(prof, orig, temp_count); 3164 PROF_MAX(prof, orig, temp_count_max); 3165 PROF_ADD(prof, orig, del_op_count); 3166 PROF_ADD(prof, orig, code_in_len); 3167 PROF_ADD(prof, orig, code_out_len); 3168 PROF_ADD(prof, orig, search_out_len); 3169 PROF_ADD(prof, orig, interm_time); 3170 PROF_ADD(prof, orig, code_time); 3171 PROF_ADD(prof, orig, la_time); 3172 PROF_ADD(prof, orig, opt_time); 3173 PROF_ADD(prof, orig, restore_count); 3174 PROF_ADD(prof, orig, restore_time); 3175 } 3176 if (table) { 3177 int i; 3178 3179 for (i = 0; i < NB_OPS; i++) { 3180 PROF_ADD(prof, orig, table_op_count[i]); 3181 } 3182 } 3183 } 3184 } 3185 3186 #undef PROF_ADD 3187 #undef PROF_MAX 3188 3189 static void tcg_profile_snapshot_counters(TCGProfile *prof) 3190 { 3191 tcg_profile_snapshot(prof, true, false); 3192 } 3193 3194 static void tcg_profile_snapshot_table(TCGProfile *prof) 3195 { 3196 tcg_profile_snapshot(prof, false, true); 3197 } 3198 3199 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 3200 { 3201 TCGProfile prof = {}; 3202 int i; 3203 3204 tcg_profile_snapshot_table(&prof); 3205 for (i = 0; i < NB_OPS; i++) { 3206 cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, 3207 prof.table_op_count[i]); 3208 } 3209 } 3210 #else 3211 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 3212 { 3213 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 3214 } 3215 #endif 3216 3217 3218 int tcg_gen_code(TCGContext *s, TranslationBlock *tb) 3219 { 3220 #ifdef CONFIG_PROFILER 3221 TCGProfile *prof = &s->prof; 3222 #endif 3223 int i, num_insns; 3224 TCGOp *op; 3225 3226 #ifdef CONFIG_PROFILER 3227 { 3228 int n; 3229 3230 QTAILQ_FOREACH(op, &s->ops, link) { 3231 n++; 3232 } 3233 atomic_set(&prof->op_count, prof->op_count + n); 3234 if (n > prof->op_count_max) { 3235 atomic_set(&prof->op_count_max, n); 3236 } 3237 3238 n = s->nb_temps; 3239 atomic_set(&prof->temp_count, prof->temp_count + n); 3240 if (n > prof->temp_count_max) { 3241 atomic_set(&prof->temp_count_max, n); 3242 } 3243 } 3244 #endif 3245 3246 #ifdef DEBUG_DISAS 3247 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 3248 && qemu_log_in_addr_range(tb->pc))) { 3249 qemu_log_lock(); 3250 qemu_log("OP:\n"); 3251 tcg_dump_ops(s); 3252 qemu_log("\n"); 3253 qemu_log_unlock(); 3254 } 3255 #endif 3256 3257 #ifdef CONFIG_PROFILER 3258 atomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 3259 #endif 3260 3261 #ifdef USE_TCG_OPTIMIZATIONS 3262 tcg_optimize(s); 3263 #endif 3264 3265 #ifdef CONFIG_PROFILER 3266 atomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 3267 atomic_set(&prof->la_time, prof->la_time - profile_getclock()); 3268 #endif 3269 3270 liveness_pass_1(s); 3271 3272 if (s->nb_indirects > 0) { 3273 #ifdef DEBUG_DISAS 3274 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 3275 && qemu_log_in_addr_range(tb->pc))) { 3276 qemu_log_lock(); 3277 qemu_log("OP before indirect lowering:\n"); 3278 tcg_dump_ops(s); 3279 qemu_log("\n"); 3280 qemu_log_unlock(); 3281 } 3282 #endif 3283 /* Replace indirect temps with direct temps. */ 3284 if (liveness_pass_2(s)) { 3285 /* If changes were made, re-run liveness. */ 3286 liveness_pass_1(s); 3287 } 3288 } 3289 3290 #ifdef CONFIG_PROFILER 3291 atomic_set(&prof->la_time, prof->la_time + profile_getclock()); 3292 #endif 3293 3294 #ifdef DEBUG_DISAS 3295 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 3296 && qemu_log_in_addr_range(tb->pc))) { 3297 qemu_log_lock(); 3298 qemu_log("OP after optimization and liveness analysis:\n"); 3299 tcg_dump_ops(s); 3300 qemu_log("\n"); 3301 qemu_log_unlock(); 3302 } 3303 #endif 3304 3305 tcg_reg_alloc_start(s); 3306 3307 s->code_buf = tb->tc.ptr; 3308 s->code_ptr = tb->tc.ptr; 3309 3310 #ifdef TCG_TARGET_NEED_LDST_LABELS 3311 s->ldst_labels = NULL; 3312 #endif 3313 #ifdef TCG_TARGET_NEED_POOL_LABELS 3314 s->pool_labels = NULL; 3315 #endif 3316 3317 num_insns = -1; 3318 QTAILQ_FOREACH(op, &s->ops, link) { 3319 TCGOpcode opc = op->opc; 3320 3321 #ifdef CONFIG_PROFILER 3322 atomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 3323 #endif 3324 3325 switch (opc) { 3326 case INDEX_op_mov_i32: 3327 case INDEX_op_mov_i64: 3328 case INDEX_op_mov_vec: 3329 tcg_reg_alloc_mov(s, op); 3330 break; 3331 case INDEX_op_movi_i32: 3332 case INDEX_op_movi_i64: 3333 case INDEX_op_dupi_vec: 3334 tcg_reg_alloc_movi(s, op); 3335 break; 3336 case INDEX_op_insn_start: 3337 if (num_insns >= 0) { 3338 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 3339 } 3340 num_insns++; 3341 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 3342 target_ulong a; 3343 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 3344 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 3345 #else 3346 a = op->args[i]; 3347 #endif 3348 s->gen_insn_data[num_insns][i] = a; 3349 } 3350 break; 3351 case INDEX_op_discard: 3352 temp_dead(s, arg_temp(op->args[0])); 3353 break; 3354 case INDEX_op_set_label: 3355 tcg_reg_alloc_bb_end(s, s->reserved_regs); 3356 tcg_out_label(s, arg_label(op->args[0]), s->code_ptr); 3357 break; 3358 case INDEX_op_call: 3359 tcg_reg_alloc_call(s, op); 3360 break; 3361 default: 3362 /* Sanity check that we've not introduced any unhandled opcodes. */ 3363 tcg_debug_assert(tcg_op_supported(opc)); 3364 /* Note: in order to speed up the code, it would be much 3365 faster to have specialized register allocator functions for 3366 some common argument patterns */ 3367 tcg_reg_alloc_op(s, op); 3368 break; 3369 } 3370 #ifdef CONFIG_DEBUG_TCG 3371 check_regs(s); 3372 #endif 3373 /* Test for (pending) buffer overflow. The assumption is that any 3374 one operation beginning below the high water mark cannot overrun 3375 the buffer completely. Thus we can test for overflow after 3376 generating code without having to check during generation. */ 3377 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 3378 return -1; 3379 } 3380 } 3381 tcg_debug_assert(num_insns >= 0); 3382 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 3383 3384 /* Generate TB finalization at the end of block */ 3385 #ifdef TCG_TARGET_NEED_LDST_LABELS 3386 if (!tcg_out_ldst_finalize(s)) { 3387 return -1; 3388 } 3389 #endif 3390 #ifdef TCG_TARGET_NEED_POOL_LABELS 3391 if (!tcg_out_pool_finalize(s)) { 3392 return -1; 3393 } 3394 #endif 3395 3396 /* flush instruction cache */ 3397 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 3398 3399 return tcg_current_code_size(s); 3400 } 3401 3402 #ifdef CONFIG_PROFILER 3403 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 3404 { 3405 TCGProfile prof = {}; 3406 const TCGProfile *s; 3407 int64_t tb_count; 3408 int64_t tb_div_count; 3409 int64_t tot; 3410 3411 tcg_profile_snapshot_counters(&prof); 3412 s = &prof; 3413 tb_count = s->tb_count; 3414 tb_div_count = tb_count ? tb_count : 1; 3415 tot = s->interm_time + s->code_time; 3416 3417 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 3418 tot, tot / 2.4e9); 3419 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 3420 tb_count, s->tb_count1 - tb_count, 3421 (double)(s->tb_count1 - s->tb_count) 3422 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 3423 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 3424 (double)s->op_count / tb_div_count, s->op_count_max); 3425 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 3426 (double)s->del_op_count / tb_div_count); 3427 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 3428 (double)s->temp_count / tb_div_count, s->temp_count_max); 3429 cpu_fprintf(f, "avg host code/TB %0.1f\n", 3430 (double)s->code_out_len / tb_div_count); 3431 cpu_fprintf(f, "avg search data/TB %0.1f\n", 3432 (double)s->search_out_len / tb_div_count); 3433 3434 cpu_fprintf(f, "cycles/op %0.1f\n", 3435 s->op_count ? (double)tot / s->op_count : 0); 3436 cpu_fprintf(f, "cycles/in byte %0.1f\n", 3437 s->code_in_len ? (double)tot / s->code_in_len : 0); 3438 cpu_fprintf(f, "cycles/out byte %0.1f\n", 3439 s->code_out_len ? (double)tot / s->code_out_len : 0); 3440 cpu_fprintf(f, "cycles/search byte %0.1f\n", 3441 s->search_out_len ? (double)tot / s->search_out_len : 0); 3442 if (tot == 0) { 3443 tot = 1; 3444 } 3445 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 3446 (double)s->interm_time / tot * 100.0); 3447 cpu_fprintf(f, " gen_code time %0.1f%%\n", 3448 (double)s->code_time / tot * 100.0); 3449 cpu_fprintf(f, "optim./code time %0.1f%%\n", 3450 (double)s->opt_time / (s->code_time ? s->code_time : 1) 3451 * 100.0); 3452 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 3453 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 3454 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 3455 s->restore_count); 3456 cpu_fprintf(f, " avg cycles %0.1f\n", 3457 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 3458 } 3459 #else 3460 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 3461 { 3462 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 3463 } 3464 #endif 3465 3466 #ifdef ELF_HOST_MACHINE 3467 /* In order to use this feature, the backend needs to do three things: 3468 3469 (1) Define ELF_HOST_MACHINE to indicate both what value to 3470 put into the ELF image and to indicate support for the feature. 3471 3472 (2) Define tcg_register_jit. This should create a buffer containing 3473 the contents of a .debug_frame section that describes the post- 3474 prologue unwind info for the tcg machine. 3475 3476 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 3477 */ 3478 3479 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 3480 typedef enum { 3481 JIT_NOACTION = 0, 3482 JIT_REGISTER_FN, 3483 JIT_UNREGISTER_FN 3484 } jit_actions_t; 3485 3486 struct jit_code_entry { 3487 struct jit_code_entry *next_entry; 3488 struct jit_code_entry *prev_entry; 3489 const void *symfile_addr; 3490 uint64_t symfile_size; 3491 }; 3492 3493 struct jit_descriptor { 3494 uint32_t version; 3495 uint32_t action_flag; 3496 struct jit_code_entry *relevant_entry; 3497 struct jit_code_entry *first_entry; 3498 }; 3499 3500 void __jit_debug_register_code(void) __attribute__((noinline)); 3501 void __jit_debug_register_code(void) 3502 { 3503 asm(""); 3504 } 3505 3506 /* Must statically initialize the version, because GDB may check 3507 the version before we can set it. */ 3508 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 3509 3510 /* End GDB interface. */ 3511 3512 static int find_string(const char *strtab, const char *str) 3513 { 3514 const char *p = strtab + 1; 3515 3516 while (1) { 3517 if (strcmp(p, str) == 0) { 3518 return p - strtab; 3519 } 3520 p += strlen(p) + 1; 3521 } 3522 } 3523 3524 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 3525 const void *debug_frame, 3526 size_t debug_frame_size) 3527 { 3528 struct __attribute__((packed)) DebugInfo { 3529 uint32_t len; 3530 uint16_t version; 3531 uint32_t abbrev; 3532 uint8_t ptr_size; 3533 uint8_t cu_die; 3534 uint16_t cu_lang; 3535 uintptr_t cu_low_pc; 3536 uintptr_t cu_high_pc; 3537 uint8_t fn_die; 3538 char fn_name[16]; 3539 uintptr_t fn_low_pc; 3540 uintptr_t fn_high_pc; 3541 uint8_t cu_eoc; 3542 }; 3543 3544 struct ElfImage { 3545 ElfW(Ehdr) ehdr; 3546 ElfW(Phdr) phdr; 3547 ElfW(Shdr) shdr[7]; 3548 ElfW(Sym) sym[2]; 3549 struct DebugInfo di; 3550 uint8_t da[24]; 3551 char str[80]; 3552 }; 3553 3554 struct ElfImage *img; 3555 3556 static const struct ElfImage img_template = { 3557 .ehdr = { 3558 .e_ident[EI_MAG0] = ELFMAG0, 3559 .e_ident[EI_MAG1] = ELFMAG1, 3560 .e_ident[EI_MAG2] = ELFMAG2, 3561 .e_ident[EI_MAG3] = ELFMAG3, 3562 .e_ident[EI_CLASS] = ELF_CLASS, 3563 .e_ident[EI_DATA] = ELF_DATA, 3564 .e_ident[EI_VERSION] = EV_CURRENT, 3565 .e_type = ET_EXEC, 3566 .e_machine = ELF_HOST_MACHINE, 3567 .e_version = EV_CURRENT, 3568 .e_phoff = offsetof(struct ElfImage, phdr), 3569 .e_shoff = offsetof(struct ElfImage, shdr), 3570 .e_ehsize = sizeof(ElfW(Shdr)), 3571 .e_phentsize = sizeof(ElfW(Phdr)), 3572 .e_phnum = 1, 3573 .e_shentsize = sizeof(ElfW(Shdr)), 3574 .e_shnum = ARRAY_SIZE(img->shdr), 3575 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 3576 #ifdef ELF_HOST_FLAGS 3577 .e_flags = ELF_HOST_FLAGS, 3578 #endif 3579 #ifdef ELF_OSABI 3580 .e_ident[EI_OSABI] = ELF_OSABI, 3581 #endif 3582 }, 3583 .phdr = { 3584 .p_type = PT_LOAD, 3585 .p_flags = PF_X, 3586 }, 3587 .shdr = { 3588 [0] = { .sh_type = SHT_NULL }, 3589 /* Trick: The contents of code_gen_buffer are not present in 3590 this fake ELF file; that got allocated elsewhere. Therefore 3591 we mark .text as SHT_NOBITS (similar to .bss) so that readers 3592 will not look for contents. We can record any address. */ 3593 [1] = { /* .text */ 3594 .sh_type = SHT_NOBITS, 3595 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 3596 }, 3597 [2] = { /* .debug_info */ 3598 .sh_type = SHT_PROGBITS, 3599 .sh_offset = offsetof(struct ElfImage, di), 3600 .sh_size = sizeof(struct DebugInfo), 3601 }, 3602 [3] = { /* .debug_abbrev */ 3603 .sh_type = SHT_PROGBITS, 3604 .sh_offset = offsetof(struct ElfImage, da), 3605 .sh_size = sizeof(img->da), 3606 }, 3607 [4] = { /* .debug_frame */ 3608 .sh_type = SHT_PROGBITS, 3609 .sh_offset = sizeof(struct ElfImage), 3610 }, 3611 [5] = { /* .symtab */ 3612 .sh_type = SHT_SYMTAB, 3613 .sh_offset = offsetof(struct ElfImage, sym), 3614 .sh_size = sizeof(img->sym), 3615 .sh_info = 1, 3616 .sh_link = ARRAY_SIZE(img->shdr) - 1, 3617 .sh_entsize = sizeof(ElfW(Sym)), 3618 }, 3619 [6] = { /* .strtab */ 3620 .sh_type = SHT_STRTAB, 3621 .sh_offset = offsetof(struct ElfImage, str), 3622 .sh_size = sizeof(img->str), 3623 } 3624 }, 3625 .sym = { 3626 [1] = { /* code_gen_buffer */ 3627 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 3628 .st_shndx = 1, 3629 } 3630 }, 3631 .di = { 3632 .len = sizeof(struct DebugInfo) - 4, 3633 .version = 2, 3634 .ptr_size = sizeof(void *), 3635 .cu_die = 1, 3636 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 3637 .fn_die = 2, 3638 .fn_name = "code_gen_buffer" 3639 }, 3640 .da = { 3641 1, /* abbrev number (the cu) */ 3642 0x11, 1, /* DW_TAG_compile_unit, has children */ 3643 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 3644 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 3645 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 3646 0, 0, /* end of abbrev */ 3647 2, /* abbrev number (the fn) */ 3648 0x2e, 0, /* DW_TAG_subprogram, no children */ 3649 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 3650 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 3651 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 3652 0, 0, /* end of abbrev */ 3653 0 /* no more abbrev */ 3654 }, 3655 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 3656 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 3657 }; 3658 3659 /* We only need a single jit entry; statically allocate it. */ 3660 static struct jit_code_entry one_entry; 3661 3662 uintptr_t buf = (uintptr_t)buf_ptr; 3663 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 3664 DebugFrameHeader *dfh; 3665 3666 img = g_malloc(img_size); 3667 *img = img_template; 3668 3669 img->phdr.p_vaddr = buf; 3670 img->phdr.p_paddr = buf; 3671 img->phdr.p_memsz = buf_size; 3672 3673 img->shdr[1].sh_name = find_string(img->str, ".text"); 3674 img->shdr[1].sh_addr = buf; 3675 img->shdr[1].sh_size = buf_size; 3676 3677 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 3678 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 3679 3680 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 3681 img->shdr[4].sh_size = debug_frame_size; 3682 3683 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 3684 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 3685 3686 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 3687 img->sym[1].st_value = buf; 3688 img->sym[1].st_size = buf_size; 3689 3690 img->di.cu_low_pc = buf; 3691 img->di.cu_high_pc = buf + buf_size; 3692 img->di.fn_low_pc = buf; 3693 img->di.fn_high_pc = buf + buf_size; 3694 3695 dfh = (DebugFrameHeader *)(img + 1); 3696 memcpy(dfh, debug_frame, debug_frame_size); 3697 dfh->fde.func_start = buf; 3698 dfh->fde.func_len = buf_size; 3699 3700 #ifdef DEBUG_JIT 3701 /* Enable this block to be able to debug the ELF image file creation. 3702 One can use readelf, objdump, or other inspection utilities. */ 3703 { 3704 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 3705 if (f) { 3706 if (fwrite(img, img_size, 1, f) != img_size) { 3707 /* Avoid stupid unused return value warning for fwrite. */ 3708 } 3709 fclose(f); 3710 } 3711 } 3712 #endif 3713 3714 one_entry.symfile_addr = img; 3715 one_entry.symfile_size = img_size; 3716 3717 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 3718 __jit_debug_descriptor.relevant_entry = &one_entry; 3719 __jit_debug_descriptor.first_entry = &one_entry; 3720 __jit_debug_register_code(); 3721 } 3722 #else 3723 /* No support for the feature. Provide the entry point expected by exec.c, 3724 and implement the internal function we declared earlier. */ 3725 3726 static void tcg_register_jit_int(void *buf, size_t size, 3727 const void *debug_frame, 3728 size_t debug_frame_size) 3729 { 3730 } 3731 3732 void tcg_register_jit(void *buf, size_t buf_size) 3733 { 3734 } 3735 #endif /* ELF_HOST_MACHINE */ 3736 3737 #if !TCG_TARGET_MAYBE_vec 3738 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 3739 { 3740 g_assert_not_reached(); 3741 } 3742 #endif 3743