Lines Matching +full:build +full:- +full:tcg +full:- +full:disabled
30 #include "qemu/error-report.h"
32 #include "qemu/host-utils.h"
33 #include "qemu/qemu-print.h"
38 #include "exec/translation-block.h"
39 #include "exec/tlb-common.h"
40 #include "tcg/startup.h"
41 #include "tcg/tcg-op-common.h"
56 #include "tcg/tcg-ldst.h"
57 #include "tcg/tcg-temp-internal.h"
58 #include "tcg-internal.h"
59 #include "tcg/perf.h"
60 #include "tcg-has.h"
62 #include "user/guest-base.h"
65 /* Forward declarations for functions declared in tcg-target.c.inc and
117 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
267 *s->code_ptr++ = v; in tcg_out8()
281 *s->code_ptr++ = v; in tcg_out16()
283 tcg_insn_unit *p = s->code_ptr; in tcg_out16()
285 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); in tcg_out16()
304 *s->code_ptr++ = v; in tcg_out32()
306 tcg_insn_unit *p = s->code_ptr; in tcg_out32()
308 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); in tcg_out32()
327 *s->code_ptr++ = v; in tcg_out64()
329 tcg_insn_unit *p = s->code_ptr; in tcg_out64()
331 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); in tcg_out64()
353 r->type = type; in tcg_out_reloc()
354 r->ptr = code_ptr; in tcg_out_reloc()
355 r->addend = addend; in tcg_out_reloc()
356 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); in tcg_out_reloc()
361 tcg_debug_assert(!l->has_value); in tcg_out_label()
362 l->has_value = 1; in tcg_out_label()
363 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); in tcg_out_label()
372 l->id = s->nb_labels++; in gen_new_label()
373 QSIMPLEQ_INIT(&l->branches); in gen_new_label()
374 QSIMPLEQ_INIT(&l->relocs); in gen_new_label()
376 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); in gen_new_label()
385 QSIMPLEQ_FOREACH(l, &s->labels, next) { in tcg_resolve_relocs()
387 uintptr_t value = l->u.value; in tcg_resolve_relocs()
389 QSIMPLEQ_FOREACH(r, &l->relocs, next) { in tcg_resolve_relocs()
390 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { in tcg_resolve_relocs()
404 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); in set_jmp_reset_offset()
413 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); in set_jmp_insn_offset()
419 * Return the read-execute version of the pointer, for the benefit in get_jmp_target_addr()
420 * of any pc-relative addressing mode. in get_jmp_target_addr()
422 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); in get_jmp_target_addr()
428 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) - in tlb_mask_table_ofs()
436 siglongjmp(s->jmp_trans, -2); in tcg_raise_tb_overflow()
457 * tcg_out_movext -- move and extend
458 * @s: tcg context
522 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src); in tcg_out_movext1_new_src()
527 tcg_out_movext1_new_src(s, i, i->src); in tcg_out_movext1()
531 * tcg_out_movext2 -- move and extend two pair
532 * @s: tcg context
535 * @scratch: temporary register, or -1 for none
544 TCGReg src1 = i1->src; in tcg_out_movext2()
545 TCGReg src2 = i2->src; in tcg_out_movext2()
547 if (i1->dst != src2) { in tcg_out_movext2()
552 if (i2->dst == src1) { in tcg_out_movext2()
553 TCGType src1_type = i1->src_type; in tcg_out_movext2()
554 TCGType src2_type = i2->src_type; in tcg_out_movext2()
558 src1 = i2->src; in tcg_out_movext2()
559 src2 = i1->src; in tcg_out_movext2()
571 * tcg_out_movext3 -- move and extend three pair
572 * @s: tcg context
576 * @scratch: temporary register, or -1 for none
586 TCGReg src1 = i1->src; in tcg_out_movext3()
587 TCGReg src2 = i2->src; in tcg_out_movext3()
588 TCGReg src3 = i3->src; in tcg_out_movext3()
590 if (i1->dst != src2 && i1->dst != src3) { in tcg_out_movext3()
595 if (i2->dst != src1 && i2->dst != src3) { in tcg_out_movext3()
600 if (i3->dst != src1 && i3->dst != src2) { in tcg_out_movext3()
608 * either "clockwise" or "anti-clockwise", and can be solved with in tcg_out_movext3()
611 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) { in tcg_out_movext3()
613 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) { in tcg_out_movext3()
614 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3); in tcg_out_movext3()
616 tcg_out_movext1_new_src(s, i1, i1->dst); in tcg_out_movext3()
617 tcg_out_movext1_new_src(s, i2, i2->dst); in tcg_out_movext3()
618 tcg_out_movext1_new_src(s, i3, i3->dst); in tcg_out_movext3()
621 tcg_out_mov(s, i1->src_type, scratch, src1); in tcg_out_movext3()
626 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) { in tcg_out_movext3()
627 /* "Anti-clockwise" */ in tcg_out_movext3()
628 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) { in tcg_out_movext3()
629 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2); in tcg_out_movext3()
631 tcg_out_movext1_new_src(s, i1, i1->dst); in tcg_out_movext3()
632 tcg_out_movext1_new_src(s, i2, i2->dst); in tcg_out_movext3()
633 tcg_out_movext1_new_src(s, i3, i3->dst); in tcg_out_movext3()
636 tcg_out_mov(s, i1->src_type, scratch, src1); in tcg_out_movext3()
656 QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next); in new_ldst_label()
680 n->label = label; in new_pool_alloc()
681 n->addend = addend; in new_pool_alloc()
682 n->rtype = rtype; in new_pool_alloc()
683 n->nlong = nlong; in new_pool_alloc()
690 int nlong = n->nlong; in new_pool_insert()
693 for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) { in new_pool_insert()
694 if (nlong > i->nlong) { in new_pool_insert()
697 if (nlong < i->nlong) { in new_pool_insert()
700 if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) { in new_pool_insert()
704 n->next = *pp; in new_pool_insert()
714 n->data[0] = d; in new_pool_label()
725 n->data[0] = d0; in new_pool_l2()
726 n->data[1] = d1; in new_pool_l2()
738 n->data[0] = d0; in new_pool_l4()
739 n->data[1] = d1; in new_pool_l4()
740 n->data[2] = d2; in new_pool_l4()
741 n->data[3] = d3; in new_pool_l4()
745 /* For v256, for 32-bit host. */
755 n->data[0] = d0; in new_pool_l8()
756 n->data[1] = d1; in new_pool_l8()
757 n->data[2] = d2; in new_pool_l8()
758 n->data[3] = d3; in new_pool_l8()
759 n->data[4] = d4; in new_pool_l8()
760 n->data[5] = d5; in new_pool_l8()
761 n->data[6] = d6; in new_pool_l8()
762 n->data[7] = d7; in new_pool_l8()
775 QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) { in tcg_out_ldst_finalize()
776 if (lb->is_ld in tcg_out_ldst_finalize()
779 return -2; in tcg_out_ldst_finalize()
788 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { in tcg_out_ldst_finalize()
789 return -1; in tcg_out_ldst_finalize()
797 TCGLabelPoolData *p = s->pool_labels; in tcg_out_pool_finalize()
809 a = (void *)ROUND_UP((uintptr_t)s->code_ptr, in tcg_out_pool_finalize()
810 sizeof(tcg_target_ulong) * p->nlong); in tcg_out_pool_finalize()
811 tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr); in tcg_out_pool_finalize()
812 s->data_gen_ptr = a; in tcg_out_pool_finalize()
814 for (; p != NULL; p = p->next) { in tcg_out_pool_finalize()
815 size_t size = sizeof(tcg_target_ulong) * p->nlong; in tcg_out_pool_finalize()
818 if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) { in tcg_out_pool_finalize()
819 if (unlikely(a > s->code_gen_highwater)) { in tcg_out_pool_finalize()
820 return -1; in tcg_out_pool_finalize()
822 memcpy(a, p->data, size); in tcg_out_pool_finalize()
827 value = (uintptr_t)tcg_splitwx_to_rx(a) - size; in tcg_out_pool_finalize()
828 if (!patch_reloc(p->label, p->rtype, value, p->addend)) { in tcg_out_pool_finalize()
829 return -2; in tcg_out_pool_finalize()
833 s->code_ptr = a; in tcg_out_pool_finalize()
867 C_Dynamic = -2,
868 C_NotImplemented = -1,
869 #include "tcg-target-con-set.h"
919 #include "tcg-target-con-set.h"
1119 #include "tcg-target.c.inc"
1123 QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
1130 * We require these functions for slow-path function calls.
1177 * This prevents trivial mistakes within each arch/tcg-target.c.inc.
1267 * All TCG threads except the parent (i.e. the one that called tcg_context_init
1268 * and registered the target's TCG globals) must register with this function
1271 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
1274 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
1275 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
1278 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
1298 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; in tcg_register_thread()
1300 s->temps[i].mem_base = &s->temps[b]; in tcg_register_thread()
1326 p->size = size; in tcg_malloc_internal()
1327 p->next = s->pool_first_large; in tcg_malloc_internal()
1328 s->pool_first_large = p; in tcg_malloc_internal()
1329 return p->data; in tcg_malloc_internal()
1331 p = s->pool_current; in tcg_malloc_internal()
1333 p = s->pool_first; in tcg_malloc_internal()
1338 if (!p->next) { in tcg_malloc_internal()
1342 p->size = pool_size; in tcg_malloc_internal()
1343 p->next = NULL; in tcg_malloc_internal()
1344 if (s->pool_current) { in tcg_malloc_internal()
1345 s->pool_current->next = p; in tcg_malloc_internal()
1347 s->pool_first = p; in tcg_malloc_internal()
1350 p = p->next; in tcg_malloc_internal()
1354 s->pool_current = p; in tcg_malloc_internal()
1355 s->pool_cur = (uintptr_t)p->data + size; in tcg_malloc_internal()
1356 s->pool_end = (uintptr_t)p->data + p->size; in tcg_malloc_internal()
1357 return p->data; in tcg_malloc_internal()
1363 for (p = s->pool_first_large; p; p = t) { in tcg_pool_reset()
1364 t = p->next; in tcg_pool_reset()
1367 s->pool_first_large = NULL; in tcg_pool_reset()
1368 s->pool_cur = s->pool_end = 0; in tcg_pool_reset()
1369 s->pool_current = NULL; in tcg_pool_reset()
1373 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
1374 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
1377 * the helpers, with the end result that it's easier to build manually.
1483 unsigned typemask = info->typemask; in init_ffi_layout()
1491 /* Ignoring the return type, find the last non-zero field. */ in init_ffi_layout()
1492 nargs = 32 - clz32(typemask >> 3); in init_ffi_layout()
1497 ca->cif.rtype = typecode_to_ffi(typemask & 7); in init_ffi_layout()
1498 ca->cif.nargs = nargs; in init_ffi_layout()
1501 ca->cif.arg_types = ca->args; in init_ffi_layout()
1504 ca->args[j] = typecode_to_ffi(typecode); in init_ffi_layout()
1508 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, in init_ffi_layout()
1509 ca->cif.rtype, ca->cif.arg_types); in init_ffi_layout()
1512 return &ca->cif; in init_ffi_layout()
1515 #define HELPER_INFO_INIT(I) (&(I)->cif)
1518 #define HELPER_INFO_INIT(I) (&(I)->init)
1535 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); in arg_slot_stk_ofs()
1550 cum->arg_slot += cum->arg_slot & 1; in layout_arg_even()
1556 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; in layout_arg_1()
1560 .arg_idx = cum->arg_idx, in layout_arg_1()
1561 .arg_slot = cum->arg_slot, in layout_arg_1()
1563 cum->info_in_idx++; in layout_arg_1()
1564 cum->arg_slot++; in layout_arg_1()
1570 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; in layout_arg_normal_n()
1576 .arg_idx = cum->arg_idx, in layout_arg_normal_n()
1578 .arg_slot = cum->arg_slot + i, in layout_arg_normal_n()
1581 cum->info_in_idx += n; in layout_arg_normal_n()
1582 cum->arg_slot += n; in layout_arg_normal_n()
1587 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; in layout_arg_by_ref()
1595 * structure pass by-reference. Therefore we must make copies. in layout_arg_by_ref()
1599 loc[0].ref_slot = cum->ref_slot; in layout_arg_by_ref()
1608 .arg_idx = cum->arg_idx, in layout_arg_by_ref()
1610 .ref_slot = cum->ref_slot + i, in layout_arg_by_ref()
1613 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */ in layout_arg_by_ref()
1614 cum->ref_slot += n; in layout_arg_by_ref()
1621 unsigned typemask = info->typemask; in init_call_layout()
1631 info->nr_out = 0; in init_call_layout()
1636 info->nr_out = 1; in init_call_layout()
1637 info->out_kind = TCG_CALL_RET_NORMAL; in init_call_layout()
1641 info->nr_out = 64 / TCG_TARGET_REG_BITS; in init_call_layout()
1642 info->out_kind = TCG_CALL_RET_NORMAL; in init_call_layout()
1644 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); in init_call_layout()
1647 info->nr_out = 128 / TCG_TARGET_REG_BITS; in init_call_layout()
1648 info->out_kind = TCG_TARGET_CALL_RET_I128; in init_call_layout()
1652 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); in init_call_layout()
1756 info->nr_in = cum.info_in_idx; in init_call_layout()
1759 assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); in init_call_layout()
1766 * which has a signed 8-bit offset encoding. in init_call_layout()
1774 ref_base = cum.arg_slot - max_reg_slots; in init_call_layout()
1783 for (int i = cum.info_in_idx - 1; i >= 0; --i) { in init_call_layout()
1784 TCGCallArgumentLoc *loc = &info->in[i]; in init_call_layout()
1785 switch (loc->kind) { in init_call_layout()
1788 loc->ref_slot += ref_base; in init_call_layout()
1810 s->nb_globals = 0; in tcg_context_init()
1831 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; in tcg_context_init()
1839 * In user-mode we simply share the init context among threads, since we in tcg_context_init()
1842 * In system-mode we will have at most max_threads TCG threads. in tcg_context_init()
1853 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); in tcg_context_init()
1875 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); in tcg_tb_alloc()
1878 if (unlikely(next > s->code_gen_highwater)) { in tcg_tb_alloc()
1884 qatomic_set(&s->code_gen_ptr, next); in tcg_tb_alloc()
1893 s->code_ptr = s->code_gen_ptr; in tcg_prologue_init()
1894 s->code_buf = s->code_gen_ptr; in tcg_prologue_init()
1895 s->data_gen_ptr = NULL; in tcg_prologue_init()
1898 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); in tcg_prologue_init()
1901 s->pool_labels = NULL; in tcg_prologue_init()
1914 perf_report_prologue(s->code_gen_ptr, prologue_size); in tcg_prologue_init()
1917 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), in tcg_prologue_init()
1918 (uintptr_t)s->code_buf, prologue_size); in tcg_prologue_init()
1925 if (s->data_gen_ptr) { in tcg_prologue_init()
1926 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; in tcg_prologue_init()
1927 size_t data_size = prologue_size - code_size; in tcg_prologue_init()
1930 disas(logfile, s->code_gen_ptr, code_size); in tcg_prologue_init()
1936 (uintptr_t)s->data_gen_ptr + i, in tcg_prologue_init()
1937 *(uint64_t *)(s->data_gen_ptr + i)); in tcg_prologue_init()
1941 (uintptr_t)s->data_gen_ptr + i, in tcg_prologue_init()
1942 *(uint32_t *)(s->data_gen_ptr + i)); in tcg_prologue_init()
1946 disas(logfile, s->code_gen_ptr, prologue_size); in tcg_prologue_init()
1968 s->nb_temps = s->nb_globals; in tcg_func_start()
1975 if (s->const_table[i]) { in tcg_func_start()
1976 g_hash_table_remove_all(s->const_table[i]); in tcg_func_start()
1980 s->nb_ops = 0; in tcg_func_start()
1981 s->nb_labels = 0; in tcg_func_start()
1982 s->current_frame_offset = s->frame_start; in tcg_func_start()
1985 s->goto_tb_issue_mask = 0; in tcg_func_start()
1988 QTAILQ_INIT(&s->ops); in tcg_func_start()
1989 QTAILQ_INIT(&s->free_ops); in tcg_func_start()
1990 s->emit_before_op = NULL; in tcg_func_start()
1991 QSIMPLEQ_INIT(&s->labels); in tcg_func_start()
1993 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); in tcg_func_start()
1998 int n = s->nb_temps++; in tcg_temp_alloc()
2003 return memset(&s->temps[n], 0, sizeof(TCGTemp)); in tcg_temp_alloc()
2010 tcg_debug_assert(s->nb_globals == s->nb_temps); in tcg_global_alloc()
2011 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); in tcg_global_alloc()
2012 s->nb_globals++; in tcg_global_alloc()
2014 ts->kind = TEMP_GLOBAL; in tcg_global_alloc()
2027 ts->base_type = type; in tcg_global_reg_new_internal()
2028 ts->type = type; in tcg_global_reg_new_internal()
2029 ts->kind = TEMP_FIXED; in tcg_global_reg_new_internal()
2030 ts->reg = reg; in tcg_global_reg_new_internal()
2031 ts->name = name; in tcg_global_reg_new_internal()
2032 tcg_regset_set_reg(s->reserved_regs, reg); in tcg_global_reg_new_internal()
2039 s->frame_start = start; in tcg_set_frame()
2040 s->frame_end = start + size; in tcg_set_frame()
2041 s->frame_temp in tcg_set_frame()
2053 switch (base_ts->kind) { in tcg_global_mem_new_internal()
2057 /* We do not support double-indirect registers. */ in tcg_global_mem_new_internal()
2058 tcg_debug_assert(!base_ts->indirect_reg); in tcg_global_mem_new_internal()
2059 base_ts->indirect_base = 1; in tcg_global_mem_new_internal()
2060 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 in tcg_global_mem_new_internal()
2072 ts->base_type = TCG_TYPE_I64; in tcg_global_mem_new_internal()
2073 ts->type = TCG_TYPE_I32; in tcg_global_mem_new_internal()
2074 ts->indirect_reg = indirect_reg; in tcg_global_mem_new_internal()
2075 ts->mem_allocated = 1; in tcg_global_mem_new_internal()
2076 ts->mem_base = base_ts; in tcg_global_mem_new_internal()
2077 ts->mem_offset = offset; in tcg_global_mem_new_internal()
2080 ts->name = strdup(buf); in tcg_global_mem_new_internal()
2083 ts2->base_type = TCG_TYPE_I64; in tcg_global_mem_new_internal()
2084 ts2->type = TCG_TYPE_I32; in tcg_global_mem_new_internal()
2085 ts2->indirect_reg = indirect_reg; in tcg_global_mem_new_internal()
2086 ts2->mem_allocated = 1; in tcg_global_mem_new_internal()
2087 ts2->mem_base = base_ts; in tcg_global_mem_new_internal()
2088 ts2->mem_offset = offset + 4; in tcg_global_mem_new_internal()
2089 ts2->temp_subindex = 1; in tcg_global_mem_new_internal()
2092 ts2->name = strdup(buf); in tcg_global_mem_new_internal()
2094 ts->base_type = type; in tcg_global_mem_new_internal()
2095 ts->type = type; in tcg_global_mem_new_internal()
2096 ts->indirect_reg = indirect_reg; in tcg_global_mem_new_internal()
2097 ts->mem_allocated = 1; in tcg_global_mem_new_internal()
2098 ts->mem_base = base_ts; in tcg_global_mem_new_internal()
2099 ts->mem_offset = offset; in tcg_global_mem_new_internal()
2100 ts->name = name; in tcg_global_mem_new_internal()
2130 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); in tcg_temp_new_internal()
2134 clear_bit(idx, s->free_temps[type].l); in tcg_temp_new_internal()
2136 ts = &s->temps[idx]; in tcg_temp_new_internal()
2137 ts->temp_allocated = 1; in tcg_temp_new_internal()
2138 tcg_debug_assert(ts->base_type == type); in tcg_temp_new_internal()
2139 tcg_debug_assert(ts->kind == kind); in tcg_temp_new_internal()
2164 ts->base_type = type; in tcg_temp_new_internal()
2165 ts->temp_allocated = 1; in tcg_temp_new_internal()
2166 ts->kind = kind; in tcg_temp_new_internal()
2169 ts->type = type; in tcg_temp_new_internal()
2171 ts->type = TCG_TYPE_REG; in tcg_temp_new_internal()
2177 ts2->base_type = type; in tcg_temp_new_internal()
2178 ts2->type = TCG_TYPE_REG; in tcg_temp_new_internal()
2179 ts2->temp_allocated = 1; in tcg_temp_new_internal()
2180 ts2->temp_subindex = i; in tcg_temp_new_internal()
2181 ts2->kind = kind; in tcg_temp_new_internal()
2256 tcg_debug_assert(t->temp_allocated != 0); in tcg_temp_new_vec_matching()
2258 t = tcg_temp_new_internal(t->base_type, TEMP_EBB); in tcg_temp_new_vec_matching()
2266 switch (ts->kind) { in tcg_temp_free_internal()
2272 tcg_debug_assert(ts->temp_allocated != 0); in tcg_temp_free_internal()
2273 ts->temp_allocated = 0; in tcg_temp_free_internal()
2274 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); in tcg_temp_free_internal()
2310 GHashTable *h = s->const_table[type]; in tcg_constant_internal()
2315 s->const_table[type] = h; in tcg_constant_internal()
2329 ts->base_type = TCG_TYPE_I64; in tcg_constant_internal()
2330 ts->type = TCG_TYPE_I32; in tcg_constant_internal()
2331 ts->kind = TEMP_CONST; in tcg_constant_internal()
2332 ts->temp_allocated = 1; in tcg_constant_internal()
2334 ts2->base_type = TCG_TYPE_I64; in tcg_constant_internal()
2335 ts2->type = TCG_TYPE_I32; in tcg_constant_internal()
2336 ts2->kind = TEMP_CONST; in tcg_constant_internal()
2337 ts2->temp_allocated = 1; in tcg_constant_internal()
2338 ts2->temp_subindex = 1; in tcg_constant_internal()
2341 * Retain the full value of the 64-bit constant in the low in tcg_constant_internal()
2349 ts->base_type = type; in tcg_constant_internal()
2350 ts->type = type; in tcg_constant_internal()
2351 ts->kind = TEMP_CONST; in tcg_constant_internal()
2352 ts->temp_allocated = 1; in tcg_constant_internal()
2353 ts->val = val; in tcg_constant_internal()
2354 val_ptr = &ts->val; in tcg_constant_internal()
2392 tcg_debug_assert(t->temp_allocated != 0); in tcg_constant_vec_matching()
2393 return tcg_constant_vec(t->base_type, vece, val); in tcg_constant_vec_matching()
2399 ptrdiff_t n = ts - tcg_ctx->temps; in temp_idx()
2400 assert(n >= 0 && n < tcg_ctx->nb_temps); in temp_idx()
2406 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps); in tcgv_i32_temp()
2408 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps); in tcgv_i32_temp()
2584 con_set = outop->static_constraint; in tcg_op_supported()
2586 con_set = outop->dynamic_constraint(type, flags); in tcg_op_supported()
2611 tcg_debug_assert(len <= width - ofs); in tcg_op_deposit_valid()
2631 total_args = info->nr_out + info->nr_in + 2; in tcg_gen_callN()
2636 if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { in tcg_gen_callN()
2637 tcg_ctx->plugin_insn->calls_helpers = true; in tcg_gen_callN()
2641 TCGOP_CALLO(op) = n = info->nr_out; in tcg_gen_callN()
2648 op->args[pi++] = temp_arg(ret); in tcg_gen_callN()
2653 tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); in tcg_gen_callN()
2654 tcg_debug_assert(ret->temp_subindex == 0); in tcg_gen_callN()
2656 op->args[pi++] = temp_arg(ret + i); in tcg_gen_callN()
2663 TCGOP_CALLI(op) = n = info->nr_in; in tcg_gen_callN()
2665 const TCGCallArgumentLoc *loc = &info->in[i]; in tcg_gen_callN()
2666 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; in tcg_gen_callN()
2668 switch (loc->kind) { in tcg_gen_callN()
2672 op->args[pi++] = temp_arg(ts); in tcg_gen_callN()
2681 if (loc->kind == TCG_CALL_ARG_EXTEND_S) { in tcg_gen_callN()
2686 op->args[pi++] = tcgv_i64_arg(temp); in tcg_gen_callN()
2695 op->args[pi++] = (uintptr_t)func; in tcg_gen_callN()
2696 op->args[pi++] = (uintptr_t)info; in tcg_gen_callN()
2699 if (tcg_ctx->emit_before_op) { in tcg_gen_callN()
2700 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); in tcg_gen_callN()
2702 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); in tcg_gen_callN()
2769 for (i = 0, n = s->nb_temps; i < n; i++) { in tcg_reg_alloc_start()
2770 TCGTemp *ts = &s->temps[i]; in tcg_reg_alloc_start()
2773 switch (ts->kind) { in tcg_reg_alloc_start()
2786 ts->mem_allocated = 0; in tcg_reg_alloc_start()
2791 ts->val_type = val; in tcg_reg_alloc_start()
2794 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); in tcg_reg_alloc_start()
2802 switch (ts->kind) { in tcg_get_arg_str_ptr()
2805 pstrcpy(buf, buf_size, ts->name); in tcg_get_arg_str_ptr()
2808 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); in tcg_get_arg_str_ptr()
2811 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); in tcg_get_arg_str_ptr()
2814 switch (ts->type) { in tcg_get_arg_str_ptr()
2816 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); in tcg_get_arg_str_ptr()
2820 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); in tcg_get_arg_str_ptr()
2827 64 << (ts->type - TCG_TYPE_V64), ts->val); in tcg_get_arg_str_ptr()
2909 "from-tb",
2910 "from-insn",
2911 "after-insn",
2912 "after-tb",
2918 return (d & (d - 1)) == 0; in tcg_regset_single()
2930 /* Return only the number of characters output -- no error return. */
2939 QTAILQ_FOREACH(op, &s->ops, link) { in tcg_dump_ops()
2945 c = op->opc; in tcg_dump_ops()
2950 col += ne_fprintf(f, "\n ----"); in tcg_dump_ops()
2963 nb_cargs = def->nb_cargs; in tcg_dump_ops()
2965 col += ne_fprintf(f, " %s ", def->name); in tcg_dump_ops()
2972 if (func == info->func) { in tcg_dump_ops()
2973 col += ne_fprintf(f, "%s", info->name); in tcg_dump_ops()
2978 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); in tcg_dump_ops()
2981 op->args[i])); in tcg_dump_ops()
2984 TCGArg arg = op->args[nb_oargs + i]; in tcg_dump_ops()
2989 if (def->flags & TCG_OPF_INT) { in tcg_dump_ops()
2991 def->name, in tcg_dump_ops()
2993 } else if (def->flags & TCG_OPF_VECTOR) { in tcg_dump_ops()
2995 def->name, in tcg_dump_ops()
2999 col += ne_fprintf(f, " %s ", def->name); in tcg_dump_ops()
3002 nb_oargs = def->nb_oargs; in tcg_dump_ops()
3003 nb_iargs = def->nb_iargs; in tcg_dump_ops()
3004 nb_cargs = def->nb_cargs; in tcg_dump_ops()
3011 op->args[k++])); in tcg_dump_ops()
3017 op->args[k++])); in tcg_dump_ops()
3028 if (op->args[k] < ARRAY_SIZE(cond_name) in tcg_dump_ops()
3029 && cond_name[op->args[k]]) { in tcg_dump_ops()
3030 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); in tcg_dump_ops()
3032 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); in tcg_dump_ops()
3042 MemOpIdx oi = op->args[k++]; in tcg_dump_ops()
3066 TCGArg flags = op->args[k]; in tcg_dump_ops()
3083 TCGArg from = op->args[k++]; in tcg_dump_ops()
3108 arg_label(op->args[k])->id); in tcg_dump_ops()
3113 TCGBar membar = op->args[k]; in tcg_dump_ops()
3195 op->args[k]); in tcg_dump_ops()
3199 if (have_prefs || op->life) { in tcg_dump_ops()
3205 if (op->life) { in tcg_dump_ops()
3206 unsigned life = op->life; in tcg_dump_ops()
3263 n = ctpop64(arg_ct->regs); in get_constraint_priority()
3269 if (n == 1 || arg_ct->oalias) { in get_constraint_priority()
3278 switch (arg_ct->pair) { in get_constraint_priority()
3283 return (arg_ct->pair_index + 1) * 2 - 1; in get_constraint_priority()
3288 return -n; in get_constraint_priority()
3302 for (i = 0; i < n - 1; i++) { in sort_constraints()
3323 int nb_oargs = tdefs->nb_oargs; in process_constraint_sets()
3324 int nb_iargs = tdefs->nb_iargs; in process_constraint_sets()
3329 const char *ct_str = tdefs->args_ct_str[i]; in process_constraint_sets()
3335 o = *ct_str - '0'; in process_constraint_sets()
3362 o = i - 1; in process_constraint_sets()
3379 o = i - 1; in process_constraint_sets()
3405 /* Include all of the target-specific constraints. */ in process_constraint_sets()
3413 #include "tcg-target-con-str.h" in process_constraint_sets()
3452 * Since [0-9pm] must be alone in the constraint string, in process_constraint_sets()
3511 TCGOpcode opc = op->opc; in opcode_args_ct()
3518 if (def->flags & TCG_OPF_NOT_PRESENT) { in opcode_args_ct()
3523 con_set = outop->static_constraint; in opcode_args_ct()
3525 con_set = outop->dynamic_constraint(type, flags); in opcode_args_ct()
3534 tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs); in opcode_args_ct()
3535 tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs); in opcode_args_ct()
3542 TCGLabel *label = arg_label(op->args[idx]); in remove_label_use()
3545 QSIMPLEQ_FOREACH(use, &label->branches, next) { in remove_label_use()
3546 if (use->op == op) { in remove_label_use()
3547 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); in remove_label_use()
3556 switch (op->opc) { in tcg_op_remove()
3570 QTAILQ_REMOVE(&s->ops, op, link); in tcg_op_remove()
3571 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); in tcg_op_remove()
3572 s->nb_ops--; in tcg_op_remove()
3593 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { in tcg_op_alloc()
3594 QTAILQ_FOREACH(op, &s->free_ops, link) { in tcg_op_alloc()
3595 if (nargs <= op->nargs) { in tcg_op_alloc()
3596 QTAILQ_REMOVE(&s->free_ops, op, link); in tcg_op_alloc()
3597 nargs = op->nargs; in tcg_op_alloc()
3609 op->opc = opc; in tcg_op_alloc()
3610 op->nargs = nargs; in tcg_op_alloc()
3613 tcg_debug_assert(op->nargs == nargs); in tcg_op_alloc()
3615 s->nb_ops++; in tcg_op_alloc()
3623 if (tcg_ctx->emit_before_op) { in tcg_emit_op()
3624 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); in tcg_emit_op()
3626 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); in tcg_emit_op()
3647 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); in tcg_op_insert_after()
3655 QSIMPLEQ_FOREACH(u, &from->branches, next) { in move_label_uses()
3656 TCGOp *op = u->op; in move_label_uses()
3657 switch (op->opc) { in move_label_uses()
3659 op->args[0] = label_arg(to); in move_label_uses()
3662 op->args[3] = label_arg(to); in move_label_uses()
3665 op->args[5] = label_arg(to); in move_label_uses()
3672 QSIMPLEQ_CONCAT(&to->branches, &from->branches); in move_label_uses()
3682 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { in reachable_code_pass()
3686 switch (op->opc) { in reachable_code_pass()
3688 label = arg_label(op->args[0]); in reachable_code_pass()
3702 if (op_prev->opc == INDEX_op_set_label) { in reachable_code_pass()
3703 move_label_uses(label, arg_label(op_prev->args[0])); in reachable_code_pass()
3715 if (op_prev->opc == INDEX_op_br && in reachable_code_pass()
3716 label == arg_label(op_prev->args[0])) { in reachable_code_pass()
3722 if (QSIMPLEQ_EMPTY(&label->branches)) { in reachable_code_pass()
3753 /* Never remove -- we need to keep these for unwind. */ in reachable_code_pass()
3776 return ts->state_ptr; in la_temp_pref()
3785 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); in la_reset_pref()
3795 s->temps[i].state = TS_DEAD | TS_MEM; in la_func_end()
3796 la_reset_pref(&s->temps[i]); in la_func_end()
3799 s->temps[i].state = TS_DEAD; in la_func_end()
3800 la_reset_pref(&s->temps[i]); in la_func_end()
3811 TCGTemp *ts = &s->temps[i]; in la_bb_end()
3814 switch (ts->kind) { in la_bb_end()
3827 ts->state = state; in la_bb_end()
3838 int state = s->temps[i].state; in la_global_sync()
3839 s->temps[i].state = state | TS_MEM; in la_global_sync()
3842 la_reset_pref(&s->temps[i]); in la_global_sync()
3849 * explicitly live-across-conditional-branch, globals and local temps
3857 TCGTemp *ts = &s->temps[i]; in la_bb_sync()
3860 switch (ts->kind) { in la_bb_sync()
3862 state = ts->state; in la_bb_sync()
3863 ts->state = state | TS_MEM; in la_bb_sync()
3874 la_reset_pref(&s->temps[i]); in la_bb_sync()
3884 s->temps[i].state = TS_DEAD | TS_MEM; in la_global_kill()
3885 la_reset_pref(&s->temps[i]); in la_global_kill()
3896 TCGTemp *ts = &s->temps[i]; in la_cross_call()
3897 if (!(ts->state & TS_DEAD)) { in la_cross_call()
3904 set = tcg_target_available_regs[ts->type] & mask; in la_cross_call()
3918 void * const multiple_ebb = (void *)(uintptr_t)-1; in liveness_pass_0()
3919 int nb_temps = s->nb_temps; in liveness_pass_0()
3922 for (int i = s->nb_globals; i < nb_temps; ++i) { in liveness_pass_0()
3923 s->temps[i].state_ptr = NULL; in liveness_pass_0()
3932 ebb = QTAILQ_FIRST(&s->ops); in liveness_pass_0()
3933 QTAILQ_FOREACH(op, &s->ops, link) { in liveness_pass_0()
3937 switch (op->opc) { in liveness_pass_0()
3948 def = &tcg_op_defs[op->opc]; in liveness_pass_0()
3949 nb_oargs = def->nb_oargs; in liveness_pass_0()
3950 nb_iargs = def->nb_iargs; in liveness_pass_0()
3955 TCGTemp *ts = arg_temp(op->args[i]); in liveness_pass_0()
3957 if (ts->kind != TEMP_TB) { in liveness_pass_0()
3960 if (ts->state_ptr == NULL) { in liveness_pass_0()
3961 ts->state_ptr = ebb; in liveness_pass_0()
3962 } else if (ts->state_ptr != ebb) { in liveness_pass_0()
3963 ts->state_ptr = multiple_ebb; in liveness_pass_0()
3972 for (int i = s->nb_globals; i < nb_temps; ++i) { in liveness_pass_0()
3973 TCGTemp *ts = &s->temps[i]; in liveness_pass_0()
3974 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { in liveness_pass_0()
3975 ts->kind = TEMP_EBB; in liveness_pass_0()
3985 * carry-out operations should always be paired with carry-in. in assert_carry_dead()
3988 tcg_debug_assert(!s->carry_live); in assert_carry_dead()
3997 int nb_globals = s->nb_globals; in liveness_pass_1()
3998 int nb_temps = s->nb_temps; in liveness_pass_1()
4004 s->temps[i].state_ptr = prefs + i; in liveness_pass_1()
4010 s->carry_live = false; in liveness_pass_1()
4011 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { in liveness_pass_1()
4016 TCGOpcode opc = op->opc; in liveness_pass_1()
4033 ts = arg_temp(op->args[i]); in liveness_pass_1()
4034 if (ts->state != TS_DEAD) { in liveness_pass_1()
4044 ts = arg_temp(op->args[i]); in liveness_pass_1()
4045 if (ts->state & TS_DEAD) { in liveness_pass_1()
4048 if (ts->state & TS_MEM) { in liveness_pass_1()
4051 ts->state = TS_DEAD; in liveness_pass_1()
4055 /* Not used -- it will be tcg_target_call_oarg_reg(). */ in liveness_pass_1()
4056 memset(op->output_pref, 0, sizeof(op->output_pref)); in liveness_pass_1()
4067 ts = arg_temp(op->args[i]); in liveness_pass_1()
4068 if (ts->state & TS_DEAD) { in liveness_pass_1()
4073 /* For all live registers, remove call-clobbered prefs. */ in liveness_pass_1()
4086 for (int i = nb_iargs - 1; i >= 0; i--) { in liveness_pass_1()
4087 const TCGCallArgumentLoc *loc = &info->in[i]; in liveness_pass_1()
4088 ts = arg_temp(op->args[nb_oargs + i]); in liveness_pass_1()
4090 if (ts->state & TS_DEAD) { in liveness_pass_1()
4091 switch (loc->kind) { in liveness_pass_1()
4095 if (arg_slot_reg_p(loc->arg_slot)) { in liveness_pass_1()
4102 tcg_target_available_regs[ts->type]; in liveness_pass_1()
4105 ts->state &= ~TS_DEAD; in liveness_pass_1()
4115 const TCGCallArgumentLoc *loc = &info->in[i]; in liveness_pass_1()
4116 ts = arg_temp(op->args[nb_oargs + i]); in liveness_pass_1()
4118 switch (loc->kind) { in liveness_pass_1()
4122 if (arg_slot_reg_p(loc->arg_slot)) { in liveness_pass_1()
4124 tcg_target_call_iarg_regs[loc->arg_slot]); in liveness_pass_1()
4138 ts = arg_temp(op->args[0]); in liveness_pass_1()
4139 ts->state = TS_DEAD; in liveness_pass_1()
4152 if (arg_temp(op->args[1])->state == TS_DEAD) { in liveness_pass_1()
4153 if (arg_temp(op->args[0])->state == TS_DEAD) { in liveness_pass_1()
4158 op->opc = opc = opc_new; in liveness_pass_1()
4159 op->args[1] = op->args[2]; in liveness_pass_1()
4160 op->args[2] = op->args[3]; in liveness_pass_1()
4161 } else if (arg_temp(op->args[0])->state == TS_DEAD && in liveness_pass_1()
4164 op->opc = opc = opc_new2; in liveness_pass_1()
4165 op->args[0] = op->args[1]; in liveness_pass_1()
4166 op->args[1] = op->args[2]; in liveness_pass_1()
4167 op->args[2] = op->args[3]; in liveness_pass_1()
4171 /* Mark the single-word operation live. */ in liveness_pass_1()
4175 if (s->carry_live) { in liveness_pass_1()
4178 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4182 if (s->carry_live) { in liveness_pass_1()
4185 op->opc = opc = INDEX_op_addci; in liveness_pass_1()
4189 if (s->carry_live) { in liveness_pass_1()
4193 op->opc = opc = INDEX_op_sub; in liveness_pass_1()
4194 ts = arg_temp(op->args[2]); in liveness_pass_1()
4195 if (ts->kind == TEMP_CONST) { in liveness_pass_1()
4196 ts = tcg_constant_internal(ts->type, -ts->val); in liveness_pass_1()
4197 if (ts->state_ptr == NULL) { in liveness_pass_1()
4200 ts->state_ptr = tcg_malloc(sizeof(TCGRegSet)); in liveness_pass_1()
4201 ts->state = TS_DEAD; in liveness_pass_1()
4204 op->args[2] = temp_arg(ts); in liveness_pass_1()
4205 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4210 if (s->carry_live) { in liveness_pass_1()
4213 op->opc = opc = INDEX_op_subbi; in liveness_pass_1()
4217 if (s->carry_live) { in liveness_pass_1()
4223 op_prev->args[0] = op->args[0]; in liveness_pass_1()
4224 op_prev->args[1] = op->args[1]; in liveness_pass_1()
4225 op_prev->args[2] = op->args[2]; in liveness_pass_1()
4226 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4227 op->args[1] = op->args[0]; in liveness_pass_1()
4228 ts = arg_temp(op->args[0]); in liveness_pass_1()
4229 ts = tcg_constant_internal(ts->type, 1); in liveness_pass_1()
4230 op->args[2] = temp_arg(ts); in liveness_pass_1()
4234 if (s->carry_live) { in liveness_pass_1()
4237 /* Lower to sub, add -1. */ in liveness_pass_1()
4240 op_prev->args[0] = op->args[0]; in liveness_pass_1()
4241 op_prev->args[1] = op->args[1]; in liveness_pass_1()
4242 op_prev->args[2] = op->args[2]; in liveness_pass_1()
4243 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4244 op->args[1] = op->args[0]; in liveness_pass_1()
4245 ts = arg_temp(op->args[0]); in liveness_pass_1()
4246 ts = tcg_constant_internal(ts->type, -1); in liveness_pass_1()
4247 op->args[2] = temp_arg(ts); in liveness_pass_1()
4258 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) { in liveness_pass_1()
4259 for (int i = def->nb_oargs - 1; i >= 0; i--) { in liveness_pass_1()
4260 if (arg_temp(op->args[i])->state != TS_DEAD) { in liveness_pass_1()
4274 nb_iargs = def->nb_iargs; in liveness_pass_1()
4275 nb_oargs = def->nb_oargs; in liveness_pass_1()
4278 ts = arg_temp(op->args[i]); in liveness_pass_1()
4281 if (i < ARRAY_SIZE(op->output_pref)) { in liveness_pass_1()
4282 op->output_pref[i] = *la_temp_pref(ts); in liveness_pass_1()
4286 if (ts->state & TS_DEAD) { in liveness_pass_1()
4289 if (ts->state & TS_MEM) { in liveness_pass_1()
4292 ts->state = TS_DEAD; in liveness_pass_1()
4297 if (def->flags & TCG_OPF_BB_EXIT) { in liveness_pass_1()
4300 } else if (def->flags & TCG_OPF_COND_BRANCH) { in liveness_pass_1()
4303 } else if (def->flags & TCG_OPF_BB_END) { in liveness_pass_1()
4306 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { in liveness_pass_1()
4309 if (def->flags & TCG_OPF_CALL_CLOBBER) { in liveness_pass_1()
4316 ts = arg_temp(op->args[i]); in liveness_pass_1()
4317 if (ts->state & TS_DEAD) { in liveness_pass_1()
4321 if (def->flags & TCG_OPF_CARRY_OUT) { in liveness_pass_1()
4322 s->carry_live = false; in liveness_pass_1()
4327 ts = arg_temp(op->args[i]); in liveness_pass_1()
4328 if (ts->state & TS_DEAD) { in liveness_pass_1()
4331 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; in liveness_pass_1()
4332 ts->state &= ~TS_DEAD; in liveness_pass_1()
4335 if (def->flags & TCG_OPF_CARRY_IN) { in liveness_pass_1()
4336 s->carry_live = true; in liveness_pass_1()
4346 *la_temp_pref(arg_temp(op->args[0])) in liveness_pass_1()
4347 = *la_temp_pref(arg_temp(op->args[1])); in liveness_pass_1()
4357 ts = arg_temp(op->args[i]); in liveness_pass_1()
4361 set &= ct->regs; in liveness_pass_1()
4362 if (ct->ialias) { in liveness_pass_1()
4363 set &= output_pref(op, ct->alias_index); in liveness_pass_1()
4367 set = ct->regs; in liveness_pass_1()
4375 op->life = arg_life; in liveness_pass_1()
4384 int nb_globals = s->nb_globals; in liveness_pass_2()
4391 TCGTemp *its = &s->temps[i]; in liveness_pass_2()
4392 if (its->indirect_reg) { in liveness_pass_2()
4394 dts->type = its->type; in liveness_pass_2()
4395 dts->base_type = its->base_type; in liveness_pass_2()
4396 dts->temp_subindex = its->temp_subindex; in liveness_pass_2()
4397 dts->kind = TEMP_EBB; in liveness_pass_2()
4398 its->state_ptr = dts; in liveness_pass_2()
4400 its->state_ptr = NULL; in liveness_pass_2()
4403 its->state = TS_DEAD; in liveness_pass_2()
4405 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { in liveness_pass_2()
4406 TCGTemp *its = &s->temps[i]; in liveness_pass_2()
4407 its->state_ptr = NULL; in liveness_pass_2()
4408 its->state = TS_DEAD; in liveness_pass_2()
4411 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { in liveness_pass_2()
4412 TCGOpcode opc = op->opc; in liveness_pass_2()
4414 TCGLifeData arg_life = op->life; in liveness_pass_2()
4423 nb_iargs = def->nb_iargs; in liveness_pass_2()
4424 nb_oargs = def->nb_oargs; in liveness_pass_2()
4427 if (def->flags & TCG_OPF_COND_BRANCH) { in liveness_pass_2()
4430 } else if (def->flags & TCG_OPF_BB_END) { in liveness_pass_2()
4433 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { in liveness_pass_2()
4445 arg_ts = arg_temp(op->args[i]); in liveness_pass_2()
4446 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4447 if (dir_ts && arg_ts->state == TS_DEAD) { in liveness_pass_2()
4449 arg_ts->type, 3); in liveness_pass_2()
4451 lop->args[0] = temp_arg(dir_ts); in liveness_pass_2()
4452 lop->args[1] = temp_arg(arg_ts->mem_base); in liveness_pass_2()
4453 lop->args[2] = arg_ts->mem_offset; in liveness_pass_2()
4456 arg_ts->state = TS_MEM; in liveness_pass_2()
4464 arg_ts = arg_temp(op->args[i]); in liveness_pass_2()
4465 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4467 op->args[i] = temp_arg(dir_ts); in liveness_pass_2()
4470 arg_ts->state = TS_DEAD; in liveness_pass_2()
4483 arg_ts = &s->temps[i]; in liveness_pass_2()
4484 tcg_debug_assert(arg_ts->state_ptr == 0 in liveness_pass_2()
4485 || arg_ts->state != 0); in liveness_pass_2()
4491 arg_ts = &s->temps[i]; in liveness_pass_2()
4492 tcg_debug_assert(arg_ts->state_ptr == 0 in liveness_pass_2()
4493 || arg_ts->state == TS_DEAD); in liveness_pass_2()
4499 arg_ts = arg_temp(op->args[0]); in liveness_pass_2()
4500 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4502 op->args[0] = temp_arg(dir_ts); in liveness_pass_2()
4506 arg_ts->state = 0; in liveness_pass_2()
4510 arg_ts->type, 3); in liveness_pass_2()
4514 out_ts = arg_temp(op->args[1]); in liveness_pass_2()
4515 arg_ts->state = TS_DEAD; in liveness_pass_2()
4518 arg_ts->state = TS_MEM; in liveness_pass_2()
4521 sop->args[0] = temp_arg(out_ts); in liveness_pass_2()
4522 sop->args[1] = temp_arg(arg_ts->mem_base); in liveness_pass_2()
4523 sop->args[2] = arg_ts->mem_offset; in liveness_pass_2()
4530 arg_ts = arg_temp(op->args[i]); in liveness_pass_2()
4531 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4535 op->args[i] = temp_arg(dir_ts); in liveness_pass_2()
4539 arg_ts->state = 0; in liveness_pass_2()
4544 arg_ts->type, 3); in liveness_pass_2()
4546 sop->args[0] = temp_arg(dir_ts); in liveness_pass_2()
4547 sop->args[1] = temp_arg(arg_ts->mem_base); in liveness_pass_2()
4548 sop->args[2] = arg_ts->mem_offset; in liveness_pass_2()
4550 arg_ts->state = TS_MEM; in liveness_pass_2()
4554 arg_ts->state = TS_DEAD; in liveness_pass_2()
4569 size = tcg_type_size(ts->base_type); in temp_allocate_frame()
4570 switch (ts->base_type) { in temp_allocate_frame()
4599 off = ROUND_UP(s->current_frame_offset, align); in temp_allocate_frame()
4602 if (off + size > s->frame_end) { in temp_allocate_frame()
4605 s->current_frame_offset = off + size; in temp_allocate_frame()
4611 if (ts->base_type != ts->type) { in temp_allocate_frame()
4612 int part_size = tcg_type_size(ts->type); in temp_allocate_frame()
4619 ts -= ts->temp_subindex; in temp_allocate_frame()
4622 ts[i].mem_base = s->frame_temp; in temp_allocate_frame()
4626 ts->mem_offset = off; in temp_allocate_frame()
4627 ts->mem_base = s->frame_temp; in temp_allocate_frame()
4628 ts->mem_allocated = 1; in temp_allocate_frame()
4635 if (ts->val_type == TEMP_VAL_REG) { in set_temp_val_reg()
4636 TCGReg old = ts->reg; in set_temp_val_reg()
4637 tcg_debug_assert(s->reg_to_temp[old] == ts); in set_temp_val_reg()
4641 s->reg_to_temp[old] = NULL; in set_temp_val_reg()
4643 tcg_debug_assert(s->reg_to_temp[reg] == NULL); in set_temp_val_reg()
4644 s->reg_to_temp[reg] = ts; in set_temp_val_reg()
4645 ts->val_type = TEMP_VAL_REG; in set_temp_val_reg()
4646 ts->reg = reg; in set_temp_val_reg()
4649 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
4653 if (ts->val_type == TEMP_VAL_REG) { in set_temp_val_nonreg()
4654 TCGReg reg = ts->reg; in set_temp_val_nonreg()
4655 tcg_debug_assert(s->reg_to_temp[reg] == ts); in set_temp_val_nonreg()
4656 s->reg_to_temp[reg] = NULL; in set_temp_val_nonreg()
4658 ts->val_type = type; in set_temp_val_nonreg()
4669 switch (ts->kind) { in temp_free_or_dead()
4696 is non-zero, subsequently release the temporary; if it is positive, the
4701 if (!temp_readonly(ts) && !ts->mem_coherent) { in temp_sync()
4702 if (!ts->mem_allocated) { in temp_sync()
4705 switch (ts->val_type) { in temp_sync()
4711 && tcg_out_sti(s, ts->type, ts->val, in temp_sync()
4712 ts->mem_base->reg, ts->mem_offset)) { in temp_sync()
4715 temp_load(s, ts, tcg_target_available_regs[ts->type], in temp_sync()
4720 tcg_out_st(s, ts->type, ts->reg, in temp_sync()
4721 ts->mem_base->reg, ts->mem_offset); in temp_sync()
4731 ts->mem_coherent = 1; in temp_sync()
4741 TCGTemp *ts = s->reg_to_temp[reg]; in tcg_reg_free()
4743 temp_sync(s, ts, allocated_regs, 0, -1); in tcg_reg_free()
4782 if (s->reg_to_temp[reg] == NULL) { in tcg_reg_alloc()
4788 if (s->reg_to_temp[reg] == NULL && in tcg_reg_alloc()
4844 for (fmin = 2; fmin >= 0; fmin--) { in tcg_reg_alloc_pair()
4852 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; in tcg_reg_alloc_pair()
4872 switch (ts->val_type) { in temp_load()
4877 preferred_regs, ts->indirect_base); in temp_load()
4878 if (ts->type <= TCG_TYPE_I64) { in temp_load()
4879 tcg_out_movi(s, ts->type, reg, ts->val); in temp_load()
4881 uint64_t val = ts->val; in temp_load()
4897 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); in temp_load()
4899 ts->mem_coherent = 0; in temp_load()
4902 if (!ts->mem_allocated) { in temp_load()
4906 preferred_regs, ts->indirect_base); in temp_load()
4907 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); in temp_load()
4908 ts->mem_coherent = 1; in temp_load()
4923 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); in temp_save()
4933 for (i = 0, n = s->nb_globals; i < n; i++) { in save_globals()
4934 temp_save(s, &s->temps[i], allocated_regs); in save_globals()
4945 for (i = 0, n = s->nb_globals; i < n; i++) { in sync_globals()
4946 TCGTemp *ts = &s->temps[i]; in sync_globals()
4947 tcg_debug_assert(ts->val_type != TEMP_VAL_REG in sync_globals()
4948 || ts->kind == TEMP_FIXED in sync_globals()
4949 || ts->mem_coherent); in sync_globals()
4958 for (int i = s->nb_globals; i < s->nb_temps; i++) { in tcg_reg_alloc_bb_end()
4959 TCGTemp *ts = &s->temps[i]; in tcg_reg_alloc_bb_end()
4961 switch (ts->kind) { in tcg_reg_alloc_bb_end()
4968 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); in tcg_reg_alloc_bb_end()
4972 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); in tcg_reg_alloc_bb_end()
4984 * explicitly live-across-conditional-branch; all globals and local
4992 for (int i = s->nb_globals; i < s->nb_temps; i++) { in tcg_reg_alloc_cbranch()
4993 TCGTemp *ts = &s->temps[i]; in tcg_reg_alloc_cbranch()
4998 switch (ts->kind) { in tcg_reg_alloc_cbranch()
5000 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); in tcg_reg_alloc_cbranch()
5023 ots->val = val; in tcg_reg_alloc_do_movi()
5024 ots->mem_coherent = 0; in tcg_reg_alloc_do_movi()
5026 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); in tcg_reg_alloc_do_movi()
5037 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_mov()
5043 allocated_regs = s->reserved_regs; in tcg_reg_alloc_mov()
5045 ots = arg_temp(op->args[0]); in tcg_reg_alloc_mov()
5046 ts = arg_temp(op->args[1]); in tcg_reg_alloc_mov()
5051 /* Note that otype != itype for no-op truncation. */ in tcg_reg_alloc_mov()
5052 otype = ots->type; in tcg_reg_alloc_mov()
5053 itype = ts->type; in tcg_reg_alloc_mov()
5055 if (ts->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_mov()
5057 tcg_target_ulong val = ts->val; in tcg_reg_alloc_mov()
5069 if (ts->val_type == TEMP_VAL_MEM) { in tcg_reg_alloc_mov()
5073 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); in tcg_reg_alloc_mov()
5074 ireg = ts->reg; in tcg_reg_alloc_mov()
5077 /* mov to a non-saved dead register makes no sense (even with in tcg_reg_alloc_mov()
5078 liveness analysis disabled). */ in tcg_reg_alloc_mov()
5080 if (!ots->mem_allocated) { in tcg_reg_alloc_mov()
5083 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); in tcg_reg_alloc_mov()
5091 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { in tcg_reg_alloc_mov()
5100 if (ots->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_mov()
5101 oreg = ots->reg; in tcg_reg_alloc_mov()
5106 preferred_regs, ots->indirect_base); in tcg_reg_alloc_mov()
5115 if (!ts->mem_allocated) { in tcg_reg_alloc_mov()
5118 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); in tcg_reg_alloc_mov()
5120 ots->mem_coherent = 1; in tcg_reg_alloc_mov()
5125 ots->mem_coherent = 0; in tcg_reg_alloc_mov()
5137 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_dup()
5146 ots = arg_temp(op->args[0]); in tcg_reg_alloc_dup()
5147 its = arg_temp(op->args[1]); in tcg_reg_alloc_dup()
5152 itype = its->type; in tcg_reg_alloc_dup()
5156 if (its->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_dup()
5157 /* Propagate constant via movi -> dupi. */ in tcg_reg_alloc_dup()
5158 tcg_target_ulong val = dup_const(vece, its->val); in tcg_reg_alloc_dup()
5171 if (ots->val_type != TEMP_VAL_REG) { in tcg_reg_alloc_dup()
5172 TCGRegSet allocated_regs = s->reserved_regs; in tcg_reg_alloc_dup()
5175 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_dup()
5177 tcg_regset_set_reg(allocated_regs, its->reg); in tcg_reg_alloc_dup()
5180 output_pref(op, 0), ots->indirect_base); in tcg_reg_alloc_dup()
5184 switch (its->val_type) { in tcg_reg_alloc_dup()
5191 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { in tcg_reg_alloc_dup()
5192 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { in tcg_reg_alloc_dup()
5197 if (!its->mem_coherent) { in tcg_reg_alloc_dup()
5200 * would be required to use memory. Attempt an integer-vector in tcg_reg_alloc_dup()
5203 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { in tcg_reg_alloc_dup()
5207 temp_sync(s, its, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup()
5214 lowpart_ofs = tcg_type_size(itype) - (1 << vece); in tcg_reg_alloc_dup()
5216 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, in tcg_reg_alloc_dup()
5217 its->mem_offset + lowpart_ofs)) { in tcg_reg_alloc_dup()
5221 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); in tcg_reg_alloc_dup()
5229 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); in tcg_reg_alloc_dup()
5233 ots->mem_coherent = 0; in tcg_reg_alloc_dup()
5238 temp_sync(s, ots, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup()
5247 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_op()
5248 const TCGOpDef * const def = &tcg_op_defs[op->opc]; in tcg_reg_alloc_op()
5261 if (def->flags & TCG_OPF_CARRY_IN) { in tcg_reg_alloc_op()
5262 tcg_debug_assert(s->carry_live); in tcg_reg_alloc_op()
5265 nb_oargs = def->nb_oargs; in tcg_reg_alloc_op()
5266 nb_iargs = def->nb_iargs; in tcg_reg_alloc_op()
5270 op->args + nb_oargs + nb_iargs, in tcg_reg_alloc_op()
5271 sizeof(TCGArg) * def->nb_cargs); in tcg_reg_alloc_op()
5273 i_allocated_regs = s->reserved_regs; in tcg_reg_alloc_op()
5274 o_allocated_regs = s->reserved_regs; in tcg_reg_alloc_op()
5276 switch (op->opc) { in tcg_reg_alloc_op()
5278 op_cond = op->args[2]; in tcg_reg_alloc_op()
5283 op_cond = op->args[3]; in tcg_reg_alloc_op()
5286 op_cond = op->args[4]; in tcg_reg_alloc_op()
5291 op_cond = op->args[5]; in tcg_reg_alloc_op()
5309 arg = op->args[i]; in tcg_reg_alloc_op()
5313 if (ts->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_op()
5315 if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) { in tcg_reg_alloc_op()
5316 /* Hardware zero register: indicate register via non-const. */ in tcg_reg_alloc_op()
5323 if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type, in tcg_reg_alloc_op()
5327 new_args[i] = ts->val; in tcg_reg_alloc_op()
5332 reg = ts->reg; in tcg_reg_alloc_op()
5334 i_required_regs = arg_ct->regs; in tcg_reg_alloc_op()
5338 switch (arg_ct->pair) { in tcg_reg_alloc_op()
5340 if (arg_ct->ialias) { in tcg_reg_alloc_op()
5341 i_preferred_regs = output_pref(op, arg_ct->alias_index); in tcg_reg_alloc_op()
5350 || args_ct[arg_ct->alias_index].newreg) { in tcg_reg_alloc_op()
5352 } else if (ts->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_op()
5364 reg = ts->reg; in tcg_reg_alloc_op()
5372 temp_load(s, ts, tcg_target_available_regs[ts->type], in tcg_reg_alloc_op()
5375 i_preferred_regs, ts->indirect_base); in tcg_reg_alloc_op()
5383 i2 = arg_ct->pair_index; in tcg_reg_alloc_op()
5384 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; in tcg_reg_alloc_op()
5390 if (arg_ct->ialias) { in tcg_reg_alloc_op()
5391 i_preferred_regs = output_pref(op, arg_ct->alias_index); in tcg_reg_alloc_op()
5395 ts->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5396 ts->reg < TCG_TARGET_NB_REGS - 1 && in tcg_reg_alloc_op()
5401 ? ts2->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5402 ts2->reg == reg + 1 && in tcg_reg_alloc_op()
5404 : s->reg_to_temp[reg + 1] == NULL)) { in tcg_reg_alloc_op()
5410 if (ts->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5411 ts2->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5412 ts2->reg == reg + 1 && in tcg_reg_alloc_op()
5418 0, ts->indirect_base); in tcg_reg_alloc_op()
5422 reg = new_args[arg_ct->pair_index] + 1; in tcg_reg_alloc_op()
5426 tcg_debug_assert(arg_ct->ialias); in tcg_reg_alloc_op()
5427 i_preferred_regs = output_pref(op, arg_ct->alias_index); in tcg_reg_alloc_op()
5431 ts->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5433 s->reg_to_temp[reg - 1] == NULL && in tcg_reg_alloc_op()
5436 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { in tcg_reg_alloc_op()
5437 tcg_regset_set_reg(i_allocated_regs, reg - 1); in tcg_reg_alloc_op()
5442 ts->indirect_base); in tcg_reg_alloc_op()
5452 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { in tcg_reg_alloc_op()
5462 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { in tcg_reg_alloc_op()
5467 tcg_debug_assert(ts->kind != TEMP_FIXED); in tcg_reg_alloc_op()
5468 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], in tcg_reg_alloc_op()
5469 t_allocated_regs, 0, ts->indirect_base); in tcg_reg_alloc_op()
5470 ok = tcg_out_mov(s, ts->type, nr, reg); in tcg_reg_alloc_op()
5475 temp_load(s, ts, tcg_target_available_regs[ts->type], in tcg_reg_alloc_op()
5484 copyto_new_reg = ts->reg != reg; in tcg_reg_alloc_op()
5493 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { in tcg_reg_alloc_op()
5499 tcg_out_ld(s, ts->type, reg, in tcg_reg_alloc_op()
5500 ts->mem_base->reg, ts->mem_offset); in tcg_reg_alloc_op()
5511 temp_dead(s, arg_temp(op->args[i])); in tcg_reg_alloc_op()
5515 if (def->flags & TCG_OPF_COND_BRANCH) { in tcg_reg_alloc_op()
5517 } else if (def->flags & TCG_OPF_BB_END) { in tcg_reg_alloc_op()
5520 if (def->flags & TCG_OPF_CALL_CLOBBER) { in tcg_reg_alloc_op()
5529 if (def->flags & TCG_OPF_SIDE_EFFECTS) { in tcg_reg_alloc_op()
5538 arg = op->args[i]; in tcg_reg_alloc_op()
5545 switch (arg_ct->pair) { in tcg_reg_alloc_op()
5547 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { in tcg_reg_alloc_op()
5548 reg = new_args[arg_ct->alias_index]; in tcg_reg_alloc_op()
5549 } else if (arg_ct->newreg) { in tcg_reg_alloc_op()
5550 reg = tcg_reg_alloc(s, arg_ct->regs, in tcg_reg_alloc_op()
5552 output_pref(op, k), ts->indirect_base); in tcg_reg_alloc_op()
5554 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, in tcg_reg_alloc_op()
5555 output_pref(op, k), ts->indirect_base); in tcg_reg_alloc_op()
5560 if (arg_ct->oalias) { in tcg_reg_alloc_op()
5561 reg = new_args[arg_ct->alias_index]; in tcg_reg_alloc_op()
5562 } else if (arg_ct->newreg) { in tcg_reg_alloc_op()
5563 reg = tcg_reg_alloc_pair(s, arg_ct->regs, in tcg_reg_alloc_op()
5566 ts->indirect_base); in tcg_reg_alloc_op()
5568 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, in tcg_reg_alloc_op()
5570 ts->indirect_base); in tcg_reg_alloc_op()
5575 if (arg_ct->oalias) { in tcg_reg_alloc_op()
5576 reg = new_args[arg_ct->alias_index]; in tcg_reg_alloc_op()
5578 reg = new_args[arg_ct->pair_index] + 1; in tcg_reg_alloc_op()
5583 tcg_debug_assert(!arg_ct->newreg); in tcg_reg_alloc_op()
5584 reg = new_args[arg_ct->pair_index] - 1; in tcg_reg_alloc_op()
5592 ts->mem_coherent = 0; in tcg_reg_alloc_op()
5599 switch (op->opc) { in tcg_reg_alloc_op()
5630 container_of(all_outop[op->opc], TCGOutOpBinary, base); in tcg_reg_alloc_op()
5635 out->out_rri(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5637 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5652 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5654 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5668 container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base); in tcg_reg_alloc_op()
5672 out->out_rii(s, type, new_args[0], in tcg_reg_alloc_op()
5675 out->out_rri(s, type, new_args[0], in tcg_reg_alloc_op()
5679 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5681 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5698 container_of(all_outop[op->opc], TCGOutOpUnary, base); in tcg_reg_alloc_op()
5702 out->out_rr(s, type, new_args[0], new_args[1]); in tcg_reg_alloc_op()
5710 container_of(all_outop[op->opc], TCGOutOpBswap, base); in tcg_reg_alloc_op()
5713 out->out_rr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5723 out->out_rri(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5728 out->out_rzr(s, type, new_args[0], new_args[2], in tcg_reg_alloc_op()
5731 out->out_rrr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5741 container_of(all_outop[op->opc], TCGOutOpDivRem, base); in tcg_reg_alloc_op()
5747 out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]); in tcg_reg_alloc_op()
5755 container_of(all_outop[op->opc], TCGOutOpExtract, base); in tcg_reg_alloc_op()
5758 out->out_rr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5769 out->out_rrr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5783 container_of(all_outop[op->opc], TCGOutOpLoad, base); in tcg_reg_alloc_op()
5786 out->out(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5794 container_of(all_outop[op->opc], TCGOutOpMul2, base); in tcg_reg_alloc_op()
5798 out->out_rrrr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5812 container_of(all_outop[op->opc], TCGOutOpStore, base); in tcg_reg_alloc_op()
5815 out->out_i(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5817 out->out_r(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5826 container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base); in tcg_reg_alloc_op()
5828 out->out(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5836 container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base); in tcg_reg_alloc_op()
5838 out->out(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5851 out->out_ri(s, type, cond, new_args[0], new_args[1], label); in tcg_reg_alloc_op()
5853 out->out_rr(s, type, cond, new_args[0], new_args[1], label); in tcg_reg_alloc_op()
5864 out->out(s, type, cond, new_args[0], in tcg_reg_alloc_op()
5875 container_of(all_outop[op->opc], TCGOutOpSetcond, base); in tcg_reg_alloc_op()
5880 out->out_rri(s, type, cond, in tcg_reg_alloc_op()
5883 out->out_rrr(s, type, cond, in tcg_reg_alloc_op()
5898 out->out(s, cond, new_args[0], new_args[1], in tcg_reg_alloc_op()
5910 out->out(s, cond, new_args[0], new_args[1], new_args[2], in tcg_reg_alloc_op()
5926 tcg_debug_assert(def->flags & TCG_OPF_VECTOR); in tcg_reg_alloc_op()
5927 tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64, in tcg_reg_alloc_op()
5932 if (def->flags & TCG_OPF_CARRY_IN) { in tcg_reg_alloc_op()
5933 s->carry_live = false; in tcg_reg_alloc_op()
5935 if (def->flags & TCG_OPF_CARRY_OUT) { in tcg_reg_alloc_op()
5936 s->carry_live = true; in tcg_reg_alloc_op()
5941 ts = arg_temp(op->args[i]); in tcg_reg_alloc_op()
5956 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_dup2()
5960 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ in tcg_reg_alloc_dup2()
5964 ots = arg_temp(op->args[0]); in tcg_reg_alloc_dup2()
5965 itsl = arg_temp(op->args[1]); in tcg_reg_alloc_dup2()
5966 itsh = arg_temp(op->args[2]); in tcg_reg_alloc_dup2()
5972 if (ots->val_type != TEMP_VAL_REG) { in tcg_reg_alloc_dup2()
5973 TCGRegSet allocated_regs = s->reserved_regs; in tcg_reg_alloc_dup2()
5978 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_dup2()
5979 tcg_regset_set_reg(allocated_regs, itsl->reg); in tcg_reg_alloc_dup2()
5981 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_dup2()
5982 tcg_regset_set_reg(allocated_regs, itsh->reg); in tcg_reg_alloc_dup2()
5986 output_pref(op, 0), ots->indirect_base); in tcg_reg_alloc_dup2()
5991 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_dup2()
5992 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); in tcg_reg_alloc_dup2()
6003 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); in tcg_reg_alloc_dup2()
6007 /* If the two inputs form one 64-bit value, try dupm_vec. */ in tcg_reg_alloc_dup2()
6008 if (itsl->temp_subindex == HOST_BIG_ENDIAN && in tcg_reg_alloc_dup2()
6009 itsh->temp_subindex == !HOST_BIG_ENDIAN && in tcg_reg_alloc_dup2()
6010 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { in tcg_reg_alloc_dup2()
6011 TCGTemp *its = itsl - HOST_BIG_ENDIAN; in tcg_reg_alloc_dup2()
6013 temp_sync(s, its + 0, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup2()
6014 temp_sync(s, its + 1, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup2()
6016 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, in tcg_reg_alloc_dup2()
6017 its->mem_base->reg, its->mem_offset)) { in tcg_reg_alloc_dup2()
6026 ots->mem_coherent = 0; in tcg_reg_alloc_dup2()
6034 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); in tcg_reg_alloc_dup2()
6044 if (ts->val_type == TEMP_VAL_REG) { in load_arg_reg()
6045 if (ts->reg != reg) { in load_arg_reg()
6047 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { in load_arg_reg()
6053 tcg_out_ld(s, ts->type, reg, in load_arg_reg()
6054 ts->mem_base->reg, ts->mem_offset); in load_arg_reg()
6071 * If there are many call-saved registers, the temp might live to in load_arg_stk()
6074 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); in load_arg_stk()
6075 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, in load_arg_stk()
6082 if (arg_slot_reg_p(l->arg_slot)) { in load_arg_normal()
6083 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; in load_arg_normal()
6087 load_arg_stk(s, l->arg_slot, ts, *allocated_regs); in load_arg_normal()
6114 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_call()
6116 TCGRegSet allocated_regs = s->reserved_regs; in tcg_reg_alloc_call()
6123 for (i = nb_iargs - 1; i >= 0; --i) { in tcg_reg_alloc_call()
6124 const TCGCallArgumentLoc *loc = &info->in[i]; in tcg_reg_alloc_call()
6125 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); in tcg_reg_alloc_call()
6127 switch (loc->kind) { in tcg_reg_alloc_call()
6134 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); in tcg_reg_alloc_call()
6135 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, in tcg_reg_alloc_call()
6136 arg_slot_stk_ofs(loc->ref_slot), in tcg_reg_alloc_call()
6140 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); in tcg_reg_alloc_call()
6150 temp_dead(s, arg_temp(op->args[i])); in tcg_reg_alloc_call()
6165 if (info->flags & TCG_CALL_NO_READ_GLOBALS) { in tcg_reg_alloc_call()
6167 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { in tcg_reg_alloc_call()
6177 if (info->out_kind == TCG_CALL_RET_BY_REF) { in tcg_reg_alloc_call()
6178 TCGTemp *ts = arg_temp(op->args[0]); in tcg_reg_alloc_call()
6180 if (!ts->mem_allocated) { in tcg_reg_alloc_call()
6183 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); in tcg_reg_alloc_call()
6189 switch (info->out_kind) { in tcg_reg_alloc_call()
6192 TCGTemp *ts = arg_temp(op->args[i]); in tcg_reg_alloc_call()
6199 ts->mem_coherent = 0; in tcg_reg_alloc_call()
6205 TCGTemp *ts = arg_temp(op->args[0]); in tcg_reg_alloc_call()
6207 tcg_debug_assert(ts->base_type == TCG_TYPE_I128); in tcg_reg_alloc_call()
6208 tcg_debug_assert(ts->temp_subindex == 0); in tcg_reg_alloc_call()
6209 if (!ts->mem_allocated) { in tcg_reg_alloc_call()
6214 ts->mem_base->reg, ts->mem_offset); in tcg_reg_alloc_call()
6221 TCGTemp *ts = arg_temp(op->args[i]); in tcg_reg_alloc_call()
6222 ts->val_type = TEMP_VAL_MEM; in tcg_reg_alloc_call()
6232 TCGTemp *ts = arg_temp(op->args[i]); in tcg_reg_alloc_call()
6234 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); in tcg_reg_alloc_call()
6243 * @s: tcg context
6261 MemOp half = size ? size - 1 : 0; in atom_and_align_for_opc()
6322 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
6353 for (i = nmov; i-- > 0; ) { in tcg_out_helper_load_slots()
6366 tcg_debug_assert(parm->ntmp != 0); in tcg_out_helper_load_slots()
6367 mov[i].dst = src = parm->tmp[0]; in tcg_out_helper_load_slots()
6389 tcg_debug_assert(parm->ntmp >= 2); in tcg_out_helper_load_slots()
6399 TCGReg scratch = parm->tmp[1]; in tcg_out_helper_load_slots()
6402 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]); in tcg_out_helper_load_slots()
6414 parm->ntmp ? parm->tmp[0] : -1); in tcg_out_helper_load_slots()
6418 parm->ntmp ? parm->tmp[0] : -1); in tcg_out_helper_load_slots()
6437 tcg_debug_assert(parm->ntmp != 0); in tcg_out_helper_load_imm()
6438 tcg_out_movi(s, type, parm->tmp[0], imm); in tcg_out_helper_load_imm()
6439 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs); in tcg_out_helper_load_imm()
6455 const TCGCallArgumentLoc *loc = &info->in[0]; in tcg_out_helper_load_common_args()
6463 ptr_mov.dst = loc->arg_slot; in tcg_out_helper_load_common_args()
6470 imm = ldst->oi; in tcg_out_helper_load_common_args()
6471 loc = &info->in[next_arg]; in tcg_out_helper_load_common_args()
6473 switch (loc->kind) { in tcg_out_helper_load_common_args()
6485 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm); in tcg_out_helper_load_common_args()
6491 loc = &info->in[next_arg]; in tcg_out_helper_load_common_args()
6492 slot = loc->arg_slot; in tcg_out_helper_load_common_args()
6493 if (parm->ra_gen) { in tcg_out_helper_load_common_args()
6494 int arg_reg = -1; in tcg_out_helper_load_common_args()
6500 ra_reg = parm->ra_gen(s, ldst, arg_reg); in tcg_out_helper_load_common_args()
6506 imm = (uintptr_t)ldst->raddr; in tcg_out_helper_load_common_args()
6521 switch (loc->kind) { in tcg_out_helper_add_mov()
6537 mov[0].dst = loc->arg_slot; in tcg_out_helper_add_mov()
6575 MemOp mop = get_memop(ldst->oi); in tcg_out_ld_helper_args()
6596 loc = &info->in[next_arg]; in tcg_out_ld_helper_args()
6597 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) { in tcg_out_ld_helper_args()
6599 * 32-bit host with 32-bit guest: zero-extend the guest address in tcg_out_ld_helper_args()
6600 * to 64-bits for the helper by storing the low part, then in tcg_out_ld_helper_args()
6605 ldst->addr_reg, -1); in tcg_out_ld_helper_args()
6612 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, in tcg_out_ld_helper_args()
6613 ldst->addr_reg, -1); in tcg_out_ld_helper_args()
6618 switch (info->out_kind) { in tcg_out_ld_helper_args()
6625 * We need memory in which to return: re-use the top of stack. in tcg_out_ld_helper_args()
6634 tcg_debug_assert(parm->ntmp != 0); in tcg_out_ld_helper_args()
6635 tcg_out_addi_ptr(s, parm->tmp[0], in tcg_out_ld_helper_args()
6637 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], in tcg_out_ld_helper_args()
6653 MemOp mop = get_memop(ldst->oi); in tcg_out_ld_helper_ret()
6657 switch (ldst->type) { in tcg_out_ld_helper_ret()
6665 mov[0].dst = ldst->datalo_reg; in tcg_out_ld_helper_ret()
6667 mov[0].dst_type = ldst->type; in tcg_out_ld_helper_ret()
6681 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) { in tcg_out_ld_helper_ret()
6704 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg, in tcg_out_ld_helper_ret()
6706 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg, in tcg_out_ld_helper_ret()
6718 mov[0].dst = ldst->datalo_reg; in tcg_out_ld_helper_ret()
6725 mov[1].dst = ldst->datahi_reg; in tcg_out_ld_helper_ret()
6732 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); in tcg_out_ld_helper_ret()
6743 MemOp mop = get_memop(ldst->oi); in tcg_out_st_helper_args()
6769 loc = &info->in[next_arg]; in tcg_out_st_helper_args()
6770 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); in tcg_out_st_helper_args()
6773 * 32-bit host (and thus 32-bit guest): zero-extend the guest address in tcg_out_st_helper_args()
6774 * to 64-bits for the helper by storing the low part. Later, in tcg_out_st_helper_args()
6780 ldst->addr_reg, -1); in tcg_out_st_helper_args()
6784 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, in tcg_out_st_helper_args()
6785 ldst->addr_reg, -1); in tcg_out_st_helper_args()
6791 loc = &info->in[next_arg]; in tcg_out_st_helper_args()
6792 switch (loc->kind) { in tcg_out_st_helper_args()
6796 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, in tcg_out_st_helper_args()
6797 ldst->datalo_reg, ldst->datahi_reg); in tcg_out_st_helper_args()
6807 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg, in tcg_out_st_helper_args()
6810 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg, in tcg_out_st_helper_args()
6815 if (arg_slot_reg_p(loc->arg_slot)) { in tcg_out_st_helper_args()
6816 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot], in tcg_out_st_helper_args()
6818 arg_slot_stk_ofs(loc->ref_slot)); in tcg_out_st_helper_args()
6820 tcg_debug_assert(parm->ntmp != 0); in tcg_out_st_helper_args()
6821 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK, in tcg_out_st_helper_args()
6822 arg_slot_stk_ofs(loc->ref_slot)); in tcg_out_st_helper_args()
6823 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], in tcg_out_st_helper_args()
6824 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot)); in tcg_out_st_helper_args()
6835 loc = &info->in[1 + !HOST_BIG_ENDIAN]; in tcg_out_st_helper_args()
6836 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm); in tcg_out_st_helper_args()
6864 QSIMPLEQ_FOREACH(l, &s->labels, next) { in tcg_gen_code()
6865 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { in tcg_gen_code()
6867 "$L%d referenced but not present.\n", l->id); in tcg_gen_code()
6884 if (s->nb_indirects > 0) { in tcg_gen_code()
6898 /* If changes were made, re-run liveness. */ in tcg_gen_code()
6915 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6916 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6917 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6918 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6924 * TODO: Move this into translate-all.c with the rest of the in tcg_gen_code()
6927 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); in tcg_gen_code()
6928 s->code_ptr = s->code_buf; in tcg_gen_code()
6929 s->data_gen_ptr = NULL; in tcg_gen_code()
6931 QSIMPLEQ_INIT(&s->ldst_labels); in tcg_gen_code()
6932 s->pool_labels = NULL; in tcg_gen_code()
6934 s->gen_insn_data = in tcg_gen_code()
6935 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS); in tcg_gen_code()
6939 num_insns = -1; in tcg_gen_code()
6940 s->carry_live = false; in tcg_gen_code()
6941 QTAILQ_FOREACH(op, &s->ops, link) { in tcg_gen_code()
6942 TCGOpcode opc = op->opc; in tcg_gen_code()
6949 * e.g. zero or sign-extended, then emit as a unary op. in tcg_gen_code()
6955 TCGLifeData arg_life = op->life; in tcg_gen_code()
6972 s->gen_insn_end_off[num_insns] = off; in tcg_gen_code()
6974 assert(s->gen_insn_end_off[num_insns] == off); in tcg_gen_code()
6978 s->gen_insn_data[num_insns * INSN_START_WORDS + i] = in tcg_gen_code()
6983 temp_dead(s, arg_temp(op->args[0])); in tcg_gen_code()
6986 tcg_reg_alloc_bb_end(s, s->reserved_regs); in tcg_gen_code()
6987 tcg_out_label(s, arg_label(op->args[0])); in tcg_gen_code()
6994 tcg_out_exit_tb(s, op->args[0]); in tcg_gen_code()
6997 tcg_out_goto_tb(s, op->args[0]); in tcg_gen_code()
7000 tcg_out_br(s, arg_label(op->args[0])); in tcg_gen_code()
7003 tcg_out_mb(s, op->args[0]); in tcg_gen_code()
7025 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { in tcg_gen_code()
7026 return -1; in tcg_gen_code()
7030 return -2; in tcg_gen_code()
7035 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount); in tcg_gen_code()
7036 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); in tcg_gen_code()
7048 return -2; in tcg_gen_code()
7053 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), in tcg_gen_code()
7054 (uintptr_t)s->code_buf, in tcg_gen_code()
7055 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); in tcg_gen_code()
7068 the contents of a .debug_frame section that describes the post-
7069 prologue unwind info for the tcg machine.
7113 return p - strtab; in find_string()
7169 .e_shnum = ARRAY_SIZE(img->shdr), in tcg_register_jit_int()
7170 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, in tcg_register_jit_int()
7200 .sh_size = sizeof(img->da), in tcg_register_jit_int()
7209 .sh_size = sizeof(img->sym), in tcg_register_jit_int()
7211 .sh_link = ARRAY_SIZE(img->shdr) - 1, in tcg_register_jit_int()
7217 .sh_size = sizeof(img->str), in tcg_register_jit_int()
7227 .len = sizeof(struct DebugInfo) - 4, in tcg_register_jit_int()
7264 img->phdr.p_vaddr = buf; in tcg_register_jit_int()
7265 img->phdr.p_paddr = buf; in tcg_register_jit_int()
7266 img->phdr.p_memsz = buf_size; in tcg_register_jit_int()
7268 img->shdr[1].sh_name = find_string(img->str, ".text"); in tcg_register_jit_int()
7269 img->shdr[1].sh_addr = buf; in tcg_register_jit_int()
7270 img->shdr[1].sh_size = buf_size; in tcg_register_jit_int()
7272 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); in tcg_register_jit_int()
7273 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); in tcg_register_jit_int()
7275 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); in tcg_register_jit_int()
7276 img->shdr[4].sh_size = debug_frame_size; in tcg_register_jit_int()
7278 img->shdr[5].sh_name = find_string(img->str, ".symtab"); in tcg_register_jit_int()
7279 img->shdr[6].sh_name = find_string(img->str, ".strtab"); in tcg_register_jit_int()
7281 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); in tcg_register_jit_int()
7282 img->sym[1].st_value = buf; in tcg_register_jit_int()
7283 img->sym[1].st_size = buf_size; in tcg_register_jit_int()
7285 img->di.cu_low_pc = buf; in tcg_register_jit_int()
7286 img->di.cu_high_pc = buf + buf_size; in tcg_register_jit_int()
7287 img->di.fn_low_pc = buf; in tcg_register_jit_int()
7288 img->di.fn_high_pc = buf + buf_size; in tcg_register_jit_int()
7292 dfh->fde.func_start = buf; in tcg_register_jit_int()
7293 dfh->fde.func_len = buf_size; in tcg_register_jit_int()