1c896fe29Sbellard /*
2c896fe29Sbellard * Tiny Code Generator for QEMU
3c896fe29Sbellard *
4c896fe29Sbellard * Copyright (c) 2008 Fabrice Bellard
5c896fe29Sbellard *
6c896fe29Sbellard * Permission is hereby granted, free of charge, to any person obtaining a copy
7c896fe29Sbellard * of this software and associated documentation files (the "Software"), to deal
8c896fe29Sbellard * in the Software without restriction, including without limitation the rights
9c896fe29Sbellard * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10c896fe29Sbellard * copies of the Software, and to permit persons to whom the Software is
11c896fe29Sbellard * furnished to do so, subject to the following conditions:
12c896fe29Sbellard *
13c896fe29Sbellard * The above copyright notice and this permission notice shall be included in
14c896fe29Sbellard * all copies or substantial portions of the Software.
15c896fe29Sbellard *
16c896fe29Sbellard * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17c896fe29Sbellard * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18c896fe29Sbellard * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19c896fe29Sbellard * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20c896fe29Sbellard * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21c896fe29Sbellard * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22c896fe29Sbellard * THE SOFTWARE.
23c896fe29Sbellard */
24c896fe29Sbellard
25757e725bSPeter Maydell #include "qemu/osdep.h"
26cca82982Saurel32
27813da627SRichard Henderson /* Define to jump the ELF file used to communicate with GDB. */
28813da627SRichard Henderson #undef DEBUG_JIT
29813da627SRichard Henderson
3072fd2efbSEmilio G. Cota #include "qemu/error-report.h"
31f348b6d1SVeronia Bahaa #include "qemu/cutils.h"
321de7afc9SPaolo Bonzini #include "qemu/host-utils.h"
33d4c51a0aSMarkus Armbruster #include "qemu/qemu-print.h"
34084cfca1SRichard Henderson #include "qemu/cacheflush.h"
35ad768e6fSPeter Maydell #include "qemu/cacheinfo.h"
36533206f0SRichard W.M. Jones #include "qemu/timer.h"
37cac9b0fdSRichard Henderson #include "exec/translation-block.h"
38d0a9bb5eSRichard Henderson #include "exec/tlb-common.h"
39d7ec12f8SRichard Henderson #include "tcg/startup.h"
40ad3d0e4dSRichard Henderson #include "tcg/tcg-op-common.h"
41813da627SRichard Henderson
42edee2579SRichard Henderson #if UINTPTR_MAX == UINT32_MAX
43813da627SRichard Henderson # define ELF_CLASS ELFCLASS32
44edee2579SRichard Henderson #else
45edee2579SRichard Henderson # define ELF_CLASS ELFCLASS64
46813da627SRichard Henderson #endif
47e03b5686SMarc-André Lureau #if HOST_BIG_ENDIAN
48813da627SRichard Henderson # define ELF_DATA ELFDATA2MSB
49813da627SRichard Henderson #else
50813da627SRichard Henderson # define ELF_DATA ELFDATA2LSB
51813da627SRichard Henderson #endif
52813da627SRichard Henderson
53c896fe29Sbellard #include "elf.h"
54508127e2SPaolo Bonzini #include "exec/log.h"
55d2ba8026SRichard Henderson #include "tcg/tcg-ldst.h"
5647f7313dSRichard Henderson #include "tcg/tcg-temp-internal.h"
575ff7258cSRichard Henderson #include "tcg-internal.h"
585584e2dbSIlya Leoshkevich #include "accel/tcg/perf.h"
597d478306SRichard Henderson #ifdef CONFIG_USER_ONLY
607d478306SRichard Henderson #include "exec/user/guest-base.h"
617d478306SRichard Henderson #endif
62c896fe29Sbellard
63139c1837SPaolo Bonzini /* Forward declarations for functions declared in tcg-target.c.inc and
64ce151109SPeter Maydell used here. */
65e4d58b41SRichard Henderson static void tcg_target_init(TCGContext *s);
66e4d58b41SRichard Henderson static void tcg_target_qemu_prologue(TCGContext *s);
676ac17786SRichard Henderson static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
682ba7fae2SRichard Henderson intptr_t value, intptr_t addend);
69c896fe29Sbellard
70497a22ebSRichard Henderson /* The CIE and FDE header definitions will be common to all hosts. */
71497a22ebSRichard Henderson typedef struct {
72497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *)))));
73497a22ebSRichard Henderson uint32_t id;
74497a22ebSRichard Henderson uint8_t version;
75497a22ebSRichard Henderson char augmentation[1];
76497a22ebSRichard Henderson uint8_t code_align;
77497a22ebSRichard Henderson uint8_t data_align;
78497a22ebSRichard Henderson uint8_t return_column;
79497a22ebSRichard Henderson } DebugFrameCIE;
80497a22ebSRichard Henderson
81497a22ebSRichard Henderson typedef struct QEMU_PACKED {
82497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *)))));
83497a22ebSRichard Henderson uint32_t cie_offset;
84edee2579SRichard Henderson uintptr_t func_start;
85edee2579SRichard Henderson uintptr_t func_len;
86497a22ebSRichard Henderson } DebugFrameFDEHeader;
87497a22ebSRichard Henderson
882c90784aSRichard Henderson typedef struct QEMU_PACKED {
892c90784aSRichard Henderson DebugFrameCIE cie;
902c90784aSRichard Henderson DebugFrameFDEHeader fde;
912c90784aSRichard Henderson } DebugFrameHeader;
922c90784aSRichard Henderson
932528f771SRichard Henderson typedef struct TCGLabelQemuLdst {
942528f771SRichard Henderson bool is_ld; /* qemu_ld: true, qemu_st: false */
952528f771SRichard Henderson MemOpIdx oi;
962528f771SRichard Henderson TCGType type; /* result type of a load */
972528f771SRichard Henderson TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */
982528f771SRichard Henderson TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */
992528f771SRichard Henderson TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
1002528f771SRichard Henderson TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
1012528f771SRichard Henderson const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */
1022528f771SRichard Henderson tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
1032528f771SRichard Henderson QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
1042528f771SRichard Henderson } TCGLabelQemuLdst;
1052528f771SRichard Henderson
106755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size,
1072c90784aSRichard Henderson const void *debug_frame,
1082c90784aSRichard Henderson size_t debug_frame_size)
109813da627SRichard Henderson __attribute__((unused));
110813da627SRichard Henderson
111139c1837SPaolo Bonzini /* Forward declarations for functions declared and used in tcg-target.c.inc. */
1129358fbbfSRichard Henderson static void tcg_out_tb_start(TCGContext *s);
1132a534affSRichard Henderson static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
114a05b5b9bSRichard Henderson intptr_t arg2);
11578113e83SRichard Henderson static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
116c0ad3001SStefan Weil static void tcg_out_movi(TCGContext *s, TCGType type,
1172a534affSRichard Henderson TCGReg ret, tcg_target_long arg);
118678155b2SRichard Henderson static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
119753e42eaSRichard Henderson static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
120d0e66c89SRichard Henderson static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
121379afdffSRichard Henderson static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
12252bf3398SRichard Henderson static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
1239ecf5f61SRichard Henderson static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
1249c6aa274SRichard Henderson static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
125b9bfe000SRichard Henderson static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
126b8b94ac6SRichard Henderson static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
127313bdea8SRichard Henderson static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
128129f1f9eSRichard Henderson static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
129b55a8d9dSRichard Henderson static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
130cf7d6b8eSRichard Henderson static void tcg_out_goto_tb(TCGContext *s, int which);
1315e8892dbSMiroslav Rezanina static void tcg_out_op(TCGContext *s, TCGOpcode opc,
1325e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS],
1335e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]);
134d2fd745fSRichard Henderson #if TCG_TARGET_MAYBE_vec
135e7632cfaSRichard Henderson static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
136e7632cfaSRichard Henderson TCGReg dst, TCGReg src);
137d6ecb4a9SRichard Henderson static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
138d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset);
1394e186175SRichard Henderson static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
1404e186175SRichard Henderson TCGReg dst, int64_t arg);
1415e8892dbSMiroslav Rezanina static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
1425e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece,
1435e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS],
1445e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]);
145d2fd745fSRichard Henderson #else
tcg_out_dup_vec(TCGContext * s,TCGType type,unsigned vece,TCGReg dst,TCGReg src)146e7632cfaSRichard Henderson static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
147e7632cfaSRichard Henderson TCGReg dst, TCGReg src)
148e7632cfaSRichard Henderson {
149e7632cfaSRichard Henderson g_assert_not_reached();
150e7632cfaSRichard Henderson }
tcg_out_dupm_vec(TCGContext * s,TCGType type,unsigned vece,TCGReg dst,TCGReg base,intptr_t offset)151d6ecb4a9SRichard Henderson static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
152d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset)
153d6ecb4a9SRichard Henderson {
154d6ecb4a9SRichard Henderson g_assert_not_reached();
155d6ecb4a9SRichard Henderson }
tcg_out_dupi_vec(TCGContext * s,TCGType type,unsigned vece,TCGReg dst,int64_t arg)1564e186175SRichard Henderson static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
1574e186175SRichard Henderson TCGReg dst, int64_t arg)
158e7632cfaSRichard Henderson {
159e7632cfaSRichard Henderson g_assert_not_reached();
160e7632cfaSRichard Henderson }
tcg_out_vec_op(TCGContext * s,TCGOpcode opc,unsigned vecl,unsigned vece,const TCGArg args[TCG_MAX_OP_ARGS],const int const_args[TCG_MAX_OP_ARGS])1615e8892dbSMiroslav Rezanina static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
1625e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece,
1635e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS],
1645e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS])
165d2fd745fSRichard Henderson {
166d2fd745fSRichard Henderson g_assert_not_reached();
167d2fd745fSRichard Henderson }
168d2fd745fSRichard Henderson #endif
1692a534affSRichard Henderson static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
170a05b5b9bSRichard Henderson intptr_t arg2);
17159d7c14eSRichard Henderson static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
17259d7c14eSRichard Henderson TCGReg base, intptr_t ofs);
1737b7d8b2dSRichard Henderson static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
174cee44b03SRichard Henderson const TCGHelperInfo *info);
1755e3d0c19SRichard Henderson static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
176ebe92db2SJiajie Chen static bool tcg_target_const_match(int64_t val, TCGType type, int ct, int vece);
177659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS
178aeee05f5SRichard Henderson static int tcg_out_ldst_finalize(TCGContext *s);
179659ef5cbSRichard Henderson #endif
180c896fe29Sbellard
18123088ca0SRichard Henderson #ifndef CONFIG_USER_ONLY
18223088ca0SRichard Henderson #define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; })
18323088ca0SRichard Henderson #endif
18423088ca0SRichard Henderson
1858429a1caSRichard Henderson typedef struct TCGLdstHelperParam {
1868429a1caSRichard Henderson TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
1878429a1caSRichard Henderson unsigned ntmp;
1888429a1caSRichard Henderson int tmp[3];
1898429a1caSRichard Henderson } TCGLdstHelperParam;
1908429a1caSRichard Henderson
1918429a1caSRichard Henderson static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
1928429a1caSRichard Henderson const TCGLdstHelperParam *p)
1938429a1caSRichard Henderson __attribute__((unused));
1948429a1caSRichard Henderson static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l,
1958429a1caSRichard Henderson bool load_sign, const TCGLdstHelperParam *p)
1968429a1caSRichard Henderson __attribute__((unused));
1978429a1caSRichard Henderson static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
1988429a1caSRichard Henderson const TCGLdstHelperParam *p)
1998429a1caSRichard Henderson __attribute__((unused));
2008429a1caSRichard Henderson
201de95016dSRichard Henderson static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = {
2020cadc1edSRichard Henderson [MO_UB] = helper_ldub_mmu,
2030cadc1edSRichard Henderson [MO_SB] = helper_ldsb_mmu,
2040cadc1edSRichard Henderson [MO_UW] = helper_lduw_mmu,
2050cadc1edSRichard Henderson [MO_SW] = helper_ldsw_mmu,
2060cadc1edSRichard Henderson [MO_UL] = helper_ldul_mmu,
2070cadc1edSRichard Henderson [MO_UQ] = helper_ldq_mmu,
2080cadc1edSRichard Henderson #if TCG_TARGET_REG_BITS == 64
2090cadc1edSRichard Henderson [MO_SL] = helper_ldsl_mmu,
210ebebea53SRichard Henderson [MO_128] = helper_ld16_mmu,
2110cadc1edSRichard Henderson #endif
2120cadc1edSRichard Henderson };
2130cadc1edSRichard Henderson
214de95016dSRichard Henderson static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = {
2150cadc1edSRichard Henderson [MO_8] = helper_stb_mmu,
2160cadc1edSRichard Henderson [MO_16] = helper_stw_mmu,
2170cadc1edSRichard Henderson [MO_32] = helper_stl_mmu,
2180cadc1edSRichard Henderson [MO_64] = helper_stq_mmu,
219ebebea53SRichard Henderson #if TCG_TARGET_REG_BITS == 64
220ebebea53SRichard Henderson [MO_128] = helper_st16_mmu,
221ebebea53SRichard Henderson #endif
2220cadc1edSRichard Henderson };
2230cadc1edSRichard Henderson
224e63b8a29SRichard Henderson typedef struct {
225e63b8a29SRichard Henderson MemOp atom; /* lg2 bits of atomicity required */
226e63b8a29SRichard Henderson MemOp align; /* lg2 bits of alignment to use */
227e63b8a29SRichard Henderson } TCGAtomAlign;
228e63b8a29SRichard Henderson
229e63b8a29SRichard Henderson static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
230e63b8a29SRichard Henderson MemOp host_atom, bool allow_two_ops)
231e63b8a29SRichard Henderson __attribute__((unused));
232e63b8a29SRichard Henderson
233397cabaaSRichard Henderson #ifdef CONFIG_USER_ONLY
234397cabaaSRichard Henderson bool tcg_use_softmmu;
235397cabaaSRichard Henderson #endif
236397cabaaSRichard Henderson
23742eb6dfcSRichard Henderson TCGContext tcg_init_ctx;
23842eb6dfcSRichard Henderson __thread TCGContext *tcg_ctx;
23942eb6dfcSRichard Henderson
2405ff7258cSRichard Henderson TCGContext **tcg_ctxs;
2410e2d61cfSRichard Henderson unsigned int tcg_cur_ctxs;
2420e2d61cfSRichard Henderson unsigned int tcg_max_ctxs;
243ad75a51eSRichard Henderson TCGv_env tcg_env;
244c8bc1168SRichard Henderson const void *tcg_code_gen_epilogue;
245db0c51a3SRichard Henderson uintptr_t tcg_splitwx_diff;
246df2cce29SEmilio G. Cota
247b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER
248b91ccb31SRichard Henderson tcg_prologue_fn *tcg_qemu_tb_exec;
249b91ccb31SRichard Henderson #endif
250b91ccb31SRichard Henderson
251d2fd745fSRichard Henderson static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
252b1d8e52eSblueswir1 static TCGRegSet tcg_target_call_clobber_regs;
253c896fe29Sbellard
2541813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE == 1
tcg_out8(TCGContext * s,uint8_t v)2554196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
256c896fe29Sbellard {
257c896fe29Sbellard *s->code_ptr++ = v;
258c896fe29Sbellard }
259c896fe29Sbellard
tcg_patch8(tcg_insn_unit * p,uint8_t v)2604196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
2614196dca6SPeter Maydell uint8_t v)
2625c53bb81SPeter Maydell {
2631813e175SRichard Henderson *p = v;
2645c53bb81SPeter Maydell }
2651813e175SRichard Henderson #endif
2665c53bb81SPeter Maydell
2671813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 2
tcg_out16(TCGContext * s,uint16_t v)2684196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
269c896fe29Sbellard {
2701813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
2711813e175SRichard Henderson *s->code_ptr++ = v;
2721813e175SRichard Henderson } else {
2731813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr;
2744387345aSPeter Maydell memcpy(p, &v, sizeof(v));
2751813e175SRichard Henderson s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
2761813e175SRichard Henderson }
277c896fe29Sbellard }
278c896fe29Sbellard
tcg_patch16(tcg_insn_unit * p,uint16_t v)2794196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
2804196dca6SPeter Maydell uint16_t v)
2815c53bb81SPeter Maydell {
2821813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
2831813e175SRichard Henderson *p = v;
2841813e175SRichard Henderson } else {
2855c53bb81SPeter Maydell memcpy(p, &v, sizeof(v));
2865c53bb81SPeter Maydell }
2871813e175SRichard Henderson }
2881813e175SRichard Henderson #endif
2895c53bb81SPeter Maydell
2901813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 4
tcg_out32(TCGContext * s,uint32_t v)2914196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
292c896fe29Sbellard {
2931813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
2941813e175SRichard Henderson *s->code_ptr++ = v;
2951813e175SRichard Henderson } else {
2961813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr;
2974387345aSPeter Maydell memcpy(p, &v, sizeof(v));
2981813e175SRichard Henderson s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
2991813e175SRichard Henderson }
300c896fe29Sbellard }
301c896fe29Sbellard
tcg_patch32(tcg_insn_unit * p,uint32_t v)3024196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
3034196dca6SPeter Maydell uint32_t v)
3045c53bb81SPeter Maydell {
3051813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
3061813e175SRichard Henderson *p = v;
3071813e175SRichard Henderson } else {
3085c53bb81SPeter Maydell memcpy(p, &v, sizeof(v));
3095c53bb81SPeter Maydell }
3101813e175SRichard Henderson }
3111813e175SRichard Henderson #endif
3125c53bb81SPeter Maydell
3131813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 8
tcg_out64(TCGContext * s,uint64_t v)3144196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
315ac26eb69SRichard Henderson {
3161813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
3171813e175SRichard Henderson *s->code_ptr++ = v;
3181813e175SRichard Henderson } else {
3191813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr;
3204387345aSPeter Maydell memcpy(p, &v, sizeof(v));
3211813e175SRichard Henderson s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
3221813e175SRichard Henderson }
323ac26eb69SRichard Henderson }
324ac26eb69SRichard Henderson
tcg_patch64(tcg_insn_unit * p,uint64_t v)3254196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
3264196dca6SPeter Maydell uint64_t v)
3275c53bb81SPeter Maydell {
3281813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
3291813e175SRichard Henderson *p = v;
3301813e175SRichard Henderson } else {
3315c53bb81SPeter Maydell memcpy(p, &v, sizeof(v));
3325c53bb81SPeter Maydell }
3331813e175SRichard Henderson }
3341813e175SRichard Henderson #endif
3355c53bb81SPeter Maydell
336c896fe29Sbellard /* label relocation processing */
337c896fe29Sbellard
tcg_out_reloc(TCGContext * s,tcg_insn_unit * code_ptr,int type,TCGLabel * l,intptr_t addend)3381813e175SRichard Henderson static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
339bec16311SRichard Henderson TCGLabel *l, intptr_t addend)
340c896fe29Sbellard {
3417ecd02a0SRichard Henderson TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
342c896fe29Sbellard
343c896fe29Sbellard r->type = type;
344c896fe29Sbellard r->ptr = code_ptr;
345c896fe29Sbellard r->addend = addend;
3467ecd02a0SRichard Henderson QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
347c896fe29Sbellard }
348c896fe29Sbellard
tcg_out_label(TCGContext * s,TCGLabel * l)34992ab8e7dSRichard Henderson static void tcg_out_label(TCGContext *s, TCGLabel *l)
350c896fe29Sbellard {
351eabb7b91SAurelien Jarno tcg_debug_assert(!l->has_value);
352c896fe29Sbellard l->has_value = 1;
35392ab8e7dSRichard Henderson l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
354c896fe29Sbellard }
355c896fe29Sbellard
gen_new_label(void)35642a268c2SRichard Henderson TCGLabel *gen_new_label(void)
357c896fe29Sbellard {
358b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx;
35951e3972cSRichard Henderson TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
360c896fe29Sbellard
3617ecd02a0SRichard Henderson memset(l, 0, sizeof(TCGLabel));
3627ecd02a0SRichard Henderson l->id = s->nb_labels++;
363f85b1fc4SRichard Henderson QSIMPLEQ_INIT(&l->branches);
3647ecd02a0SRichard Henderson QSIMPLEQ_INIT(&l->relocs);
3657ecd02a0SRichard Henderson
366bef16ab4SRichard Henderson QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
36742a268c2SRichard Henderson
36842a268c2SRichard Henderson return l;
369c896fe29Sbellard }
370c896fe29Sbellard
tcg_resolve_relocs(TCGContext * s)3717ecd02a0SRichard Henderson static bool tcg_resolve_relocs(TCGContext *s)
3727ecd02a0SRichard Henderson {
3737ecd02a0SRichard Henderson TCGLabel *l;
3747ecd02a0SRichard Henderson
3757ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) {
3767ecd02a0SRichard Henderson TCGRelocation *r;
3777ecd02a0SRichard Henderson uintptr_t value = l->u.value;
3787ecd02a0SRichard Henderson
3797ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(r, &l->relocs, next) {
3807ecd02a0SRichard Henderson if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
3817ecd02a0SRichard Henderson return false;
3827ecd02a0SRichard Henderson }
3837ecd02a0SRichard Henderson }
3847ecd02a0SRichard Henderson }
3857ecd02a0SRichard Henderson return true;
3867ecd02a0SRichard Henderson }
3877ecd02a0SRichard Henderson
set_jmp_reset_offset(TCGContext * s,int which)3889f754620SRichard Henderson static void set_jmp_reset_offset(TCGContext *s, int which)
3899f754620SRichard Henderson {
390f14bed3fSRichard Henderson /*
391f14bed3fSRichard Henderson * We will check for overflow at the end of the opcode loop in
392f14bed3fSRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
393f14bed3fSRichard Henderson */
394b7e4afbdSRichard Henderson s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
3959f754620SRichard Henderson }
3969f754620SRichard Henderson
set_jmp_insn_offset(TCGContext * s,int which)397b52a2c03SRichard Henderson static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
398b52a2c03SRichard Henderson {
399b52a2c03SRichard Henderson /*
400b52a2c03SRichard Henderson * We will check for overflow at the end of the opcode loop in
401b52a2c03SRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
402b52a2c03SRichard Henderson */
4039da6079bSRichard Henderson s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
404b52a2c03SRichard Henderson }
405b52a2c03SRichard Henderson
get_jmp_target_addr(TCGContext * s,int which)406becc452aSRichard Henderson static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
407becc452aSRichard Henderson {
408becc452aSRichard Henderson /*
409becc452aSRichard Henderson * Return the read-execute version of the pointer, for the benefit
410becc452aSRichard Henderson * of any pc-relative addressing mode.
411becc452aSRichard Henderson */
4129da6079bSRichard Henderson return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
413becc452aSRichard Henderson }
414becc452aSRichard Henderson
415397cabaaSRichard Henderson static int __attribute__((unused))
tlb_mask_table_ofs(TCGContext * s,int which)416397cabaaSRichard Henderson tlb_mask_table_ofs(TCGContext *s, int which)
417d0a9bb5eSRichard Henderson {
4187857ee11SRichard Henderson return (offsetof(CPUNegativeOffsetState, tlb.f[which]) -
4197857ee11SRichard Henderson sizeof(CPUNegativeOffsetState));
420d0a9bb5eSRichard Henderson }
421d0a9bb5eSRichard Henderson
422db6b7d0cSRichard Henderson /* Signal overflow, starting over with fewer guest insns. */
4238905770bSMarc-André Lureau static G_NORETURN
tcg_raise_tb_overflow(TCGContext * s)4248905770bSMarc-André Lureau void tcg_raise_tb_overflow(TCGContext *s)
425db6b7d0cSRichard Henderson {
426db6b7d0cSRichard Henderson siglongjmp(s->jmp_trans, -2);
427db6b7d0cSRichard Henderson }
428db6b7d0cSRichard Henderson
4298429a1caSRichard Henderson /*
4308429a1caSRichard Henderson * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext.
4318429a1caSRichard Henderson * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg.
4328429a1caSRichard Henderson *
4338429a1caSRichard Henderson * However, tcg_out_helper_load_slots reuses this field to hold an
4348429a1caSRichard Henderson * argument slot number (which may designate a argument register or an
4358429a1caSRichard Henderson * argument stack slot), converting to TCGReg once all arguments that
4368429a1caSRichard Henderson * are destined for the stack are processed.
4378429a1caSRichard Henderson */
438129f1f9eSRichard Henderson typedef struct TCGMovExtend {
4398429a1caSRichard Henderson unsigned dst;
440129f1f9eSRichard Henderson TCGReg src;
441129f1f9eSRichard Henderson TCGType dst_type;
442129f1f9eSRichard Henderson TCGType src_type;
443129f1f9eSRichard Henderson MemOp src_ext;
444129f1f9eSRichard Henderson } TCGMovExtend;
445129f1f9eSRichard Henderson
446b3dfd5fcSRichard Henderson /**
447b3dfd5fcSRichard Henderson * tcg_out_movext -- move and extend
448b3dfd5fcSRichard Henderson * @s: tcg context
449b3dfd5fcSRichard Henderson * @dst_type: integral type for destination
450b3dfd5fcSRichard Henderson * @dst: destination register
451b3dfd5fcSRichard Henderson * @src_type: integral type for source
452b3dfd5fcSRichard Henderson * @src_ext: extension to apply to source
453b3dfd5fcSRichard Henderson * @src: source register
454b3dfd5fcSRichard Henderson *
455b3dfd5fcSRichard Henderson * Move or extend @src into @dst, depending on @src_ext and the types.
456b3dfd5fcSRichard Henderson */
tcg_out_movext(TCGContext * s,TCGType dst_type,TCGReg dst,TCGType src_type,MemOp src_ext,TCGReg src)457129f1f9eSRichard Henderson static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
458b3dfd5fcSRichard Henderson TCGType src_type, MemOp src_ext, TCGReg src)
459b3dfd5fcSRichard Henderson {
460b3dfd5fcSRichard Henderson switch (src_ext) {
461b3dfd5fcSRichard Henderson case MO_UB:
462b3dfd5fcSRichard Henderson tcg_out_ext8u(s, dst, src);
463b3dfd5fcSRichard Henderson break;
464b3dfd5fcSRichard Henderson case MO_SB:
465b3dfd5fcSRichard Henderson tcg_out_ext8s(s, dst_type, dst, src);
466b3dfd5fcSRichard Henderson break;
467b3dfd5fcSRichard Henderson case MO_UW:
468b3dfd5fcSRichard Henderson tcg_out_ext16u(s, dst, src);
469b3dfd5fcSRichard Henderson break;
470b3dfd5fcSRichard Henderson case MO_SW:
471b3dfd5fcSRichard Henderson tcg_out_ext16s(s, dst_type, dst, src);
472b3dfd5fcSRichard Henderson break;
473b3dfd5fcSRichard Henderson case MO_UL:
474b3dfd5fcSRichard Henderson case MO_SL:
475b3dfd5fcSRichard Henderson if (dst_type == TCG_TYPE_I32) {
476b3dfd5fcSRichard Henderson if (src_type == TCG_TYPE_I32) {
477b3dfd5fcSRichard Henderson tcg_out_mov(s, TCG_TYPE_I32, dst, src);
478b3dfd5fcSRichard Henderson } else {
479b3dfd5fcSRichard Henderson tcg_out_extrl_i64_i32(s, dst, src);
480b3dfd5fcSRichard Henderson }
481b3dfd5fcSRichard Henderson } else if (src_type == TCG_TYPE_I32) {
482b3dfd5fcSRichard Henderson if (src_ext & MO_SIGN) {
483b3dfd5fcSRichard Henderson tcg_out_exts_i32_i64(s, dst, src);
484b3dfd5fcSRichard Henderson } else {
485b3dfd5fcSRichard Henderson tcg_out_extu_i32_i64(s, dst, src);
486b3dfd5fcSRichard Henderson }
487b3dfd5fcSRichard Henderson } else {
488b3dfd5fcSRichard Henderson if (src_ext & MO_SIGN) {
489b3dfd5fcSRichard Henderson tcg_out_ext32s(s, dst, src);
490b3dfd5fcSRichard Henderson } else {
491b3dfd5fcSRichard Henderson tcg_out_ext32u(s, dst, src);
492b3dfd5fcSRichard Henderson }
493b3dfd5fcSRichard Henderson }
494b3dfd5fcSRichard Henderson break;
495b3dfd5fcSRichard Henderson case MO_UQ:
496b3dfd5fcSRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
497b3dfd5fcSRichard Henderson if (dst_type == TCG_TYPE_I32) {
498b3dfd5fcSRichard Henderson tcg_out_extrl_i64_i32(s, dst, src);
499b3dfd5fcSRichard Henderson } else {
500b3dfd5fcSRichard Henderson tcg_out_mov(s, TCG_TYPE_I64, dst, src);
501b3dfd5fcSRichard Henderson }
502b3dfd5fcSRichard Henderson break;
503b3dfd5fcSRichard Henderson default:
504b3dfd5fcSRichard Henderson g_assert_not_reached();
505b3dfd5fcSRichard Henderson }
506b3dfd5fcSRichard Henderson }
507b3dfd5fcSRichard Henderson
508129f1f9eSRichard Henderson /* Minor variations on a theme, using a structure. */
tcg_out_movext1_new_src(TCGContext * s,const TCGMovExtend * i,TCGReg src)509129f1f9eSRichard Henderson static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i,
510129f1f9eSRichard Henderson TCGReg src)
511129f1f9eSRichard Henderson {
512129f1f9eSRichard Henderson tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
513129f1f9eSRichard Henderson }
514129f1f9eSRichard Henderson
tcg_out_movext1(TCGContext * s,const TCGMovExtend * i)515129f1f9eSRichard Henderson static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i)
516129f1f9eSRichard Henderson {
517129f1f9eSRichard Henderson tcg_out_movext1_new_src(s, i, i->src);
518129f1f9eSRichard Henderson }
519129f1f9eSRichard Henderson
520129f1f9eSRichard Henderson /**
521129f1f9eSRichard Henderson * tcg_out_movext2 -- move and extend two pair
522129f1f9eSRichard Henderson * @s: tcg context
523129f1f9eSRichard Henderson * @i1: first move description
524129f1f9eSRichard Henderson * @i2: second move description
525129f1f9eSRichard Henderson * @scratch: temporary register, or -1 for none
526129f1f9eSRichard Henderson *
527129f1f9eSRichard Henderson * As tcg_out_movext, for both @i1 and @i2, caring for overlap
528129f1f9eSRichard Henderson * between the sources and destinations.
529129f1f9eSRichard Henderson */
530129f1f9eSRichard Henderson
tcg_out_movext2(TCGContext * s,const TCGMovExtend * i1,const TCGMovExtend * i2,int scratch)5318429a1caSRichard Henderson static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1,
532129f1f9eSRichard Henderson const TCGMovExtend *i2, int scratch)
533129f1f9eSRichard Henderson {
534129f1f9eSRichard Henderson TCGReg src1 = i1->src;
535129f1f9eSRichard Henderson TCGReg src2 = i2->src;
536129f1f9eSRichard Henderson
537129f1f9eSRichard Henderson if (i1->dst != src2) {
538129f1f9eSRichard Henderson tcg_out_movext1(s, i1);
539129f1f9eSRichard Henderson tcg_out_movext1(s, i2);
540129f1f9eSRichard Henderson return;
541129f1f9eSRichard Henderson }
542129f1f9eSRichard Henderson if (i2->dst == src1) {
543129f1f9eSRichard Henderson TCGType src1_type = i1->src_type;
544129f1f9eSRichard Henderson TCGType src2_type = i2->src_type;
545129f1f9eSRichard Henderson
546129f1f9eSRichard Henderson if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
547129f1f9eSRichard Henderson /* The data is now in the correct registers, now extend. */
548129f1f9eSRichard Henderson src1 = i2->src;
549129f1f9eSRichard Henderson src2 = i1->src;
550129f1f9eSRichard Henderson } else {
551129f1f9eSRichard Henderson tcg_debug_assert(scratch >= 0);
552129f1f9eSRichard Henderson tcg_out_mov(s, src1_type, scratch, src1);
553129f1f9eSRichard Henderson src1 = scratch;
554129f1f9eSRichard Henderson }
555129f1f9eSRichard Henderson }
556129f1f9eSRichard Henderson tcg_out_movext1_new_src(s, i2, src2);
557129f1f9eSRichard Henderson tcg_out_movext1_new_src(s, i1, src1);
558129f1f9eSRichard Henderson }
559129f1f9eSRichard Henderson
5602462e30eSRichard Henderson /**
5612462e30eSRichard Henderson * tcg_out_movext3 -- move and extend three pair
5622462e30eSRichard Henderson * @s: tcg context
5632462e30eSRichard Henderson * @i1: first move description
5642462e30eSRichard Henderson * @i2: second move description
5652462e30eSRichard Henderson * @i3: third move description
5662462e30eSRichard Henderson * @scratch: temporary register, or -1 for none
5672462e30eSRichard Henderson *
5682462e30eSRichard Henderson * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap
5692462e30eSRichard Henderson * between the sources and destinations.
5702462e30eSRichard Henderson */
5712462e30eSRichard Henderson
tcg_out_movext3(TCGContext * s,const TCGMovExtend * i1,const TCGMovExtend * i2,const TCGMovExtend * i3,int scratch)5722462e30eSRichard Henderson static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1,
5732462e30eSRichard Henderson const TCGMovExtend *i2, const TCGMovExtend *i3,
5742462e30eSRichard Henderson int scratch)
5752462e30eSRichard Henderson {
5762462e30eSRichard Henderson TCGReg src1 = i1->src;
5772462e30eSRichard Henderson TCGReg src2 = i2->src;
5782462e30eSRichard Henderson TCGReg src3 = i3->src;
5792462e30eSRichard Henderson
5802462e30eSRichard Henderson if (i1->dst != src2 && i1->dst != src3) {
5812462e30eSRichard Henderson tcg_out_movext1(s, i1);
5822462e30eSRichard Henderson tcg_out_movext2(s, i2, i3, scratch);
5832462e30eSRichard Henderson return;
5842462e30eSRichard Henderson }
5852462e30eSRichard Henderson if (i2->dst != src1 && i2->dst != src3) {
5862462e30eSRichard Henderson tcg_out_movext1(s, i2);
5872462e30eSRichard Henderson tcg_out_movext2(s, i1, i3, scratch);
5882462e30eSRichard Henderson return;
5892462e30eSRichard Henderson }
5902462e30eSRichard Henderson if (i3->dst != src1 && i3->dst != src2) {
5912462e30eSRichard Henderson tcg_out_movext1(s, i3);
5922462e30eSRichard Henderson tcg_out_movext2(s, i1, i2, scratch);
5932462e30eSRichard Henderson return;
5942462e30eSRichard Henderson }
5952462e30eSRichard Henderson
5962462e30eSRichard Henderson /*
5972462e30eSRichard Henderson * There is a cycle. Since there are only 3 nodes, the cycle is
5982462e30eSRichard Henderson * either "clockwise" or "anti-clockwise", and can be solved with
5992462e30eSRichard Henderson * a single scratch or two xchg.
6002462e30eSRichard Henderson */
6012462e30eSRichard Henderson if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
6022462e30eSRichard Henderson /* "Clockwise" */
6032462e30eSRichard Henderson if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
6042462e30eSRichard Henderson tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
6052462e30eSRichard Henderson /* The data is now in the correct registers, now extend. */
6062462e30eSRichard Henderson tcg_out_movext1_new_src(s, i1, i1->dst);
6072462e30eSRichard Henderson tcg_out_movext1_new_src(s, i2, i2->dst);
6082462e30eSRichard Henderson tcg_out_movext1_new_src(s, i3, i3->dst);
6092462e30eSRichard Henderson } else {
6102462e30eSRichard Henderson tcg_debug_assert(scratch >= 0);
6112462e30eSRichard Henderson tcg_out_mov(s, i1->src_type, scratch, src1);
6122462e30eSRichard Henderson tcg_out_movext1(s, i3);
6132462e30eSRichard Henderson tcg_out_movext1(s, i2);
6142462e30eSRichard Henderson tcg_out_movext1_new_src(s, i1, scratch);
6152462e30eSRichard Henderson }
6162462e30eSRichard Henderson } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
6172462e30eSRichard Henderson /* "Anti-clockwise" */
6182462e30eSRichard Henderson if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
6192462e30eSRichard Henderson tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
6202462e30eSRichard Henderson /* The data is now in the correct registers, now extend. */
6212462e30eSRichard Henderson tcg_out_movext1_new_src(s, i1, i1->dst);
6222462e30eSRichard Henderson tcg_out_movext1_new_src(s, i2, i2->dst);
6232462e30eSRichard Henderson tcg_out_movext1_new_src(s, i3, i3->dst);
6242462e30eSRichard Henderson } else {
6252462e30eSRichard Henderson tcg_debug_assert(scratch >= 0);
6262462e30eSRichard Henderson tcg_out_mov(s, i1->src_type, scratch, src1);
6272462e30eSRichard Henderson tcg_out_movext1(s, i2);
6282462e30eSRichard Henderson tcg_out_movext1(s, i3);
6292462e30eSRichard Henderson tcg_out_movext1_new_src(s, i1, scratch);
6302462e30eSRichard Henderson }
6312462e30eSRichard Henderson } else {
6322462e30eSRichard Henderson g_assert_not_reached();
6332462e30eSRichard Henderson }
6342462e30eSRichard Henderson }
6352462e30eSRichard Henderson
6364c22e840SRichard Henderson #define C_PFX1(P, A) P##A
6374c22e840SRichard Henderson #define C_PFX2(P, A, B) P##A##_##B
6384c22e840SRichard Henderson #define C_PFX3(P, A, B, C) P##A##_##B##_##C
6394c22e840SRichard Henderson #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
6404c22e840SRichard Henderson #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
6414c22e840SRichard Henderson #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
6424c22e840SRichard Henderson
6434c22e840SRichard Henderson /* Define an enumeration for the various combinations. */
6444c22e840SRichard Henderson
6454c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
6464c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
6474c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
6484c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
6494c22e840SRichard Henderson
6504c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
6514c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
6524c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
6534c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
6544c22e840SRichard Henderson
6554c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
656*b8819108SRichard Henderson #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1),
657fa645b48SRichard Henderson #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1),
6584c22e840SRichard Henderson
6594c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
6604c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
6614c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
6624c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
66322d2e535SIlya Leoshkevich #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
6644c22e840SRichard Henderson
6654c22e840SRichard Henderson typedef enum {
6664c22e840SRichard Henderson #include "tcg-target-con-set.h"
6674c22e840SRichard Henderson } TCGConstraintSetIndex;
6684c22e840SRichard Henderson
6694c22e840SRichard Henderson static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
6704c22e840SRichard Henderson
6714c22e840SRichard Henderson #undef C_O0_I1
6724c22e840SRichard Henderson #undef C_O0_I2
6734c22e840SRichard Henderson #undef C_O0_I3
6744c22e840SRichard Henderson #undef C_O0_I4
6754c22e840SRichard Henderson #undef C_O1_I1
6764c22e840SRichard Henderson #undef C_O1_I2
6774c22e840SRichard Henderson #undef C_O1_I3
6784c22e840SRichard Henderson #undef C_O1_I4
6794c22e840SRichard Henderson #undef C_N1_I2
680*b8819108SRichard Henderson #undef C_N1O1_I1
681fa645b48SRichard Henderson #undef C_N2_I1
6824c22e840SRichard Henderson #undef C_O2_I1
6834c22e840SRichard Henderson #undef C_O2_I2
6844c22e840SRichard Henderson #undef C_O2_I3
6854c22e840SRichard Henderson #undef C_O2_I4
68622d2e535SIlya Leoshkevich #undef C_N1_O1_I4
6874c22e840SRichard Henderson
6884c22e840SRichard Henderson /* Put all of the constraint sets into an array, indexed by the enum. */
6894c22e840SRichard Henderson
6904c22e840SRichard Henderson #define C_O0_I1(I1) { .args_ct_str = { #I1 } },
6914c22e840SRichard Henderson #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
6924c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
6934c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
6944c22e840SRichard Henderson
6954c22e840SRichard Henderson #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
6964c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
6974c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
6984c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
6994c22e840SRichard Henderson
7004c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
701*b8819108SRichard Henderson #define C_N1O1_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, #O2, #I1 } },
702fa645b48SRichard Henderson #define C_N2_I1(O1, O2, I1) { .args_ct_str = { "&" #O1, "&" #O2, #I1 } },
7034c22e840SRichard Henderson
7044c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
7054c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
7064c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
7074c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
70822d2e535SIlya Leoshkevich #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
7094c22e840SRichard Henderson
7104c22e840SRichard Henderson static const TCGTargetOpDef constraint_sets[] = {
7114c22e840SRichard Henderson #include "tcg-target-con-set.h"
7124c22e840SRichard Henderson };
7134c22e840SRichard Henderson
7144c22e840SRichard Henderson
7154c22e840SRichard Henderson #undef C_O0_I1
7164c22e840SRichard Henderson #undef C_O0_I2
7174c22e840SRichard Henderson #undef C_O0_I3
7184c22e840SRichard Henderson #undef C_O0_I4
7194c22e840SRichard Henderson #undef C_O1_I1
7204c22e840SRichard Henderson #undef C_O1_I2
7214c22e840SRichard Henderson #undef C_O1_I3
7224c22e840SRichard Henderson #undef C_O1_I4
7234c22e840SRichard Henderson #undef C_N1_I2
724*b8819108SRichard Henderson #undef C_N1O1_I1
725fa645b48SRichard Henderson #undef C_N2_I1
7264c22e840SRichard Henderson #undef C_O2_I1
7274c22e840SRichard Henderson #undef C_O2_I2
7284c22e840SRichard Henderson #undef C_O2_I3
7294c22e840SRichard Henderson #undef C_O2_I4
73022d2e535SIlya Leoshkevich #undef C_N1_O1_I4
7314c22e840SRichard Henderson
7324c22e840SRichard Henderson /* Expand the enumerator to be returned from tcg_target_op_def(). */
7334c22e840SRichard Henderson
7344c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
7354c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
7364c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
7374c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
7384c22e840SRichard Henderson
7394c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
7404c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
7414c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
7424c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
7434c22e840SRichard Henderson
7444c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
745*b8819108SRichard Henderson #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1)
746fa645b48SRichard Henderson #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1)
7474c22e840SRichard Henderson
7484c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
7494c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
7504c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
7514c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
75222d2e535SIlya Leoshkevich #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
7534c22e840SRichard Henderson
754139c1837SPaolo Bonzini #include "tcg-target.c.inc"
755c896fe29Sbellard
7567857ee11SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER
7577857ee11SRichard Henderson /* Validate CPUTLBDescFast placement. */
7587857ee11SRichard Henderson QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
7597857ee11SRichard Henderson sizeof(CPUNegativeOffsetState))
7607857ee11SRichard Henderson < MIN_TLB_MASK_TABLE_OFS);
7617857ee11SRichard Henderson #endif
7627857ee11SRichard Henderson
alloc_tcg_plugin_context(TCGContext * s)76338b47b19SEmilio G. Cota static void alloc_tcg_plugin_context(TCGContext *s)
76438b47b19SEmilio G. Cota {
76538b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN
76638b47b19SEmilio G. Cota s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
76738b47b19SEmilio G. Cota s->plugin_tb->insns =
76838b47b19SEmilio G. Cota g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
76938b47b19SEmilio G. Cota #endif
77038b47b19SEmilio G. Cota }
77138b47b19SEmilio G. Cota
772e8feb96fSEmilio G. Cota /*
7733468b59eSEmilio G. Cota * All TCG threads except the parent (i.e. the one that called tcg_context_init
7743468b59eSEmilio G. Cota * and registered the target's TCG globals) must register with this function
7753468b59eSEmilio G. Cota * before initiating translation.
7763468b59eSEmilio G. Cota *
7773468b59eSEmilio G. Cota * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
7783468b59eSEmilio G. Cota * of tcg_region_init() for the reasoning behind this.
7793468b59eSEmilio G. Cota *
7807893e42dSPhilippe Mathieu-Daudé * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
7817893e42dSPhilippe Mathieu-Daudé * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
7823468b59eSEmilio G. Cota * is not used anymore for translation once this function is called.
7833468b59eSEmilio G. Cota *
7847893e42dSPhilippe Mathieu-Daudé * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
7857893e42dSPhilippe Mathieu-Daudé * iterates over the array (e.g. tcg_code_size() the same for both system/user
7867893e42dSPhilippe Mathieu-Daudé * modes.
7873468b59eSEmilio G. Cota */
7883468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY
tcg_register_thread(void)7893468b59eSEmilio G. Cota void tcg_register_thread(void)
7903468b59eSEmilio G. Cota {
7913468b59eSEmilio G. Cota tcg_ctx = &tcg_init_ctx;
7923468b59eSEmilio G. Cota }
7933468b59eSEmilio G. Cota #else
tcg_register_thread(void)7943468b59eSEmilio G. Cota void tcg_register_thread(void)
7953468b59eSEmilio G. Cota {
7963468b59eSEmilio G. Cota TCGContext *s = g_malloc(sizeof(*s));
7973468b59eSEmilio G. Cota unsigned int i, n;
7983468b59eSEmilio G. Cota
7993468b59eSEmilio G. Cota *s = tcg_init_ctx;
8003468b59eSEmilio G. Cota
8013468b59eSEmilio G. Cota /* Relink mem_base. */
8023468b59eSEmilio G. Cota for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
8033468b59eSEmilio G. Cota if (tcg_init_ctx.temps[i].mem_base) {
8043468b59eSEmilio G. Cota ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
8053468b59eSEmilio G. Cota tcg_debug_assert(b >= 0 && b < n);
8063468b59eSEmilio G. Cota s->temps[i].mem_base = &s->temps[b];
8073468b59eSEmilio G. Cota }
8083468b59eSEmilio G. Cota }
8093468b59eSEmilio G. Cota
8103468b59eSEmilio G. Cota /* Claim an entry in tcg_ctxs */
8110e2d61cfSRichard Henderson n = qatomic_fetch_inc(&tcg_cur_ctxs);
8120e2d61cfSRichard Henderson g_assert(n < tcg_max_ctxs);
813d73415a3SStefan Hajnoczi qatomic_set(&tcg_ctxs[n], s);
8143468b59eSEmilio G. Cota
81538b47b19SEmilio G. Cota if (n > 0) {
81638b47b19SEmilio G. Cota alloc_tcg_plugin_context(s);
817bf042e8eSRichard Henderson tcg_region_initial_alloc(s);
81838b47b19SEmilio G. Cota }
81938b47b19SEmilio G. Cota
8203468b59eSEmilio G. Cota tcg_ctx = s;
8213468b59eSEmilio G. Cota }
8223468b59eSEmilio G. Cota #endif /* !CONFIG_USER_ONLY */
8233468b59eSEmilio G. Cota
824c896fe29Sbellard /* pool based memory allocation */
tcg_malloc_internal(TCGContext * s,int size)825c896fe29Sbellard void *tcg_malloc_internal(TCGContext *s, int size)
826c896fe29Sbellard {
827c896fe29Sbellard TCGPool *p;
828c896fe29Sbellard int pool_size;
829c896fe29Sbellard
830c896fe29Sbellard if (size > TCG_POOL_CHUNK_SIZE) {
831c896fe29Sbellard /* big malloc: insert a new pool (XXX: could optimize) */
8327267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + size);
833c896fe29Sbellard p->size = size;
8344055299eSKirill Batuzov p->next = s->pool_first_large;
8354055299eSKirill Batuzov s->pool_first_large = p;
8364055299eSKirill Batuzov return p->data;
837c896fe29Sbellard } else {
838c896fe29Sbellard p = s->pool_current;
839c896fe29Sbellard if (!p) {
840c896fe29Sbellard p = s->pool_first;
841c896fe29Sbellard if (!p)
842c896fe29Sbellard goto new_pool;
843c896fe29Sbellard } else {
844c896fe29Sbellard if (!p->next) {
845c896fe29Sbellard new_pool:
846c896fe29Sbellard pool_size = TCG_POOL_CHUNK_SIZE;
8477267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + pool_size);
848c896fe29Sbellard p->size = pool_size;
849c896fe29Sbellard p->next = NULL;
850a813e36fSRichard Henderson if (s->pool_current) {
851c896fe29Sbellard s->pool_current->next = p;
852a813e36fSRichard Henderson } else {
853c896fe29Sbellard s->pool_first = p;
854a813e36fSRichard Henderson }
855c896fe29Sbellard } else {
856c896fe29Sbellard p = p->next;
857c896fe29Sbellard }
858c896fe29Sbellard }
859c896fe29Sbellard }
860c896fe29Sbellard s->pool_current = p;
861c896fe29Sbellard s->pool_cur = p->data + size;
862c896fe29Sbellard s->pool_end = p->data + p->size;
863c896fe29Sbellard return p->data;
864c896fe29Sbellard }
865c896fe29Sbellard
tcg_pool_reset(TCGContext * s)866c896fe29Sbellard void tcg_pool_reset(TCGContext *s)
867c896fe29Sbellard {
8684055299eSKirill Batuzov TCGPool *p, *t;
8694055299eSKirill Batuzov for (p = s->pool_first_large; p; p = t) {
8704055299eSKirill Batuzov t = p->next;
8714055299eSKirill Batuzov g_free(p);
8724055299eSKirill Batuzov }
8734055299eSKirill Batuzov s->pool_first_large = NULL;
874c896fe29Sbellard s->pool_cur = s->pool_end = NULL;
875c896fe29Sbellard s->pool_current = NULL;
876c896fe29Sbellard }
877c896fe29Sbellard
8788429a1caSRichard Henderson /*
8798429a1caSRichard Henderson * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
8808429a1caSRichard Henderson * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
8818429a1caSRichard Henderson * We only use these for layout in tcg_out_ld_helper_ret and
8828429a1caSRichard Henderson * tcg_out_st_helper_args, and share them between several of
8838429a1caSRichard Henderson * the helpers, with the end result that it's easier to build manually.
8848429a1caSRichard Henderson */
8858429a1caSRichard Henderson
8868429a1caSRichard Henderson #if TCG_TARGET_REG_BITS == 32
8878429a1caSRichard Henderson # define dh_typecode_ttl dh_typecode_i32
8888429a1caSRichard Henderson #else
8898429a1caSRichard Henderson # define dh_typecode_ttl dh_typecode_i64
8908429a1caSRichard Henderson #endif
8918429a1caSRichard Henderson
8928429a1caSRichard Henderson static TCGHelperInfo info_helper_ld32_mmu = {
8938429a1caSRichard Henderson .flags = TCG_CALL_NO_WG,
8948429a1caSRichard Henderson .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */
8958429a1caSRichard Henderson | dh_typemask(env, 1)
89624e46e6cSRichard Henderson | dh_typemask(i64, 2) /* uint64_t addr */
8978429a1caSRichard Henderson | dh_typemask(i32, 3) /* unsigned oi */
8988429a1caSRichard Henderson | dh_typemask(ptr, 4) /* uintptr_t ra */
8998429a1caSRichard Henderson };
9008429a1caSRichard Henderson
9018429a1caSRichard Henderson static TCGHelperInfo info_helper_ld64_mmu = {
9028429a1caSRichard Henderson .flags = TCG_CALL_NO_WG,
9038429a1caSRichard Henderson .typemask = dh_typemask(i64, 0) /* return uint64_t */
9048429a1caSRichard Henderson | dh_typemask(env, 1)
90524e46e6cSRichard Henderson | dh_typemask(i64, 2) /* uint64_t addr */
9068429a1caSRichard Henderson | dh_typemask(i32, 3) /* unsigned oi */
9078429a1caSRichard Henderson | dh_typemask(ptr, 4) /* uintptr_t ra */
9088429a1caSRichard Henderson };
9098429a1caSRichard Henderson
910ebebea53SRichard Henderson static TCGHelperInfo info_helper_ld128_mmu = {
911ebebea53SRichard Henderson .flags = TCG_CALL_NO_WG,
912ebebea53SRichard Henderson .typemask = dh_typemask(i128, 0) /* return Int128 */
913ebebea53SRichard Henderson | dh_typemask(env, 1)
91424e46e6cSRichard Henderson | dh_typemask(i64, 2) /* uint64_t addr */
915ebebea53SRichard Henderson | dh_typemask(i32, 3) /* unsigned oi */
916ebebea53SRichard Henderson | dh_typemask(ptr, 4) /* uintptr_t ra */
917ebebea53SRichard Henderson };
918ebebea53SRichard Henderson
9198429a1caSRichard Henderson static TCGHelperInfo info_helper_st32_mmu = {
9208429a1caSRichard Henderson .flags = TCG_CALL_NO_WG,
9218429a1caSRichard Henderson .typemask = dh_typemask(void, 0)
9228429a1caSRichard Henderson | dh_typemask(env, 1)
92324e46e6cSRichard Henderson | dh_typemask(i64, 2) /* uint64_t addr */
9248429a1caSRichard Henderson | dh_typemask(i32, 3) /* uint32_t data */
9258429a1caSRichard Henderson | dh_typemask(i32, 4) /* unsigned oi */
9268429a1caSRichard Henderson | dh_typemask(ptr, 5) /* uintptr_t ra */
9278429a1caSRichard Henderson };
9288429a1caSRichard Henderson
9298429a1caSRichard Henderson static TCGHelperInfo info_helper_st64_mmu = {
9308429a1caSRichard Henderson .flags = TCG_CALL_NO_WG,
9318429a1caSRichard Henderson .typemask = dh_typemask(void, 0)
9328429a1caSRichard Henderson | dh_typemask(env, 1)
93324e46e6cSRichard Henderson | dh_typemask(i64, 2) /* uint64_t addr */
9348429a1caSRichard Henderson | dh_typemask(i64, 3) /* uint64_t data */
9358429a1caSRichard Henderson | dh_typemask(i32, 4) /* unsigned oi */
9368429a1caSRichard Henderson | dh_typemask(ptr, 5) /* uintptr_t ra */
9378429a1caSRichard Henderson };
9388429a1caSRichard Henderson
939ebebea53SRichard Henderson static TCGHelperInfo info_helper_st128_mmu = {
940ebebea53SRichard Henderson .flags = TCG_CALL_NO_WG,
941ebebea53SRichard Henderson .typemask = dh_typemask(void, 0)
942ebebea53SRichard Henderson | dh_typemask(env, 1)
94324e46e6cSRichard Henderson | dh_typemask(i64, 2) /* uint64_t addr */
944ebebea53SRichard Henderson | dh_typemask(i128, 3) /* Int128 data */
945ebebea53SRichard Henderson | dh_typemask(i32, 4) /* unsigned oi */
946ebebea53SRichard Henderson | dh_typemask(ptr, 5) /* uintptr_t ra */
947ebebea53SRichard Henderson };
948ebebea53SRichard Henderson
94922f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER
typecode_to_ffi(int argmask)950c6ef8c7bSPhilippe Mathieu-Daudé static ffi_type *typecode_to_ffi(int argmask)
951c6ef8c7bSPhilippe Mathieu-Daudé {
952e9709e17SRichard Henderson /*
953e9709e17SRichard Henderson * libffi does not support __int128_t, so we have forced Int128
954e9709e17SRichard Henderson * to use the structure definition instead of the builtin type.
955e9709e17SRichard Henderson */
956e9709e17SRichard Henderson static ffi_type *ffi_type_i128_elements[3] = {
957e9709e17SRichard Henderson &ffi_type_uint64,
958e9709e17SRichard Henderson &ffi_type_uint64,
959e9709e17SRichard Henderson NULL
960e9709e17SRichard Henderson };
961e9709e17SRichard Henderson static ffi_type ffi_type_i128 = {
962e9709e17SRichard Henderson .size = 16,
963e9709e17SRichard Henderson .alignment = __alignof__(Int128),
964e9709e17SRichard Henderson .type = FFI_TYPE_STRUCT,
965e9709e17SRichard Henderson .elements = ffi_type_i128_elements,
966e9709e17SRichard Henderson };
967e9709e17SRichard Henderson
968c6ef8c7bSPhilippe Mathieu-Daudé switch (argmask) {
969c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_void:
970c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_void;
971c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i32:
972c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint32;
973c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s32:
974c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint32;
975c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i64:
976c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint64;
977c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s64:
978c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint64;
979c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_ptr:
980c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_pointer;
981e9709e17SRichard Henderson case dh_typecode_i128:
982e9709e17SRichard Henderson return &ffi_type_i128;
983c6ef8c7bSPhilippe Mathieu-Daudé }
984c6ef8c7bSPhilippe Mathieu-Daudé g_assert_not_reached();
985c6ef8c7bSPhilippe Mathieu-Daudé }
9860c22e176SPhilippe Mathieu-Daudé
init_ffi_layout(TCGHelperInfo * info)987d53106c9SRichard Henderson static ffi_cif *init_ffi_layout(TCGHelperInfo *info)
9880c22e176SPhilippe Mathieu-Daudé {
989f9c4bb80SRichard Henderson unsigned typemask = info->typemask;
9900c22e176SPhilippe Mathieu-Daudé struct {
9910c22e176SPhilippe Mathieu-Daudé ffi_cif cif;
9920c22e176SPhilippe Mathieu-Daudé ffi_type *args[];
9930c22e176SPhilippe Mathieu-Daudé } *ca;
9940c22e176SPhilippe Mathieu-Daudé ffi_status status;
9950c22e176SPhilippe Mathieu-Daudé int nargs;
9960c22e176SPhilippe Mathieu-Daudé
9970c22e176SPhilippe Mathieu-Daudé /* Ignoring the return type, find the last non-zero field. */
9980c22e176SPhilippe Mathieu-Daudé nargs = 32 - clz32(typemask >> 3);
9990c22e176SPhilippe Mathieu-Daudé nargs = DIV_ROUND_UP(nargs, 3);
1000e9709e17SRichard Henderson assert(nargs <= MAX_CALL_IARGS);
10010c22e176SPhilippe Mathieu-Daudé
10020c22e176SPhilippe Mathieu-Daudé ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
10030c22e176SPhilippe Mathieu-Daudé ca->cif.rtype = typecode_to_ffi(typemask & 7);
10040c22e176SPhilippe Mathieu-Daudé ca->cif.nargs = nargs;
10050c22e176SPhilippe Mathieu-Daudé
10060c22e176SPhilippe Mathieu-Daudé if (nargs != 0) {
10070c22e176SPhilippe Mathieu-Daudé ca->cif.arg_types = ca->args;
10080c22e176SPhilippe Mathieu-Daudé for (int j = 0; j < nargs; ++j) {
10090c22e176SPhilippe Mathieu-Daudé int typecode = extract32(typemask, (j + 1) * 3, 3);
10100c22e176SPhilippe Mathieu-Daudé ca->args[j] = typecode_to_ffi(typecode);
10110c22e176SPhilippe Mathieu-Daudé }
10120c22e176SPhilippe Mathieu-Daudé }
10130c22e176SPhilippe Mathieu-Daudé
10140c22e176SPhilippe Mathieu-Daudé status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
10150c22e176SPhilippe Mathieu-Daudé ca->cif.rtype, ca->cif.arg_types);
10160c22e176SPhilippe Mathieu-Daudé assert(status == FFI_OK);
10170c22e176SPhilippe Mathieu-Daudé
1018d53106c9SRichard Henderson return &ca->cif;
10190c22e176SPhilippe Mathieu-Daudé }
1020f9c4bb80SRichard Henderson
1021d53106c9SRichard Henderson #define HELPER_INFO_INIT(I) (&(I)->cif)
1022d53106c9SRichard Henderson #define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I)
1023d53106c9SRichard Henderson #else
1024d53106c9SRichard Henderson #define HELPER_INFO_INIT(I) (&(I)->init)
1025d53106c9SRichard Henderson #define HELPER_INFO_INIT_VAL(I) 1
10260c22e176SPhilippe Mathieu-Daudé #endif /* CONFIG_TCG_INTERPRETER */
102722f15579SRichard Henderson
arg_slot_reg_p(unsigned arg_slot)1028338b61e9SRichard Henderson static inline bool arg_slot_reg_p(unsigned arg_slot)
1029338b61e9SRichard Henderson {
1030338b61e9SRichard Henderson /*
1031338b61e9SRichard Henderson * Split the sizeof away from the comparison to avoid Werror from
1032338b61e9SRichard Henderson * "unsigned < 0 is always false", when iarg_regs is empty.
1033338b61e9SRichard Henderson */
1034338b61e9SRichard Henderson unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs);
1035338b61e9SRichard Henderson return arg_slot < nreg;
1036338b61e9SRichard Henderson }
1037338b61e9SRichard Henderson
arg_slot_stk_ofs(unsigned arg_slot)1038d78e4a4fSRichard Henderson static inline int arg_slot_stk_ofs(unsigned arg_slot)
1039d78e4a4fSRichard Henderson {
1040d78e4a4fSRichard Henderson unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1041d78e4a4fSRichard Henderson unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
1042d78e4a4fSRichard Henderson
1043d78e4a4fSRichard Henderson tcg_debug_assert(stk_slot < max);
1044d78e4a4fSRichard Henderson return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long);
1045d78e4a4fSRichard Henderson }
1046d78e4a4fSRichard Henderson
104739004a71SRichard Henderson typedef struct TCGCumulativeArgs {
104839004a71SRichard Henderson int arg_idx; /* tcg_gen_callN args[] */
104939004a71SRichard Henderson int info_in_idx; /* TCGHelperInfo in[] */
105039004a71SRichard Henderson int arg_slot; /* regs+stack slot */
105139004a71SRichard Henderson int ref_slot; /* stack slots for references */
105239004a71SRichard Henderson } TCGCumulativeArgs;
105339004a71SRichard Henderson
layout_arg_even(TCGCumulativeArgs * cum)105439004a71SRichard Henderson static void layout_arg_even(TCGCumulativeArgs *cum)
105539004a71SRichard Henderson {
105639004a71SRichard Henderson cum->arg_slot += cum->arg_slot & 1;
105739004a71SRichard Henderson }
105839004a71SRichard Henderson
layout_arg_1(TCGCumulativeArgs * cum,TCGHelperInfo * info,TCGCallArgumentKind kind)105939004a71SRichard Henderson static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
106039004a71SRichard Henderson TCGCallArgumentKind kind)
106139004a71SRichard Henderson {
106239004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
106339004a71SRichard Henderson
106439004a71SRichard Henderson *loc = (TCGCallArgumentLoc){
106539004a71SRichard Henderson .kind = kind,
106639004a71SRichard Henderson .arg_idx = cum->arg_idx,
106739004a71SRichard Henderson .arg_slot = cum->arg_slot,
106839004a71SRichard Henderson };
106939004a71SRichard Henderson cum->info_in_idx++;
107039004a71SRichard Henderson cum->arg_slot++;
107139004a71SRichard Henderson }
107239004a71SRichard Henderson
layout_arg_normal_n(TCGCumulativeArgs * cum,TCGHelperInfo * info,int n)107339004a71SRichard Henderson static void layout_arg_normal_n(TCGCumulativeArgs *cum,
107439004a71SRichard Henderson TCGHelperInfo *info, int n)
107539004a71SRichard Henderson {
107639004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
107739004a71SRichard Henderson
107839004a71SRichard Henderson for (int i = 0; i < n; ++i) {
107939004a71SRichard Henderson /* Layout all using the same arg_idx, adjusting the subindex. */
108039004a71SRichard Henderson loc[i] = (TCGCallArgumentLoc){
108139004a71SRichard Henderson .kind = TCG_CALL_ARG_NORMAL,
108239004a71SRichard Henderson .arg_idx = cum->arg_idx,
108339004a71SRichard Henderson .tmp_subindex = i,
108439004a71SRichard Henderson .arg_slot = cum->arg_slot + i,
108539004a71SRichard Henderson };
108639004a71SRichard Henderson }
108739004a71SRichard Henderson cum->info_in_idx += n;
108839004a71SRichard Henderson cum->arg_slot += n;
108939004a71SRichard Henderson }
109039004a71SRichard Henderson
layout_arg_by_ref(TCGCumulativeArgs * cum,TCGHelperInfo * info)1091313bdea8SRichard Henderson static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
1092313bdea8SRichard Henderson {
1093313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1094313bdea8SRichard Henderson int n = 128 / TCG_TARGET_REG_BITS;
1095313bdea8SRichard Henderson
1096313bdea8SRichard Henderson /* The first subindex carries the pointer. */
1097313bdea8SRichard Henderson layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
1098313bdea8SRichard Henderson
1099313bdea8SRichard Henderson /*
1100313bdea8SRichard Henderson * The callee is allowed to clobber memory associated with
1101313bdea8SRichard Henderson * structure pass by-reference. Therefore we must make copies.
1102313bdea8SRichard Henderson * Allocate space from "ref_slot", which will be adjusted to
1103313bdea8SRichard Henderson * follow the parameters on the stack.
1104313bdea8SRichard Henderson */
1105313bdea8SRichard Henderson loc[0].ref_slot = cum->ref_slot;
1106313bdea8SRichard Henderson
1107313bdea8SRichard Henderson /*
1108313bdea8SRichard Henderson * Subsequent words also go into the reference slot, but
1109313bdea8SRichard Henderson * do not accumulate into the regular arguments.
1110313bdea8SRichard Henderson */
1111313bdea8SRichard Henderson for (int i = 1; i < n; ++i) {
1112313bdea8SRichard Henderson loc[i] = (TCGCallArgumentLoc){
1113313bdea8SRichard Henderson .kind = TCG_CALL_ARG_BY_REF_N,
1114313bdea8SRichard Henderson .arg_idx = cum->arg_idx,
1115313bdea8SRichard Henderson .tmp_subindex = i,
1116313bdea8SRichard Henderson .ref_slot = cum->ref_slot + i,
1117313bdea8SRichard Henderson };
1118313bdea8SRichard Henderson }
1119e18ed26cSRichard Henderson cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
1120313bdea8SRichard Henderson cum->ref_slot += n;
1121313bdea8SRichard Henderson }
1122313bdea8SRichard Henderson
init_call_layout(TCGHelperInfo * info)112339004a71SRichard Henderson static void init_call_layout(TCGHelperInfo *info)
112439004a71SRichard Henderson {
112539004a71SRichard Henderson int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
112639004a71SRichard Henderson int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
112739004a71SRichard Henderson unsigned typemask = info->typemask;
112839004a71SRichard Henderson unsigned typecode;
112939004a71SRichard Henderson TCGCumulativeArgs cum = { };
113039004a71SRichard Henderson
113139004a71SRichard Henderson /*
113239004a71SRichard Henderson * Parse and place any function return value.
113339004a71SRichard Henderson */
113439004a71SRichard Henderson typecode = typemask & 7;
113539004a71SRichard Henderson switch (typecode) {
113639004a71SRichard Henderson case dh_typecode_void:
113739004a71SRichard Henderson info->nr_out = 0;
113839004a71SRichard Henderson break;
113939004a71SRichard Henderson case dh_typecode_i32:
114039004a71SRichard Henderson case dh_typecode_s32:
114139004a71SRichard Henderson case dh_typecode_ptr:
114239004a71SRichard Henderson info->nr_out = 1;
114339004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL;
114439004a71SRichard Henderson break;
114539004a71SRichard Henderson case dh_typecode_i64:
114639004a71SRichard Henderson case dh_typecode_s64:
114739004a71SRichard Henderson info->nr_out = 64 / TCG_TARGET_REG_BITS;
114839004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL;
11495e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */
11505e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1151466d3759SRichard Henderson break;
1152466d3759SRichard Henderson case dh_typecode_i128:
1153466d3759SRichard Henderson info->nr_out = 128 / TCG_TARGET_REG_BITS;
11545427a9a7SRichard Henderson info->out_kind = TCG_TARGET_CALL_RET_I128;
11555427a9a7SRichard Henderson switch (TCG_TARGET_CALL_RET_I128) {
1156466d3759SRichard Henderson case TCG_CALL_RET_NORMAL:
11575e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */
11585e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1159466d3759SRichard Henderson break;
1160c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC:
1161c6556aa0SRichard Henderson /* Query the single register now to trigger any assert early. */
1162c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
1163c6556aa0SRichard Henderson break;
1164313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF:
1165313bdea8SRichard Henderson /*
1166313bdea8SRichard Henderson * Allocate the first argument to the output.
1167313bdea8SRichard Henderson * We don't need to store this anywhere, just make it
1168313bdea8SRichard Henderson * unavailable for use in the input loop below.
1169313bdea8SRichard Henderson */
1170313bdea8SRichard Henderson cum.arg_slot = 1;
1171313bdea8SRichard Henderson break;
1172466d3759SRichard Henderson default:
1173466d3759SRichard Henderson qemu_build_not_reached();
1174466d3759SRichard Henderson }
117539004a71SRichard Henderson break;
117639004a71SRichard Henderson default:
117739004a71SRichard Henderson g_assert_not_reached();
117839004a71SRichard Henderson }
117939004a71SRichard Henderson
118039004a71SRichard Henderson /*
118139004a71SRichard Henderson * Parse and place function arguments.
118239004a71SRichard Henderson */
118339004a71SRichard Henderson for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
118439004a71SRichard Henderson TCGCallArgumentKind kind;
118539004a71SRichard Henderson TCGType type;
118639004a71SRichard Henderson
118739004a71SRichard Henderson typecode = typemask & 7;
118839004a71SRichard Henderson switch (typecode) {
118939004a71SRichard Henderson case dh_typecode_i32:
119039004a71SRichard Henderson case dh_typecode_s32:
119139004a71SRichard Henderson type = TCG_TYPE_I32;
119239004a71SRichard Henderson break;
119339004a71SRichard Henderson case dh_typecode_i64:
119439004a71SRichard Henderson case dh_typecode_s64:
119539004a71SRichard Henderson type = TCG_TYPE_I64;
119639004a71SRichard Henderson break;
119739004a71SRichard Henderson case dh_typecode_ptr:
119839004a71SRichard Henderson type = TCG_TYPE_PTR;
119939004a71SRichard Henderson break;
1200466d3759SRichard Henderson case dh_typecode_i128:
1201466d3759SRichard Henderson type = TCG_TYPE_I128;
1202466d3759SRichard Henderson break;
120339004a71SRichard Henderson default:
120439004a71SRichard Henderson g_assert_not_reached();
120539004a71SRichard Henderson }
120639004a71SRichard Henderson
120739004a71SRichard Henderson switch (type) {
120839004a71SRichard Henderson case TCG_TYPE_I32:
120939004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I32) {
121039004a71SRichard Henderson case TCG_CALL_ARG_EVEN:
121139004a71SRichard Henderson layout_arg_even(&cum);
121239004a71SRichard Henderson /* fall through */
121339004a71SRichard Henderson case TCG_CALL_ARG_NORMAL:
121439004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
121539004a71SRichard Henderson break;
121639004a71SRichard Henderson case TCG_CALL_ARG_EXTEND:
121739004a71SRichard Henderson kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
121839004a71SRichard Henderson layout_arg_1(&cum, info, kind);
121939004a71SRichard Henderson break;
122039004a71SRichard Henderson default:
122139004a71SRichard Henderson qemu_build_not_reached();
122239004a71SRichard Henderson }
122339004a71SRichard Henderson break;
122439004a71SRichard Henderson
122539004a71SRichard Henderson case TCG_TYPE_I64:
122639004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I64) {
122739004a71SRichard Henderson case TCG_CALL_ARG_EVEN:
122839004a71SRichard Henderson layout_arg_even(&cum);
122939004a71SRichard Henderson /* fall through */
123039004a71SRichard Henderson case TCG_CALL_ARG_NORMAL:
123139004a71SRichard Henderson if (TCG_TARGET_REG_BITS == 32) {
123239004a71SRichard Henderson layout_arg_normal_n(&cum, info, 2);
123339004a71SRichard Henderson } else {
123439004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
123539004a71SRichard Henderson }
123639004a71SRichard Henderson break;
123739004a71SRichard Henderson default:
123839004a71SRichard Henderson qemu_build_not_reached();
123939004a71SRichard Henderson }
124039004a71SRichard Henderson break;
124139004a71SRichard Henderson
1242466d3759SRichard Henderson case TCG_TYPE_I128:
12435427a9a7SRichard Henderson switch (TCG_TARGET_CALL_ARG_I128) {
1244466d3759SRichard Henderson case TCG_CALL_ARG_EVEN:
1245466d3759SRichard Henderson layout_arg_even(&cum);
1246466d3759SRichard Henderson /* fall through */
1247466d3759SRichard Henderson case TCG_CALL_ARG_NORMAL:
1248466d3759SRichard Henderson layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
1249466d3759SRichard Henderson break;
1250313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF:
1251313bdea8SRichard Henderson layout_arg_by_ref(&cum, info);
1252313bdea8SRichard Henderson break;
1253466d3759SRichard Henderson default:
1254466d3759SRichard Henderson qemu_build_not_reached();
1255466d3759SRichard Henderson }
1256466d3759SRichard Henderson break;
1257466d3759SRichard Henderson
125839004a71SRichard Henderson default:
125939004a71SRichard Henderson g_assert_not_reached();
126039004a71SRichard Henderson }
126139004a71SRichard Henderson }
126239004a71SRichard Henderson info->nr_in = cum.info_in_idx;
126339004a71SRichard Henderson
126439004a71SRichard Henderson /* Validate that we didn't overrun the input array. */
126539004a71SRichard Henderson assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
126639004a71SRichard Henderson /* Validate the backend has enough argument space. */
126739004a71SRichard Henderson assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
1268313bdea8SRichard Henderson
1269313bdea8SRichard Henderson /*
1270313bdea8SRichard Henderson * Relocate the "ref_slot" area to the end of the parameters.
1271313bdea8SRichard Henderson * Minimizing this stack offset helps code size for x86,
1272313bdea8SRichard Henderson * which has a signed 8-bit offset encoding.
1273313bdea8SRichard Henderson */
1274313bdea8SRichard Henderson if (cum.ref_slot != 0) {
1275313bdea8SRichard Henderson int ref_base = 0;
1276313bdea8SRichard Henderson
1277313bdea8SRichard Henderson if (cum.arg_slot > max_reg_slots) {
1278313bdea8SRichard Henderson int align = __alignof(Int128) / sizeof(tcg_target_long);
1279313bdea8SRichard Henderson
1280313bdea8SRichard Henderson ref_base = cum.arg_slot - max_reg_slots;
1281313bdea8SRichard Henderson if (align > 1) {
1282313bdea8SRichard Henderson ref_base = ROUND_UP(ref_base, align);
1283313bdea8SRichard Henderson }
1284313bdea8SRichard Henderson }
1285313bdea8SRichard Henderson assert(ref_base + cum.ref_slot <= max_stk_slots);
1286d78e4a4fSRichard Henderson ref_base += max_reg_slots;
1287313bdea8SRichard Henderson
1288313bdea8SRichard Henderson if (ref_base != 0) {
1289313bdea8SRichard Henderson for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1290313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[i];
1291313bdea8SRichard Henderson switch (loc->kind) {
1292313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF:
1293313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N:
1294313bdea8SRichard Henderson loc->ref_slot += ref_base;
1295313bdea8SRichard Henderson break;
1296313bdea8SRichard Henderson default:
1297313bdea8SRichard Henderson break;
1298313bdea8SRichard Henderson }
1299313bdea8SRichard Henderson }
1300313bdea8SRichard Henderson }
1301313bdea8SRichard Henderson }
130239004a71SRichard Henderson }
130339004a71SRichard Henderson
130491478cefSRichard Henderson static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
1305f69d277eSRichard Henderson static void process_op_defs(TCGContext *s);
13061c2adb95SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
13071c2adb95SRichard Henderson TCGReg reg, const char *name);
130891478cefSRichard Henderson
tcg_context_init(unsigned max_cpus)130943b972b7SRichard Henderson static void tcg_context_init(unsigned max_cpus)
1310c896fe29Sbellard {
1311a76aabd3SRichard Henderson TCGContext *s = &tcg_init_ctx;
1312100b5e01SRichard Henderson int op, total_args, n, i;
1313c896fe29Sbellard TCGOpDef *def;
1314c896fe29Sbellard TCGArgConstraint *args_ct;
13151c2adb95SRichard Henderson TCGTemp *ts;
1316c896fe29Sbellard
1317c896fe29Sbellard memset(s, 0, sizeof(*s));
1318c896fe29Sbellard s->nb_globals = 0;
1319c896fe29Sbellard
1320c896fe29Sbellard /* Count total number of arguments and allocate the corresponding
1321c896fe29Sbellard space */
1322c896fe29Sbellard total_args = 0;
1323c896fe29Sbellard for(op = 0; op < NB_OPS; op++) {
1324c896fe29Sbellard def = &tcg_op_defs[op];
1325c896fe29Sbellard n = def->nb_iargs + def->nb_oargs;
1326c896fe29Sbellard total_args += n;
1327c896fe29Sbellard }
1328c896fe29Sbellard
1329bc2b17e6SRichard Henderson args_ct = g_new0(TCGArgConstraint, total_args);
1330c896fe29Sbellard
1331c896fe29Sbellard for(op = 0; op < NB_OPS; op++) {
1332c896fe29Sbellard def = &tcg_op_defs[op];
1333c896fe29Sbellard def->args_ct = args_ct;
1334c896fe29Sbellard n = def->nb_iargs + def->nb_oargs;
1335c896fe29Sbellard args_ct += n;
1336c896fe29Sbellard }
1337c896fe29Sbellard
13388429a1caSRichard Henderson init_call_layout(&info_helper_ld32_mmu);
13398429a1caSRichard Henderson init_call_layout(&info_helper_ld64_mmu);
1340ebebea53SRichard Henderson init_call_layout(&info_helper_ld128_mmu);
13418429a1caSRichard Henderson init_call_layout(&info_helper_st32_mmu);
13428429a1caSRichard Henderson init_call_layout(&info_helper_st64_mmu);
1343ebebea53SRichard Henderson init_call_layout(&info_helper_st128_mmu);
13448429a1caSRichard Henderson
1345c896fe29Sbellard tcg_target_init(s);
1346f69d277eSRichard Henderson process_op_defs(s);
134791478cefSRichard Henderson
134891478cefSRichard Henderson /* Reverse the order of the saved registers, assuming they're all at
134991478cefSRichard Henderson the start of tcg_target_reg_alloc_order. */
135091478cefSRichard Henderson for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
135191478cefSRichard Henderson int r = tcg_target_reg_alloc_order[n];
135291478cefSRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
135391478cefSRichard Henderson break;
135491478cefSRichard Henderson }
135591478cefSRichard Henderson }
135691478cefSRichard Henderson for (i = 0; i < n; ++i) {
135791478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
135891478cefSRichard Henderson }
135991478cefSRichard Henderson for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
136091478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
136191478cefSRichard Henderson }
1362b1311c4aSEmilio G. Cota
136338b47b19SEmilio G. Cota alloc_tcg_plugin_context(s);
136438b47b19SEmilio G. Cota
1365b1311c4aSEmilio G. Cota tcg_ctx = s;
13663468b59eSEmilio G. Cota /*
13673468b59eSEmilio G. Cota * In user-mode we simply share the init context among threads, since we
13683468b59eSEmilio G. Cota * use a single region. See the documentation tcg_region_init() for the
13693468b59eSEmilio G. Cota * reasoning behind this.
13707893e42dSPhilippe Mathieu-Daudé * In system-mode we will have at most max_cpus TCG threads.
13713468b59eSEmilio G. Cota */
13723468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY
1373df2cce29SEmilio G. Cota tcg_ctxs = &tcg_ctx;
13740e2d61cfSRichard Henderson tcg_cur_ctxs = 1;
13750e2d61cfSRichard Henderson tcg_max_ctxs = 1;
13763468b59eSEmilio G. Cota #else
13770e2d61cfSRichard Henderson tcg_max_ctxs = max_cpus;
13780e2d61cfSRichard Henderson tcg_ctxs = g_new0(TCGContext *, max_cpus);
13793468b59eSEmilio G. Cota #endif
13801c2adb95SRichard Henderson
13811c2adb95SRichard Henderson tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
13821c2adb95SRichard Henderson ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1383ad75a51eSRichard Henderson tcg_env = temp_tcgv_ptr(ts);
13849002ec79SRichard Henderson }
1385b03cce8eSbellard
tcg_init(size_t tb_size,int splitwx,unsigned max_cpus)138643b972b7SRichard Henderson void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
1387a76aabd3SRichard Henderson {
138843b972b7SRichard Henderson tcg_context_init(max_cpus);
138943b972b7SRichard Henderson tcg_region_init(tb_size, splitwx, max_cpus);
1390a76aabd3SRichard Henderson }
1391a76aabd3SRichard Henderson
13926e3b2bfdSEmilio G. Cota /*
13936e3b2bfdSEmilio G. Cota * Allocate TBs right before their corresponding translated code, making
13946e3b2bfdSEmilio G. Cota * sure that TBs and code are on different cache lines.
13956e3b2bfdSEmilio G. Cota */
tcg_tb_alloc(TCGContext * s)13966e3b2bfdSEmilio G. Cota TranslationBlock *tcg_tb_alloc(TCGContext *s)
13976e3b2bfdSEmilio G. Cota {
13986e3b2bfdSEmilio G. Cota uintptr_t align = qemu_icache_linesize;
13996e3b2bfdSEmilio G. Cota TranslationBlock *tb;
14006e3b2bfdSEmilio G. Cota void *next;
14016e3b2bfdSEmilio G. Cota
1402e8feb96fSEmilio G. Cota retry:
14036e3b2bfdSEmilio G. Cota tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
14046e3b2bfdSEmilio G. Cota next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
14056e3b2bfdSEmilio G. Cota
14066e3b2bfdSEmilio G. Cota if (unlikely(next > s->code_gen_highwater)) {
1407e8feb96fSEmilio G. Cota if (tcg_region_alloc(s)) {
14086e3b2bfdSEmilio G. Cota return NULL;
14096e3b2bfdSEmilio G. Cota }
1410e8feb96fSEmilio G. Cota goto retry;
1411e8feb96fSEmilio G. Cota }
1412d73415a3SStefan Hajnoczi qatomic_set(&s->code_gen_ptr, next);
141357a26946SRichard Henderson s->data_gen_ptr = NULL;
14146e3b2bfdSEmilio G. Cota return tb;
14156e3b2bfdSEmilio G. Cota }
14166e3b2bfdSEmilio G. Cota
tcg_prologue_init(void)1417935f75aeSRichard Henderson void tcg_prologue_init(void)
14189002ec79SRichard Henderson {
1419935f75aeSRichard Henderson TCGContext *s = tcg_ctx;
1420b0a0794aSRichard Henderson size_t prologue_size;
14218163b749SRichard Henderson
1422b0a0794aSRichard Henderson s->code_ptr = s->code_gen_ptr;
1423b0a0794aSRichard Henderson s->code_buf = s->code_gen_ptr;
14245b38ee31SRichard Henderson s->data_gen_ptr = NULL;
1425b91ccb31SRichard Henderson
1426b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER
1427b0a0794aSRichard Henderson tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
1428b91ccb31SRichard Henderson #endif
14298163b749SRichard Henderson
14305b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS
14315b38ee31SRichard Henderson s->pool_labels = NULL;
14325b38ee31SRichard Henderson #endif
14335b38ee31SRichard Henderson
1434653b87ebSRoman Bolshakov qemu_thread_jit_write();
14358163b749SRichard Henderson /* Generate the prologue. */
1436b03cce8eSbellard tcg_target_qemu_prologue(s);
14375b38ee31SRichard Henderson
14385b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS
14395b38ee31SRichard Henderson /* Allow the prologue to put e.g. guest_base into a pool entry. */
14405b38ee31SRichard Henderson {
14411768987bSRichard Henderson int result = tcg_out_pool_finalize(s);
14421768987bSRichard Henderson tcg_debug_assert(result == 0);
14435b38ee31SRichard Henderson }
14445b38ee31SRichard Henderson #endif
14455b38ee31SRichard Henderson
1446b0a0794aSRichard Henderson prologue_size = tcg_current_code_size(s);
14475584e2dbSIlya Leoshkevich perf_report_prologue(s->code_gen_ptr, prologue_size);
1448b0a0794aSRichard Henderson
1449df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER
1450b0a0794aSRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1451b0a0794aSRichard Henderson (uintptr_t)s->code_buf, prologue_size);
1452df5d2b16SRichard Henderson #endif
14538163b749SRichard Henderson
1454d6b64b2bSRichard Henderson if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
1455c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock();
145678b54858SRichard Henderson if (logfile) {
145778b54858SRichard Henderson fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
14585b38ee31SRichard Henderson if (s->data_gen_ptr) {
1459b0a0794aSRichard Henderson size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
14605b38ee31SRichard Henderson size_t data_size = prologue_size - code_size;
14615b38ee31SRichard Henderson size_t i;
14625b38ee31SRichard Henderson
146378b54858SRichard Henderson disas(logfile, s->code_gen_ptr, code_size);
14645b38ee31SRichard Henderson
14655b38ee31SRichard Henderson for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
14665b38ee31SRichard Henderson if (sizeof(tcg_target_ulong) == 8) {
146778b54858SRichard Henderson fprintf(logfile,
146878b54858SRichard Henderson "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
14695b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i,
14705b38ee31SRichard Henderson *(uint64_t *)(s->data_gen_ptr + i));
14715b38ee31SRichard Henderson } else {
147278b54858SRichard Henderson fprintf(logfile,
147378b54858SRichard Henderson "0x%08" PRIxPTR ": .long 0x%08x\n",
14745b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i,
14755b38ee31SRichard Henderson *(uint32_t *)(s->data_gen_ptr + i));
14765b38ee31SRichard Henderson }
14775b38ee31SRichard Henderson }
14785b38ee31SRichard Henderson } else {
147978b54858SRichard Henderson disas(logfile, s->code_gen_ptr, prologue_size);
14805b38ee31SRichard Henderson }
148178b54858SRichard Henderson fprintf(logfile, "\n");
1482fc59d2d8SRobert Foley qemu_log_unlock(logfile);
1483d6b64b2bSRichard Henderson }
148478b54858SRichard Henderson }
1485cedbcb01SEmilio G. Cota
14866eea0434SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER
14876eea0434SRichard Henderson /*
14886eea0434SRichard Henderson * Assert that goto_ptr is implemented completely, setting an epilogue.
14896eea0434SRichard Henderson * For tci, we use NULL as the signal to return from the interpreter,
14906eea0434SRichard Henderson * so skip this check.
14916eea0434SRichard Henderson */
14928b5c2b62SRichard Henderson tcg_debug_assert(tcg_code_gen_epilogue != NULL);
14936eea0434SRichard Henderson #endif
1494d1c74ab3SRichard Henderson
1495d1c74ab3SRichard Henderson tcg_region_prologue_set(s);
1496c896fe29Sbellard }
1497c896fe29Sbellard
tcg_func_start(TCGContext * s)1498c896fe29Sbellard void tcg_func_start(TCGContext *s)
1499c896fe29Sbellard {
1500c896fe29Sbellard tcg_pool_reset(s);
1501c896fe29Sbellard s->nb_temps = s->nb_globals;
15020ec9eabcSRichard Henderson
15030ec9eabcSRichard Henderson /* No temps have been previously allocated for size or locality. */
15040ec9eabcSRichard Henderson memset(s->free_temps, 0, sizeof(s->free_temps));
15050ec9eabcSRichard Henderson
1506c0522136SRichard Henderson /* No constant temps have been previously allocated. */
1507c0522136SRichard Henderson for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1508c0522136SRichard Henderson if (s->const_table[i]) {
1509c0522136SRichard Henderson g_hash_table_remove_all(s->const_table[i]);
1510c0522136SRichard Henderson }
1511c0522136SRichard Henderson }
1512c0522136SRichard Henderson
1513abebf925SRichard Henderson s->nb_ops = 0;
1514c896fe29Sbellard s->nb_labels = 0;
1515c896fe29Sbellard s->current_frame_offset = s->frame_start;
1516c896fe29Sbellard
15170a209d4bSRichard Henderson #ifdef CONFIG_DEBUG_TCG
15180a209d4bSRichard Henderson s->goto_tb_issue_mask = 0;
15190a209d4bSRichard Henderson #endif
15200a209d4bSRichard Henderson
152115fa08f8SRichard Henderson QTAILQ_INIT(&s->ops);
152215fa08f8SRichard Henderson QTAILQ_INIT(&s->free_ops);
1523bef16ab4SRichard Henderson QSIMPLEQ_INIT(&s->labels);
15244baf3978SRichard Henderson
15254baf3978SRichard Henderson tcg_debug_assert(s->addr_type == TCG_TYPE_I32 ||
15264baf3978SRichard Henderson s->addr_type == TCG_TYPE_I64);
1527d0a9bb5eSRichard Henderson
1528747bd69dSRichard Henderson tcg_debug_assert(s->insn_start_words > 0);
1529c896fe29Sbellard }
1530c896fe29Sbellard
tcg_temp_alloc(TCGContext * s)1531ae30e866SRichard Henderson static TCGTemp *tcg_temp_alloc(TCGContext *s)
15327ca4b752SRichard Henderson {
15337ca4b752SRichard Henderson int n = s->nb_temps++;
1534ae30e866SRichard Henderson
1535ae30e866SRichard Henderson if (n >= TCG_MAX_TEMPS) {
1536db6b7d0cSRichard Henderson tcg_raise_tb_overflow(s);
1537ae30e866SRichard Henderson }
15387ca4b752SRichard Henderson return memset(&s->temps[n], 0, sizeof(TCGTemp));
15397ca4b752SRichard Henderson }
15407ca4b752SRichard Henderson
tcg_global_alloc(TCGContext * s)1541ae30e866SRichard Henderson static TCGTemp *tcg_global_alloc(TCGContext *s)
15427ca4b752SRichard Henderson {
1543fa477d25SRichard Henderson TCGTemp *ts;
1544fa477d25SRichard Henderson
15457ca4b752SRichard Henderson tcg_debug_assert(s->nb_globals == s->nb_temps);
1546ae30e866SRichard Henderson tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
15477ca4b752SRichard Henderson s->nb_globals++;
1548fa477d25SRichard Henderson ts = tcg_temp_alloc(s);
1549ee17db83SRichard Henderson ts->kind = TEMP_GLOBAL;
1550fa477d25SRichard Henderson
1551fa477d25SRichard Henderson return ts;
1552c896fe29Sbellard }
1553c896fe29Sbellard
tcg_global_reg_new_internal(TCGContext * s,TCGType type,TCGReg reg,const char * name)1554085272b3SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1555b6638662SRichard Henderson TCGReg reg, const char *name)
1556c896fe29Sbellard {
1557c896fe29Sbellard TCGTemp *ts;
1558c896fe29Sbellard
15591a057554SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
15607ca4b752SRichard Henderson
15617ca4b752SRichard Henderson ts = tcg_global_alloc(s);
1562c896fe29Sbellard ts->base_type = type;
1563c896fe29Sbellard ts->type = type;
1564ee17db83SRichard Henderson ts->kind = TEMP_FIXED;
1565c896fe29Sbellard ts->reg = reg;
1566c896fe29Sbellard ts->name = name;
1567c896fe29Sbellard tcg_regset_set_reg(s->reserved_regs, reg);
15687ca4b752SRichard Henderson
1569085272b3SRichard Henderson return ts;
1570a7812ae4Spbrook }
1571a7812ae4Spbrook
tcg_set_frame(TCGContext * s,TCGReg reg,intptr_t start,intptr_t size)1572b6638662SRichard Henderson void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
1573a7812ae4Spbrook {
1574b3a62939SRichard Henderson s->frame_start = start;
1575b3a62939SRichard Henderson s->frame_end = start + size;
1576085272b3SRichard Henderson s->frame_temp
1577085272b3SRichard Henderson = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
1578b3a62939SRichard Henderson }
1579a7812ae4Spbrook
tcg_global_mem_new_internal(TCGv_ptr base,intptr_t offset,const char * name,TCGType type)15804643f3e0SRichard Henderson static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
15814643f3e0SRichard Henderson const char *name, TCGType type)
1582c896fe29Sbellard {
1583b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx;
1584dc41aa7dSRichard Henderson TCGTemp *base_ts = tcgv_ptr_temp(base);
15857ca4b752SRichard Henderson TCGTemp *ts = tcg_global_alloc(s);
1586aef85402SRichard Henderson int indirect_reg = 0;
1587c896fe29Sbellard
1588c0522136SRichard Henderson switch (base_ts->kind) {
1589c0522136SRichard Henderson case TEMP_FIXED:
1590c0522136SRichard Henderson break;
1591c0522136SRichard Henderson case TEMP_GLOBAL:
15925a18407fSRichard Henderson /* We do not support double-indirect registers. */
15935a18407fSRichard Henderson tcg_debug_assert(!base_ts->indirect_reg);
1594b3915dbbSRichard Henderson base_ts->indirect_base = 1;
15955a18407fSRichard Henderson s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
15965a18407fSRichard Henderson ? 2 : 1);
15975a18407fSRichard Henderson indirect_reg = 1;
1598c0522136SRichard Henderson break;
1599c0522136SRichard Henderson default:
1600c0522136SRichard Henderson g_assert_not_reached();
1601b3915dbbSRichard Henderson }
1602b3915dbbSRichard Henderson
16037ca4b752SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
16047ca4b752SRichard Henderson TCGTemp *ts2 = tcg_global_alloc(s);
1605c896fe29Sbellard char buf[64];
16067ca4b752SRichard Henderson
16077ca4b752SRichard Henderson ts->base_type = TCG_TYPE_I64;
1608c896fe29Sbellard ts->type = TCG_TYPE_I32;
1609b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg;
1610c896fe29Sbellard ts->mem_allocated = 1;
1611b3a62939SRichard Henderson ts->mem_base = base_ts;
1612aef85402SRichard Henderson ts->mem_offset = offset;
1613c896fe29Sbellard pstrcpy(buf, sizeof(buf), name);
1614c896fe29Sbellard pstrcat(buf, sizeof(buf), "_0");
1615c896fe29Sbellard ts->name = strdup(buf);
1616c896fe29Sbellard
16177ca4b752SRichard Henderson tcg_debug_assert(ts2 == ts + 1);
16187ca4b752SRichard Henderson ts2->base_type = TCG_TYPE_I64;
16197ca4b752SRichard Henderson ts2->type = TCG_TYPE_I32;
1620b3915dbbSRichard Henderson ts2->indirect_reg = indirect_reg;
16217ca4b752SRichard Henderson ts2->mem_allocated = 1;
16227ca4b752SRichard Henderson ts2->mem_base = base_ts;
1623aef85402SRichard Henderson ts2->mem_offset = offset + 4;
1624fac87bd2SRichard Henderson ts2->temp_subindex = 1;
1625c896fe29Sbellard pstrcpy(buf, sizeof(buf), name);
1626c896fe29Sbellard pstrcat(buf, sizeof(buf), "_1");
1627120c1084SRichard Henderson ts2->name = strdup(buf);
16287ca4b752SRichard Henderson } else {
1629c896fe29Sbellard ts->base_type = type;
1630c896fe29Sbellard ts->type = type;
1631b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg;
1632c896fe29Sbellard ts->mem_allocated = 1;
1633b3a62939SRichard Henderson ts->mem_base = base_ts;
1634c896fe29Sbellard ts->mem_offset = offset;
1635c896fe29Sbellard ts->name = name;
1636c896fe29Sbellard }
1637085272b3SRichard Henderson return ts;
1638c896fe29Sbellard }
1639c896fe29Sbellard
tcg_global_mem_new_i32(TCGv_ptr reg,intptr_t off,const char * name)16404643f3e0SRichard Henderson TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
16414643f3e0SRichard Henderson {
16424643f3e0SRichard Henderson TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
16434643f3e0SRichard Henderson return temp_tcgv_i32(ts);
16444643f3e0SRichard Henderson }
16454643f3e0SRichard Henderson
tcg_global_mem_new_i64(TCGv_ptr reg,intptr_t off,const char * name)16464643f3e0SRichard Henderson TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
16474643f3e0SRichard Henderson {
16484643f3e0SRichard Henderson TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
16494643f3e0SRichard Henderson return temp_tcgv_i64(ts);
16504643f3e0SRichard Henderson }
16514643f3e0SRichard Henderson
tcg_global_mem_new_ptr(TCGv_ptr reg,intptr_t off,const char * name)16524643f3e0SRichard Henderson TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
16534643f3e0SRichard Henderson {
16544643f3e0SRichard Henderson TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
16554643f3e0SRichard Henderson return temp_tcgv_ptr(ts);
16564643f3e0SRichard Henderson }
16574643f3e0SRichard Henderson
tcg_temp_new_internal(TCGType type,TCGTempKind kind)16584643f3e0SRichard Henderson static TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
1659c896fe29Sbellard {
1660b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx;
1661c896fe29Sbellard TCGTemp *ts;
1662e1c08b00SRichard Henderson int n;
1663c896fe29Sbellard
1664e1c08b00SRichard Henderson if (kind == TEMP_EBB) {
1665e1c08b00SRichard Henderson int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
1666e1c08b00SRichard Henderson
16670ec9eabcSRichard Henderson if (idx < TCG_MAX_TEMPS) {
16680ec9eabcSRichard Henderson /* There is already an available temp with the right type. */
1669e1c08b00SRichard Henderson clear_bit(idx, s->free_temps[type].l);
16700ec9eabcSRichard Henderson
1671e8996ee0Sbellard ts = &s->temps[idx];
1672e8996ee0Sbellard ts->temp_allocated = 1;
16737ca4b752SRichard Henderson tcg_debug_assert(ts->base_type == type);
1674ee17db83SRichard Henderson tcg_debug_assert(ts->kind == kind);
16752f2e911dSRichard Henderson return ts;
1676e1c08b00SRichard Henderson }
1677e8996ee0Sbellard } else {
1678e1c08b00SRichard Henderson tcg_debug_assert(kind == TEMP_TB);
1679e1c08b00SRichard Henderson }
168043eef72fSRichard Henderson
168143eef72fSRichard Henderson switch (type) {
168243eef72fSRichard Henderson case TCG_TYPE_I32:
168343eef72fSRichard Henderson case TCG_TYPE_V64:
168443eef72fSRichard Henderson case TCG_TYPE_V128:
168543eef72fSRichard Henderson case TCG_TYPE_V256:
168643eef72fSRichard Henderson n = 1;
168743eef72fSRichard Henderson break;
168843eef72fSRichard Henderson case TCG_TYPE_I64:
168943eef72fSRichard Henderson n = 64 / TCG_TARGET_REG_BITS;
169043eef72fSRichard Henderson break;
169143eef72fSRichard Henderson case TCG_TYPE_I128:
169243eef72fSRichard Henderson n = 128 / TCG_TARGET_REG_BITS;
169343eef72fSRichard Henderson break;
169443eef72fSRichard Henderson default:
169543eef72fSRichard Henderson g_assert_not_reached();
169643eef72fSRichard Henderson }
169743eef72fSRichard Henderson
16987ca4b752SRichard Henderson ts = tcg_temp_alloc(s);
169943eef72fSRichard Henderson ts->base_type = type;
170043eef72fSRichard Henderson ts->temp_allocated = 1;
170143eef72fSRichard Henderson ts->kind = kind;
170243eef72fSRichard Henderson
170343eef72fSRichard Henderson if (n == 1) {
170443eef72fSRichard Henderson ts->type = type;
170543eef72fSRichard Henderson } else {
170643eef72fSRichard Henderson ts->type = TCG_TYPE_REG;
170743eef72fSRichard Henderson
1708e1c08b00SRichard Henderson for (int i = 1; i < n; ++i) {
17097ca4b752SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s);
17107ca4b752SRichard Henderson
171143eef72fSRichard Henderson tcg_debug_assert(ts2 == ts + i);
171243eef72fSRichard Henderson ts2->base_type = type;
171343eef72fSRichard Henderson ts2->type = TCG_TYPE_REG;
17147ca4b752SRichard Henderson ts2->temp_allocated = 1;
171543eef72fSRichard Henderson ts2->temp_subindex = i;
1716ee17db83SRichard Henderson ts2->kind = kind;
171743eef72fSRichard Henderson }
1718c896fe29Sbellard }
1719085272b3SRichard Henderson return ts;
1720c896fe29Sbellard }
1721c896fe29Sbellard
tcg_temp_new_i32(void)17224643f3e0SRichard Henderson TCGv_i32 tcg_temp_new_i32(void)
17234643f3e0SRichard Henderson {
17244643f3e0SRichard Henderson return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
17254643f3e0SRichard Henderson }
17264643f3e0SRichard Henderson
tcg_temp_ebb_new_i32(void)17274643f3e0SRichard Henderson TCGv_i32 tcg_temp_ebb_new_i32(void)
17284643f3e0SRichard Henderson {
17294643f3e0SRichard Henderson return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
17304643f3e0SRichard Henderson }
17314643f3e0SRichard Henderson
tcg_temp_new_i64(void)17324643f3e0SRichard Henderson TCGv_i64 tcg_temp_new_i64(void)
17334643f3e0SRichard Henderson {
17344643f3e0SRichard Henderson return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
17354643f3e0SRichard Henderson }
17364643f3e0SRichard Henderson
tcg_temp_ebb_new_i64(void)17374643f3e0SRichard Henderson TCGv_i64 tcg_temp_ebb_new_i64(void)
17384643f3e0SRichard Henderson {
17394643f3e0SRichard Henderson return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
17404643f3e0SRichard Henderson }
17414643f3e0SRichard Henderson
tcg_temp_new_ptr(void)17424643f3e0SRichard Henderson TCGv_ptr tcg_temp_new_ptr(void)
17434643f3e0SRichard Henderson {
17444643f3e0SRichard Henderson return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
17454643f3e0SRichard Henderson }
17464643f3e0SRichard Henderson
tcg_temp_ebb_new_ptr(void)17474643f3e0SRichard Henderson TCGv_ptr tcg_temp_ebb_new_ptr(void)
17484643f3e0SRichard Henderson {
17494643f3e0SRichard Henderson return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
17504643f3e0SRichard Henderson }
17514643f3e0SRichard Henderson
tcg_temp_new_i128(void)17524643f3e0SRichard Henderson TCGv_i128 tcg_temp_new_i128(void)
17534643f3e0SRichard Henderson {
17544643f3e0SRichard Henderson return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
17554643f3e0SRichard Henderson }
17564643f3e0SRichard Henderson
tcg_temp_ebb_new_i128(void)17574643f3e0SRichard Henderson TCGv_i128 tcg_temp_ebb_new_i128(void)
17584643f3e0SRichard Henderson {
17594643f3e0SRichard Henderson return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
17604643f3e0SRichard Henderson }
17614643f3e0SRichard Henderson
tcg_temp_new_vec(TCGType type)1762d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec(TCGType type)
1763d2fd745fSRichard Henderson {
1764d2fd745fSRichard Henderson TCGTemp *t;
1765d2fd745fSRichard Henderson
1766d2fd745fSRichard Henderson #ifdef CONFIG_DEBUG_TCG
1767d2fd745fSRichard Henderson switch (type) {
1768d2fd745fSRichard Henderson case TCG_TYPE_V64:
1769d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v64);
1770d2fd745fSRichard Henderson break;
1771d2fd745fSRichard Henderson case TCG_TYPE_V128:
1772d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v128);
1773d2fd745fSRichard Henderson break;
1774d2fd745fSRichard Henderson case TCG_TYPE_V256:
1775d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v256);
1776d2fd745fSRichard Henderson break;
1777d2fd745fSRichard Henderson default:
1778d2fd745fSRichard Henderson g_assert_not_reached();
1779d2fd745fSRichard Henderson }
1780d2fd745fSRichard Henderson #endif
1781d2fd745fSRichard Henderson
1782bbf989bfSRichard Henderson t = tcg_temp_new_internal(type, TEMP_EBB);
1783d2fd745fSRichard Henderson return temp_tcgv_vec(t);
1784d2fd745fSRichard Henderson }
1785d2fd745fSRichard Henderson
1786d2fd745fSRichard Henderson /* Create a new temp of the same type as an existing temp. */
tcg_temp_new_vec_matching(TCGv_vec match)1787d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1788d2fd745fSRichard Henderson {
1789d2fd745fSRichard Henderson TCGTemp *t = tcgv_vec_temp(match);
1790d2fd745fSRichard Henderson
1791d2fd745fSRichard Henderson tcg_debug_assert(t->temp_allocated != 0);
1792d2fd745fSRichard Henderson
1793bbf989bfSRichard Henderson t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
1794d2fd745fSRichard Henderson return temp_tcgv_vec(t);
1795d2fd745fSRichard Henderson }
1796d2fd745fSRichard Henderson
tcg_temp_free_internal(TCGTemp * ts)17975bfa8034SRichard Henderson void tcg_temp_free_internal(TCGTemp *ts)
1798c896fe29Sbellard {
1799b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx;
1800c896fe29Sbellard
1801c7482438SRichard Henderson switch (ts->kind) {
1802c7482438SRichard Henderson case TEMP_CONST:
1803f57c6915SRichard Henderson case TEMP_TB:
18042f2e911dSRichard Henderson /* Silently ignore free. */
1805c7482438SRichard Henderson break;
18062f2e911dSRichard Henderson case TEMP_EBB:
1807eabb7b91SAurelien Jarno tcg_debug_assert(ts->temp_allocated != 0);
1808e8996ee0Sbellard ts->temp_allocated = 0;
18092f2e911dSRichard Henderson set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
18102f2e911dSRichard Henderson break;
18112f2e911dSRichard Henderson default:
18122f2e911dSRichard Henderson /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
18132f2e911dSRichard Henderson g_assert_not_reached();
1814e1c08b00SRichard Henderson }
1815e8996ee0Sbellard }
1816e8996ee0Sbellard
tcg_temp_free_i32(TCGv_i32 arg)181758b79713SRichard Henderson void tcg_temp_free_i32(TCGv_i32 arg)
181858b79713SRichard Henderson {
181958b79713SRichard Henderson tcg_temp_free_internal(tcgv_i32_temp(arg));
182058b79713SRichard Henderson }
182158b79713SRichard Henderson
tcg_temp_free_i64(TCGv_i64 arg)182258b79713SRichard Henderson void tcg_temp_free_i64(TCGv_i64 arg)
182358b79713SRichard Henderson {
182458b79713SRichard Henderson tcg_temp_free_internal(tcgv_i64_temp(arg));
182558b79713SRichard Henderson }
182658b79713SRichard Henderson
tcg_temp_free_i128(TCGv_i128 arg)182758b79713SRichard Henderson void tcg_temp_free_i128(TCGv_i128 arg)
182858b79713SRichard Henderson {
182958b79713SRichard Henderson tcg_temp_free_internal(tcgv_i128_temp(arg));
183058b79713SRichard Henderson }
183158b79713SRichard Henderson
tcg_temp_free_ptr(TCGv_ptr arg)183258b79713SRichard Henderson void tcg_temp_free_ptr(TCGv_ptr arg)
183358b79713SRichard Henderson {
183458b79713SRichard Henderson tcg_temp_free_internal(tcgv_ptr_temp(arg));
183558b79713SRichard Henderson }
183658b79713SRichard Henderson
tcg_temp_free_vec(TCGv_vec arg)183758b79713SRichard Henderson void tcg_temp_free_vec(TCGv_vec arg)
183858b79713SRichard Henderson {
183958b79713SRichard Henderson tcg_temp_free_internal(tcgv_vec_temp(arg));
184058b79713SRichard Henderson }
184158b79713SRichard Henderson
tcg_constant_internal(TCGType type,int64_t val)1842c0522136SRichard Henderson TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1843c0522136SRichard Henderson {
1844c0522136SRichard Henderson TCGContext *s = tcg_ctx;
1845c0522136SRichard Henderson GHashTable *h = s->const_table[type];
1846c0522136SRichard Henderson TCGTemp *ts;
1847c0522136SRichard Henderson
1848c0522136SRichard Henderson if (h == NULL) {
1849c0522136SRichard Henderson h = g_hash_table_new(g_int64_hash, g_int64_equal);
1850c0522136SRichard Henderson s->const_table[type] = h;
1851c0522136SRichard Henderson }
1852c0522136SRichard Henderson
1853c0522136SRichard Henderson ts = g_hash_table_lookup(h, &val);
1854c0522136SRichard Henderson if (ts == NULL) {
1855aef85402SRichard Henderson int64_t *val_ptr;
1856aef85402SRichard Henderson
1857c0522136SRichard Henderson ts = tcg_temp_alloc(s);
1858c0522136SRichard Henderson
1859c0522136SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1860c0522136SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s);
1861c0522136SRichard Henderson
1862aef85402SRichard Henderson tcg_debug_assert(ts2 == ts + 1);
1863aef85402SRichard Henderson
1864c0522136SRichard Henderson ts->base_type = TCG_TYPE_I64;
1865c0522136SRichard Henderson ts->type = TCG_TYPE_I32;
1866c0522136SRichard Henderson ts->kind = TEMP_CONST;
1867c0522136SRichard Henderson ts->temp_allocated = 1;
1868c0522136SRichard Henderson
1869c0522136SRichard Henderson ts2->base_type = TCG_TYPE_I64;
1870c0522136SRichard Henderson ts2->type = TCG_TYPE_I32;
1871c0522136SRichard Henderson ts2->kind = TEMP_CONST;
1872c0522136SRichard Henderson ts2->temp_allocated = 1;
1873fac87bd2SRichard Henderson ts2->temp_subindex = 1;
1874aef85402SRichard Henderson
1875aef85402SRichard Henderson /*
1876aef85402SRichard Henderson * Retain the full value of the 64-bit constant in the low
1877aef85402SRichard Henderson * part, so that the hash table works. Actual uses will
1878aef85402SRichard Henderson * truncate the value to the low part.
1879aef85402SRichard Henderson */
1880aef85402SRichard Henderson ts[HOST_BIG_ENDIAN].val = val;
1881aef85402SRichard Henderson ts[!HOST_BIG_ENDIAN].val = val >> 32;
1882aef85402SRichard Henderson val_ptr = &ts[HOST_BIG_ENDIAN].val;
1883c0522136SRichard Henderson } else {
1884c0522136SRichard Henderson ts->base_type = type;
1885c0522136SRichard Henderson ts->type = type;
1886c0522136SRichard Henderson ts->kind = TEMP_CONST;
1887c0522136SRichard Henderson ts->temp_allocated = 1;
1888c0522136SRichard Henderson ts->val = val;
1889aef85402SRichard Henderson val_ptr = &ts->val;
1890c0522136SRichard Henderson }
1891aef85402SRichard Henderson g_hash_table_insert(h, val_ptr, ts);
1892c0522136SRichard Henderson }
1893c0522136SRichard Henderson
1894c0522136SRichard Henderson return ts;
1895c0522136SRichard Henderson }
1896c0522136SRichard Henderson
tcg_constant_i32(int32_t val)189716edaee7SRichard Henderson TCGv_i32 tcg_constant_i32(int32_t val)
189816edaee7SRichard Henderson {
189916edaee7SRichard Henderson return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
190016edaee7SRichard Henderson }
190116edaee7SRichard Henderson
tcg_constant_i64(int64_t val)190216edaee7SRichard Henderson TCGv_i64 tcg_constant_i64(int64_t val)
190316edaee7SRichard Henderson {
190416edaee7SRichard Henderson return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
190516edaee7SRichard Henderson }
190616edaee7SRichard Henderson
tcg_constant_ptr_int(intptr_t val)190716edaee7SRichard Henderson TCGv_ptr tcg_constant_ptr_int(intptr_t val)
190816edaee7SRichard Henderson {
190916edaee7SRichard Henderson return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
191016edaee7SRichard Henderson }
191116edaee7SRichard Henderson
tcg_constant_vec(TCGType type,unsigned vece,int64_t val)1912c0522136SRichard Henderson TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1913c0522136SRichard Henderson {
1914c0522136SRichard Henderson val = dup_const(vece, val);
1915c0522136SRichard Henderson return temp_tcgv_vec(tcg_constant_internal(type, val));
1916c0522136SRichard Henderson }
1917c0522136SRichard Henderson
tcg_constant_vec_matching(TCGv_vec match,unsigned vece,int64_t val)191888d4005bSRichard Henderson TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
191988d4005bSRichard Henderson {
192088d4005bSRichard Henderson TCGTemp *t = tcgv_vec_temp(match);
192188d4005bSRichard Henderson
192288d4005bSRichard Henderson tcg_debug_assert(t->temp_allocated != 0);
192388d4005bSRichard Henderson return tcg_constant_vec(t->base_type, vece, val);
192488d4005bSRichard Henderson }
192588d4005bSRichard Henderson
1926177f648fSRichard Henderson #ifdef CONFIG_DEBUG_TCG
temp_idx(TCGTemp * ts)1927177f648fSRichard Henderson size_t temp_idx(TCGTemp *ts)
1928177f648fSRichard Henderson {
1929177f648fSRichard Henderson ptrdiff_t n = ts - tcg_ctx->temps;
1930177f648fSRichard Henderson assert(n >= 0 && n < tcg_ctx->nb_temps);
1931177f648fSRichard Henderson return n;
1932177f648fSRichard Henderson }
1933177f648fSRichard Henderson
tcgv_i32_temp(TCGv_i32 v)1934177f648fSRichard Henderson TCGTemp *tcgv_i32_temp(TCGv_i32 v)
1935177f648fSRichard Henderson {
1936177f648fSRichard Henderson uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
1937177f648fSRichard Henderson
1938177f648fSRichard Henderson assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
1939177f648fSRichard Henderson assert(o % sizeof(TCGTemp) == 0);
1940177f648fSRichard Henderson
1941177f648fSRichard Henderson return (void *)tcg_ctx + (uintptr_t)v;
1942177f648fSRichard Henderson }
1943177f648fSRichard Henderson #endif /* CONFIG_DEBUG_TCG */
1944177f648fSRichard Henderson
1945be0f34b5SRichard Henderson /* Return true if OP may appear in the opcode stream.
1946be0f34b5SRichard Henderson Test the runtime variable that controls each opcode. */
tcg_op_supported(TCGOpcode op)1947be0f34b5SRichard Henderson bool tcg_op_supported(TCGOpcode op)
1948be0f34b5SRichard Henderson {
1949d2fd745fSRichard Henderson const bool have_vec
1950d2fd745fSRichard Henderson = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1951d2fd745fSRichard Henderson
1952be0f34b5SRichard Henderson switch (op) {
1953be0f34b5SRichard Henderson case INDEX_op_discard:
1954be0f34b5SRichard Henderson case INDEX_op_set_label:
1955be0f34b5SRichard Henderson case INDEX_op_call:
1956be0f34b5SRichard Henderson case INDEX_op_br:
1957be0f34b5SRichard Henderson case INDEX_op_mb:
1958be0f34b5SRichard Henderson case INDEX_op_insn_start:
1959be0f34b5SRichard Henderson case INDEX_op_exit_tb:
1960be0f34b5SRichard Henderson case INDEX_op_goto_tb:
1961f4e01e30SRichard Henderson case INDEX_op_goto_ptr:
1962fecccfccSRichard Henderson case INDEX_op_qemu_ld_a32_i32:
1963fecccfccSRichard Henderson case INDEX_op_qemu_ld_a64_i32:
1964fecccfccSRichard Henderson case INDEX_op_qemu_st_a32_i32:
1965fecccfccSRichard Henderson case INDEX_op_qemu_st_a64_i32:
1966fecccfccSRichard Henderson case INDEX_op_qemu_ld_a32_i64:
1967fecccfccSRichard Henderson case INDEX_op_qemu_ld_a64_i64:
1968fecccfccSRichard Henderson case INDEX_op_qemu_st_a32_i64:
1969fecccfccSRichard Henderson case INDEX_op_qemu_st_a64_i64:
1970be0f34b5SRichard Henderson return true;
1971be0f34b5SRichard Henderson
1972fecccfccSRichard Henderson case INDEX_op_qemu_st8_a32_i32:
1973fecccfccSRichard Henderson case INDEX_op_qemu_st8_a64_i32:
197407ce0b05SRichard Henderson return TCG_TARGET_HAS_qemu_st8_i32;
197507ce0b05SRichard Henderson
1976fecccfccSRichard Henderson case INDEX_op_qemu_ld_a32_i128:
1977fecccfccSRichard Henderson case INDEX_op_qemu_ld_a64_i128:
1978fecccfccSRichard Henderson case INDEX_op_qemu_st_a32_i128:
1979fecccfccSRichard Henderson case INDEX_op_qemu_st_a64_i128:
198012fde9bcSRichard Henderson return TCG_TARGET_HAS_qemu_ldst_i128;
198112fde9bcSRichard Henderson
1982be0f34b5SRichard Henderson case INDEX_op_mov_i32:
1983be0f34b5SRichard Henderson case INDEX_op_setcond_i32:
1984be0f34b5SRichard Henderson case INDEX_op_brcond_i32:
19853871be75SRichard Henderson case INDEX_op_movcond_i32:
1986be0f34b5SRichard Henderson case INDEX_op_ld8u_i32:
1987be0f34b5SRichard Henderson case INDEX_op_ld8s_i32:
1988be0f34b5SRichard Henderson case INDEX_op_ld16u_i32:
1989be0f34b5SRichard Henderson case INDEX_op_ld16s_i32:
1990be0f34b5SRichard Henderson case INDEX_op_ld_i32:
1991be0f34b5SRichard Henderson case INDEX_op_st8_i32:
1992be0f34b5SRichard Henderson case INDEX_op_st16_i32:
1993be0f34b5SRichard Henderson case INDEX_op_st_i32:
1994be0f34b5SRichard Henderson case INDEX_op_add_i32:
1995be0f34b5SRichard Henderson case INDEX_op_sub_i32:
1996b701f195SRichard Henderson case INDEX_op_neg_i32:
1997be0f34b5SRichard Henderson case INDEX_op_mul_i32:
1998be0f34b5SRichard Henderson case INDEX_op_and_i32:
1999be0f34b5SRichard Henderson case INDEX_op_or_i32:
2000be0f34b5SRichard Henderson case INDEX_op_xor_i32:
2001be0f34b5SRichard Henderson case INDEX_op_shl_i32:
2002be0f34b5SRichard Henderson case INDEX_op_shr_i32:
2003be0f34b5SRichard Henderson case INDEX_op_sar_i32:
2004be0f34b5SRichard Henderson return true;
2005be0f34b5SRichard Henderson
20063635502dSRichard Henderson case INDEX_op_negsetcond_i32:
20073635502dSRichard Henderson return TCG_TARGET_HAS_negsetcond_i32;
2008be0f34b5SRichard Henderson case INDEX_op_div_i32:
2009be0f34b5SRichard Henderson case INDEX_op_divu_i32:
2010be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i32;
2011be0f34b5SRichard Henderson case INDEX_op_rem_i32:
2012be0f34b5SRichard Henderson case INDEX_op_remu_i32:
2013be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i32;
2014be0f34b5SRichard Henderson case INDEX_op_div2_i32:
2015be0f34b5SRichard Henderson case INDEX_op_divu2_i32:
2016be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i32;
2017be0f34b5SRichard Henderson case INDEX_op_rotl_i32:
2018be0f34b5SRichard Henderson case INDEX_op_rotr_i32:
2019be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i32;
2020be0f34b5SRichard Henderson case INDEX_op_deposit_i32:
2021be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i32;
2022be0f34b5SRichard Henderson case INDEX_op_extract_i32:
2023be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i32;
2024be0f34b5SRichard Henderson case INDEX_op_sextract_i32:
2025be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i32;
2026fce1296fSRichard Henderson case INDEX_op_extract2_i32:
2027fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i32;
2028be0f34b5SRichard Henderson case INDEX_op_add2_i32:
2029be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i32;
2030be0f34b5SRichard Henderson case INDEX_op_sub2_i32:
2031be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i32;
2032be0f34b5SRichard Henderson case INDEX_op_mulu2_i32:
2033be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i32;
2034be0f34b5SRichard Henderson case INDEX_op_muls2_i32:
2035be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i32;
2036be0f34b5SRichard Henderson case INDEX_op_muluh_i32:
2037be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i32;
2038be0f34b5SRichard Henderson case INDEX_op_mulsh_i32:
2039be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i32;
2040be0f34b5SRichard Henderson case INDEX_op_ext8s_i32:
2041be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i32;
2042be0f34b5SRichard Henderson case INDEX_op_ext16s_i32:
2043be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i32;
2044be0f34b5SRichard Henderson case INDEX_op_ext8u_i32:
2045be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i32;
2046be0f34b5SRichard Henderson case INDEX_op_ext16u_i32:
2047be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i32;
2048be0f34b5SRichard Henderson case INDEX_op_bswap16_i32:
2049be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i32;
2050be0f34b5SRichard Henderson case INDEX_op_bswap32_i32:
2051be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i32;
2052be0f34b5SRichard Henderson case INDEX_op_not_i32:
2053be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i32;
2054be0f34b5SRichard Henderson case INDEX_op_andc_i32:
2055be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i32;
2056be0f34b5SRichard Henderson case INDEX_op_orc_i32:
2057be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i32;
2058be0f34b5SRichard Henderson case INDEX_op_eqv_i32:
2059be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i32;
2060be0f34b5SRichard Henderson case INDEX_op_nand_i32:
2061be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i32;
2062be0f34b5SRichard Henderson case INDEX_op_nor_i32:
2063be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i32;
2064be0f34b5SRichard Henderson case INDEX_op_clz_i32:
2065be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i32;
2066be0f34b5SRichard Henderson case INDEX_op_ctz_i32:
2067be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i32;
2068be0f34b5SRichard Henderson case INDEX_op_ctpop_i32:
2069be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i32;
2070be0f34b5SRichard Henderson
2071be0f34b5SRichard Henderson case INDEX_op_brcond2_i32:
2072be0f34b5SRichard Henderson case INDEX_op_setcond2_i32:
2073be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 32;
2074be0f34b5SRichard Henderson
2075be0f34b5SRichard Henderson case INDEX_op_mov_i64:
2076be0f34b5SRichard Henderson case INDEX_op_setcond_i64:
2077be0f34b5SRichard Henderson case INDEX_op_brcond_i64:
20783871be75SRichard Henderson case INDEX_op_movcond_i64:
2079be0f34b5SRichard Henderson case INDEX_op_ld8u_i64:
2080be0f34b5SRichard Henderson case INDEX_op_ld8s_i64:
2081be0f34b5SRichard Henderson case INDEX_op_ld16u_i64:
2082be0f34b5SRichard Henderson case INDEX_op_ld16s_i64:
2083be0f34b5SRichard Henderson case INDEX_op_ld32u_i64:
2084be0f34b5SRichard Henderson case INDEX_op_ld32s_i64:
2085be0f34b5SRichard Henderson case INDEX_op_ld_i64:
2086be0f34b5SRichard Henderson case INDEX_op_st8_i64:
2087be0f34b5SRichard Henderson case INDEX_op_st16_i64:
2088be0f34b5SRichard Henderson case INDEX_op_st32_i64:
2089be0f34b5SRichard Henderson case INDEX_op_st_i64:
2090be0f34b5SRichard Henderson case INDEX_op_add_i64:
2091be0f34b5SRichard Henderson case INDEX_op_sub_i64:
2092b701f195SRichard Henderson case INDEX_op_neg_i64:
2093be0f34b5SRichard Henderson case INDEX_op_mul_i64:
2094be0f34b5SRichard Henderson case INDEX_op_and_i64:
2095be0f34b5SRichard Henderson case INDEX_op_or_i64:
2096be0f34b5SRichard Henderson case INDEX_op_xor_i64:
2097be0f34b5SRichard Henderson case INDEX_op_shl_i64:
2098be0f34b5SRichard Henderson case INDEX_op_shr_i64:
2099be0f34b5SRichard Henderson case INDEX_op_sar_i64:
2100be0f34b5SRichard Henderson case INDEX_op_ext_i32_i64:
2101be0f34b5SRichard Henderson case INDEX_op_extu_i32_i64:
2102be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 64;
2103be0f34b5SRichard Henderson
21043635502dSRichard Henderson case INDEX_op_negsetcond_i64:
21053635502dSRichard Henderson return TCG_TARGET_HAS_negsetcond_i64;
2106be0f34b5SRichard Henderson case INDEX_op_div_i64:
2107be0f34b5SRichard Henderson case INDEX_op_divu_i64:
2108be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i64;
2109be0f34b5SRichard Henderson case INDEX_op_rem_i64:
2110be0f34b5SRichard Henderson case INDEX_op_remu_i64:
2111be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i64;
2112be0f34b5SRichard Henderson case INDEX_op_div2_i64:
2113be0f34b5SRichard Henderson case INDEX_op_divu2_i64:
2114be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i64;
2115be0f34b5SRichard Henderson case INDEX_op_rotl_i64:
2116be0f34b5SRichard Henderson case INDEX_op_rotr_i64:
2117be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i64;
2118be0f34b5SRichard Henderson case INDEX_op_deposit_i64:
2119be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i64;
2120be0f34b5SRichard Henderson case INDEX_op_extract_i64:
2121be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i64;
2122be0f34b5SRichard Henderson case INDEX_op_sextract_i64:
2123be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i64;
2124fce1296fSRichard Henderson case INDEX_op_extract2_i64:
2125fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i64;
2126be0f34b5SRichard Henderson case INDEX_op_extrl_i64_i32:
2127be0f34b5SRichard Henderson case INDEX_op_extrh_i64_i32:
212813d885b0SRichard Henderson return TCG_TARGET_HAS_extr_i64_i32;
2129be0f34b5SRichard Henderson case INDEX_op_ext8s_i64:
2130be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i64;
2131be0f34b5SRichard Henderson case INDEX_op_ext16s_i64:
2132be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i64;
2133be0f34b5SRichard Henderson case INDEX_op_ext32s_i64:
2134be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32s_i64;
2135be0f34b5SRichard Henderson case INDEX_op_ext8u_i64:
2136be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i64;
2137be0f34b5SRichard Henderson case INDEX_op_ext16u_i64:
2138be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i64;
2139be0f34b5SRichard Henderson case INDEX_op_ext32u_i64:
2140be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32u_i64;
2141be0f34b5SRichard Henderson case INDEX_op_bswap16_i64:
2142be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i64;
2143be0f34b5SRichard Henderson case INDEX_op_bswap32_i64:
2144be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i64;
2145be0f34b5SRichard Henderson case INDEX_op_bswap64_i64:
2146be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap64_i64;
2147be0f34b5SRichard Henderson case INDEX_op_not_i64:
2148be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i64;
2149be0f34b5SRichard Henderson case INDEX_op_andc_i64:
2150be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i64;
2151be0f34b5SRichard Henderson case INDEX_op_orc_i64:
2152be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i64;
2153be0f34b5SRichard Henderson case INDEX_op_eqv_i64:
2154be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i64;
2155be0f34b5SRichard Henderson case INDEX_op_nand_i64:
2156be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i64;
2157be0f34b5SRichard Henderson case INDEX_op_nor_i64:
2158be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i64;
2159be0f34b5SRichard Henderson case INDEX_op_clz_i64:
2160be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i64;
2161be0f34b5SRichard Henderson case INDEX_op_ctz_i64:
2162be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i64;
2163be0f34b5SRichard Henderson case INDEX_op_ctpop_i64:
2164be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i64;
2165be0f34b5SRichard Henderson case INDEX_op_add2_i64:
2166be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i64;
2167be0f34b5SRichard Henderson case INDEX_op_sub2_i64:
2168be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i64;
2169be0f34b5SRichard Henderson case INDEX_op_mulu2_i64:
2170be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i64;
2171be0f34b5SRichard Henderson case INDEX_op_muls2_i64:
2172be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i64;
2173be0f34b5SRichard Henderson case INDEX_op_muluh_i64:
2174be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i64;
2175be0f34b5SRichard Henderson case INDEX_op_mulsh_i64:
2176be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i64;
2177be0f34b5SRichard Henderson
2178d2fd745fSRichard Henderson case INDEX_op_mov_vec:
2179d2fd745fSRichard Henderson case INDEX_op_dup_vec:
218037ee55a0SRichard Henderson case INDEX_op_dupm_vec:
2181d2fd745fSRichard Henderson case INDEX_op_ld_vec:
2182d2fd745fSRichard Henderson case INDEX_op_st_vec:
2183d2fd745fSRichard Henderson case INDEX_op_add_vec:
2184d2fd745fSRichard Henderson case INDEX_op_sub_vec:
2185d2fd745fSRichard Henderson case INDEX_op_and_vec:
2186d2fd745fSRichard Henderson case INDEX_op_or_vec:
2187d2fd745fSRichard Henderson case INDEX_op_xor_vec:
2188212be173SRichard Henderson case INDEX_op_cmp_vec:
2189d2fd745fSRichard Henderson return have_vec;
2190d2fd745fSRichard Henderson case INDEX_op_dup2_vec:
2191d2fd745fSRichard Henderson return have_vec && TCG_TARGET_REG_BITS == 32;
2192d2fd745fSRichard Henderson case INDEX_op_not_vec:
2193d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_not_vec;
2194d2fd745fSRichard Henderson case INDEX_op_neg_vec:
2195d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_neg_vec;
2196bcefc902SRichard Henderson case INDEX_op_abs_vec:
2197bcefc902SRichard Henderson return have_vec && TCG_TARGET_HAS_abs_vec;
2198d2fd745fSRichard Henderson case INDEX_op_andc_vec:
2199d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_andc_vec;
2200d2fd745fSRichard Henderson case INDEX_op_orc_vec:
2201d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_orc_vec;
2202ed523473SRichard Henderson case INDEX_op_nand_vec:
2203ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nand_vec;
2204ed523473SRichard Henderson case INDEX_op_nor_vec:
2205ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nor_vec;
2206ed523473SRichard Henderson case INDEX_op_eqv_vec:
2207ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_eqv_vec;
22083774030aSRichard Henderson case INDEX_op_mul_vec:
22093774030aSRichard Henderson return have_vec && TCG_TARGET_HAS_mul_vec;
2210d0ec9796SRichard Henderson case INDEX_op_shli_vec:
2211d0ec9796SRichard Henderson case INDEX_op_shri_vec:
2212d0ec9796SRichard Henderson case INDEX_op_sari_vec:
2213d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shi_vec;
2214d0ec9796SRichard Henderson case INDEX_op_shls_vec:
2215d0ec9796SRichard Henderson case INDEX_op_shrs_vec:
2216d0ec9796SRichard Henderson case INDEX_op_sars_vec:
2217d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shs_vec;
2218d0ec9796SRichard Henderson case INDEX_op_shlv_vec:
2219d0ec9796SRichard Henderson case INDEX_op_shrv_vec:
2220d0ec9796SRichard Henderson case INDEX_op_sarv_vec:
2221d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shv_vec;
2222b0f7e744SRichard Henderson case INDEX_op_rotli_vec:
2223b0f7e744SRichard Henderson return have_vec && TCG_TARGET_HAS_roti_vec;
222423850a74SRichard Henderson case INDEX_op_rotls_vec:
222523850a74SRichard Henderson return have_vec && TCG_TARGET_HAS_rots_vec;
22265d0ceda9SRichard Henderson case INDEX_op_rotlv_vec:
22275d0ceda9SRichard Henderson case INDEX_op_rotrv_vec:
22285d0ceda9SRichard Henderson return have_vec && TCG_TARGET_HAS_rotv_vec;
22298afaf050SRichard Henderson case INDEX_op_ssadd_vec:
22308afaf050SRichard Henderson case INDEX_op_usadd_vec:
22318afaf050SRichard Henderson case INDEX_op_sssub_vec:
22328afaf050SRichard Henderson case INDEX_op_ussub_vec:
22338afaf050SRichard Henderson return have_vec && TCG_TARGET_HAS_sat_vec;
2234dd0a0fcdSRichard Henderson case INDEX_op_smin_vec:
2235dd0a0fcdSRichard Henderson case INDEX_op_umin_vec:
2236dd0a0fcdSRichard Henderson case INDEX_op_smax_vec:
2237dd0a0fcdSRichard Henderson case INDEX_op_umax_vec:
2238dd0a0fcdSRichard Henderson return have_vec && TCG_TARGET_HAS_minmax_vec;
223938dc1294SRichard Henderson case INDEX_op_bitsel_vec:
224038dc1294SRichard Henderson return have_vec && TCG_TARGET_HAS_bitsel_vec;
2241f75da298SRichard Henderson case INDEX_op_cmpsel_vec:
2242f75da298SRichard Henderson return have_vec && TCG_TARGET_HAS_cmpsel_vec;
2243d2fd745fSRichard Henderson
2244db432672SRichard Henderson default:
2245db432672SRichard Henderson tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
2246db432672SRichard Henderson return true;
2247be0f34b5SRichard Henderson }
2248be0f34b5SRichard Henderson }
2249be0f34b5SRichard Henderson
225039004a71SRichard Henderson static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
225139004a71SRichard Henderson
tcg_gen_callN(TCGHelperInfo * info,TCGTemp * ret,TCGTemp ** args)2252a3a692b8SRichard Henderson static void tcg_gen_callN(TCGHelperInfo *info, TCGTemp *ret, TCGTemp **args)
2253c896fe29Sbellard {
225439004a71SRichard Henderson TCGv_i64 extend_free[MAX_CALL_IARGS];
225539004a71SRichard Henderson int n_extend = 0;
225675e8b9b7SRichard Henderson TCGOp *op;
225739004a71SRichard Henderson int i, n, pi = 0, total_args;
2258afb49896SRichard Henderson
2259d53106c9SRichard Henderson if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
2260d53106c9SRichard Henderson init_call_layout(info);
2261d53106c9SRichard Henderson g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
2262d53106c9SRichard Henderson }
2263d53106c9SRichard Henderson
226439004a71SRichard Henderson total_args = info->nr_out + info->nr_in + 2;
226539004a71SRichard Henderson op = tcg_op_alloc(INDEX_op_call, total_args);
22662bece2c8SRichard Henderson
226738b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN
226817083f6fSEmilio Cota /* Flag helpers that may affect guest state */
226917083f6fSEmilio Cota if (tcg_ctx->plugin_insn &&
227017083f6fSEmilio Cota !(info->flags & TCG_CALL_PLUGIN) &&
227117083f6fSEmilio Cota !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
227238b47b19SEmilio G. Cota tcg_ctx->plugin_insn->calls_helpers = true;
227338b47b19SEmilio G. Cota }
227438b47b19SEmilio G. Cota #endif
227538b47b19SEmilio G. Cota
227639004a71SRichard Henderson TCGOP_CALLO(op) = n = info->nr_out;
227739004a71SRichard Henderson switch (n) {
227839004a71SRichard Henderson case 0:
227939004a71SRichard Henderson tcg_debug_assert(ret == NULL);
228039004a71SRichard Henderson break;
228139004a71SRichard Henderson case 1:
228239004a71SRichard Henderson tcg_debug_assert(ret != NULL);
228339004a71SRichard Henderson op->args[pi++] = temp_arg(ret);
228439004a71SRichard Henderson break;
228539004a71SRichard Henderson case 2:
2286466d3759SRichard Henderson case 4:
228739004a71SRichard Henderson tcg_debug_assert(ret != NULL);
2288466d3759SRichard Henderson tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
228939004a71SRichard Henderson tcg_debug_assert(ret->temp_subindex == 0);
2290466d3759SRichard Henderson for (i = 0; i < n; ++i) {
2291466d3759SRichard Henderson op->args[pi++] = temp_arg(ret + i);
2292466d3759SRichard Henderson }
229339004a71SRichard Henderson break;
229439004a71SRichard Henderson default:
229539004a71SRichard Henderson g_assert_not_reached();
229639004a71SRichard Henderson }
22977319d83aSRichard Henderson
229839004a71SRichard Henderson TCGOP_CALLI(op) = n = info->nr_in;
229939004a71SRichard Henderson for (i = 0; i < n; i++) {
230039004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i];
230139004a71SRichard Henderson TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
230239004a71SRichard Henderson
230339004a71SRichard Henderson switch (loc->kind) {
230439004a71SRichard Henderson case TCG_CALL_ARG_NORMAL:
2305313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF:
2306313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N:
230739004a71SRichard Henderson op->args[pi++] = temp_arg(ts);
230839004a71SRichard Henderson break;
230939004a71SRichard Henderson
231039004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U:
231139004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S:
231239004a71SRichard Henderson {
23135dd48602SRichard Henderson TCGv_i64 temp = tcg_temp_ebb_new_i64();
231439004a71SRichard Henderson TCGv_i32 orig = temp_tcgv_i32(ts);
231539004a71SRichard Henderson
231639004a71SRichard Henderson if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
231718cf3d07SRichard Henderson tcg_gen_ext_i32_i64(temp, orig);
23182bece2c8SRichard Henderson } else {
231918cf3d07SRichard Henderson tcg_gen_extu_i32_i64(temp, orig);
23202bece2c8SRichard Henderson }
232139004a71SRichard Henderson op->args[pi++] = tcgv_i64_arg(temp);
232239004a71SRichard Henderson extend_free[n_extend++] = temp;
23232bece2c8SRichard Henderson }
232439004a71SRichard Henderson break;
23252bece2c8SRichard Henderson
2326e2a9dd6bSRichard Henderson default:
2327e2a9dd6bSRichard Henderson g_assert_not_reached();
2328e2a9dd6bSRichard Henderson }
2329c896fe29Sbellard }
2330d53106c9SRichard Henderson op->args[pi++] = (uintptr_t)info->func;
23313e92aa34SRichard Henderson op->args[pi++] = (uintptr_t)info;
233239004a71SRichard Henderson tcg_debug_assert(pi == total_args);
2333a7812ae4Spbrook
233439004a71SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
23352bece2c8SRichard Henderson
233639004a71SRichard Henderson tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
233739004a71SRichard Henderson for (i = 0; i < n_extend; ++i) {
233839004a71SRichard Henderson tcg_temp_free_i64(extend_free[i]);
2339eb8b0224SRichard Henderson }
2340a7812ae4Spbrook }
2341c896fe29Sbellard
tcg_gen_call0(TCGHelperInfo * info,TCGTemp * ret)2342a3a692b8SRichard Henderson void tcg_gen_call0(TCGHelperInfo *info, TCGTemp *ret)
2343a3a692b8SRichard Henderson {
2344a3a692b8SRichard Henderson tcg_gen_callN(info, ret, NULL);
2345a3a692b8SRichard Henderson }
2346a3a692b8SRichard Henderson
tcg_gen_call1(TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1)2347a3a692b8SRichard Henderson void tcg_gen_call1(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
2348a3a692b8SRichard Henderson {
2349a3a692b8SRichard Henderson tcg_gen_callN(info, ret, &t1);
2350a3a692b8SRichard Henderson }
2351a3a692b8SRichard Henderson
tcg_gen_call2(TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2)2352a3a692b8SRichard Henderson void tcg_gen_call2(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2)
2353a3a692b8SRichard Henderson {
2354a3a692b8SRichard Henderson TCGTemp *args[2] = { t1, t2 };
2355a3a692b8SRichard Henderson tcg_gen_callN(info, ret, args);
2356a3a692b8SRichard Henderson }
2357a3a692b8SRichard Henderson
tcg_gen_call3(TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3)2358a3a692b8SRichard Henderson void tcg_gen_call3(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2359a3a692b8SRichard Henderson TCGTemp *t2, TCGTemp *t3)
2360a3a692b8SRichard Henderson {
2361a3a692b8SRichard Henderson TCGTemp *args[3] = { t1, t2, t3 };
2362a3a692b8SRichard Henderson tcg_gen_callN(info, ret, args);
2363a3a692b8SRichard Henderson }
2364a3a692b8SRichard Henderson
tcg_gen_call4(TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4)2365a3a692b8SRichard Henderson void tcg_gen_call4(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2366a3a692b8SRichard Henderson TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
2367a3a692b8SRichard Henderson {
2368a3a692b8SRichard Henderson TCGTemp *args[4] = { t1, t2, t3, t4 };
2369a3a692b8SRichard Henderson tcg_gen_callN(info, ret, args);
2370a3a692b8SRichard Henderson }
2371a3a692b8SRichard Henderson
tcg_gen_call5(TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4,TCGTemp * t5)2372a3a692b8SRichard Henderson void tcg_gen_call5(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2373a3a692b8SRichard Henderson TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
2374a3a692b8SRichard Henderson {
2375a3a692b8SRichard Henderson TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
2376a3a692b8SRichard Henderson tcg_gen_callN(info, ret, args);
2377a3a692b8SRichard Henderson }
2378a3a692b8SRichard Henderson
tcg_gen_call6(TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4,TCGTemp * t5,TCGTemp * t6)2379a3a692b8SRichard Henderson void tcg_gen_call6(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2,
2380a3a692b8SRichard Henderson TCGTemp *t3, TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
2381a3a692b8SRichard Henderson {
2382a3a692b8SRichard Henderson TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
2383a3a692b8SRichard Henderson tcg_gen_callN(info, ret, args);
2384a3a692b8SRichard Henderson }
2385a3a692b8SRichard Henderson
tcg_gen_call7(TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4,TCGTemp * t5,TCGTemp * t6,TCGTemp * t7)2386a3a692b8SRichard Henderson void tcg_gen_call7(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2387a3a692b8SRichard Henderson TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
2388a3a692b8SRichard Henderson TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
2389a3a692b8SRichard Henderson {
2390a3a692b8SRichard Henderson TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
2391a3a692b8SRichard Henderson tcg_gen_callN(info, ret, args);
2392a3a692b8SRichard Henderson }
2393a3a692b8SRichard Henderson
tcg_reg_alloc_start(TCGContext * s)23948fcd3692Sblueswir1 static void tcg_reg_alloc_start(TCGContext *s)
2395c896fe29Sbellard {
2396ac3b8891SRichard Henderson int i, n;
2397ac3b8891SRichard Henderson
2398ee17db83SRichard Henderson for (i = 0, n = s->nb_temps; i < n; i++) {
2399ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i];
2400ee17db83SRichard Henderson TCGTempVal val = TEMP_VAL_MEM;
2401ee17db83SRichard Henderson
2402ee17db83SRichard Henderson switch (ts->kind) {
2403c0522136SRichard Henderson case TEMP_CONST:
2404c0522136SRichard Henderson val = TEMP_VAL_CONST;
2405c0522136SRichard Henderson break;
2406ee17db83SRichard Henderson case TEMP_FIXED:
2407ee17db83SRichard Henderson val = TEMP_VAL_REG;
2408ee17db83SRichard Henderson break;
2409ee17db83SRichard Henderson case TEMP_GLOBAL:
2410ee17db83SRichard Henderson break;
2411c7482438SRichard Henderson case TEMP_EBB:
2412ee17db83SRichard Henderson val = TEMP_VAL_DEAD;
2413ee17db83SRichard Henderson /* fall through */
2414f57c6915SRichard Henderson case TEMP_TB:
2415e8996ee0Sbellard ts->mem_allocated = 0;
2416ee17db83SRichard Henderson break;
2417ee17db83SRichard Henderson default:
2418ee17db83SRichard Henderson g_assert_not_reached();
2419ee17db83SRichard Henderson }
2420ee17db83SRichard Henderson ts->val_type = val;
2421e8996ee0Sbellard }
2422f8b2f202SRichard Henderson
2423f8b2f202SRichard Henderson memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
2424c896fe29Sbellard }
2425c896fe29Sbellard
tcg_get_arg_str_ptr(TCGContext * s,char * buf,int buf_size,TCGTemp * ts)2426f8b2f202SRichard Henderson static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
2427f8b2f202SRichard Henderson TCGTemp *ts)
2428c896fe29Sbellard {
24291807f4c4SRichard Henderson int idx = temp_idx(ts);
2430ac56dd48Spbrook
2431ee17db83SRichard Henderson switch (ts->kind) {
2432ee17db83SRichard Henderson case TEMP_FIXED:
2433ee17db83SRichard Henderson case TEMP_GLOBAL:
2434ac56dd48Spbrook pstrcpy(buf, buf_size, ts->name);
2435ee17db83SRichard Henderson break;
2436f57c6915SRichard Henderson case TEMP_TB:
2437641d5fbeSbellard snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
2438ee17db83SRichard Henderson break;
2439c7482438SRichard Henderson case TEMP_EBB:
2440ac56dd48Spbrook snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
2441ee17db83SRichard Henderson break;
2442c0522136SRichard Henderson case TEMP_CONST:
2443c0522136SRichard Henderson switch (ts->type) {
2444c0522136SRichard Henderson case TCG_TYPE_I32:
2445c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2446c0522136SRichard Henderson break;
2447c0522136SRichard Henderson #if TCG_TARGET_REG_BITS > 32
2448c0522136SRichard Henderson case TCG_TYPE_I64:
2449c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2450c0522136SRichard Henderson break;
2451c0522136SRichard Henderson #endif
2452c0522136SRichard Henderson case TCG_TYPE_V64:
2453c0522136SRichard Henderson case TCG_TYPE_V128:
2454c0522136SRichard Henderson case TCG_TYPE_V256:
2455c0522136SRichard Henderson snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2456c0522136SRichard Henderson 64 << (ts->type - TCG_TYPE_V64), ts->val);
2457c0522136SRichard Henderson break;
2458c0522136SRichard Henderson default:
2459c0522136SRichard Henderson g_assert_not_reached();
2460c0522136SRichard Henderson }
2461c0522136SRichard Henderson break;
2462c896fe29Sbellard }
2463c896fe29Sbellard return buf;
2464c896fe29Sbellard }
2465c896fe29Sbellard
tcg_get_arg_str(TCGContext * s,char * buf,int buf_size,TCGArg arg)246643439139SRichard Henderson static char *tcg_get_arg_str(TCGContext *s, char *buf,
246743439139SRichard Henderson int buf_size, TCGArg arg)
2468f8b2f202SRichard Henderson {
246943439139SRichard Henderson return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
2470f8b2f202SRichard Henderson }
2471f8b2f202SRichard Henderson
2472f48f3edeSblueswir1 static const char * const cond_name[] =
2473f48f3edeSblueswir1 {
24740aed257fSRichard Henderson [TCG_COND_NEVER] = "never",
24750aed257fSRichard Henderson [TCG_COND_ALWAYS] = "always",
2476f48f3edeSblueswir1 [TCG_COND_EQ] = "eq",
2477f48f3edeSblueswir1 [TCG_COND_NE] = "ne",
2478f48f3edeSblueswir1 [TCG_COND_LT] = "lt",
2479f48f3edeSblueswir1 [TCG_COND_GE] = "ge",
2480f48f3edeSblueswir1 [TCG_COND_LE] = "le",
2481f48f3edeSblueswir1 [TCG_COND_GT] = "gt",
2482f48f3edeSblueswir1 [TCG_COND_LTU] = "ltu",
2483f48f3edeSblueswir1 [TCG_COND_GEU] = "geu",
2484f48f3edeSblueswir1 [TCG_COND_LEU] = "leu",
2485f48f3edeSblueswir1 [TCG_COND_GTU] = "gtu"
2486f48f3edeSblueswir1 };
2487f48f3edeSblueswir1
248812fde9bcSRichard Henderson static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] =
2489f713d6adSRichard Henderson {
2490f713d6adSRichard Henderson [MO_UB] = "ub",
2491f713d6adSRichard Henderson [MO_SB] = "sb",
2492f713d6adSRichard Henderson [MO_LEUW] = "leuw",
2493f713d6adSRichard Henderson [MO_LESW] = "lesw",
2494f713d6adSRichard Henderson [MO_LEUL] = "leul",
2495f713d6adSRichard Henderson [MO_LESL] = "lesl",
2496fc313c64SFrédéric Pétrot [MO_LEUQ] = "leq",
2497f713d6adSRichard Henderson [MO_BEUW] = "beuw",
2498f713d6adSRichard Henderson [MO_BESW] = "besw",
2499f713d6adSRichard Henderson [MO_BEUL] = "beul",
2500f713d6adSRichard Henderson [MO_BESL] = "besl",
2501fc313c64SFrédéric Pétrot [MO_BEUQ] = "beq",
250212fde9bcSRichard Henderson [MO_128 + MO_BE] = "beo",
250312fde9bcSRichard Henderson [MO_128 + MO_LE] = "leo",
2504f713d6adSRichard Henderson };
2505f713d6adSRichard Henderson
25061f00b27fSSergey Sorokin static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
25071f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "un+",
25081f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "al+",
25091f00b27fSSergey Sorokin [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
25101f00b27fSSergey Sorokin [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
25111f00b27fSSergey Sorokin [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
25121f00b27fSSergey Sorokin [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
25131f00b27fSSergey Sorokin [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
25141f00b27fSSergey Sorokin [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
25151f00b27fSSergey Sorokin };
25161f00b27fSSergey Sorokin
251737031fefSRichard Henderson static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = {
251837031fefSRichard Henderson [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "",
251937031fefSRichard Henderson [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+",
252037031fefSRichard Henderson [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+",
252137031fefSRichard Henderson [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+",
252237031fefSRichard Henderson [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+",
252337031fefSRichard Henderson [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+",
252437031fefSRichard Henderson };
252537031fefSRichard Henderson
2526587195bdSRichard Henderson static const char bswap_flag_name[][6] = {
2527587195bdSRichard Henderson [TCG_BSWAP_IZ] = "iz",
2528587195bdSRichard Henderson [TCG_BSWAP_OZ] = "oz",
2529587195bdSRichard Henderson [TCG_BSWAP_OS] = "os",
2530587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
2531587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
2532587195bdSRichard Henderson };
2533587195bdSRichard Henderson
tcg_regset_single(TCGRegSet d)2534b016486eSRichard Henderson static inline bool tcg_regset_single(TCGRegSet d)
2535b016486eSRichard Henderson {
2536b016486eSRichard Henderson return (d & (d - 1)) == 0;
2537b016486eSRichard Henderson }
2538b016486eSRichard Henderson
tcg_regset_first(TCGRegSet d)2539b016486eSRichard Henderson static inline TCGReg tcg_regset_first(TCGRegSet d)
2540b016486eSRichard Henderson {
2541b016486eSRichard Henderson if (TCG_TARGET_NB_REGS <= 32) {
2542b016486eSRichard Henderson return ctz32(d);
2543b016486eSRichard Henderson } else {
2544b016486eSRichard Henderson return ctz64(d);
2545b016486eSRichard Henderson }
2546b016486eSRichard Henderson }
2547b016486eSRichard Henderson
2548b7a83ff8SRichard Henderson /* Return only the number of characters output -- no error return. */
2549b7a83ff8SRichard Henderson #define ne_fprintf(...) \
2550b7a83ff8SRichard Henderson ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2551b7a83ff8SRichard Henderson
tcg_dump_ops(TCGContext * s,FILE * f,bool have_prefs)2552b7a83ff8SRichard Henderson static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
2553c896fe29Sbellard {
2554c896fe29Sbellard char buf[128];
2555c45cb8bbSRichard Henderson TCGOp *op;
2556c896fe29Sbellard
255715fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) {
2558c45cb8bbSRichard Henderson int i, k, nb_oargs, nb_iargs, nb_cargs;
2559c45cb8bbSRichard Henderson const TCGOpDef *def;
2560c45cb8bbSRichard Henderson TCGOpcode c;
2561bdfb460eSRichard Henderson int col = 0;
2562c45cb8bbSRichard Henderson
2563c45cb8bbSRichard Henderson c = op->opc;
2564c896fe29Sbellard def = &tcg_op_defs[c];
2565c45cb8bbSRichard Henderson
2566765b842aSRichard Henderson if (c == INDEX_op_insn_start) {
2567b016486eSRichard Henderson nb_oargs = 0;
2568b7a83ff8SRichard Henderson col += ne_fprintf(f, "\n ----");
25699aef40edSRichard Henderson
2570747bd69dSRichard Henderson for (i = 0, k = s->insn_start_words; i < k; ++i) {
2571c9ad8d27SRichard Henderson col += ne_fprintf(f, " %016" PRIx64,
2572c9ad8d27SRichard Henderson tcg_get_insn_start_param(op, i));
2573eeacee4dSBlue Swirl }
25747e4597d7Sbellard } else if (c == INDEX_op_call) {
25753e92aa34SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op);
2576fa52e660SRichard Henderson void *func = tcg_call_func(op);
25773e92aa34SRichard Henderson
2578c896fe29Sbellard /* variable number of arguments */
2579cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op);
2580cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op);
2581c896fe29Sbellard nb_cargs = def->nb_cargs;
2582b03cce8eSbellard
2583b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name);
25843e92aa34SRichard Henderson
25853e92aa34SRichard Henderson /*
25863e92aa34SRichard Henderson * Print the function name from TCGHelperInfo, if available.
25873e92aa34SRichard Henderson * Note that plugins have a template function for the info,
25883e92aa34SRichard Henderson * but the actual function pointer comes from the plugin.
25893e92aa34SRichard Henderson */
25903e92aa34SRichard Henderson if (func == info->func) {
2591b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s", info->name);
25923e92aa34SRichard Henderson } else {
2593b7a83ff8SRichard Henderson col += ne_fprintf(f, "plugin(%p)", func);
25943e92aa34SRichard Henderson }
25953e92aa34SRichard Henderson
2596b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
2597b03cce8eSbellard for (i = 0; i < nb_oargs; i++) {
2598b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2599efee3746SRichard Henderson op->args[i]));
2600b03cce8eSbellard }
2601cf066674SRichard Henderson for (i = 0; i < nb_iargs; i++) {
2602efee3746SRichard Henderson TCGArg arg = op->args[nb_oargs + i];
260339004a71SRichard Henderson const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
2604b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", t);
2605e8996ee0Sbellard }
2606b03cce8eSbellard } else {
2607b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name);
2608c45cb8bbSRichard Henderson
2609c896fe29Sbellard nb_oargs = def->nb_oargs;
2610c896fe29Sbellard nb_iargs = def->nb_iargs;
2611c896fe29Sbellard nb_cargs = def->nb_cargs;
2612c896fe29Sbellard
2613d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) {
2614b7a83ff8SRichard Henderson col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
2615d2fd745fSRichard Henderson 8 << TCGOP_VECE(op));
2616d2fd745fSRichard Henderson }
2617d2fd745fSRichard Henderson
2618c896fe29Sbellard k = 0;
2619c896fe29Sbellard for (i = 0; i < nb_oargs; i++) {
2620b7a83ff8SRichard Henderson const char *sep = k ? "," : "";
2621b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep,
2622b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf),
2623efee3746SRichard Henderson op->args[k++]));
2624c896fe29Sbellard }
2625c896fe29Sbellard for (i = 0; i < nb_iargs; i++) {
2626b7a83ff8SRichard Henderson const char *sep = k ? "," : "";
2627b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep,
2628b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf),
2629efee3746SRichard Henderson op->args[k++]));
2630c896fe29Sbellard }
2631be210acbSRichard Henderson switch (c) {
2632be210acbSRichard Henderson case INDEX_op_brcond_i32:
2633ffc5ea09SRichard Henderson case INDEX_op_setcond_i32:
26343635502dSRichard Henderson case INDEX_op_negsetcond_i32:
2635ffc5ea09SRichard Henderson case INDEX_op_movcond_i32:
2636be210acbSRichard Henderson case INDEX_op_brcond2_i32:
2637be210acbSRichard Henderson case INDEX_op_setcond2_i32:
2638ffc5ea09SRichard Henderson case INDEX_op_brcond_i64:
2639be210acbSRichard Henderson case INDEX_op_setcond_i64:
26403635502dSRichard Henderson case INDEX_op_negsetcond_i64:
2641ffc5ea09SRichard Henderson case INDEX_op_movcond_i64:
2642212be173SRichard Henderson case INDEX_op_cmp_vec:
2643f75da298SRichard Henderson case INDEX_op_cmpsel_vec:
2644efee3746SRichard Henderson if (op->args[k] < ARRAY_SIZE(cond_name)
2645efee3746SRichard Henderson && cond_name[op->args[k]]) {
2646b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
2647eeacee4dSBlue Swirl } else {
2648b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
2649eeacee4dSBlue Swirl }
2650f48f3edeSblueswir1 i = 1;
2651be210acbSRichard Henderson break;
2652fecccfccSRichard Henderson case INDEX_op_qemu_ld_a32_i32:
2653fecccfccSRichard Henderson case INDEX_op_qemu_ld_a64_i32:
2654fecccfccSRichard Henderson case INDEX_op_qemu_st_a32_i32:
2655fecccfccSRichard Henderson case INDEX_op_qemu_st_a64_i32:
2656fecccfccSRichard Henderson case INDEX_op_qemu_st8_a32_i32:
2657fecccfccSRichard Henderson case INDEX_op_qemu_st8_a64_i32:
2658fecccfccSRichard Henderson case INDEX_op_qemu_ld_a32_i64:
2659fecccfccSRichard Henderson case INDEX_op_qemu_ld_a64_i64:
2660fecccfccSRichard Henderson case INDEX_op_qemu_st_a32_i64:
2661fecccfccSRichard Henderson case INDEX_op_qemu_st_a64_i64:
2662fecccfccSRichard Henderson case INDEX_op_qemu_ld_a32_i128:
2663fecccfccSRichard Henderson case INDEX_op_qemu_ld_a64_i128:
2664fecccfccSRichard Henderson case INDEX_op_qemu_st_a32_i128:
2665fecccfccSRichard Henderson case INDEX_op_qemu_st_a64_i128:
266659227d5dSRichard Henderson {
266737031fefSRichard Henderson const char *s_al, *s_op, *s_at;
26689002ffcbSRichard Henderson MemOpIdx oi = op->args[k++];
26699a239c6eSPhilippe Mathieu-Daudé MemOp mop = get_memop(oi);
267059227d5dSRichard Henderson unsigned ix = get_mmuidx(oi);
267159227d5dSRichard Henderson
26729a239c6eSPhilippe Mathieu-Daudé s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
26739a239c6eSPhilippe Mathieu-Daudé s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)];
26749a239c6eSPhilippe Mathieu-Daudé s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
26759a239c6eSPhilippe Mathieu-Daudé mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
267637031fefSRichard Henderson
267737031fefSRichard Henderson /* If all fields are accounted for, print symbolically. */
26789a239c6eSPhilippe Mathieu-Daudé if (!mop && s_al && s_op && s_at) {
267937031fefSRichard Henderson col += ne_fprintf(f, ",%s%s%s,%u",
268037031fefSRichard Henderson s_at, s_al, s_op, ix);
268137031fefSRichard Henderson } else {
26829a239c6eSPhilippe Mathieu-Daudé mop = get_memop(oi);
26839a239c6eSPhilippe Mathieu-Daudé col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
2684f713d6adSRichard Henderson }
2685f713d6adSRichard Henderson i = 1;
268659227d5dSRichard Henderson }
2687f713d6adSRichard Henderson break;
2688587195bdSRichard Henderson case INDEX_op_bswap16_i32:
2689587195bdSRichard Henderson case INDEX_op_bswap16_i64:
2690587195bdSRichard Henderson case INDEX_op_bswap32_i32:
2691587195bdSRichard Henderson case INDEX_op_bswap32_i64:
2692587195bdSRichard Henderson case INDEX_op_bswap64_i64:
2693587195bdSRichard Henderson {
2694587195bdSRichard Henderson TCGArg flags = op->args[k];
2695587195bdSRichard Henderson const char *name = NULL;
2696587195bdSRichard Henderson
2697587195bdSRichard Henderson if (flags < ARRAY_SIZE(bswap_flag_name)) {
2698587195bdSRichard Henderson name = bswap_flag_name[flags];
2699587195bdSRichard Henderson }
2700587195bdSRichard Henderson if (name) {
2701b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", name);
2702587195bdSRichard Henderson } else {
2703b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
2704587195bdSRichard Henderson }
2705587195bdSRichard Henderson i = k = 1;
2706587195bdSRichard Henderson }
2707587195bdSRichard Henderson break;
2708be210acbSRichard Henderson default:
2709f48f3edeSblueswir1 i = 0;
2710be210acbSRichard Henderson break;
2711be210acbSRichard Henderson }
271251e3972cSRichard Henderson switch (c) {
271351e3972cSRichard Henderson case INDEX_op_set_label:
271451e3972cSRichard Henderson case INDEX_op_br:
271551e3972cSRichard Henderson case INDEX_op_brcond_i32:
271651e3972cSRichard Henderson case INDEX_op_brcond_i64:
271751e3972cSRichard Henderson case INDEX_op_brcond2_i32:
2718b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2719efee3746SRichard Henderson arg_label(op->args[k])->id);
272051e3972cSRichard Henderson i++, k++;
272151e3972cSRichard Henderson break;
27223470867bSRichard Henderson case INDEX_op_mb:
27233470867bSRichard Henderson {
27243470867bSRichard Henderson TCGBar membar = op->args[k];
27253470867bSRichard Henderson const char *b_op, *m_op;
27263470867bSRichard Henderson
27273470867bSRichard Henderson switch (membar & TCG_BAR_SC) {
27283470867bSRichard Henderson case 0:
27293470867bSRichard Henderson b_op = "none";
27303470867bSRichard Henderson break;
27313470867bSRichard Henderson case TCG_BAR_LDAQ:
27323470867bSRichard Henderson b_op = "acq";
27333470867bSRichard Henderson break;
27343470867bSRichard Henderson case TCG_BAR_STRL:
27353470867bSRichard Henderson b_op = "rel";
27363470867bSRichard Henderson break;
27373470867bSRichard Henderson case TCG_BAR_SC:
27383470867bSRichard Henderson b_op = "seq";
27393470867bSRichard Henderson break;
27403470867bSRichard Henderson default:
27413470867bSRichard Henderson g_assert_not_reached();
27423470867bSRichard Henderson }
27433470867bSRichard Henderson
27443470867bSRichard Henderson switch (membar & TCG_MO_ALL) {
27453470867bSRichard Henderson case 0:
27463470867bSRichard Henderson m_op = "none";
27473470867bSRichard Henderson break;
27483470867bSRichard Henderson case TCG_MO_LD_LD:
27493470867bSRichard Henderson m_op = "rr";
27503470867bSRichard Henderson break;
27513470867bSRichard Henderson case TCG_MO_LD_ST:
27523470867bSRichard Henderson m_op = "rw";
27533470867bSRichard Henderson break;
27543470867bSRichard Henderson case TCG_MO_ST_LD:
27553470867bSRichard Henderson m_op = "wr";
27563470867bSRichard Henderson break;
27573470867bSRichard Henderson case TCG_MO_ST_ST:
27583470867bSRichard Henderson m_op = "ww";
27593470867bSRichard Henderson break;
27603470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST:
27613470867bSRichard Henderson m_op = "rr+rw";
27623470867bSRichard Henderson break;
27633470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD:
27643470867bSRichard Henderson m_op = "rr+wr";
27653470867bSRichard Henderson break;
27663470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_ST:
27673470867bSRichard Henderson m_op = "rr+ww";
27683470867bSRichard Henderson break;
27693470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD:
27703470867bSRichard Henderson m_op = "rw+wr";
27713470867bSRichard Henderson break;
27723470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_ST:
27733470867bSRichard Henderson m_op = "rw+ww";
27743470867bSRichard Henderson break;
27753470867bSRichard Henderson case TCG_MO_ST_LD | TCG_MO_ST_ST:
27763470867bSRichard Henderson m_op = "wr+ww";
27773470867bSRichard Henderson break;
27783470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
27793470867bSRichard Henderson m_op = "rr+rw+wr";
27803470867bSRichard Henderson break;
27813470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
27823470867bSRichard Henderson m_op = "rr+rw+ww";
27833470867bSRichard Henderson break;
27843470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
27853470867bSRichard Henderson m_op = "rr+wr+ww";
27863470867bSRichard Henderson break;
27873470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
27883470867bSRichard Henderson m_op = "rw+wr+ww";
27893470867bSRichard Henderson break;
27903470867bSRichard Henderson case TCG_MO_ALL:
27913470867bSRichard Henderson m_op = "all";
27923470867bSRichard Henderson break;
27933470867bSRichard Henderson default:
27943470867bSRichard Henderson g_assert_not_reached();
27953470867bSRichard Henderson }
27963470867bSRichard Henderson
27973470867bSRichard Henderson col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
27983470867bSRichard Henderson i++, k++;
27993470867bSRichard Henderson }
28003470867bSRichard Henderson break;
280151e3972cSRichard Henderson default:
280251e3972cSRichard Henderson break;
2803eeacee4dSBlue Swirl }
280451e3972cSRichard Henderson for (; i < nb_cargs; i++, k++) {
2805b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2806b7a83ff8SRichard Henderson op->args[k]);
2807bdfb460eSRichard Henderson }
2808bdfb460eSRichard Henderson }
2809bdfb460eSRichard Henderson
28101894f69aSRichard Henderson if (have_prefs || op->life) {
28111894f69aSRichard Henderson for (; col < 40; ++col) {
2812b7a83ff8SRichard Henderson putc(' ', f);
2813bdfb460eSRichard Henderson }
28141894f69aSRichard Henderson }
28151894f69aSRichard Henderson
28161894f69aSRichard Henderson if (op->life) {
28171894f69aSRichard Henderson unsigned life = op->life;
2818bdfb460eSRichard Henderson
2819bdfb460eSRichard Henderson if (life & (SYNC_ARG * 3)) {
2820b7a83ff8SRichard Henderson ne_fprintf(f, " sync:");
2821bdfb460eSRichard Henderson for (i = 0; i < 2; ++i) {
2822bdfb460eSRichard Henderson if (life & (SYNC_ARG << i)) {
2823b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i);
2824bdfb460eSRichard Henderson }
2825bdfb460eSRichard Henderson }
2826bdfb460eSRichard Henderson }
2827bdfb460eSRichard Henderson life /= DEAD_ARG;
2828bdfb460eSRichard Henderson if (life) {
2829b7a83ff8SRichard Henderson ne_fprintf(f, " dead:");
2830bdfb460eSRichard Henderson for (i = 0; life; ++i, life >>= 1) {
2831bdfb460eSRichard Henderson if (life & 1) {
2832b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i);
2833bdfb460eSRichard Henderson }
2834bdfb460eSRichard Henderson }
2835c896fe29Sbellard }
2836b03cce8eSbellard }
28371894f69aSRichard Henderson
28381894f69aSRichard Henderson if (have_prefs) {
28391894f69aSRichard Henderson for (i = 0; i < nb_oargs; ++i) {
284031fd884bSRichard Henderson TCGRegSet set = output_pref(op, i);
28411894f69aSRichard Henderson
28421894f69aSRichard Henderson if (i == 0) {
2843b7a83ff8SRichard Henderson ne_fprintf(f, " pref=");
28441894f69aSRichard Henderson } else {
2845b7a83ff8SRichard Henderson ne_fprintf(f, ",");
28461894f69aSRichard Henderson }
28471894f69aSRichard Henderson if (set == 0) {
2848b7a83ff8SRichard Henderson ne_fprintf(f, "none");
28491894f69aSRichard Henderson } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2850b7a83ff8SRichard Henderson ne_fprintf(f, "all");
28511894f69aSRichard Henderson #ifdef CONFIG_DEBUG_TCG
28521894f69aSRichard Henderson } else if (tcg_regset_single(set)) {
28531894f69aSRichard Henderson TCGReg reg = tcg_regset_first(set);
2854b7a83ff8SRichard Henderson ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
28551894f69aSRichard Henderson #endif
28561894f69aSRichard Henderson } else if (TCG_TARGET_NB_REGS <= 32) {
2857b7a83ff8SRichard Henderson ne_fprintf(f, "0x%x", (uint32_t)set);
28581894f69aSRichard Henderson } else {
2859b7a83ff8SRichard Henderson ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
28601894f69aSRichard Henderson }
28611894f69aSRichard Henderson }
28621894f69aSRichard Henderson }
28631894f69aSRichard Henderson
2864b7a83ff8SRichard Henderson putc('\n', f);
2865c896fe29Sbellard }
2866c896fe29Sbellard }
2867c896fe29Sbellard
2868c896fe29Sbellard /* we give more priority to constraints with less registers */
get_constraint_priority(const TCGOpDef * def,int k)2869c896fe29Sbellard static int get_constraint_priority(const TCGOpDef *def, int k)
2870c896fe29Sbellard {
287174a11790SRichard Henderson const TCGArgConstraint *arg_ct = &def->args_ct[k];
287229f5e925SRichard Henderson int n = ctpop64(arg_ct->regs);
2873c896fe29Sbellard
287429f5e925SRichard Henderson /*
287529f5e925SRichard Henderson * Sort constraints of a single register first, which includes output
287629f5e925SRichard Henderson * aliases (which must exactly match the input already allocated).
287729f5e925SRichard Henderson */
287829f5e925SRichard Henderson if (n == 1 || arg_ct->oalias) {
287929f5e925SRichard Henderson return INT_MAX;
2880c896fe29Sbellard }
288129f5e925SRichard Henderson
288229f5e925SRichard Henderson /*
288329f5e925SRichard Henderson * Sort register pairs next, first then second immediately after.
288429f5e925SRichard Henderson * Arbitrarily sort multiple pairs by the index of the first reg;
288529f5e925SRichard Henderson * there shouldn't be many pairs.
288629f5e925SRichard Henderson */
288729f5e925SRichard Henderson switch (arg_ct->pair) {
288829f5e925SRichard Henderson case 1:
288929f5e925SRichard Henderson case 3:
289029f5e925SRichard Henderson return (k + 1) * 2;
289129f5e925SRichard Henderson case 2:
289229f5e925SRichard Henderson return (arg_ct->pair_index + 1) * 2 - 1;
289329f5e925SRichard Henderson }
289429f5e925SRichard Henderson
289529f5e925SRichard Henderson /* Finally, sort by decreasing register count. */
289629f5e925SRichard Henderson assert(n > 1);
289729f5e925SRichard Henderson return -n;
2898c896fe29Sbellard }
2899c896fe29Sbellard
2900c896fe29Sbellard /* sort from highest priority to lowest */
sort_constraints(TCGOpDef * def,int start,int n)2901c896fe29Sbellard static void sort_constraints(TCGOpDef *def, int start, int n)
2902c896fe29Sbellard {
290366792f90SRichard Henderson int i, j;
290466792f90SRichard Henderson TCGArgConstraint *a = def->args_ct;
2905c896fe29Sbellard
290666792f90SRichard Henderson for (i = 0; i < n; i++) {
290766792f90SRichard Henderson a[start + i].sort_index = start + i;
290866792f90SRichard Henderson }
290966792f90SRichard Henderson if (n <= 1) {
2910c896fe29Sbellard return;
291166792f90SRichard Henderson }
2912c896fe29Sbellard for (i = 0; i < n - 1; i++) {
2913c896fe29Sbellard for (j = i + 1; j < n; j++) {
291466792f90SRichard Henderson int p1 = get_constraint_priority(def, a[start + i].sort_index);
291566792f90SRichard Henderson int p2 = get_constraint_priority(def, a[start + j].sort_index);
2916c896fe29Sbellard if (p1 < p2) {
291766792f90SRichard Henderson int tmp = a[start + i].sort_index;
291866792f90SRichard Henderson a[start + i].sort_index = a[start + j].sort_index;
291966792f90SRichard Henderson a[start + j].sort_index = tmp;
2920c896fe29Sbellard }
2921c896fe29Sbellard }
2922c896fe29Sbellard }
2923c896fe29Sbellard }
2924c896fe29Sbellard
process_op_defs(TCGContext * s)2925f69d277eSRichard Henderson static void process_op_defs(TCGContext *s)
2926c896fe29Sbellard {
2927a9751609SRichard Henderson TCGOpcode op;
2928c896fe29Sbellard
2929f69d277eSRichard Henderson for (op = 0; op < NB_OPS; op++) {
2930f69d277eSRichard Henderson TCGOpDef *def = &tcg_op_defs[op];
2931f69d277eSRichard Henderson const TCGTargetOpDef *tdefs;
293229f5e925SRichard Henderson bool saw_alias_pair = false;
293329f5e925SRichard Henderson int i, o, i2, o2, nb_args;
2934f69d277eSRichard Henderson
2935f69d277eSRichard Henderson if (def->flags & TCG_OPF_NOT_PRESENT) {
2936f69d277eSRichard Henderson continue;
2937f69d277eSRichard Henderson }
2938f69d277eSRichard Henderson
2939c896fe29Sbellard nb_args = def->nb_iargs + def->nb_oargs;
2940f69d277eSRichard Henderson if (nb_args == 0) {
2941f69d277eSRichard Henderson continue;
2942f69d277eSRichard Henderson }
2943f69d277eSRichard Henderson
29444c22e840SRichard Henderson /*
29454c22e840SRichard Henderson * Macro magic should make it impossible, but double-check that
29464c22e840SRichard Henderson * the array index is in range. Since the signness of an enum
29474c22e840SRichard Henderson * is implementation defined, force the result to unsigned.
29484c22e840SRichard Henderson */
29494c22e840SRichard Henderson unsigned con_set = tcg_target_op_def(op);
29504c22e840SRichard Henderson tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
29514c22e840SRichard Henderson tdefs = &constraint_sets[con_set];
2952f69d277eSRichard Henderson
2953c896fe29Sbellard for (i = 0; i < nb_args; i++) {
2954f69d277eSRichard Henderson const char *ct_str = tdefs->args_ct_str[i];
29558940ea0dSPhilippe Mathieu-Daudé bool input_p = i >= def->nb_oargs;
29568940ea0dSPhilippe Mathieu-Daudé
2957f69d277eSRichard Henderson /* Incomplete TCGTargetOpDef entry. */
2958eabb7b91SAurelien Jarno tcg_debug_assert(ct_str != NULL);
2959f69d277eSRichard Henderson
296017280ff4SRichard Henderson switch (*ct_str) {
296117280ff4SRichard Henderson case '0' ... '9':
29628940ea0dSPhilippe Mathieu-Daudé o = *ct_str - '0';
29638940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(input_p);
29648940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(o < def->nb_oargs);
29658940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(def->args_ct[o].regs != 0);
29668940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!def->args_ct[o].oalias);
29678940ea0dSPhilippe Mathieu-Daudé def->args_ct[i] = def->args_ct[o];
2968bc2b17e6SRichard Henderson /* The output sets oalias. */
29698940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].oalias = 1;
29708940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].alias_index = i;
2971bc2b17e6SRichard Henderson /* The input sets ialias. */
29728940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].ialias = 1;
29738940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].alias_index = o;
297429f5e925SRichard Henderson if (def->args_ct[i].pair) {
297529f5e925SRichard Henderson saw_alias_pair = true;
297629f5e925SRichard Henderson }
29778940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(ct_str[1] == '\0');
29788940ea0dSPhilippe Mathieu-Daudé continue;
29798940ea0dSPhilippe Mathieu-Daudé
298082790a87SRichard Henderson case '&':
29818940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!input_p);
2982bc2b17e6SRichard Henderson def->args_ct[i].newreg = true;
298382790a87SRichard Henderson ct_str++;
298482790a87SRichard Henderson break;
298529f5e925SRichard Henderson
298629f5e925SRichard Henderson case 'p': /* plus */
298729f5e925SRichard Henderson /* Allocate to the register after the previous. */
298829f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
298929f5e925SRichard Henderson o = i - 1;
299029f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair);
299129f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct);
299229f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){
299329f5e925SRichard Henderson .pair = 2,
299429f5e925SRichard Henderson .pair_index = o,
299529f5e925SRichard Henderson .regs = def->args_ct[o].regs << 1,
2996*b8819108SRichard Henderson .newreg = def->args_ct[o].newreg,
299729f5e925SRichard Henderson };
299829f5e925SRichard Henderson def->args_ct[o].pair = 1;
299929f5e925SRichard Henderson def->args_ct[o].pair_index = i;
300029f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0');
300129f5e925SRichard Henderson continue;
300229f5e925SRichard Henderson
300329f5e925SRichard Henderson case 'm': /* minus */
300429f5e925SRichard Henderson /* Allocate to the register before the previous. */
300529f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
300629f5e925SRichard Henderson o = i - 1;
300729f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair);
300829f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct);
300929f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){
301029f5e925SRichard Henderson .pair = 1,
301129f5e925SRichard Henderson .pair_index = o,
301229f5e925SRichard Henderson .regs = def->args_ct[o].regs >> 1,
3013*b8819108SRichard Henderson .newreg = def->args_ct[o].newreg,
301429f5e925SRichard Henderson };
301529f5e925SRichard Henderson def->args_ct[o].pair = 2;
301629f5e925SRichard Henderson def->args_ct[o].pair_index = i;
301729f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0');
301829f5e925SRichard Henderson continue;
30198940ea0dSPhilippe Mathieu-Daudé }
30208940ea0dSPhilippe Mathieu-Daudé
30218940ea0dSPhilippe Mathieu-Daudé do {
30228940ea0dSPhilippe Mathieu-Daudé switch (*ct_str) {
3023c896fe29Sbellard case 'i':
3024c896fe29Sbellard def->args_ct[i].ct |= TCG_CT_CONST;
3025c896fe29Sbellard break;
3026358b4923SRichard Henderson
3027358b4923SRichard Henderson /* Include all of the target-specific constraints. */
3028358b4923SRichard Henderson
3029358b4923SRichard Henderson #undef CONST
3030358b4923SRichard Henderson #define CONST(CASE, MASK) \
30318940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].ct |= MASK; break;
3032358b4923SRichard Henderson #define REGS(CASE, MASK) \
30338940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].regs |= MASK; break;
3034358b4923SRichard Henderson
3035358b4923SRichard Henderson #include "tcg-target-con-str.h"
3036358b4923SRichard Henderson
3037358b4923SRichard Henderson #undef REGS
3038358b4923SRichard Henderson #undef CONST
3039c896fe29Sbellard default:
30408940ea0dSPhilippe Mathieu-Daudé case '0' ... '9':
30418940ea0dSPhilippe Mathieu-Daudé case '&':
304229f5e925SRichard Henderson case 'p':
304329f5e925SRichard Henderson case 'm':
3044358b4923SRichard Henderson /* Typo in TCGTargetOpDef constraint. */
3045358b4923SRichard Henderson g_assert_not_reached();
3046358b4923SRichard Henderson }
30478940ea0dSPhilippe Mathieu-Daudé } while (*++ct_str != '\0');
3048c896fe29Sbellard }
3049c896fe29Sbellard
3050c68aaa18SStefan Weil /* TCGTargetOpDef entry with too much information? */
3051eabb7b91SAurelien Jarno tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
3052c68aaa18SStefan Weil
305329f5e925SRichard Henderson /*
305429f5e925SRichard Henderson * Fix up output pairs that are aliased with inputs.
305529f5e925SRichard Henderson * When we created the alias, we copied pair from the output.
305629f5e925SRichard Henderson * There are three cases:
305729f5e925SRichard Henderson * (1a) Pairs of inputs alias pairs of outputs.
305829f5e925SRichard Henderson * (1b) One input aliases the first of a pair of outputs.
305929f5e925SRichard Henderson * (2) One input aliases the second of a pair of outputs.
306029f5e925SRichard Henderson *
306129f5e925SRichard Henderson * Case 1a is handled by making sure that the pair_index'es are
306229f5e925SRichard Henderson * properly updated so that they appear the same as a pair of inputs.
306329f5e925SRichard Henderson *
306429f5e925SRichard Henderson * Case 1b is handled by setting the pair_index of the input to
306529f5e925SRichard Henderson * itself, simply so it doesn't point to an unrelated argument.
306629f5e925SRichard Henderson * Since we don't encounter the "second" during the input allocation
306729f5e925SRichard Henderson * phase, nothing happens with the second half of the input pair.
306829f5e925SRichard Henderson *
306929f5e925SRichard Henderson * Case 2 is handled by setting the second input to pair=3, the
307029f5e925SRichard Henderson * first output to pair=3, and the pair_index'es to match.
307129f5e925SRichard Henderson */
307229f5e925SRichard Henderson if (saw_alias_pair) {
307329f5e925SRichard Henderson for (i = def->nb_oargs; i < nb_args; i++) {
307429f5e925SRichard Henderson /*
307529f5e925SRichard Henderson * Since [0-9pm] must be alone in the constraint string,
307629f5e925SRichard Henderson * the only way they can both be set is if the pair comes
307729f5e925SRichard Henderson * from the output alias.
307829f5e925SRichard Henderson */
307929f5e925SRichard Henderson if (!def->args_ct[i].ialias) {
308029f5e925SRichard Henderson continue;
308129f5e925SRichard Henderson }
308229f5e925SRichard Henderson switch (def->args_ct[i].pair) {
308329f5e925SRichard Henderson case 0:
308429f5e925SRichard Henderson break;
308529f5e925SRichard Henderson case 1:
308629f5e925SRichard Henderson o = def->args_ct[i].alias_index;
308729f5e925SRichard Henderson o2 = def->args_ct[o].pair_index;
308829f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 1);
308929f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 2);
309029f5e925SRichard Henderson if (def->args_ct[o2].oalias) {
309129f5e925SRichard Henderson /* Case 1a */
309229f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index;
309329f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 2);
309429f5e925SRichard Henderson def->args_ct[i2].pair_index = i;
309529f5e925SRichard Henderson def->args_ct[i].pair_index = i2;
309629f5e925SRichard Henderson } else {
309729f5e925SRichard Henderson /* Case 1b */
309829f5e925SRichard Henderson def->args_ct[i].pair_index = i;
309929f5e925SRichard Henderson }
310029f5e925SRichard Henderson break;
310129f5e925SRichard Henderson case 2:
310229f5e925SRichard Henderson o = def->args_ct[i].alias_index;
310329f5e925SRichard Henderson o2 = def->args_ct[o].pair_index;
310429f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 2);
310529f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 1);
310629f5e925SRichard Henderson if (def->args_ct[o2].oalias) {
310729f5e925SRichard Henderson /* Case 1a */
310829f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index;
310929f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 1);
311029f5e925SRichard Henderson def->args_ct[i2].pair_index = i;
311129f5e925SRichard Henderson def->args_ct[i].pair_index = i2;
311229f5e925SRichard Henderson } else {
311329f5e925SRichard Henderson /* Case 2 */
311429f5e925SRichard Henderson def->args_ct[i].pair = 3;
311529f5e925SRichard Henderson def->args_ct[o2].pair = 3;
311629f5e925SRichard Henderson def->args_ct[i].pair_index = o2;
311729f5e925SRichard Henderson def->args_ct[o2].pair_index = i;
311829f5e925SRichard Henderson }
311929f5e925SRichard Henderson break;
312029f5e925SRichard Henderson default:
312129f5e925SRichard Henderson g_assert_not_reached();
312229f5e925SRichard Henderson }
312329f5e925SRichard Henderson }
312429f5e925SRichard Henderson }
312529f5e925SRichard Henderson
3126c896fe29Sbellard /* sort the constraints (XXX: this is just an heuristic) */
3127c896fe29Sbellard sort_constraints(def, 0, def->nb_oargs);
3128c896fe29Sbellard sort_constraints(def, def->nb_oargs, def->nb_iargs);
3129c896fe29Sbellard }
3130c896fe29Sbellard }
3131c896fe29Sbellard
remove_label_use(TCGOp * op,int idx)3132f85b1fc4SRichard Henderson static void remove_label_use(TCGOp *op, int idx)
3133f85b1fc4SRichard Henderson {
3134f85b1fc4SRichard Henderson TCGLabel *label = arg_label(op->args[idx]);
3135f85b1fc4SRichard Henderson TCGLabelUse *use;
3136f85b1fc4SRichard Henderson
3137f85b1fc4SRichard Henderson QSIMPLEQ_FOREACH(use, &label->branches, next) {
3138f85b1fc4SRichard Henderson if (use->op == op) {
3139f85b1fc4SRichard Henderson QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
3140f85b1fc4SRichard Henderson return;
3141f85b1fc4SRichard Henderson }
3142f85b1fc4SRichard Henderson }
3143f85b1fc4SRichard Henderson g_assert_not_reached();
3144f85b1fc4SRichard Henderson }
3145f85b1fc4SRichard Henderson
tcg_op_remove(TCGContext * s,TCGOp * op)31460c627cdcSRichard Henderson void tcg_op_remove(TCGContext *s, TCGOp *op)
31470c627cdcSRichard Henderson {
3148d88a117eSRichard Henderson switch (op->opc) {
3149d88a117eSRichard Henderson case INDEX_op_br:
3150f85b1fc4SRichard Henderson remove_label_use(op, 0);
3151d88a117eSRichard Henderson break;
3152d88a117eSRichard Henderson case INDEX_op_brcond_i32:
3153d88a117eSRichard Henderson case INDEX_op_brcond_i64:
3154f85b1fc4SRichard Henderson remove_label_use(op, 3);
3155d88a117eSRichard Henderson break;
3156d88a117eSRichard Henderson case INDEX_op_brcond2_i32:
3157f85b1fc4SRichard Henderson remove_label_use(op, 5);
3158d88a117eSRichard Henderson break;
3159d88a117eSRichard Henderson default:
3160d88a117eSRichard Henderson break;
3161d88a117eSRichard Henderson }
3162d88a117eSRichard Henderson
316315fa08f8SRichard Henderson QTAILQ_REMOVE(&s->ops, op, link);
316415fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
3165abebf925SRichard Henderson s->nb_ops--;
31660c627cdcSRichard Henderson }
31670c627cdcSRichard Henderson
tcg_remove_ops_after(TCGOp * op)3168a80cdd31SRichard Henderson void tcg_remove_ops_after(TCGOp *op)
3169a80cdd31SRichard Henderson {
3170a80cdd31SRichard Henderson TCGContext *s = tcg_ctx;
3171a80cdd31SRichard Henderson
3172a80cdd31SRichard Henderson while (true) {
3173a80cdd31SRichard Henderson TCGOp *last = tcg_last_op();
3174a80cdd31SRichard Henderson if (last == op) {
3175a80cdd31SRichard Henderson return;
3176a80cdd31SRichard Henderson }
3177a80cdd31SRichard Henderson tcg_op_remove(s, last);
3178a80cdd31SRichard Henderson }
3179a80cdd31SRichard Henderson }
3180a80cdd31SRichard Henderson
tcg_op_alloc(TCGOpcode opc,unsigned nargs)3181d4478943SPhilippe Mathieu-Daudé static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
318215fa08f8SRichard Henderson {
318315fa08f8SRichard Henderson TCGContext *s = tcg_ctx;
3184cb10bc63SRichard Henderson TCGOp *op = NULL;
318515fa08f8SRichard Henderson
3186cb10bc63SRichard Henderson if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
3187cb10bc63SRichard Henderson QTAILQ_FOREACH(op, &s->free_ops, link) {
3188cb10bc63SRichard Henderson if (nargs <= op->nargs) {
318915fa08f8SRichard Henderson QTAILQ_REMOVE(&s->free_ops, op, link);
3190cb10bc63SRichard Henderson nargs = op->nargs;
3191cb10bc63SRichard Henderson goto found;
319215fa08f8SRichard Henderson }
3193cb10bc63SRichard Henderson }
3194cb10bc63SRichard Henderson }
3195cb10bc63SRichard Henderson
3196cb10bc63SRichard Henderson /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
3197cb10bc63SRichard Henderson nargs = MAX(4, nargs);
3198cb10bc63SRichard Henderson op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
3199cb10bc63SRichard Henderson
3200cb10bc63SRichard Henderson found:
320115fa08f8SRichard Henderson memset(op, 0, offsetof(TCGOp, link));
320215fa08f8SRichard Henderson op->opc = opc;
3203cb10bc63SRichard Henderson op->nargs = nargs;
320415fa08f8SRichard Henderson
3205cb10bc63SRichard Henderson /* Check for bitfield overflow. */
3206cb10bc63SRichard Henderson tcg_debug_assert(op->nargs == nargs);
3207cb10bc63SRichard Henderson
3208cb10bc63SRichard Henderson s->nb_ops++;
320915fa08f8SRichard Henderson return op;
321015fa08f8SRichard Henderson }
321115fa08f8SRichard Henderson
tcg_emit_op(TCGOpcode opc,unsigned nargs)3212d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
321315fa08f8SRichard Henderson {
3214d4478943SPhilippe Mathieu-Daudé TCGOp *op = tcg_op_alloc(opc, nargs);
321515fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
321615fa08f8SRichard Henderson return op;
321715fa08f8SRichard Henderson }
321815fa08f8SRichard Henderson
tcg_op_insert_before(TCGContext * s,TCGOp * old_op,TCGOpcode opc,unsigned nargs)3219d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
3220d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs)
32215a18407fSRichard Henderson {
3222d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs);
322315fa08f8SRichard Henderson QTAILQ_INSERT_BEFORE(old_op, new_op, link);
32245a18407fSRichard Henderson return new_op;
32255a18407fSRichard Henderson }
32265a18407fSRichard Henderson
tcg_op_insert_after(TCGContext * s,TCGOp * old_op,TCGOpcode opc,unsigned nargs)3227d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
3228d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs)
32295a18407fSRichard Henderson {
3230d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs);
323115fa08f8SRichard Henderson QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
32325a18407fSRichard Henderson return new_op;
32335a18407fSRichard Henderson }
32345a18407fSRichard Henderson
move_label_uses(TCGLabel * to,TCGLabel * from)3235968f305eSRichard Henderson static void move_label_uses(TCGLabel *to, TCGLabel *from)
3236968f305eSRichard Henderson {
3237968f305eSRichard Henderson TCGLabelUse *u;
3238968f305eSRichard Henderson
3239968f305eSRichard Henderson QSIMPLEQ_FOREACH(u, &from->branches, next) {
3240968f305eSRichard Henderson TCGOp *op = u->op;
3241968f305eSRichard Henderson switch (op->opc) {
3242968f305eSRichard Henderson case INDEX_op_br:
3243968f305eSRichard Henderson op->args[0] = label_arg(to);
3244968f305eSRichard Henderson break;
3245968f305eSRichard Henderson case INDEX_op_brcond_i32:
3246968f305eSRichard Henderson case INDEX_op_brcond_i64:
3247968f305eSRichard Henderson op->args[3] = label_arg(to);
3248968f305eSRichard Henderson break;
3249968f305eSRichard Henderson case INDEX_op_brcond2_i32:
3250968f305eSRichard Henderson op->args[5] = label_arg(to);
3251968f305eSRichard Henderson break;
3252968f305eSRichard Henderson default:
3253968f305eSRichard Henderson g_assert_not_reached();
3254968f305eSRichard Henderson }
3255968f305eSRichard Henderson }
3256968f305eSRichard Henderson
3257968f305eSRichard Henderson QSIMPLEQ_CONCAT(&to->branches, &from->branches);
3258968f305eSRichard Henderson }
3259968f305eSRichard Henderson
3260b4fc67c7SRichard Henderson /* Reachable analysis : remove unreachable code. */
32619bbee4c0SRichard Henderson static void __attribute__((noinline))
reachable_code_pass(TCGContext * s)32629bbee4c0SRichard Henderson reachable_code_pass(TCGContext *s)
3263b4fc67c7SRichard Henderson {
32644d89d0bbSRichard Henderson TCGOp *op, *op_next, *op_prev;
3265b4fc67c7SRichard Henderson bool dead = false;
3266b4fc67c7SRichard Henderson
3267b4fc67c7SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3268b4fc67c7SRichard Henderson bool remove = dead;
3269b4fc67c7SRichard Henderson TCGLabel *label;
3270b4fc67c7SRichard Henderson
3271b4fc67c7SRichard Henderson switch (op->opc) {
3272b4fc67c7SRichard Henderson case INDEX_op_set_label:
3273b4fc67c7SRichard Henderson label = arg_label(op->args[0]);
32744d89d0bbSRichard Henderson
32754d89d0bbSRichard Henderson /*
3276968f305eSRichard Henderson * Note that the first op in the TB is always a load,
3277968f305eSRichard Henderson * so there is always something before a label.
3278968f305eSRichard Henderson */
3279968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link);
3280968f305eSRichard Henderson
3281968f305eSRichard Henderson /*
3282968f305eSRichard Henderson * If we find two sequential labels, move all branches to
3283968f305eSRichard Henderson * reference the second label and remove the first label.
3284968f305eSRichard Henderson * Do this before branch to next optimization, so that the
3285968f305eSRichard Henderson * middle label is out of the way.
3286968f305eSRichard Henderson */
3287968f305eSRichard Henderson if (op_prev->opc == INDEX_op_set_label) {
3288968f305eSRichard Henderson move_label_uses(label, arg_label(op_prev->args[0]));
3289968f305eSRichard Henderson tcg_op_remove(s, op_prev);
3290968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link);
3291968f305eSRichard Henderson }
3292968f305eSRichard Henderson
3293968f305eSRichard Henderson /*
32944d89d0bbSRichard Henderson * Optimization can fold conditional branches to unconditional.
32954d89d0bbSRichard Henderson * If we find a label which is preceded by an unconditional
32964d89d0bbSRichard Henderson * branch to next, remove the branch. We couldn't do this when
32974d89d0bbSRichard Henderson * processing the branch because any dead code between the branch
32984d89d0bbSRichard Henderson * and label had not yet been removed.
32994d89d0bbSRichard Henderson */
33004d89d0bbSRichard Henderson if (op_prev->opc == INDEX_op_br &&
33014d89d0bbSRichard Henderson label == arg_label(op_prev->args[0])) {
33024d89d0bbSRichard Henderson tcg_op_remove(s, op_prev);
33034d89d0bbSRichard Henderson /* Fall through means insns become live again. */
33044d89d0bbSRichard Henderson dead = false;
33054d89d0bbSRichard Henderson }
33064d89d0bbSRichard Henderson
3307f85b1fc4SRichard Henderson if (QSIMPLEQ_EMPTY(&label->branches)) {
3308b4fc67c7SRichard Henderson /*
3309b4fc67c7SRichard Henderson * While there is an occasional backward branch, virtually
3310b4fc67c7SRichard Henderson * all branches generated by the translators are forward.
3311b4fc67c7SRichard Henderson * Which means that generally we will have already removed
3312b4fc67c7SRichard Henderson * all references to the label that will be, and there is
3313b4fc67c7SRichard Henderson * little to be gained by iterating.
3314b4fc67c7SRichard Henderson */
3315b4fc67c7SRichard Henderson remove = true;
3316b4fc67c7SRichard Henderson } else {
3317b4fc67c7SRichard Henderson /* Once we see a label, insns become live again. */
3318b4fc67c7SRichard Henderson dead = false;
3319b4fc67c7SRichard Henderson remove = false;
3320b4fc67c7SRichard Henderson }
3321b4fc67c7SRichard Henderson break;
3322b4fc67c7SRichard Henderson
3323b4fc67c7SRichard Henderson case INDEX_op_br:
3324b4fc67c7SRichard Henderson case INDEX_op_exit_tb:
3325b4fc67c7SRichard Henderson case INDEX_op_goto_ptr:
3326b4fc67c7SRichard Henderson /* Unconditional branches; everything following is dead. */
3327b4fc67c7SRichard Henderson dead = true;
3328b4fc67c7SRichard Henderson break;
3329b4fc67c7SRichard Henderson
3330b4fc67c7SRichard Henderson case INDEX_op_call:
3331b4fc67c7SRichard Henderson /* Notice noreturn helper calls, raising exceptions. */
333290163900SRichard Henderson if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
3333b4fc67c7SRichard Henderson dead = true;
3334b4fc67c7SRichard Henderson }
3335b4fc67c7SRichard Henderson break;
3336b4fc67c7SRichard Henderson
3337b4fc67c7SRichard Henderson case INDEX_op_insn_start:
3338b4fc67c7SRichard Henderson /* Never remove -- we need to keep these for unwind. */
3339b4fc67c7SRichard Henderson remove = false;
3340b4fc67c7SRichard Henderson break;
3341b4fc67c7SRichard Henderson
3342b4fc67c7SRichard Henderson default:
3343b4fc67c7SRichard Henderson break;
3344b4fc67c7SRichard Henderson }
3345b4fc67c7SRichard Henderson
3346b4fc67c7SRichard Henderson if (remove) {
3347b4fc67c7SRichard Henderson tcg_op_remove(s, op);
3348b4fc67c7SRichard Henderson }
3349b4fc67c7SRichard Henderson }
3350b4fc67c7SRichard Henderson }
3351b4fc67c7SRichard Henderson
3352c70fbf0aSRichard Henderson #define TS_DEAD 1
3353c70fbf0aSRichard Henderson #define TS_MEM 2
3354c70fbf0aSRichard Henderson
33555a18407fSRichard Henderson #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
33565a18407fSRichard Henderson #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
33575a18407fSRichard Henderson
335825f49c5fSRichard Henderson /* For liveness_pass_1, the register preferences for a given temp. */
la_temp_pref(TCGTemp * ts)335925f49c5fSRichard Henderson static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
336025f49c5fSRichard Henderson {
336125f49c5fSRichard Henderson return ts->state_ptr;
336225f49c5fSRichard Henderson }
336325f49c5fSRichard Henderson
336425f49c5fSRichard Henderson /* For liveness_pass_1, reset the preferences for a given temp to the
336525f49c5fSRichard Henderson * maximal regset for its type.
336625f49c5fSRichard Henderson */
la_reset_pref(TCGTemp * ts)336725f49c5fSRichard Henderson static inline void la_reset_pref(TCGTemp *ts)
336825f49c5fSRichard Henderson {
336925f49c5fSRichard Henderson *la_temp_pref(ts)
337025f49c5fSRichard Henderson = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
337125f49c5fSRichard Henderson }
337225f49c5fSRichard Henderson
33739c43b68dSAurelien Jarno /* liveness analysis: end of function: all temps are dead, and globals
33749c43b68dSAurelien Jarno should be in memory. */
la_func_end(TCGContext * s,int ng,int nt)33752616c808SRichard Henderson static void la_func_end(TCGContext *s, int ng, int nt)
3376c896fe29Sbellard {
3377b83eabeaSRichard Henderson int i;
3378b83eabeaSRichard Henderson
3379b83eabeaSRichard Henderson for (i = 0; i < ng; ++i) {
3380b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM;
338125f49c5fSRichard Henderson la_reset_pref(&s->temps[i]);
3382b83eabeaSRichard Henderson }
3383b83eabeaSRichard Henderson for (i = ng; i < nt; ++i) {
3384b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD;
338525f49c5fSRichard Henderson la_reset_pref(&s->temps[i]);
3386b83eabeaSRichard Henderson }
3387c896fe29Sbellard }
3388c896fe29Sbellard
33899c43b68dSAurelien Jarno /* liveness analysis: end of basic block: all temps are dead, globals
33909c43b68dSAurelien Jarno and local temps should be in memory. */
la_bb_end(TCGContext * s,int ng,int nt)33912616c808SRichard Henderson static void la_bb_end(TCGContext *s, int ng, int nt)
3392641d5fbeSbellard {
3393b83eabeaSRichard Henderson int i;
3394641d5fbeSbellard
3395ee17db83SRichard Henderson for (i = 0; i < nt; ++i) {
3396ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i];
3397ee17db83SRichard Henderson int state;
3398ee17db83SRichard Henderson
3399ee17db83SRichard Henderson switch (ts->kind) {
3400ee17db83SRichard Henderson case TEMP_FIXED:
3401ee17db83SRichard Henderson case TEMP_GLOBAL:
3402f57c6915SRichard Henderson case TEMP_TB:
3403ee17db83SRichard Henderson state = TS_DEAD | TS_MEM;
3404ee17db83SRichard Henderson break;
3405c7482438SRichard Henderson case TEMP_EBB:
3406c0522136SRichard Henderson case TEMP_CONST:
3407ee17db83SRichard Henderson state = TS_DEAD;
3408ee17db83SRichard Henderson break;
3409ee17db83SRichard Henderson default:
3410ee17db83SRichard Henderson g_assert_not_reached();
3411c70fbf0aSRichard Henderson }
3412ee17db83SRichard Henderson ts->state = state;
3413ee17db83SRichard Henderson la_reset_pref(ts);
3414641d5fbeSbellard }
3415641d5fbeSbellard }
3416641d5fbeSbellard
3417f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory. */
la_global_sync(TCGContext * s,int ng)3418f65a061cSRichard Henderson static void la_global_sync(TCGContext *s, int ng)
3419f65a061cSRichard Henderson {
3420f65a061cSRichard Henderson int i;
3421f65a061cSRichard Henderson
3422f65a061cSRichard Henderson for (i = 0; i < ng; ++i) {
342325f49c5fSRichard Henderson int state = s->temps[i].state;
342425f49c5fSRichard Henderson s->temps[i].state = state | TS_MEM;
342525f49c5fSRichard Henderson if (state == TS_DEAD) {
342625f49c5fSRichard Henderson /* If the global was previously dead, reset prefs. */
342725f49c5fSRichard Henderson la_reset_pref(&s->temps[i]);
342825f49c5fSRichard Henderson }
3429f65a061cSRichard Henderson }
3430f65a061cSRichard Henderson }
3431f65a061cSRichard Henderson
3432b4cb76e6SRichard Henderson /*
3433c7482438SRichard Henderson * liveness analysis: conditional branch: all temps are dead unless
3434c7482438SRichard Henderson * explicitly live-across-conditional-branch, globals and local temps
3435c7482438SRichard Henderson * should be synced.
3436b4cb76e6SRichard Henderson */
la_bb_sync(TCGContext * s,int ng,int nt)3437b4cb76e6SRichard Henderson static void la_bb_sync(TCGContext *s, int ng, int nt)
3438b4cb76e6SRichard Henderson {
3439b4cb76e6SRichard Henderson la_global_sync(s, ng);
3440b4cb76e6SRichard Henderson
3441b4cb76e6SRichard Henderson for (int i = ng; i < nt; ++i) {
3442c0522136SRichard Henderson TCGTemp *ts = &s->temps[i];
3443c0522136SRichard Henderson int state;
3444c0522136SRichard Henderson
3445c0522136SRichard Henderson switch (ts->kind) {
3446f57c6915SRichard Henderson case TEMP_TB:
3447c0522136SRichard Henderson state = ts->state;
3448c0522136SRichard Henderson ts->state = state | TS_MEM;
3449b4cb76e6SRichard Henderson if (state != TS_DEAD) {
3450b4cb76e6SRichard Henderson continue;
3451b4cb76e6SRichard Henderson }
3452c0522136SRichard Henderson break;
3453c7482438SRichard Henderson case TEMP_EBB:
3454c0522136SRichard Henderson case TEMP_CONST:
3455c0522136SRichard Henderson continue;
3456c0522136SRichard Henderson default:
3457c0522136SRichard Henderson g_assert_not_reached();
3458b4cb76e6SRichard Henderson }
3459b4cb76e6SRichard Henderson la_reset_pref(&s->temps[i]);
3460b4cb76e6SRichard Henderson }
3461b4cb76e6SRichard Henderson }
3462b4cb76e6SRichard Henderson
3463f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory and kill. */
la_global_kill(TCGContext * s,int ng)3464f65a061cSRichard Henderson static void la_global_kill(TCGContext *s, int ng)
3465f65a061cSRichard Henderson {
3466f65a061cSRichard Henderson int i;
3467f65a061cSRichard Henderson
3468f65a061cSRichard Henderson for (i = 0; i < ng; i++) {
3469f65a061cSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM;
347025f49c5fSRichard Henderson la_reset_pref(&s->temps[i]);
347125f49c5fSRichard Henderson }
347225f49c5fSRichard Henderson }
347325f49c5fSRichard Henderson
347425f49c5fSRichard Henderson /* liveness analysis: note live globals crossing calls. */
la_cross_call(TCGContext * s,int nt)347525f49c5fSRichard Henderson static void la_cross_call(TCGContext *s, int nt)
347625f49c5fSRichard Henderson {
347725f49c5fSRichard Henderson TCGRegSet mask = ~tcg_target_call_clobber_regs;
347825f49c5fSRichard Henderson int i;
347925f49c5fSRichard Henderson
348025f49c5fSRichard Henderson for (i = 0; i < nt; i++) {
348125f49c5fSRichard Henderson TCGTemp *ts = &s->temps[i];
348225f49c5fSRichard Henderson if (!(ts->state & TS_DEAD)) {
348325f49c5fSRichard Henderson TCGRegSet *pset = la_temp_pref(ts);
348425f49c5fSRichard Henderson TCGRegSet set = *pset;
348525f49c5fSRichard Henderson
348625f49c5fSRichard Henderson set &= mask;
348725f49c5fSRichard Henderson /* If the combination is not possible, restart. */
348825f49c5fSRichard Henderson if (set == 0) {
348925f49c5fSRichard Henderson set = tcg_target_available_regs[ts->type] & mask;
349025f49c5fSRichard Henderson }
349125f49c5fSRichard Henderson *pset = set;
349225f49c5fSRichard Henderson }
3493f65a061cSRichard Henderson }
3494f65a061cSRichard Henderson }
3495f65a061cSRichard Henderson
3496874b8574SRichard Henderson /*
3497874b8574SRichard Henderson * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
3498874b8574SRichard Henderson * to TEMP_EBB, if possible.
3499874b8574SRichard Henderson */
3500874b8574SRichard Henderson static void __attribute__((noinline))
liveness_pass_0(TCGContext * s)3501874b8574SRichard Henderson liveness_pass_0(TCGContext *s)
3502874b8574SRichard Henderson {
3503874b8574SRichard Henderson void * const multiple_ebb = (void *)(uintptr_t)-1;
3504874b8574SRichard Henderson int nb_temps = s->nb_temps;
3505874b8574SRichard Henderson TCGOp *op, *ebb;
3506874b8574SRichard Henderson
3507874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) {
3508874b8574SRichard Henderson s->temps[i].state_ptr = NULL;
3509874b8574SRichard Henderson }
3510874b8574SRichard Henderson
3511874b8574SRichard Henderson /*
3512874b8574SRichard Henderson * Represent each EBB by the op at which it begins. In the case of
3513874b8574SRichard Henderson * the first EBB, this is the first op, otherwise it is a label.
3514874b8574SRichard Henderson * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
3515874b8574SRichard Henderson * within a single EBB, else MULTIPLE_EBB.
3516874b8574SRichard Henderson */
3517874b8574SRichard Henderson ebb = QTAILQ_FIRST(&s->ops);
3518874b8574SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) {
3519874b8574SRichard Henderson const TCGOpDef *def;
3520874b8574SRichard Henderson int nb_oargs, nb_iargs;
3521874b8574SRichard Henderson
3522874b8574SRichard Henderson switch (op->opc) {
3523874b8574SRichard Henderson case INDEX_op_set_label:
3524874b8574SRichard Henderson ebb = op;
3525874b8574SRichard Henderson continue;
3526874b8574SRichard Henderson case INDEX_op_discard:
3527874b8574SRichard Henderson continue;
3528874b8574SRichard Henderson case INDEX_op_call:
3529874b8574SRichard Henderson nb_oargs = TCGOP_CALLO(op);
3530874b8574SRichard Henderson nb_iargs = TCGOP_CALLI(op);
3531874b8574SRichard Henderson break;
3532874b8574SRichard Henderson default:
3533874b8574SRichard Henderson def = &tcg_op_defs[op->opc];
3534874b8574SRichard Henderson nb_oargs = def->nb_oargs;
3535874b8574SRichard Henderson nb_iargs = def->nb_iargs;
3536874b8574SRichard Henderson break;
3537874b8574SRichard Henderson }
3538874b8574SRichard Henderson
3539874b8574SRichard Henderson for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
3540874b8574SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]);
3541874b8574SRichard Henderson
3542874b8574SRichard Henderson if (ts->kind != TEMP_TB) {
3543874b8574SRichard Henderson continue;
3544874b8574SRichard Henderson }
3545874b8574SRichard Henderson if (ts->state_ptr == NULL) {
3546874b8574SRichard Henderson ts->state_ptr = ebb;
3547874b8574SRichard Henderson } else if (ts->state_ptr != ebb) {
3548874b8574SRichard Henderson ts->state_ptr = multiple_ebb;
3549874b8574SRichard Henderson }
3550874b8574SRichard Henderson }
3551874b8574SRichard Henderson }
3552874b8574SRichard Henderson
3553874b8574SRichard Henderson /*
3554874b8574SRichard Henderson * For TEMP_TB that turned out not to be used beyond one EBB,
3555874b8574SRichard Henderson * reduce the liveness to TEMP_EBB.
3556874b8574SRichard Henderson */
3557874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) {
3558874b8574SRichard Henderson TCGTemp *ts = &s->temps[i];
3559874b8574SRichard Henderson if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3560874b8574SRichard Henderson ts->kind = TEMP_EBB;
3561874b8574SRichard Henderson }
3562874b8574SRichard Henderson }
3563874b8574SRichard Henderson }
3564874b8574SRichard Henderson
3565a1b3c48dSRichard Henderson /* Liveness analysis : update the opc_arg_life array to tell if a
3566c896fe29Sbellard given input arguments is dead. Instructions updating dead
3567c896fe29Sbellard temporaries are removed. */
35689bbee4c0SRichard Henderson static void __attribute__((noinline))
liveness_pass_1(TCGContext * s)35699bbee4c0SRichard Henderson liveness_pass_1(TCGContext *s)
3570c896fe29Sbellard {
3571c70fbf0aSRichard Henderson int nb_globals = s->nb_globals;
35722616c808SRichard Henderson int nb_temps = s->nb_temps;
357315fa08f8SRichard Henderson TCGOp *op, *op_prev;
357425f49c5fSRichard Henderson TCGRegSet *prefs;
357525f49c5fSRichard Henderson int i;
357625f49c5fSRichard Henderson
357725f49c5fSRichard Henderson prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
357825f49c5fSRichard Henderson for (i = 0; i < nb_temps; ++i) {
357925f49c5fSRichard Henderson s->temps[i].state_ptr = prefs + i;
358025f49c5fSRichard Henderson }
3581c896fe29Sbellard
3582ae36a246SRichard Henderson /* ??? Should be redundant with the exit_tb that ends the TB. */
35832616c808SRichard Henderson la_func_end(s, nb_globals, nb_temps);
3584c896fe29Sbellard
3585eae3eb3eSPaolo Bonzini QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
358625f49c5fSRichard Henderson int nb_iargs, nb_oargs;
3587c45cb8bbSRichard Henderson TCGOpcode opc_new, opc_new2;
3588c45cb8bbSRichard Henderson bool have_opc_new2;
3589a1b3c48dSRichard Henderson TCGLifeData arg_life = 0;
359025f49c5fSRichard Henderson TCGTemp *ts;
3591c45cb8bbSRichard Henderson TCGOpcode opc = op->opc;
3592c45cb8bbSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc];
3593c45cb8bbSRichard Henderson
3594c45cb8bbSRichard Henderson switch (opc) {
3595c896fe29Sbellard case INDEX_op_call:
3596c6e113f5Sbellard {
359739004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op);
359839004a71SRichard Henderson int call_flags = tcg_call_flags(op);
3599c6e113f5Sbellard
3600cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op);
3601cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op);
3602c6e113f5Sbellard
3603c45cb8bbSRichard Henderson /* pure functions can be removed if their result is unused */
360478505279SAurelien Jarno if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
3605c6e113f5Sbellard for (i = 0; i < nb_oargs; i++) {
360625f49c5fSRichard Henderson ts = arg_temp(op->args[i]);
360725f49c5fSRichard Henderson if (ts->state != TS_DEAD) {
3608c6e113f5Sbellard goto do_not_remove_call;
3609c6e113f5Sbellard }
36109c43b68dSAurelien Jarno }
3611c45cb8bbSRichard Henderson goto do_remove;
3612152c35aaSRichard Henderson }
3613c6e113f5Sbellard do_not_remove_call:
3614c896fe29Sbellard
361525f49c5fSRichard Henderson /* Output args are dead. */
3616c896fe29Sbellard for (i = 0; i < nb_oargs; i++) {
361725f49c5fSRichard Henderson ts = arg_temp(op->args[i]);
361825f49c5fSRichard Henderson if (ts->state & TS_DEAD) {
3619a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i;
36206b64b624SAurelien Jarno }
362125f49c5fSRichard Henderson if (ts->state & TS_MEM) {
3622a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i;
36239c43b68dSAurelien Jarno }
362425f49c5fSRichard Henderson ts->state = TS_DEAD;
362525f49c5fSRichard Henderson la_reset_pref(ts);
3626c896fe29Sbellard }
3627c896fe29Sbellard
362831fd884bSRichard Henderson /* Not used -- it will be tcg_target_call_oarg_reg(). */
362931fd884bSRichard Henderson memset(op->output_pref, 0, sizeof(op->output_pref));
363031fd884bSRichard Henderson
363178505279SAurelien Jarno if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
363278505279SAurelien Jarno TCG_CALL_NO_READ_GLOBALS))) {
3633f65a061cSRichard Henderson la_global_kill(s, nb_globals);
3634c70fbf0aSRichard Henderson } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
3635f65a061cSRichard Henderson la_global_sync(s, nb_globals);
3636b9c18f56Saurel32 }
3637c896fe29Sbellard
363825f49c5fSRichard Henderson /* Record arguments that die in this helper. */
3639866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
364025f49c5fSRichard Henderson ts = arg_temp(op->args[i]);
364139004a71SRichard Henderson if (ts->state & TS_DEAD) {
3642a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i;
3643c896fe29Sbellard }
3644c896fe29Sbellard }
364525f49c5fSRichard Henderson
364625f49c5fSRichard Henderson /* For all live registers, remove call-clobbered prefs. */
364725f49c5fSRichard Henderson la_cross_call(s, nb_temps);
364825f49c5fSRichard Henderson
364939004a71SRichard Henderson /*
365039004a71SRichard Henderson * Input arguments are live for preceding opcodes.
365139004a71SRichard Henderson *
365239004a71SRichard Henderson * For those arguments that die, and will be allocated in
365339004a71SRichard Henderson * registers, clear the register set for that arg, to be
365439004a71SRichard Henderson * filled in below. For args that will be on the stack,
365539004a71SRichard Henderson * reset to any available reg. Process arguments in reverse
365639004a71SRichard Henderson * order so that if a temp is used more than once, the stack
365739004a71SRichard Henderson * reset to max happens before the register reset to 0.
365825f49c5fSRichard Henderson */
365939004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; i--) {
366039004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i];
366139004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]);
366239004a71SRichard Henderson
366339004a71SRichard Henderson if (ts->state & TS_DEAD) {
366439004a71SRichard Henderson switch (loc->kind) {
366539004a71SRichard Henderson case TCG_CALL_ARG_NORMAL:
366639004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U:
366739004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S:
3668338b61e9SRichard Henderson if (arg_slot_reg_p(loc->arg_slot)) {
366939004a71SRichard Henderson *la_temp_pref(ts) = 0;
367039004a71SRichard Henderson break;
367139004a71SRichard Henderson }
367239004a71SRichard Henderson /* fall through */
367339004a71SRichard Henderson default:
367439004a71SRichard Henderson *la_temp_pref(ts) =
367539004a71SRichard Henderson tcg_target_available_regs[ts->type];
367639004a71SRichard Henderson break;
367739004a71SRichard Henderson }
367825f49c5fSRichard Henderson ts->state &= ~TS_DEAD;
367925f49c5fSRichard Henderson }
368025f49c5fSRichard Henderson }
368125f49c5fSRichard Henderson
368239004a71SRichard Henderson /*
368339004a71SRichard Henderson * For each input argument, add its input register to prefs.
368439004a71SRichard Henderson * If a temp is used once, this produces a single set bit;
368539004a71SRichard Henderson * if a temp is used multiple times, this produces a set.
368639004a71SRichard Henderson */
368739004a71SRichard Henderson for (i = 0; i < nb_iargs; i++) {
368839004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i];
368939004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]);
369039004a71SRichard Henderson
369139004a71SRichard Henderson switch (loc->kind) {
369239004a71SRichard Henderson case TCG_CALL_ARG_NORMAL:
369339004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U:
369439004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S:
3695338b61e9SRichard Henderson if (arg_slot_reg_p(loc->arg_slot)) {
369625f49c5fSRichard Henderson tcg_regset_set_reg(*la_temp_pref(ts),
369739004a71SRichard Henderson tcg_target_call_iarg_regs[loc->arg_slot]);
369839004a71SRichard Henderson }
369939004a71SRichard Henderson break;
370039004a71SRichard Henderson default:
370139004a71SRichard Henderson break;
3702c70fbf0aSRichard Henderson }
3703c19f47bfSAurelien Jarno }
3704c6e113f5Sbellard }
3705c896fe29Sbellard break;
3706765b842aSRichard Henderson case INDEX_op_insn_start:
3707c896fe29Sbellard break;
37085ff9d6a4Sbellard case INDEX_op_discard:
37095ff9d6a4Sbellard /* mark the temporary as dead */
371025f49c5fSRichard Henderson ts = arg_temp(op->args[0]);
371125f49c5fSRichard Henderson ts->state = TS_DEAD;
371225f49c5fSRichard Henderson la_reset_pref(ts);
37135ff9d6a4Sbellard break;
37141305c451SRichard Henderson
37151305c451SRichard Henderson case INDEX_op_add2_i32:
3716c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i32;
3717f1fae40cSRichard Henderson goto do_addsub2;
37181305c451SRichard Henderson case INDEX_op_sub2_i32:
3719c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i32;
3720f1fae40cSRichard Henderson goto do_addsub2;
3721f1fae40cSRichard Henderson case INDEX_op_add2_i64:
3722c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i64;
3723f1fae40cSRichard Henderson goto do_addsub2;
3724f1fae40cSRichard Henderson case INDEX_op_sub2_i64:
3725c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i64;
3726f1fae40cSRichard Henderson do_addsub2:
37271305c451SRichard Henderson nb_iargs = 4;
37281305c451SRichard Henderson nb_oargs = 2;
37291305c451SRichard Henderson /* Test if the high part of the operation is dead, but not
37301305c451SRichard Henderson the low part. The result can be optimized to a simple
37311305c451SRichard Henderson add or sub. This happens often for x86_64 guest when the
37321305c451SRichard Henderson cpu mode is set to 32 bit. */
3733b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) {
3734b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) {
37351305c451SRichard Henderson goto do_remove;
37361305c451SRichard Henderson }
3737c45cb8bbSRichard Henderson /* Replace the opcode and adjust the args in place,
3738c45cb8bbSRichard Henderson leaving 3 unused args at the end. */
3739c45cb8bbSRichard Henderson op->opc = opc = opc_new;
3740efee3746SRichard Henderson op->args[1] = op->args[2];
3741efee3746SRichard Henderson op->args[2] = op->args[4];
37421305c451SRichard Henderson /* Fall through and mark the single-word operation live. */
37431305c451SRichard Henderson nb_iargs = 2;
37441305c451SRichard Henderson nb_oargs = 1;
37451305c451SRichard Henderson }
37461305c451SRichard Henderson goto do_not_remove;
37471305c451SRichard Henderson
37481414968aSRichard Henderson case INDEX_op_mulu2_i32:
3749c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32;
3750c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i32;
3751c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
375203271524SRichard Henderson goto do_mul2;
3753f1fae40cSRichard Henderson case INDEX_op_muls2_i32:
3754c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32;
3755c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i32;
3756c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
3757f1fae40cSRichard Henderson goto do_mul2;
3758f1fae40cSRichard Henderson case INDEX_op_mulu2_i64:
3759c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64;
3760c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i64;
3761c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
376203271524SRichard Henderson goto do_mul2;
3763f1fae40cSRichard Henderson case INDEX_op_muls2_i64:
3764c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64;
3765c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i64;
3766c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
376703271524SRichard Henderson goto do_mul2;
3768f1fae40cSRichard Henderson do_mul2:
37691414968aSRichard Henderson nb_iargs = 2;
37701414968aSRichard Henderson nb_oargs = 2;
3771b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) {
3772b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) {
377303271524SRichard Henderson /* Both parts of the operation are dead. */
37741414968aSRichard Henderson goto do_remove;
37751414968aSRichard Henderson }
377603271524SRichard Henderson /* The high part of the operation is dead; generate the low. */
3777c45cb8bbSRichard Henderson op->opc = opc = opc_new;
3778efee3746SRichard Henderson op->args[1] = op->args[2];
3779efee3746SRichard Henderson op->args[2] = op->args[3];
3780b83eabeaSRichard Henderson } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
378103271524SRichard Henderson /* The low part of the operation is dead; generate the high. */
3782c45cb8bbSRichard Henderson op->opc = opc = opc_new2;
3783efee3746SRichard Henderson op->args[0] = op->args[1];
3784efee3746SRichard Henderson op->args[1] = op->args[2];
3785efee3746SRichard Henderson op->args[2] = op->args[3];
378603271524SRichard Henderson } else {
378703271524SRichard Henderson goto do_not_remove;
378803271524SRichard Henderson }
378903271524SRichard Henderson /* Mark the single-word operation live. */
37901414968aSRichard Henderson nb_oargs = 1;
37911414968aSRichard Henderson goto do_not_remove;
37921414968aSRichard Henderson
3793c896fe29Sbellard default:
37941305c451SRichard Henderson /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
3795c896fe29Sbellard nb_iargs = def->nb_iargs;
3796c896fe29Sbellard nb_oargs = def->nb_oargs;
3797c896fe29Sbellard
3798c896fe29Sbellard /* Test if the operation can be removed because all
37995ff9d6a4Sbellard its outputs are dead. We assume that nb_oargs == 0
38005ff9d6a4Sbellard implies side effects */
38015ff9d6a4Sbellard if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
3802c896fe29Sbellard for (i = 0; i < nb_oargs; i++) {
3803b83eabeaSRichard Henderson if (arg_temp(op->args[i])->state != TS_DEAD) {
3804c896fe29Sbellard goto do_not_remove;
3805c896fe29Sbellard }
38069c43b68dSAurelien Jarno }
3807152c35aaSRichard Henderson goto do_remove;
3808152c35aaSRichard Henderson }
3809152c35aaSRichard Henderson goto do_not_remove;
3810152c35aaSRichard Henderson
38111305c451SRichard Henderson do_remove:
38120c627cdcSRichard Henderson tcg_op_remove(s, op);
3813152c35aaSRichard Henderson break;
3814152c35aaSRichard Henderson
3815c896fe29Sbellard do_not_remove:
3816c896fe29Sbellard for (i = 0; i < nb_oargs; i++) {
381725f49c5fSRichard Henderson ts = arg_temp(op->args[i]);
381825f49c5fSRichard Henderson
381925f49c5fSRichard Henderson /* Remember the preference of the uses that followed. */
382031fd884bSRichard Henderson if (i < ARRAY_SIZE(op->output_pref)) {
382125f49c5fSRichard Henderson op->output_pref[i] = *la_temp_pref(ts);
382231fd884bSRichard Henderson }
382325f49c5fSRichard Henderson
382425f49c5fSRichard Henderson /* Output args are dead. */
382525f49c5fSRichard Henderson if (ts->state & TS_DEAD) {
3826a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i;
38276b64b624SAurelien Jarno }
382825f49c5fSRichard Henderson if (ts->state & TS_MEM) {
3829a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i;
38309c43b68dSAurelien Jarno }
383125f49c5fSRichard Henderson ts->state = TS_DEAD;
383225f49c5fSRichard Henderson la_reset_pref(ts);
3833c896fe29Sbellard }
3834c896fe29Sbellard
383525f49c5fSRichard Henderson /* If end of basic block, update. */
3836ae36a246SRichard Henderson if (def->flags & TCG_OPF_BB_EXIT) {
3837ae36a246SRichard Henderson la_func_end(s, nb_globals, nb_temps);
3838b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_COND_BRANCH) {
3839b4cb76e6SRichard Henderson la_bb_sync(s, nb_globals, nb_temps);
3840ae36a246SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) {
38412616c808SRichard Henderson la_bb_end(s, nb_globals, nb_temps);
38423d5c5f87SAurelien Jarno } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3843f65a061cSRichard Henderson la_global_sync(s, nb_globals);
384425f49c5fSRichard Henderson if (def->flags & TCG_OPF_CALL_CLOBBER) {
384525f49c5fSRichard Henderson la_cross_call(s, nb_temps);
384625f49c5fSRichard Henderson }
3847c896fe29Sbellard }
3848c896fe29Sbellard
384925f49c5fSRichard Henderson /* Record arguments that die in this opcode. */
3850866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
385125f49c5fSRichard Henderson ts = arg_temp(op->args[i]);
385225f49c5fSRichard Henderson if (ts->state & TS_DEAD) {
3853a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i;
3854c896fe29Sbellard }
3855c19f47bfSAurelien Jarno }
385625f49c5fSRichard Henderson
385725f49c5fSRichard Henderson /* Input arguments are live for preceding opcodes. */
3858c19f47bfSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
385925f49c5fSRichard Henderson ts = arg_temp(op->args[i]);
386025f49c5fSRichard Henderson if (ts->state & TS_DEAD) {
386125f49c5fSRichard Henderson /* For operands that were dead, initially allow
386225f49c5fSRichard Henderson all regs for the type. */
386325f49c5fSRichard Henderson *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
386425f49c5fSRichard Henderson ts->state &= ~TS_DEAD;
386525f49c5fSRichard Henderson }
386625f49c5fSRichard Henderson }
386725f49c5fSRichard Henderson
386825f49c5fSRichard Henderson /* Incorporate constraints for this operand. */
386925f49c5fSRichard Henderson switch (opc) {
387025f49c5fSRichard Henderson case INDEX_op_mov_i32:
387125f49c5fSRichard Henderson case INDEX_op_mov_i64:
387225f49c5fSRichard Henderson /* Note that these are TCG_OPF_NOT_PRESENT and do not
387325f49c5fSRichard Henderson have proper constraints. That said, special case
387425f49c5fSRichard Henderson moves to propagate preferences backward. */
387525f49c5fSRichard Henderson if (IS_DEAD_ARG(1)) {
387625f49c5fSRichard Henderson *la_temp_pref(arg_temp(op->args[0]))
387725f49c5fSRichard Henderson = *la_temp_pref(arg_temp(op->args[1]));
387825f49c5fSRichard Henderson }
387925f49c5fSRichard Henderson break;
388025f49c5fSRichard Henderson
388125f49c5fSRichard Henderson default:
388225f49c5fSRichard Henderson for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
388325f49c5fSRichard Henderson const TCGArgConstraint *ct = &def->args_ct[i];
388425f49c5fSRichard Henderson TCGRegSet set, *pset;
388525f49c5fSRichard Henderson
388625f49c5fSRichard Henderson ts = arg_temp(op->args[i]);
388725f49c5fSRichard Henderson pset = la_temp_pref(ts);
388825f49c5fSRichard Henderson set = *pset;
388925f49c5fSRichard Henderson
38909be0d080SRichard Henderson set &= ct->regs;
3891bc2b17e6SRichard Henderson if (ct->ialias) {
389231fd884bSRichard Henderson set &= output_pref(op, ct->alias_index);
389325f49c5fSRichard Henderson }
389425f49c5fSRichard Henderson /* If the combination is not possible, restart. */
389525f49c5fSRichard Henderson if (set == 0) {
38969be0d080SRichard Henderson set = ct->regs;
389725f49c5fSRichard Henderson }
389825f49c5fSRichard Henderson *pset = set;
389925f49c5fSRichard Henderson }
390025f49c5fSRichard Henderson break;
3901c896fe29Sbellard }
3902c896fe29Sbellard break;
3903c896fe29Sbellard }
3904bee158cbSRichard Henderson op->life = arg_life;
3905c896fe29Sbellard }
39061ff0a2c5SEvgeny Voevodin }
3907c896fe29Sbellard
39085a18407fSRichard Henderson /* Liveness analysis: Convert indirect regs to direct temporaries. */
39099bbee4c0SRichard Henderson static bool __attribute__((noinline))
liveness_pass_2(TCGContext * s)39109bbee4c0SRichard Henderson liveness_pass_2(TCGContext *s)
39115a18407fSRichard Henderson {
39125a18407fSRichard Henderson int nb_globals = s->nb_globals;
391315fa08f8SRichard Henderson int nb_temps, i;
39145a18407fSRichard Henderson bool changes = false;
391515fa08f8SRichard Henderson TCGOp *op, *op_next;
39165a18407fSRichard Henderson
39175a18407fSRichard Henderson /* Create a temporary for each indirect global. */
39185a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) {
39195a18407fSRichard Henderson TCGTemp *its = &s->temps[i];
39205a18407fSRichard Henderson if (its->indirect_reg) {
39215a18407fSRichard Henderson TCGTemp *dts = tcg_temp_alloc(s);
39225a18407fSRichard Henderson dts->type = its->type;
39235a18407fSRichard Henderson dts->base_type = its->base_type;
3924e1e64652SRichard Henderson dts->temp_subindex = its->temp_subindex;
3925c7482438SRichard Henderson dts->kind = TEMP_EBB;
3926b83eabeaSRichard Henderson its->state_ptr = dts;
3927b83eabeaSRichard Henderson } else {
3928b83eabeaSRichard Henderson its->state_ptr = NULL;
39295a18407fSRichard Henderson }
3930b83eabeaSRichard Henderson /* All globals begin dead. */
3931b83eabeaSRichard Henderson its->state = TS_DEAD;
39325a18407fSRichard Henderson }
3933b83eabeaSRichard Henderson for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3934b83eabeaSRichard Henderson TCGTemp *its = &s->temps[i];
3935b83eabeaSRichard Henderson its->state_ptr = NULL;
3936b83eabeaSRichard Henderson its->state = TS_DEAD;
3937b83eabeaSRichard Henderson }
39385a18407fSRichard Henderson
393915fa08f8SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
39405a18407fSRichard Henderson TCGOpcode opc = op->opc;
39415a18407fSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc];
39425a18407fSRichard Henderson TCGLifeData arg_life = op->life;
39435a18407fSRichard Henderson int nb_iargs, nb_oargs, call_flags;
3944b83eabeaSRichard Henderson TCGTemp *arg_ts, *dir_ts;
39455a18407fSRichard Henderson
39465a18407fSRichard Henderson if (opc == INDEX_op_call) {
3947cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op);
3948cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op);
394990163900SRichard Henderson call_flags = tcg_call_flags(op);
39505a18407fSRichard Henderson } else {
39515a18407fSRichard Henderson nb_iargs = def->nb_iargs;
39525a18407fSRichard Henderson nb_oargs = def->nb_oargs;
39535a18407fSRichard Henderson
39545a18407fSRichard Henderson /* Set flags similar to how calls require. */
3955b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) {
3956b4cb76e6SRichard Henderson /* Like reading globals: sync_globals */
3957b4cb76e6SRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3958b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) {
39595a18407fSRichard Henderson /* Like writing globals: save_globals */
39605a18407fSRichard Henderson call_flags = 0;
39615a18407fSRichard Henderson } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
39625a18407fSRichard Henderson /* Like reading globals: sync_globals */
39635a18407fSRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS;
39645a18407fSRichard Henderson } else {
39655a18407fSRichard Henderson /* No effect on globals. */
39665a18407fSRichard Henderson call_flags = (TCG_CALL_NO_READ_GLOBALS |
39675a18407fSRichard Henderson TCG_CALL_NO_WRITE_GLOBALS);
39685a18407fSRichard Henderson }
39695a18407fSRichard Henderson }
39705a18407fSRichard Henderson
39715a18407fSRichard Henderson /* Make sure that input arguments are available. */
39725a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3973b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]);
3974b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr;
3975b83eabeaSRichard Henderson if (dir_ts && arg_ts->state == TS_DEAD) {
3976b83eabeaSRichard Henderson TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
39775a18407fSRichard Henderson ? INDEX_op_ld_i32
39785a18407fSRichard Henderson : INDEX_op_ld_i64);
3979d4478943SPhilippe Mathieu-Daudé TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
39805a18407fSRichard Henderson
3981b83eabeaSRichard Henderson lop->args[0] = temp_arg(dir_ts);
3982b83eabeaSRichard Henderson lop->args[1] = temp_arg(arg_ts->mem_base);
3983b83eabeaSRichard Henderson lop->args[2] = arg_ts->mem_offset;
39845a18407fSRichard Henderson
39855a18407fSRichard Henderson /* Loaded, but synced with memory. */
3986b83eabeaSRichard Henderson arg_ts->state = TS_MEM;
39875a18407fSRichard Henderson }
39885a18407fSRichard Henderson }
39895a18407fSRichard Henderson
39905a18407fSRichard Henderson /* Perform input replacement, and mark inputs that became dead.
39915a18407fSRichard Henderson No action is required except keeping temp_state up to date
39925a18407fSRichard Henderson so that we reload when needed. */
39935a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3994b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]);
3995b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr;
3996b83eabeaSRichard Henderson if (dir_ts) {
3997b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts);
39985a18407fSRichard Henderson changes = true;
39995a18407fSRichard Henderson if (IS_DEAD_ARG(i)) {
4000b83eabeaSRichard Henderson arg_ts->state = TS_DEAD;
40015a18407fSRichard Henderson }
40025a18407fSRichard Henderson }
40035a18407fSRichard Henderson }
40045a18407fSRichard Henderson
40055a18407fSRichard Henderson /* Liveness analysis should ensure that the following are
40065a18407fSRichard Henderson all correct, for call sites and basic block end points. */
40075a18407fSRichard Henderson if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
40085a18407fSRichard Henderson /* Nothing to do */
40095a18407fSRichard Henderson } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
40105a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) {
40115a18407fSRichard Henderson /* Liveness should see that globals are synced back,
40125a18407fSRichard Henderson that is, either TS_DEAD or TS_MEM. */
4013b83eabeaSRichard Henderson arg_ts = &s->temps[i];
4014b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0
4015b83eabeaSRichard Henderson || arg_ts->state != 0);
40165a18407fSRichard Henderson }
40175a18407fSRichard Henderson } else {
40185a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) {
40195a18407fSRichard Henderson /* Liveness should see that globals are saved back,
40205a18407fSRichard Henderson that is, TS_DEAD, waiting to be reloaded. */
4021b83eabeaSRichard Henderson arg_ts = &s->temps[i];
4022b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0
4023b83eabeaSRichard Henderson || arg_ts->state == TS_DEAD);
40245a18407fSRichard Henderson }
40255a18407fSRichard Henderson }
40265a18407fSRichard Henderson
40275a18407fSRichard Henderson /* Outputs become available. */
402861f15c48SRichard Henderson if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
402961f15c48SRichard Henderson arg_ts = arg_temp(op->args[0]);
403061f15c48SRichard Henderson dir_ts = arg_ts->state_ptr;
403161f15c48SRichard Henderson if (dir_ts) {
403261f15c48SRichard Henderson op->args[0] = temp_arg(dir_ts);
403361f15c48SRichard Henderson changes = true;
403461f15c48SRichard Henderson
403561f15c48SRichard Henderson /* The output is now live and modified. */
403661f15c48SRichard Henderson arg_ts->state = 0;
403761f15c48SRichard Henderson
403861f15c48SRichard Henderson if (NEED_SYNC_ARG(0)) {
403961f15c48SRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
404061f15c48SRichard Henderson ? INDEX_op_st_i32
404161f15c48SRichard Henderson : INDEX_op_st_i64);
4042d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
404361f15c48SRichard Henderson TCGTemp *out_ts = dir_ts;
404461f15c48SRichard Henderson
404561f15c48SRichard Henderson if (IS_DEAD_ARG(0)) {
404661f15c48SRichard Henderson out_ts = arg_temp(op->args[1]);
404761f15c48SRichard Henderson arg_ts->state = TS_DEAD;
404861f15c48SRichard Henderson tcg_op_remove(s, op);
404961f15c48SRichard Henderson } else {
405061f15c48SRichard Henderson arg_ts->state = TS_MEM;
405161f15c48SRichard Henderson }
405261f15c48SRichard Henderson
405361f15c48SRichard Henderson sop->args[0] = temp_arg(out_ts);
405461f15c48SRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base);
405561f15c48SRichard Henderson sop->args[2] = arg_ts->mem_offset;
405661f15c48SRichard Henderson } else {
405761f15c48SRichard Henderson tcg_debug_assert(!IS_DEAD_ARG(0));
405861f15c48SRichard Henderson }
405961f15c48SRichard Henderson }
406061f15c48SRichard Henderson } else {
40615a18407fSRichard Henderson for (i = 0; i < nb_oargs; i++) {
4062b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]);
4063b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr;
4064b83eabeaSRichard Henderson if (!dir_ts) {
40655a18407fSRichard Henderson continue;
40665a18407fSRichard Henderson }
4067b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts);
40685a18407fSRichard Henderson changes = true;
40695a18407fSRichard Henderson
40705a18407fSRichard Henderson /* The output is now live and modified. */
4071b83eabeaSRichard Henderson arg_ts->state = 0;
40725a18407fSRichard Henderson
40735a18407fSRichard Henderson /* Sync outputs upon their last write. */
40745a18407fSRichard Henderson if (NEED_SYNC_ARG(i)) {
4075b83eabeaSRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
40765a18407fSRichard Henderson ? INDEX_op_st_i32
40775a18407fSRichard Henderson : INDEX_op_st_i64);
4078d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
40795a18407fSRichard Henderson
4080b83eabeaSRichard Henderson sop->args[0] = temp_arg(dir_ts);
4081b83eabeaSRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base);
4082b83eabeaSRichard Henderson sop->args[2] = arg_ts->mem_offset;
40835a18407fSRichard Henderson
4084b83eabeaSRichard Henderson arg_ts->state = TS_MEM;
40855a18407fSRichard Henderson }
40865a18407fSRichard Henderson /* Drop outputs that are dead. */
40875a18407fSRichard Henderson if (IS_DEAD_ARG(i)) {
4088b83eabeaSRichard Henderson arg_ts->state = TS_DEAD;
40895a18407fSRichard Henderson }
40905a18407fSRichard Henderson }
40915a18407fSRichard Henderson }
409261f15c48SRichard Henderson }
40935a18407fSRichard Henderson
40945a18407fSRichard Henderson return changes;
40955a18407fSRichard Henderson }
40965a18407fSRichard Henderson
temp_allocate_frame(TCGContext * s,TCGTemp * ts)40972272e4a7SRichard Henderson static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
4098c896fe29Sbellard {
409931c96417SRichard Henderson intptr_t off;
4100273eb50cSRichard Henderson int size, align;
4101c1c09194SRichard Henderson
4102273eb50cSRichard Henderson /* When allocating an object, look at the full type. */
4103273eb50cSRichard Henderson size = tcg_type_size(ts->base_type);
4104273eb50cSRichard Henderson switch (ts->base_type) {
4105c1c09194SRichard Henderson case TCG_TYPE_I32:
410631c96417SRichard Henderson align = 4;
4107c1c09194SRichard Henderson break;
4108c1c09194SRichard Henderson case TCG_TYPE_I64:
4109c1c09194SRichard Henderson case TCG_TYPE_V64:
411031c96417SRichard Henderson align = 8;
4111c1c09194SRichard Henderson break;
411243eef72fSRichard Henderson case TCG_TYPE_I128:
4113c1c09194SRichard Henderson case TCG_TYPE_V128:
4114c1c09194SRichard Henderson case TCG_TYPE_V256:
411543eef72fSRichard Henderson /*
411643eef72fSRichard Henderson * Note that we do not require aligned storage for V256,
411743eef72fSRichard Henderson * and that we provide alignment for I128 to match V128,
411843eef72fSRichard Henderson * even if that's above what the host ABI requires.
411943eef72fSRichard Henderson */
412031c96417SRichard Henderson align = 16;
4121c1c09194SRichard Henderson break;
4122c1c09194SRichard Henderson default:
4123c1c09194SRichard Henderson g_assert_not_reached();
4124b591dc59SBlue Swirl }
4125c1c09194SRichard Henderson
4126b9537d59SRichard Henderson /*
4127b9537d59SRichard Henderson * Assume the stack is sufficiently aligned.
4128b9537d59SRichard Henderson * This affects e.g. ARM NEON, where we have 8 byte stack alignment
4129b9537d59SRichard Henderson * and do not require 16 byte vector alignment. This seems slightly
4130b9537d59SRichard Henderson * easier than fully parameterizing the above switch statement.
4131b9537d59SRichard Henderson */
4132b9537d59SRichard Henderson align = MIN(TCG_TARGET_STACK_ALIGN, align);
4133c1c09194SRichard Henderson off = ROUND_UP(s->current_frame_offset, align);
4134732d5897SRichard Henderson
4135732d5897SRichard Henderson /* If we've exhausted the stack frame, restart with a smaller TB. */
4136732d5897SRichard Henderson if (off + size > s->frame_end) {
4137732d5897SRichard Henderson tcg_raise_tb_overflow(s);
4138732d5897SRichard Henderson }
4139c1c09194SRichard Henderson s->current_frame_offset = off + size;
41409defd1bdSRichard Henderson #if defined(__sparc__)
4141273eb50cSRichard Henderson off += TCG_TARGET_STACK_BIAS;
41429defd1bdSRichard Henderson #endif
4143273eb50cSRichard Henderson
4144273eb50cSRichard Henderson /* If the object was subdivided, assign memory to all the parts. */
4145273eb50cSRichard Henderson if (ts->base_type != ts->type) {
4146273eb50cSRichard Henderson int part_size = tcg_type_size(ts->type);
4147273eb50cSRichard Henderson int part_count = size / part_size;
4148273eb50cSRichard Henderson
4149273eb50cSRichard Henderson /*
4150273eb50cSRichard Henderson * Each part is allocated sequentially in tcg_temp_new_internal.
4151273eb50cSRichard Henderson * Jump back to the first part by subtracting the current index.
4152273eb50cSRichard Henderson */
4153273eb50cSRichard Henderson ts -= ts->temp_subindex;
4154273eb50cSRichard Henderson for (int i = 0; i < part_count; ++i) {
4155273eb50cSRichard Henderson ts[i].mem_offset = off + i * part_size;
4156273eb50cSRichard Henderson ts[i].mem_base = s->frame_temp;
4157273eb50cSRichard Henderson ts[i].mem_allocated = 1;
4158273eb50cSRichard Henderson }
4159273eb50cSRichard Henderson } else {
4160273eb50cSRichard Henderson ts->mem_offset = off;
4161b3a62939SRichard Henderson ts->mem_base = s->frame_temp;
4162c896fe29Sbellard ts->mem_allocated = 1;
4163c896fe29Sbellard }
4164273eb50cSRichard Henderson }
4165c896fe29Sbellard
4166098859f1SRichard Henderson /* Assign @reg to @ts, and update reg_to_temp[]. */
set_temp_val_reg(TCGContext * s,TCGTemp * ts,TCGReg reg)4167098859f1SRichard Henderson static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
4168098859f1SRichard Henderson {
4169098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) {
4170098859f1SRichard Henderson TCGReg old = ts->reg;
4171098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[old] == ts);
4172098859f1SRichard Henderson if (old == reg) {
4173098859f1SRichard Henderson return;
4174098859f1SRichard Henderson }
4175098859f1SRichard Henderson s->reg_to_temp[old] = NULL;
4176098859f1SRichard Henderson }
4177098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4178098859f1SRichard Henderson s->reg_to_temp[reg] = ts;
4179098859f1SRichard Henderson ts->val_type = TEMP_VAL_REG;
4180098859f1SRichard Henderson ts->reg = reg;
4181098859f1SRichard Henderson }
4182098859f1SRichard Henderson
4183098859f1SRichard Henderson /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
set_temp_val_nonreg(TCGContext * s,TCGTemp * ts,TCGTempVal type)4184098859f1SRichard Henderson static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
4185098859f1SRichard Henderson {
4186098859f1SRichard Henderson tcg_debug_assert(type != TEMP_VAL_REG);
4187098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) {
4188098859f1SRichard Henderson TCGReg reg = ts->reg;
4189098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == ts);
4190098859f1SRichard Henderson s->reg_to_temp[reg] = NULL;
4191098859f1SRichard Henderson }
4192098859f1SRichard Henderson ts->val_type = type;
4193098859f1SRichard Henderson }
4194098859f1SRichard Henderson
4195b722452aSRichard Henderson static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
4196b3915dbbSRichard Henderson
419759d7c14eSRichard Henderson /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
419859d7c14eSRichard Henderson mark it free; otherwise mark it dead. */
temp_free_or_dead(TCGContext * s,TCGTemp * ts,int free_or_dead)419959d7c14eSRichard Henderson static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
4200c896fe29Sbellard {
4201c0522136SRichard Henderson TCGTempVal new_type;
4202c0522136SRichard Henderson
4203c0522136SRichard Henderson switch (ts->kind) {
4204c0522136SRichard Henderson case TEMP_FIXED:
420559d7c14eSRichard Henderson return;
4206c0522136SRichard Henderson case TEMP_GLOBAL:
4207f57c6915SRichard Henderson case TEMP_TB:
4208c0522136SRichard Henderson new_type = TEMP_VAL_MEM;
4209c0522136SRichard Henderson break;
4210c7482438SRichard Henderson case TEMP_EBB:
4211c0522136SRichard Henderson new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
4212c0522136SRichard Henderson break;
4213c0522136SRichard Henderson case TEMP_CONST:
4214c0522136SRichard Henderson new_type = TEMP_VAL_CONST;
4215c0522136SRichard Henderson break;
4216c0522136SRichard Henderson default:
4217c0522136SRichard Henderson g_assert_not_reached();
421859d7c14eSRichard Henderson }
4219098859f1SRichard Henderson set_temp_val_nonreg(s, ts, new_type);
422059d7c14eSRichard Henderson }
4221c896fe29Sbellard
422259d7c14eSRichard Henderson /* Mark a temporary as dead. */
temp_dead(TCGContext * s,TCGTemp * ts)422359d7c14eSRichard Henderson static inline void temp_dead(TCGContext *s, TCGTemp *ts)
422459d7c14eSRichard Henderson {
422559d7c14eSRichard Henderson temp_free_or_dead(s, ts, 1);
422659d7c14eSRichard Henderson }
422759d7c14eSRichard Henderson
422859d7c14eSRichard Henderson /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
422959d7c14eSRichard Henderson registers needs to be allocated to store a constant. If 'free_or_dead'
423059d7c14eSRichard Henderson is non-zero, subsequently release the temporary; if it is positive, the
423159d7c14eSRichard Henderson temp is dead; if it is negative, the temp is free. */
temp_sync(TCGContext * s,TCGTemp * ts,TCGRegSet allocated_regs,TCGRegSet preferred_regs,int free_or_dead)423298b4e186SRichard Henderson static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
423398b4e186SRichard Henderson TCGRegSet preferred_regs, int free_or_dead)
423459d7c14eSRichard Henderson {
4235c0522136SRichard Henderson if (!temp_readonly(ts) && !ts->mem_coherent) {
42367f6ceedfSAurelien Jarno if (!ts->mem_allocated) {
42372272e4a7SRichard Henderson temp_allocate_frame(s, ts);
423859d7c14eSRichard Henderson }
423959d7c14eSRichard Henderson switch (ts->val_type) {
424059d7c14eSRichard Henderson case TEMP_VAL_CONST:
424159d7c14eSRichard Henderson /* If we're going to free the temp immediately, then we won't
424259d7c14eSRichard Henderson require it later in a register, so attempt to store the
424359d7c14eSRichard Henderson constant to memory directly. */
424459d7c14eSRichard Henderson if (free_or_dead
424559d7c14eSRichard Henderson && tcg_out_sti(s, ts->type, ts->val,
424659d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset)) {
424759d7c14eSRichard Henderson break;
424859d7c14eSRichard Henderson }
424959d7c14eSRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type],
425098b4e186SRichard Henderson allocated_regs, preferred_regs);
425159d7c14eSRichard Henderson /* fallthrough */
425259d7c14eSRichard Henderson
425359d7c14eSRichard Henderson case TEMP_VAL_REG:
425459d7c14eSRichard Henderson tcg_out_st(s, ts->type, ts->reg,
425559d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset);
425659d7c14eSRichard Henderson break;
425759d7c14eSRichard Henderson
425859d7c14eSRichard Henderson case TEMP_VAL_MEM:
425959d7c14eSRichard Henderson break;
426059d7c14eSRichard Henderson
426159d7c14eSRichard Henderson case TEMP_VAL_DEAD:
426259d7c14eSRichard Henderson default:
4263732e89f4SRichard Henderson g_assert_not_reached();
4264c896fe29Sbellard }
42657f6ceedfSAurelien Jarno ts->mem_coherent = 1;
42667f6ceedfSAurelien Jarno }
426759d7c14eSRichard Henderson if (free_or_dead) {
426859d7c14eSRichard Henderson temp_free_or_dead(s, ts, free_or_dead);
426959d7c14eSRichard Henderson }
427059d7c14eSRichard Henderson }
42717f6ceedfSAurelien Jarno
42727f6ceedfSAurelien Jarno /* free register 'reg' by spilling the corresponding temporary if necessary */
tcg_reg_free(TCGContext * s,TCGReg reg,TCGRegSet allocated_regs)4273b3915dbbSRichard Henderson static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
42747f6ceedfSAurelien Jarno {
4275f8b2f202SRichard Henderson TCGTemp *ts = s->reg_to_temp[reg];
4276f8b2f202SRichard Henderson if (ts != NULL) {
427798b4e186SRichard Henderson temp_sync(s, ts, allocated_regs, 0, -1);
4278c896fe29Sbellard }
4279c896fe29Sbellard }
4280c896fe29Sbellard
4281b016486eSRichard Henderson /**
4282b016486eSRichard Henderson * tcg_reg_alloc:
4283b016486eSRichard Henderson * @required_regs: Set of registers in which we must allocate.
4284b016486eSRichard Henderson * @allocated_regs: Set of registers which must be avoided.
4285b016486eSRichard Henderson * @preferred_regs: Set of registers we should prefer.
4286b016486eSRichard Henderson * @rev: True if we search the registers in "indirect" order.
4287b016486eSRichard Henderson *
4288b016486eSRichard Henderson * The allocated register must be in @required_regs & ~@allocated_regs,
4289b016486eSRichard Henderson * but if we can put it in @preferred_regs we may save a move later.
4290b016486eSRichard Henderson */
tcg_reg_alloc(TCGContext * s,TCGRegSet required_regs,TCGRegSet allocated_regs,TCGRegSet preferred_regs,bool rev)4291b016486eSRichard Henderson static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
4292b016486eSRichard Henderson TCGRegSet allocated_regs,
4293b016486eSRichard Henderson TCGRegSet preferred_regs, bool rev)
4294c896fe29Sbellard {
4295b016486eSRichard Henderson int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4296b016486eSRichard Henderson TCGRegSet reg_ct[2];
429791478cefSRichard Henderson const int *order;
4298c896fe29Sbellard
4299b016486eSRichard Henderson reg_ct[1] = required_regs & ~allocated_regs;
4300b016486eSRichard Henderson tcg_debug_assert(reg_ct[1] != 0);
4301b016486eSRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs;
4302b016486eSRichard Henderson
4303b016486eSRichard Henderson /* Skip the preferred_regs option if it cannot be satisfied,
4304b016486eSRichard Henderson or if the preference made no difference. */
4305b016486eSRichard Henderson f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4306b016486eSRichard Henderson
430791478cefSRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4308c896fe29Sbellard
4309b016486eSRichard Henderson /* Try free registers, preferences first. */
4310b016486eSRichard Henderson for (j = f; j < 2; j++) {
4311b016486eSRichard Henderson TCGRegSet set = reg_ct[j];
4312b016486eSRichard Henderson
4313b016486eSRichard Henderson if (tcg_regset_single(set)) {
4314b016486eSRichard Henderson /* One register in the set. */
4315b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set);
4316b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL) {
4317c896fe29Sbellard return reg;
4318c896fe29Sbellard }
4319b016486eSRichard Henderson } else {
432091478cefSRichard Henderson for (i = 0; i < n; i++) {
4321b016486eSRichard Henderson TCGReg reg = order[i];
4322b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL &&
4323b016486eSRichard Henderson tcg_regset_test_reg(set, reg)) {
4324b016486eSRichard Henderson return reg;
4325b016486eSRichard Henderson }
4326b016486eSRichard Henderson }
4327b016486eSRichard Henderson }
4328b016486eSRichard Henderson }
4329b016486eSRichard Henderson
4330b016486eSRichard Henderson /* We must spill something. */
4331b016486eSRichard Henderson for (j = f; j < 2; j++) {
4332b016486eSRichard Henderson TCGRegSet set = reg_ct[j];
4333b016486eSRichard Henderson
4334b016486eSRichard Henderson if (tcg_regset_single(set)) {
4335b016486eSRichard Henderson /* One register in the set. */
4336b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set);
4337b3915dbbSRichard Henderson tcg_reg_free(s, reg, allocated_regs);
4338c896fe29Sbellard return reg;
4339b016486eSRichard Henderson } else {
4340b016486eSRichard Henderson for (i = 0; i < n; i++) {
4341b016486eSRichard Henderson TCGReg reg = order[i];
4342b016486eSRichard Henderson if (tcg_regset_test_reg(set, reg)) {
4343b016486eSRichard Henderson tcg_reg_free(s, reg, allocated_regs);
4344b016486eSRichard Henderson return reg;
4345b016486eSRichard Henderson }
4346b016486eSRichard Henderson }
4347c896fe29Sbellard }
4348c896fe29Sbellard }
4349c896fe29Sbellard
4350732e89f4SRichard Henderson g_assert_not_reached();
4351c896fe29Sbellard }
4352c896fe29Sbellard
tcg_reg_alloc_pair(TCGContext * s,TCGRegSet required_regs,TCGRegSet allocated_regs,TCGRegSet preferred_regs,bool rev)435329f5e925SRichard Henderson static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
435429f5e925SRichard Henderson TCGRegSet allocated_regs,
435529f5e925SRichard Henderson TCGRegSet preferred_regs, bool rev)
435629f5e925SRichard Henderson {
435729f5e925SRichard Henderson int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
435829f5e925SRichard Henderson TCGRegSet reg_ct[2];
435929f5e925SRichard Henderson const int *order;
436029f5e925SRichard Henderson
436129f5e925SRichard Henderson /* Ensure that if I is not in allocated_regs, I+1 is not either. */
436229f5e925SRichard Henderson reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
436329f5e925SRichard Henderson tcg_debug_assert(reg_ct[1] != 0);
436429f5e925SRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs;
436529f5e925SRichard Henderson
436629f5e925SRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
436729f5e925SRichard Henderson
436829f5e925SRichard Henderson /*
436929f5e925SRichard Henderson * Skip the preferred_regs option if it cannot be satisfied,
437029f5e925SRichard Henderson * or if the preference made no difference.
437129f5e925SRichard Henderson */
437229f5e925SRichard Henderson k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
437329f5e925SRichard Henderson
437429f5e925SRichard Henderson /*
437529f5e925SRichard Henderson * Minimize the number of flushes by looking for 2 free registers first,
437629f5e925SRichard Henderson * then a single flush, then two flushes.
437729f5e925SRichard Henderson */
437829f5e925SRichard Henderson for (fmin = 2; fmin >= 0; fmin--) {
437929f5e925SRichard Henderson for (j = k; j < 2; j++) {
438029f5e925SRichard Henderson TCGRegSet set = reg_ct[j];
438129f5e925SRichard Henderson
438229f5e925SRichard Henderson for (i = 0; i < n; i++) {
438329f5e925SRichard Henderson TCGReg reg = order[i];
438429f5e925SRichard Henderson
438529f5e925SRichard Henderson if (tcg_regset_test_reg(set, reg)) {
438629f5e925SRichard Henderson int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
438729f5e925SRichard Henderson if (f >= fmin) {
438829f5e925SRichard Henderson tcg_reg_free(s, reg, allocated_regs);
438929f5e925SRichard Henderson tcg_reg_free(s, reg + 1, allocated_regs);
439029f5e925SRichard Henderson return reg;
439129f5e925SRichard Henderson }
439229f5e925SRichard Henderson }
439329f5e925SRichard Henderson }
439429f5e925SRichard Henderson }
439529f5e925SRichard Henderson }
4396732e89f4SRichard Henderson g_assert_not_reached();
439729f5e925SRichard Henderson }
439829f5e925SRichard Henderson
439940ae5c62SRichard Henderson /* Make sure the temporary is in a register. If needed, allocate the register
440040ae5c62SRichard Henderson from DESIRED while avoiding ALLOCATED. */
temp_load(TCGContext * s,TCGTemp * ts,TCGRegSet desired_regs,TCGRegSet allocated_regs,TCGRegSet preferred_regs)440140ae5c62SRichard Henderson static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
4402b722452aSRichard Henderson TCGRegSet allocated_regs, TCGRegSet preferred_regs)
440340ae5c62SRichard Henderson {
440440ae5c62SRichard Henderson TCGReg reg;
440540ae5c62SRichard Henderson
440640ae5c62SRichard Henderson switch (ts->val_type) {
440740ae5c62SRichard Henderson case TEMP_VAL_REG:
440840ae5c62SRichard Henderson return;
440940ae5c62SRichard Henderson case TEMP_VAL_CONST:
4410b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4411b722452aSRichard Henderson preferred_regs, ts->indirect_base);
44120a6a8bc8SRichard Henderson if (ts->type <= TCG_TYPE_I64) {
441340ae5c62SRichard Henderson tcg_out_movi(s, ts->type, reg, ts->val);
44140a6a8bc8SRichard Henderson } else {
44154e186175SRichard Henderson uint64_t val = ts->val;
44164e186175SRichard Henderson MemOp vece = MO_64;
44174e186175SRichard Henderson
44184e186175SRichard Henderson /*
44194e186175SRichard Henderson * Find the minimal vector element that matches the constant.
44204e186175SRichard Henderson * The targets will, in general, have to do this search anyway,
44214e186175SRichard Henderson * do this generically.
44224e186175SRichard Henderson */
44234e186175SRichard Henderson if (val == dup_const(MO_8, val)) {
44244e186175SRichard Henderson vece = MO_8;
44254e186175SRichard Henderson } else if (val == dup_const(MO_16, val)) {
44264e186175SRichard Henderson vece = MO_16;
44270b4286ddSRichard Henderson } else if (val == dup_const(MO_32, val)) {
44284e186175SRichard Henderson vece = MO_32;
44294e186175SRichard Henderson }
44304e186175SRichard Henderson
44314e186175SRichard Henderson tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
44320a6a8bc8SRichard Henderson }
443340ae5c62SRichard Henderson ts->mem_coherent = 0;
443440ae5c62SRichard Henderson break;
443540ae5c62SRichard Henderson case TEMP_VAL_MEM:
4436b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4437b722452aSRichard Henderson preferred_regs, ts->indirect_base);
443840ae5c62SRichard Henderson tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
443940ae5c62SRichard Henderson ts->mem_coherent = 1;
444040ae5c62SRichard Henderson break;
444140ae5c62SRichard Henderson case TEMP_VAL_DEAD:
444240ae5c62SRichard Henderson default:
4443732e89f4SRichard Henderson g_assert_not_reached();
444440ae5c62SRichard Henderson }
4445098859f1SRichard Henderson set_temp_val_reg(s, ts, reg);
444640ae5c62SRichard Henderson }
444740ae5c62SRichard Henderson
444859d7c14eSRichard Henderson /* Save a temporary to memory. 'allocated_regs' is used in case a
4449e8996ee0Sbellard temporary registers needs to be allocated to store a constant. */
temp_save(TCGContext * s,TCGTemp * ts,TCGRegSet allocated_regs)445059d7c14eSRichard Henderson static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
44511ad80729SAurelien Jarno {
44522c0366f0SAurelien Jarno /* The liveness analysis already ensures that globals are back
4453eabb7b91SAurelien Jarno in memory. Keep an tcg_debug_assert for safety. */
4454e01fa97dSRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
44551ad80729SAurelien Jarno }
44561ad80729SAurelien Jarno
44579814dd27SDong Xu Wang /* save globals to their canonical location and assume they can be
4458641d5fbeSbellard modified be the following code. 'allocated_regs' is used in case a
4459641d5fbeSbellard temporary registers needs to be allocated to store a constant. */
save_globals(TCGContext * s,TCGRegSet allocated_regs)4460641d5fbeSbellard static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
4461641d5fbeSbellard {
4462ac3b8891SRichard Henderson int i, n;
4463641d5fbeSbellard
4464ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) {
4465b13eb728SRichard Henderson temp_save(s, &s->temps[i], allocated_regs);
4466641d5fbeSbellard }
4467e5097dc8Sbellard }
4468e5097dc8Sbellard
44693d5c5f87SAurelien Jarno /* sync globals to their canonical location and assume they can be
44703d5c5f87SAurelien Jarno read by the following code. 'allocated_regs' is used in case a
44713d5c5f87SAurelien Jarno temporary registers needs to be allocated to store a constant. */
sync_globals(TCGContext * s,TCGRegSet allocated_regs)44723d5c5f87SAurelien Jarno static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
44733d5c5f87SAurelien Jarno {
4474ac3b8891SRichard Henderson int i, n;
44753d5c5f87SAurelien Jarno
4476ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) {
447712b9b11aSRichard Henderson TCGTemp *ts = &s->temps[i];
447812b9b11aSRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG
4479ee17db83SRichard Henderson || ts->kind == TEMP_FIXED
448012b9b11aSRichard Henderson || ts->mem_coherent);
44813d5c5f87SAurelien Jarno }
44823d5c5f87SAurelien Jarno }
44833d5c5f87SAurelien Jarno
4484e5097dc8Sbellard /* at the end of a basic block, we assume all temporaries are dead and
4485e8996ee0Sbellard all globals are stored at their canonical location. */
tcg_reg_alloc_bb_end(TCGContext * s,TCGRegSet allocated_regs)4486e8996ee0Sbellard static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
4487e5097dc8Sbellard {
4488e5097dc8Sbellard int i;
4489e5097dc8Sbellard
4490c896fe29Sbellard for (i = s->nb_globals; i < s->nb_temps; i++) {
4491b13eb728SRichard Henderson TCGTemp *ts = &s->temps[i];
4492c0522136SRichard Henderson
4493c0522136SRichard Henderson switch (ts->kind) {
4494f57c6915SRichard Henderson case TEMP_TB:
4495b13eb728SRichard Henderson temp_save(s, ts, allocated_regs);
4496c0522136SRichard Henderson break;
4497c7482438SRichard Henderson case TEMP_EBB:
44982c0366f0SAurelien Jarno /* The liveness analysis already ensures that temps are dead.
4499eabb7b91SAurelien Jarno Keep an tcg_debug_assert for safety. */
4500eabb7b91SAurelien Jarno tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
4501c0522136SRichard Henderson break;
4502c0522136SRichard Henderson case TEMP_CONST:
4503c0522136SRichard Henderson /* Similarly, we should have freed any allocated register. */
4504c0522136SRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4505c0522136SRichard Henderson break;
4506c0522136SRichard Henderson default:
4507c0522136SRichard Henderson g_assert_not_reached();
4508c896fe29Sbellard }
4509641d5fbeSbellard }
4510e8996ee0Sbellard
4511e8996ee0Sbellard save_globals(s, allocated_regs);
4512c896fe29Sbellard }
4513c896fe29Sbellard
4514bab1671fSRichard Henderson /*
4515c7482438SRichard Henderson * At a conditional branch, we assume all temporaries are dead unless
4516c7482438SRichard Henderson * explicitly live-across-conditional-branch; all globals and local
4517c7482438SRichard Henderson * temps are synced to their location.
4518b4cb76e6SRichard Henderson */
tcg_reg_alloc_cbranch(TCGContext * s,TCGRegSet allocated_regs)4519b4cb76e6SRichard Henderson static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
4520b4cb76e6SRichard Henderson {
4521b4cb76e6SRichard Henderson sync_globals(s, allocated_regs);
4522b4cb76e6SRichard Henderson
4523b4cb76e6SRichard Henderson for (int i = s->nb_globals; i < s->nb_temps; i++) {
4524b4cb76e6SRichard Henderson TCGTemp *ts = &s->temps[i];
4525b4cb76e6SRichard Henderson /*
4526b4cb76e6SRichard Henderson * The liveness analysis already ensures that temps are dead.
4527b4cb76e6SRichard Henderson * Keep tcg_debug_asserts for safety.
4528b4cb76e6SRichard Henderson */
4529c0522136SRichard Henderson switch (ts->kind) {
4530f57c6915SRichard Henderson case TEMP_TB:
4531b4cb76e6SRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
4532c0522136SRichard Henderson break;
4533c7482438SRichard Henderson case TEMP_EBB:
4534c0522136SRichard Henderson case TEMP_CONST:
4535c0522136SRichard Henderson break;
4536c0522136SRichard Henderson default:
4537c0522136SRichard Henderson g_assert_not_reached();
4538b4cb76e6SRichard Henderson }
4539b4cb76e6SRichard Henderson }
4540b4cb76e6SRichard Henderson }
4541b4cb76e6SRichard Henderson
4542b4cb76e6SRichard Henderson /*
4543c58f4c97SRichard Henderson * Specialized code generation for INDEX_op_mov_* with a constant.
4544bab1671fSRichard Henderson */
tcg_reg_alloc_do_movi(TCGContext * s,TCGTemp * ots,tcg_target_ulong val,TCGLifeData arg_life,TCGRegSet preferred_regs)45450fe4fca4SPaolo Bonzini static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
4546ba87719cSRichard Henderson tcg_target_ulong val, TCGLifeData arg_life,
4547ba87719cSRichard Henderson TCGRegSet preferred_regs)
4548e8996ee0Sbellard {
4549d63e3b6eSRichard Henderson /* ENV should not be modified. */
4550e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots));
455159d7c14eSRichard Henderson
455259d7c14eSRichard Henderson /* The movi is not explicitly generated here. */
4553098859f1SRichard Henderson set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
4554e8996ee0Sbellard ots->val = val;
455559d7c14eSRichard Henderson ots->mem_coherent = 0;
4556ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) {
4557ba87719cSRichard Henderson temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
455859d7c14eSRichard Henderson } else if (IS_DEAD_ARG(0)) {
4559f8bf00f1SRichard Henderson temp_dead(s, ots);
45604c4e1ab2SAurelien Jarno }
4561e8996ee0Sbellard }
4562e8996ee0Sbellard
4563bab1671fSRichard Henderson /*
4564bab1671fSRichard Henderson * Specialized code generation for INDEX_op_mov_*.
4565bab1671fSRichard Henderson */
tcg_reg_alloc_mov(TCGContext * s,const TCGOp * op)4566dd186292SRichard Henderson static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
4567c896fe29Sbellard {
4568dd186292SRichard Henderson const TCGLifeData arg_life = op->life;
456969e3706dSRichard Henderson TCGRegSet allocated_regs, preferred_regs;
4570c896fe29Sbellard TCGTemp *ts, *ots;
4571450445d5SRichard Henderson TCGType otype, itype;
4572098859f1SRichard Henderson TCGReg oreg, ireg;
4573c896fe29Sbellard
4574d21369f5SRichard Henderson allocated_regs = s->reserved_regs;
457531fd884bSRichard Henderson preferred_regs = output_pref(op, 0);
457643439139SRichard Henderson ots = arg_temp(op->args[0]);
457743439139SRichard Henderson ts = arg_temp(op->args[1]);
4578450445d5SRichard Henderson
4579d63e3b6eSRichard Henderson /* ENV should not be modified. */
4580e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots));
4581d63e3b6eSRichard Henderson
4582450445d5SRichard Henderson /* Note that otype != itype for no-op truncation. */
4583450445d5SRichard Henderson otype = ots->type;
4584450445d5SRichard Henderson itype = ts->type;
4585c896fe29Sbellard
45860fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_CONST) {
45870fe4fca4SPaolo Bonzini /* propagate constant or generate sti */
45880fe4fca4SPaolo Bonzini tcg_target_ulong val = ts->val;
45890fe4fca4SPaolo Bonzini if (IS_DEAD_ARG(1)) {
45900fe4fca4SPaolo Bonzini temp_dead(s, ts);
45910fe4fca4SPaolo Bonzini }
459269e3706dSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
45930fe4fca4SPaolo Bonzini return;
45940fe4fca4SPaolo Bonzini }
45950fe4fca4SPaolo Bonzini
45960fe4fca4SPaolo Bonzini /* If the source value is in memory we're going to be forced
45970fe4fca4SPaolo Bonzini to have it in a register in order to perform the copy. Copy
45980fe4fca4SPaolo Bonzini the SOURCE value into its own register first, that way we
45990fe4fca4SPaolo Bonzini don't have to reload SOURCE the next time it is used. */
46000fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_MEM) {
460169e3706dSRichard Henderson temp_load(s, ts, tcg_target_available_regs[itype],
460269e3706dSRichard Henderson allocated_regs, preferred_regs);
4603c29c1d7eSAurelien Jarno }
46040fe4fca4SPaolo Bonzini tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
4605098859f1SRichard Henderson ireg = ts->reg;
4606098859f1SRichard Henderson
4607d63e3b6eSRichard Henderson if (IS_DEAD_ARG(0)) {
4608c29c1d7eSAurelien Jarno /* mov to a non-saved dead register makes no sense (even with
4609c29c1d7eSAurelien Jarno liveness analysis disabled). */
4610eabb7b91SAurelien Jarno tcg_debug_assert(NEED_SYNC_ARG(0));
4611c29c1d7eSAurelien Jarno if (!ots->mem_allocated) {
46122272e4a7SRichard Henderson temp_allocate_frame(s, ots);
4613c29c1d7eSAurelien Jarno }
4614098859f1SRichard Henderson tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
4615c29c1d7eSAurelien Jarno if (IS_DEAD_ARG(1)) {
4616f8bf00f1SRichard Henderson temp_dead(s, ts);
4617c29c1d7eSAurelien Jarno }
4618f8bf00f1SRichard Henderson temp_dead(s, ots);
4619098859f1SRichard Henderson return;
4620098859f1SRichard Henderson }
4621098859f1SRichard Henderson
4622ee17db83SRichard Henderson if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
4623098859f1SRichard Henderson /*
4624098859f1SRichard Henderson * The mov can be suppressed. Kill input first, so that it
4625098859f1SRichard Henderson * is unlinked from reg_to_temp, then set the output to the
4626098859f1SRichard Henderson * reg that we saved from the input.
4627098859f1SRichard Henderson */
4628f8bf00f1SRichard Henderson temp_dead(s, ts);
4629098859f1SRichard Henderson oreg = ireg;
4630c29c1d7eSAurelien Jarno } else {
4631098859f1SRichard Henderson if (ots->val_type == TEMP_VAL_REG) {
4632098859f1SRichard Henderson oreg = ots->reg;
4633098859f1SRichard Henderson } else {
4634098859f1SRichard Henderson /* Make sure to not spill the input register during allocation. */
4635098859f1SRichard Henderson oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
4636098859f1SRichard Henderson allocated_regs | ((TCGRegSet)1 << ireg),
4637098859f1SRichard Henderson preferred_regs, ots->indirect_base);
4638c29c1d7eSAurelien Jarno }
4639098859f1SRichard Henderson if (!tcg_out_mov(s, otype, oreg, ireg)) {
4640240c08d0SRichard Henderson /*
4641240c08d0SRichard Henderson * Cross register class move not supported.
4642240c08d0SRichard Henderson * Store the source register into the destination slot
4643240c08d0SRichard Henderson * and leave the destination temp as TEMP_VAL_MEM.
4644240c08d0SRichard Henderson */
4645e01fa97dSRichard Henderson assert(!temp_readonly(ots));
4646240c08d0SRichard Henderson if (!ts->mem_allocated) {
4647240c08d0SRichard Henderson temp_allocate_frame(s, ots);
4648240c08d0SRichard Henderson }
4649098859f1SRichard Henderson tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
4650098859f1SRichard Henderson set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
4651240c08d0SRichard Henderson ots->mem_coherent = 1;
4652240c08d0SRichard Henderson return;
465378113e83SRichard Henderson }
4654c29c1d7eSAurelien Jarno }
4655098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg);
4656c896fe29Sbellard ots->mem_coherent = 0;
4657098859f1SRichard Henderson
4658ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) {
465998b4e186SRichard Henderson temp_sync(s, ots, allocated_regs, 0, 0);
4660c29c1d7eSAurelien Jarno }
4661ec7a869dSAurelien Jarno }
4662c896fe29Sbellard
4663bab1671fSRichard Henderson /*
4664bab1671fSRichard Henderson * Specialized code generation for INDEX_op_dup_vec.
4665bab1671fSRichard Henderson */
tcg_reg_alloc_dup(TCGContext * s,const TCGOp * op)4666bab1671fSRichard Henderson static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
4667bab1671fSRichard Henderson {
4668bab1671fSRichard Henderson const TCGLifeData arg_life = op->life;
4669bab1671fSRichard Henderson TCGRegSet dup_out_regs, dup_in_regs;
4670bab1671fSRichard Henderson TCGTemp *its, *ots;
4671bab1671fSRichard Henderson TCGType itype, vtype;
4672bab1671fSRichard Henderson unsigned vece;
467331c96417SRichard Henderson int lowpart_ofs;
4674bab1671fSRichard Henderson bool ok;
4675bab1671fSRichard Henderson
4676bab1671fSRichard Henderson ots = arg_temp(op->args[0]);
4677bab1671fSRichard Henderson its = arg_temp(op->args[1]);
4678bab1671fSRichard Henderson
4679bab1671fSRichard Henderson /* ENV should not be modified. */
4680e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots));
4681bab1671fSRichard Henderson
4682bab1671fSRichard Henderson itype = its->type;
4683bab1671fSRichard Henderson vece = TCGOP_VECE(op);
4684bab1671fSRichard Henderson vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4685bab1671fSRichard Henderson
4686bab1671fSRichard Henderson if (its->val_type == TEMP_VAL_CONST) {
4687bab1671fSRichard Henderson /* Propagate constant via movi -> dupi. */
4688bab1671fSRichard Henderson tcg_target_ulong val = its->val;
4689bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) {
4690bab1671fSRichard Henderson temp_dead(s, its);
4691bab1671fSRichard Henderson }
469231fd884bSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
4693bab1671fSRichard Henderson return;
4694bab1671fSRichard Henderson }
4695bab1671fSRichard Henderson
46969be0d080SRichard Henderson dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
46979be0d080SRichard Henderson dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
4698bab1671fSRichard Henderson
4699bab1671fSRichard Henderson /* Allocate the output register now. */
4700bab1671fSRichard Henderson if (ots->val_type != TEMP_VAL_REG) {
4701bab1671fSRichard Henderson TCGRegSet allocated_regs = s->reserved_regs;
4702098859f1SRichard Henderson TCGReg oreg;
4703bab1671fSRichard Henderson
4704bab1671fSRichard Henderson if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
4705bab1671fSRichard Henderson /* Make sure to not spill the input register. */
4706bab1671fSRichard Henderson tcg_regset_set_reg(allocated_regs, its->reg);
4707bab1671fSRichard Henderson }
4708098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
470931fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base);
4710098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg);
4711bab1671fSRichard Henderson }
4712bab1671fSRichard Henderson
4713bab1671fSRichard Henderson switch (its->val_type) {
4714bab1671fSRichard Henderson case TEMP_VAL_REG:
4715bab1671fSRichard Henderson /*
4716bab1671fSRichard Henderson * The dup constriaints must be broad, covering all possible VECE.
4717bab1671fSRichard Henderson * However, tcg_op_dup_vec() gets to see the VECE and we allow it
4718bab1671fSRichard Henderson * to fail, indicating that extra moves are required for that case.
4719bab1671fSRichard Henderson */
4720bab1671fSRichard Henderson if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
4721bab1671fSRichard Henderson if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
4722bab1671fSRichard Henderson goto done;
4723bab1671fSRichard Henderson }
4724bab1671fSRichard Henderson /* Try again from memory or a vector input register. */
4725bab1671fSRichard Henderson }
4726bab1671fSRichard Henderson if (!its->mem_coherent) {
4727bab1671fSRichard Henderson /*
4728bab1671fSRichard Henderson * The input register is not synced, and so an extra store
4729bab1671fSRichard Henderson * would be required to use memory. Attempt an integer-vector
4730bab1671fSRichard Henderson * register move first. We do not have a TCGRegSet for this.
4731bab1671fSRichard Henderson */
4732bab1671fSRichard Henderson if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
4733bab1671fSRichard Henderson break;
4734bab1671fSRichard Henderson }
4735bab1671fSRichard Henderson /* Sync the temp back to its slot and load from there. */
4736bab1671fSRichard Henderson temp_sync(s, its, s->reserved_regs, 0, 0);
4737bab1671fSRichard Henderson }
4738bab1671fSRichard Henderson /* fall through */
4739bab1671fSRichard Henderson
4740bab1671fSRichard Henderson case TEMP_VAL_MEM:
474131c96417SRichard Henderson lowpart_ofs = 0;
474231c96417SRichard Henderson if (HOST_BIG_ENDIAN) {
474331c96417SRichard Henderson lowpart_ofs = tcg_type_size(itype) - (1 << vece);
474431c96417SRichard Henderson }
4745d6ecb4a9SRichard Henderson if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
474631c96417SRichard Henderson its->mem_offset + lowpart_ofs)) {
4747d6ecb4a9SRichard Henderson goto done;
4748d6ecb4a9SRichard Henderson }
4749098859f1SRichard Henderson /* Load the input into the destination vector register. */
4750bab1671fSRichard Henderson tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
4751bab1671fSRichard Henderson break;
4752bab1671fSRichard Henderson
4753bab1671fSRichard Henderson default:
4754bab1671fSRichard Henderson g_assert_not_reached();
4755bab1671fSRichard Henderson }
4756bab1671fSRichard Henderson
4757bab1671fSRichard Henderson /* We now have a vector input register, so dup must succeed. */
4758bab1671fSRichard Henderson ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
4759bab1671fSRichard Henderson tcg_debug_assert(ok);
4760bab1671fSRichard Henderson
4761bab1671fSRichard Henderson done:
476236f5539cSRichard Henderson ots->mem_coherent = 0;
4763bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) {
4764bab1671fSRichard Henderson temp_dead(s, its);
4765bab1671fSRichard Henderson }
4766bab1671fSRichard Henderson if (NEED_SYNC_ARG(0)) {
4767bab1671fSRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, 0);
4768bab1671fSRichard Henderson }
4769bab1671fSRichard Henderson if (IS_DEAD_ARG(0)) {
4770bab1671fSRichard Henderson temp_dead(s, ots);
4771bab1671fSRichard Henderson }
4772bab1671fSRichard Henderson }
4773bab1671fSRichard Henderson
tcg_reg_alloc_op(TCGContext * s,const TCGOp * op)4774dd186292SRichard Henderson static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
4775c896fe29Sbellard {
4776dd186292SRichard Henderson const TCGLifeData arg_life = op->life;
4777dd186292SRichard Henderson const TCGOpDef * const def = &tcg_op_defs[op->opc];
477882790a87SRichard Henderson TCGRegSet i_allocated_regs;
477982790a87SRichard Henderson TCGRegSet o_allocated_regs;
4780b6638662SRichard Henderson int i, k, nb_iargs, nb_oargs;
4781b6638662SRichard Henderson TCGReg reg;
4782c896fe29Sbellard TCGArg arg;
4783c896fe29Sbellard const TCGArgConstraint *arg_ct;
4784c896fe29Sbellard TCGTemp *ts;
4785c896fe29Sbellard TCGArg new_args[TCG_MAX_OP_ARGS];
4786c896fe29Sbellard int const_args[TCG_MAX_OP_ARGS];
4787c896fe29Sbellard
4788c896fe29Sbellard nb_oargs = def->nb_oargs;
4789c896fe29Sbellard nb_iargs = def->nb_iargs;
4790c896fe29Sbellard
4791c896fe29Sbellard /* copy constants */
4792c896fe29Sbellard memcpy(new_args + nb_oargs + nb_iargs,
4793dd186292SRichard Henderson op->args + nb_oargs + nb_iargs,
4794c896fe29Sbellard sizeof(TCGArg) * def->nb_cargs);
4795c896fe29Sbellard
4796d21369f5SRichard Henderson i_allocated_regs = s->reserved_regs;
4797d21369f5SRichard Henderson o_allocated_regs = s->reserved_regs;
479882790a87SRichard Henderson
4799c896fe29Sbellard /* satisfy input constraints */
4800c896fe29Sbellard for (k = 0; k < nb_iargs; k++) {
480129f5e925SRichard Henderson TCGRegSet i_preferred_regs, i_required_regs;
480229f5e925SRichard Henderson bool allocate_new_reg, copyto_new_reg;
480329f5e925SRichard Henderson TCGTemp *ts2;
480429f5e925SRichard Henderson int i1, i2;
4805d62816f2SRichard Henderson
480666792f90SRichard Henderson i = def->args_ct[nb_oargs + k].sort_index;
4807dd186292SRichard Henderson arg = op->args[i];
4808c896fe29Sbellard arg_ct = &def->args_ct[i];
480943439139SRichard Henderson ts = arg_temp(arg);
481040ae5c62SRichard Henderson
481140ae5c62SRichard Henderson if (ts->val_type == TEMP_VAL_CONST
4812ebe92db2SJiajie Chen && tcg_target_const_match(ts->val, ts->type, arg_ct->ct, TCGOP_VECE(op))) {
4813c896fe29Sbellard /* constant is OK for instruction */
4814c896fe29Sbellard const_args[i] = 1;
4815c896fe29Sbellard new_args[i] = ts->val;
4816d62816f2SRichard Henderson continue;
4817c896fe29Sbellard }
481840ae5c62SRichard Henderson
48191c1824dcSRichard Henderson reg = ts->reg;
48201c1824dcSRichard Henderson i_preferred_regs = 0;
482129f5e925SRichard Henderson i_required_regs = arg_ct->regs;
48221c1824dcSRichard Henderson allocate_new_reg = false;
482329f5e925SRichard Henderson copyto_new_reg = false;
48241c1824dcSRichard Henderson
482529f5e925SRichard Henderson switch (arg_ct->pair) {
482629f5e925SRichard Henderson case 0: /* not paired */
4827bc2b17e6SRichard Henderson if (arg_ct->ialias) {
482831fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index);
4829c0522136SRichard Henderson
4830c0522136SRichard Henderson /*
4831c0522136SRichard Henderson * If the input is readonly, then it cannot also be an
4832c0522136SRichard Henderson * output and aliased to itself. If the input is not
4833c0522136SRichard Henderson * dead after the instruction, we must allocate a new
4834c0522136SRichard Henderson * register and move it.
4835c0522136SRichard Henderson */
483622d2e535SIlya Leoshkevich if (temp_readonly(ts) || !IS_DEAD_ARG(i)
483722d2e535SIlya Leoshkevich || def->args_ct[arg_ct->alias_index].newreg) {
48381c1824dcSRichard Henderson allocate_new_reg = true;
48391c1824dcSRichard Henderson } else if (ts->val_type == TEMP_VAL_REG) {
4840c0522136SRichard Henderson /*
48411c1824dcSRichard Henderson * Check if the current register has already been
48421c1824dcSRichard Henderson * allocated for another input.
4843c0522136SRichard Henderson */
484429f5e925SRichard Henderson allocate_new_reg =
484529f5e925SRichard Henderson tcg_regset_test_reg(i_allocated_regs, reg);
48467e1df267SAurelien Jarno }
48477e1df267SAurelien Jarno }
48481c1824dcSRichard Henderson if (!allocate_new_reg) {
484929f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs,
485029f5e925SRichard Henderson i_preferred_regs);
4851c896fe29Sbellard reg = ts->reg;
485229f5e925SRichard Henderson allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
48531c1824dcSRichard Henderson }
48541c1824dcSRichard Henderson if (allocate_new_reg) {
4855c0522136SRichard Henderson /*
4856c0522136SRichard Henderson * Allocate a new register matching the constraint
4857c0522136SRichard Henderson * and move the temporary register into it.
4858c0522136SRichard Henderson */
4859d62816f2SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type],
4860d62816f2SRichard Henderson i_allocated_regs, 0);
486129f5e925SRichard Henderson reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
48621c1824dcSRichard Henderson i_preferred_regs, ts->indirect_base);
486329f5e925SRichard Henderson copyto_new_reg = true;
486429f5e925SRichard Henderson }
486529f5e925SRichard Henderson break;
486629f5e925SRichard Henderson
486729f5e925SRichard Henderson case 1:
486829f5e925SRichard Henderson /* First of an input pair; if i1 == i2, the second is an output. */
486929f5e925SRichard Henderson i1 = i;
487029f5e925SRichard Henderson i2 = arg_ct->pair_index;
487129f5e925SRichard Henderson ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
487229f5e925SRichard Henderson
487329f5e925SRichard Henderson /*
487429f5e925SRichard Henderson * It is easier to default to allocating a new pair
487529f5e925SRichard Henderson * and to identify a few cases where it's not required.
487629f5e925SRichard Henderson */
487729f5e925SRichard Henderson if (arg_ct->ialias) {
487831fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index);
487929f5e925SRichard Henderson if (IS_DEAD_ARG(i1) &&
488029f5e925SRichard Henderson IS_DEAD_ARG(i2) &&
488129f5e925SRichard Henderson !temp_readonly(ts) &&
488229f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG &&
488329f5e925SRichard Henderson ts->reg < TCG_TARGET_NB_REGS - 1 &&
488429f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) &&
488529f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) &&
488629f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
488729f5e925SRichard Henderson (ts2
488829f5e925SRichard Henderson ? ts2->val_type == TEMP_VAL_REG &&
488929f5e925SRichard Henderson ts2->reg == reg + 1 &&
489029f5e925SRichard Henderson !temp_readonly(ts2)
489129f5e925SRichard Henderson : s->reg_to_temp[reg + 1] == NULL)) {
489229f5e925SRichard Henderson break;
489329f5e925SRichard Henderson }
489429f5e925SRichard Henderson } else {
489529f5e925SRichard Henderson /* Without aliasing, the pair must also be an input. */
489629f5e925SRichard Henderson tcg_debug_assert(ts2);
489729f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG &&
489829f5e925SRichard Henderson ts2->val_type == TEMP_VAL_REG &&
489929f5e925SRichard Henderson ts2->reg == reg + 1 &&
490029f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg)) {
490129f5e925SRichard Henderson break;
490229f5e925SRichard Henderson }
490329f5e925SRichard Henderson }
490429f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
490529f5e925SRichard Henderson 0, ts->indirect_base);
490629f5e925SRichard Henderson goto do_pair;
490729f5e925SRichard Henderson
490829f5e925SRichard Henderson case 2: /* pair second */
490929f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1;
491029f5e925SRichard Henderson goto do_pair;
491129f5e925SRichard Henderson
491229f5e925SRichard Henderson case 3: /* ialias with second output, no first input */
491329f5e925SRichard Henderson tcg_debug_assert(arg_ct->ialias);
491431fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index);
491529f5e925SRichard Henderson
491629f5e925SRichard Henderson if (IS_DEAD_ARG(i) &&
491729f5e925SRichard Henderson !temp_readonly(ts) &&
491829f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG &&
491929f5e925SRichard Henderson reg > 0 &&
492029f5e925SRichard Henderson s->reg_to_temp[reg - 1] == NULL &&
492129f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) &&
492229f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) &&
492329f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
492429f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg - 1);
492529f5e925SRichard Henderson break;
492629f5e925SRichard Henderson }
492729f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
492829f5e925SRichard Henderson i_allocated_regs, 0,
492929f5e925SRichard Henderson ts->indirect_base);
493029f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg);
493129f5e925SRichard Henderson reg += 1;
493229f5e925SRichard Henderson goto do_pair;
493329f5e925SRichard Henderson
493429f5e925SRichard Henderson do_pair:
493529f5e925SRichard Henderson /*
493629f5e925SRichard Henderson * If an aliased input is not dead after the instruction,
493729f5e925SRichard Henderson * we must allocate a new register and move it.
493829f5e925SRichard Henderson */
493929f5e925SRichard Henderson if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
494029f5e925SRichard Henderson TCGRegSet t_allocated_regs = i_allocated_regs;
494129f5e925SRichard Henderson
494229f5e925SRichard Henderson /*
494329f5e925SRichard Henderson * Because of the alias, and the continued life, make sure
494429f5e925SRichard Henderson * that the temp is somewhere *other* than the reg pair,
494529f5e925SRichard Henderson * and we get a copy in reg.
494629f5e925SRichard Henderson */
494729f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg);
494829f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg + 1);
494929f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
495029f5e925SRichard Henderson /* If ts was already in reg, copy it somewhere else. */
495129f5e925SRichard Henderson TCGReg nr;
495229f5e925SRichard Henderson bool ok;
495329f5e925SRichard Henderson
495429f5e925SRichard Henderson tcg_debug_assert(ts->kind != TEMP_FIXED);
495529f5e925SRichard Henderson nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
495629f5e925SRichard Henderson t_allocated_regs, 0, ts->indirect_base);
495729f5e925SRichard Henderson ok = tcg_out_mov(s, ts->type, nr, reg);
495829f5e925SRichard Henderson tcg_debug_assert(ok);
495929f5e925SRichard Henderson
496029f5e925SRichard Henderson set_temp_val_reg(s, ts, nr);
496129f5e925SRichard Henderson } else {
496229f5e925SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type],
496329f5e925SRichard Henderson t_allocated_regs, 0);
496429f5e925SRichard Henderson copyto_new_reg = true;
496529f5e925SRichard Henderson }
496629f5e925SRichard Henderson } else {
496729f5e925SRichard Henderson /* Preferably allocate to reg, otherwise copy. */
496829f5e925SRichard Henderson i_required_regs = (TCGRegSet)1 << reg;
496929f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs,
497029f5e925SRichard Henderson i_preferred_regs);
497129f5e925SRichard Henderson copyto_new_reg = ts->reg != reg;
497229f5e925SRichard Henderson }
497329f5e925SRichard Henderson break;
497429f5e925SRichard Henderson
497529f5e925SRichard Henderson default:
497629f5e925SRichard Henderson g_assert_not_reached();
497729f5e925SRichard Henderson }
497829f5e925SRichard Henderson
497929f5e925SRichard Henderson if (copyto_new_reg) {
498078113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4981240c08d0SRichard Henderson /*
4982240c08d0SRichard Henderson * Cross register class move not supported. Sync the
4983240c08d0SRichard Henderson * temp back to its slot and load from there.
4984240c08d0SRichard Henderson */
4985240c08d0SRichard Henderson temp_sync(s, ts, i_allocated_regs, 0, 0);
4986240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg,
4987240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset);
498878113e83SRichard Henderson }
4989c896fe29Sbellard }
4990c896fe29Sbellard new_args[i] = reg;
4991c896fe29Sbellard const_args[i] = 0;
499282790a87SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg);
4993c896fe29Sbellard }
4994c896fe29Sbellard
4995c896fe29Sbellard /* mark dead temporaries and free the associated registers */
4996866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4997866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) {
499843439139SRichard Henderson temp_dead(s, arg_temp(op->args[i]));
4999c896fe29Sbellard }
5000c896fe29Sbellard }
5001c896fe29Sbellard
5002b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) {
5003b4cb76e6SRichard Henderson tcg_reg_alloc_cbranch(s, i_allocated_regs);
5004b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) {
500582790a87SRichard Henderson tcg_reg_alloc_bb_end(s, i_allocated_regs);
5006a52ad07eSAurelien Jarno } else {
5007c896fe29Sbellard if (def->flags & TCG_OPF_CALL_CLOBBER) {
5008b03cce8eSbellard /* XXX: permit generic clobber register list ? */
5009c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5010c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
501182790a87SRichard Henderson tcg_reg_free(s, i, i_allocated_regs);
5012c896fe29Sbellard }
5013c896fe29Sbellard }
50143d5c5f87SAurelien Jarno }
50153d5c5f87SAurelien Jarno if (def->flags & TCG_OPF_SIDE_EFFECTS) {
50163d5c5f87SAurelien Jarno /* sync globals if the op has side effects and might trigger
50173d5c5f87SAurelien Jarno an exception. */
501882790a87SRichard Henderson sync_globals(s, i_allocated_regs);
5019c896fe29Sbellard }
5020c896fe29Sbellard
5021c896fe29Sbellard /* satisfy the output constraints */
5022c896fe29Sbellard for(k = 0; k < nb_oargs; k++) {
502366792f90SRichard Henderson i = def->args_ct[k].sort_index;
5024dd186292SRichard Henderson arg = op->args[i];
5025c896fe29Sbellard arg_ct = &def->args_ct[i];
502643439139SRichard Henderson ts = arg_temp(arg);
5027d63e3b6eSRichard Henderson
5028d63e3b6eSRichard Henderson /* ENV should not be modified. */
5029e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts));
5030d63e3b6eSRichard Henderson
503129f5e925SRichard Henderson switch (arg_ct->pair) {
503229f5e925SRichard Henderson case 0: /* not paired */
5033bc2b17e6SRichard Henderson if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
50345ff9d6a4Sbellard reg = new_args[arg_ct->alias_index];
5035bc2b17e6SRichard Henderson } else if (arg_ct->newreg) {
50369be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs,
503782790a87SRichard Henderson i_allocated_regs | o_allocated_regs,
503831fd884bSRichard Henderson output_pref(op, k), ts->indirect_base);
5039c896fe29Sbellard } else {
50409be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
504131fd884bSRichard Henderson output_pref(op, k), ts->indirect_base);
5042c896fe29Sbellard }
504329f5e925SRichard Henderson break;
504429f5e925SRichard Henderson
504529f5e925SRichard Henderson case 1: /* first of pair */
504629f5e925SRichard Henderson if (arg_ct->oalias) {
504729f5e925SRichard Henderson reg = new_args[arg_ct->alias_index];
5048*b8819108SRichard Henderson } else if (arg_ct->newreg) {
5049*b8819108SRichard Henderson reg = tcg_reg_alloc_pair(s, arg_ct->regs,
5050*b8819108SRichard Henderson i_allocated_regs | o_allocated_regs,
5051*b8819108SRichard Henderson output_pref(op, k),
5052*b8819108SRichard Henderson ts->indirect_base);
5053*b8819108SRichard Henderson } else {
505429f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
5055*b8819108SRichard Henderson output_pref(op, k),
5056*b8819108SRichard Henderson ts->indirect_base);
5057*b8819108SRichard Henderson }
505829f5e925SRichard Henderson break;
505929f5e925SRichard Henderson
506029f5e925SRichard Henderson case 2: /* second of pair */
506129f5e925SRichard Henderson if (arg_ct->oalias) {
506229f5e925SRichard Henderson reg = new_args[arg_ct->alias_index];
506329f5e925SRichard Henderson } else {
506429f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1;
506529f5e925SRichard Henderson }
506629f5e925SRichard Henderson break;
506729f5e925SRichard Henderson
506829f5e925SRichard Henderson case 3: /* first of pair, aliasing with a second input */
506929f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg);
507029f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] - 1;
507129f5e925SRichard Henderson break;
507229f5e925SRichard Henderson
507329f5e925SRichard Henderson default:
507429f5e925SRichard Henderson g_assert_not_reached();
507529f5e925SRichard Henderson }
507682790a87SRichard Henderson tcg_regset_set_reg(o_allocated_regs, reg);
5077098859f1SRichard Henderson set_temp_val_reg(s, ts, reg);
5078c896fe29Sbellard ts->mem_coherent = 0;
5079c896fe29Sbellard new_args[i] = reg;
5080c896fe29Sbellard }
5081e8996ee0Sbellard }
5082c896fe29Sbellard
5083c896fe29Sbellard /* emit instruction */
5084678155b2SRichard Henderson switch (op->opc) {
5085678155b2SRichard Henderson case INDEX_op_ext8s_i32:
5086678155b2SRichard Henderson tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
5087678155b2SRichard Henderson break;
5088678155b2SRichard Henderson case INDEX_op_ext8s_i64:
5089678155b2SRichard Henderson tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
5090678155b2SRichard Henderson break;
5091d0e66c89SRichard Henderson case INDEX_op_ext8u_i32:
5092d0e66c89SRichard Henderson case INDEX_op_ext8u_i64:
5093d0e66c89SRichard Henderson tcg_out_ext8u(s, new_args[0], new_args[1]);
5094d0e66c89SRichard Henderson break;
5095753e42eaSRichard Henderson case INDEX_op_ext16s_i32:
5096753e42eaSRichard Henderson tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]);
5097753e42eaSRichard Henderson break;
5098753e42eaSRichard Henderson case INDEX_op_ext16s_i64:
5099753e42eaSRichard Henderson tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]);
5100753e42eaSRichard Henderson break;
5101379afdffSRichard Henderson case INDEX_op_ext16u_i32:
5102379afdffSRichard Henderson case INDEX_op_ext16u_i64:
5103379afdffSRichard Henderson tcg_out_ext16u(s, new_args[0], new_args[1]);
5104379afdffSRichard Henderson break;
510552bf3398SRichard Henderson case INDEX_op_ext32s_i64:
510652bf3398SRichard Henderson tcg_out_ext32s(s, new_args[0], new_args[1]);
510752bf3398SRichard Henderson break;
51089ecf5f61SRichard Henderson case INDEX_op_ext32u_i64:
51099ecf5f61SRichard Henderson tcg_out_ext32u(s, new_args[0], new_args[1]);
51109ecf5f61SRichard Henderson break;
51119c6aa274SRichard Henderson case INDEX_op_ext_i32_i64:
51129c6aa274SRichard Henderson tcg_out_exts_i32_i64(s, new_args[0], new_args[1]);
51139c6aa274SRichard Henderson break;
5114b9bfe000SRichard Henderson case INDEX_op_extu_i32_i64:
5115b9bfe000SRichard Henderson tcg_out_extu_i32_i64(s, new_args[0], new_args[1]);
5116b9bfe000SRichard Henderson break;
5117b8b94ac6SRichard Henderson case INDEX_op_extrl_i64_i32:
5118b8b94ac6SRichard Henderson tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]);
5119b8b94ac6SRichard Henderson break;
5120678155b2SRichard Henderson default:
5121d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) {
5122d2fd745fSRichard Henderson tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
5123d2fd745fSRichard Henderson new_args, const_args);
5124d2fd745fSRichard Henderson } else {
5125dd186292SRichard Henderson tcg_out_op(s, op->opc, new_args, const_args);
5126d2fd745fSRichard Henderson }
5127678155b2SRichard Henderson break;
5128678155b2SRichard Henderson }
5129c896fe29Sbellard
5130c896fe29Sbellard /* move the outputs in the correct register if needed */
5131c896fe29Sbellard for(i = 0; i < nb_oargs; i++) {
513243439139SRichard Henderson ts = arg_temp(op->args[i]);
5133d63e3b6eSRichard Henderson
5134d63e3b6eSRichard Henderson /* ENV should not be modified. */
5135e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts));
5136d63e3b6eSRichard Henderson
5137ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) {
513898b4e186SRichard Henderson temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
513959d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) {
5140f8bf00f1SRichard Henderson temp_dead(s, ts);
5141ec7a869dSAurelien Jarno }
5142c896fe29Sbellard }
5143c896fe29Sbellard }
5144c896fe29Sbellard
tcg_reg_alloc_dup2(TCGContext * s,const TCGOp * op)5145efe86b21SRichard Henderson static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
5146efe86b21SRichard Henderson {
5147efe86b21SRichard Henderson const TCGLifeData arg_life = op->life;
5148efe86b21SRichard Henderson TCGTemp *ots, *itsl, *itsh;
5149efe86b21SRichard Henderson TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
5150efe86b21SRichard Henderson
5151efe86b21SRichard Henderson /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
5152efe86b21SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
5153efe86b21SRichard Henderson tcg_debug_assert(TCGOP_VECE(op) == MO_64);
5154efe86b21SRichard Henderson
5155efe86b21SRichard Henderson ots = arg_temp(op->args[0]);
5156efe86b21SRichard Henderson itsl = arg_temp(op->args[1]);
5157efe86b21SRichard Henderson itsh = arg_temp(op->args[2]);
5158efe86b21SRichard Henderson
5159efe86b21SRichard Henderson /* ENV should not be modified. */
5160efe86b21SRichard Henderson tcg_debug_assert(!temp_readonly(ots));
5161efe86b21SRichard Henderson
5162efe86b21SRichard Henderson /* Allocate the output register now. */
5163efe86b21SRichard Henderson if (ots->val_type != TEMP_VAL_REG) {
5164efe86b21SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs;
5165efe86b21SRichard Henderson TCGRegSet dup_out_regs =
5166efe86b21SRichard Henderson tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
5167098859f1SRichard Henderson TCGReg oreg;
5168efe86b21SRichard Henderson
5169efe86b21SRichard Henderson /* Make sure to not spill the input registers. */
5170efe86b21SRichard Henderson if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
5171efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsl->reg);
5172efe86b21SRichard Henderson }
5173efe86b21SRichard Henderson if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
5174efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsh->reg);
5175efe86b21SRichard Henderson }
5176efe86b21SRichard Henderson
5177098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
517831fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base);
5179098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg);
5180efe86b21SRichard Henderson }
5181efe86b21SRichard Henderson
5182efe86b21SRichard Henderson /* Promote dup2 of immediates to dupi_vec. */
5183efe86b21SRichard Henderson if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
5184efe86b21SRichard Henderson uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
5185efe86b21SRichard Henderson MemOp vece = MO_64;
5186efe86b21SRichard Henderson
5187efe86b21SRichard Henderson if (val == dup_const(MO_8, val)) {
5188efe86b21SRichard Henderson vece = MO_8;
5189efe86b21SRichard Henderson } else if (val == dup_const(MO_16, val)) {
5190efe86b21SRichard Henderson vece = MO_16;
5191efe86b21SRichard Henderson } else if (val == dup_const(MO_32, val)) {
5192efe86b21SRichard Henderson vece = MO_32;
5193efe86b21SRichard Henderson }
5194efe86b21SRichard Henderson
5195efe86b21SRichard Henderson tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
5196efe86b21SRichard Henderson goto done;
5197efe86b21SRichard Henderson }
5198efe86b21SRichard Henderson
5199efe86b21SRichard Henderson /* If the two inputs form one 64-bit value, try dupm_vec. */
5200aef85402SRichard Henderson if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
5201aef85402SRichard Henderson itsh->temp_subindex == !HOST_BIG_ENDIAN &&
5202aef85402SRichard Henderson itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
5203aef85402SRichard Henderson TCGTemp *its = itsl - HOST_BIG_ENDIAN;
5204aef85402SRichard Henderson
5205aef85402SRichard Henderson temp_sync(s, its + 0, s->reserved_regs, 0, 0);
5206aef85402SRichard Henderson temp_sync(s, its + 1, s->reserved_regs, 0, 0);
5207aef85402SRichard Henderson
5208efe86b21SRichard Henderson if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
5209efe86b21SRichard Henderson its->mem_base->reg, its->mem_offset)) {
5210efe86b21SRichard Henderson goto done;
5211efe86b21SRichard Henderson }
5212efe86b21SRichard Henderson }
5213efe86b21SRichard Henderson
5214efe86b21SRichard Henderson /* Fall back to generic expansion. */
5215efe86b21SRichard Henderson return false;
5216efe86b21SRichard Henderson
5217efe86b21SRichard Henderson done:
521836f5539cSRichard Henderson ots->mem_coherent = 0;
5219efe86b21SRichard Henderson if (IS_DEAD_ARG(1)) {
5220efe86b21SRichard Henderson temp_dead(s, itsl);
5221efe86b21SRichard Henderson }
5222efe86b21SRichard Henderson if (IS_DEAD_ARG(2)) {
5223efe86b21SRichard Henderson temp_dead(s, itsh);
5224efe86b21SRichard Henderson }
5225efe86b21SRichard Henderson if (NEED_SYNC_ARG(0)) {
5226efe86b21SRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
5227efe86b21SRichard Henderson } else if (IS_DEAD_ARG(0)) {
5228efe86b21SRichard Henderson temp_dead(s, ots);
5229efe86b21SRichard Henderson }
5230efe86b21SRichard Henderson return true;
5231efe86b21SRichard Henderson }
5232efe86b21SRichard Henderson
load_arg_reg(TCGContext * s,TCGReg reg,TCGTemp * ts,TCGRegSet allocated_regs)523339004a71SRichard Henderson static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
523439004a71SRichard Henderson TCGRegSet allocated_regs)
5235c896fe29Sbellard {
5236c896fe29Sbellard if (ts->val_type == TEMP_VAL_REG) {
5237c896fe29Sbellard if (ts->reg != reg) {
52384250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs);
523978113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
5240240c08d0SRichard Henderson /*
5241240c08d0SRichard Henderson * Cross register class move not supported. Sync the
5242240c08d0SRichard Henderson * temp back to its slot and load from there.
5243240c08d0SRichard Henderson */
5244240c08d0SRichard Henderson temp_sync(s, ts, allocated_regs, 0, 0);
5245240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg,
5246240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset);
524778113e83SRichard Henderson }
5248c896fe29Sbellard }
5249c896fe29Sbellard } else {
5250ccb1bb66SRichard Henderson TCGRegSet arg_set = 0;
525140ae5c62SRichard Henderson
52524250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs);
525340ae5c62SRichard Henderson tcg_regset_set_reg(arg_set, reg);
5254b722452aSRichard Henderson temp_load(s, ts, arg_set, allocated_regs, 0);
5255c896fe29Sbellard }
525639004a71SRichard Henderson }
525740ae5c62SRichard Henderson
load_arg_stk(TCGContext * s,unsigned arg_slot,TCGTemp * ts,TCGRegSet allocated_regs)5258d78e4a4fSRichard Henderson static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts,
525939004a71SRichard Henderson TCGRegSet allocated_regs)
526039004a71SRichard Henderson {
526139004a71SRichard Henderson /*
526239004a71SRichard Henderson * When the destination is on the stack, load up the temp and store.
526339004a71SRichard Henderson * If there are many call-saved registers, the temp might live to
526439004a71SRichard Henderson * see another use; otherwise it'll be discarded.
526539004a71SRichard Henderson */
526639004a71SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
526739004a71SRichard Henderson tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
5268d78e4a4fSRichard Henderson arg_slot_stk_ofs(arg_slot));
526939004a71SRichard Henderson }
527039004a71SRichard Henderson
load_arg_normal(TCGContext * s,const TCGCallArgumentLoc * l,TCGTemp * ts,TCGRegSet * allocated_regs)527139004a71SRichard Henderson static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
527239004a71SRichard Henderson TCGTemp *ts, TCGRegSet *allocated_regs)
527339004a71SRichard Henderson {
5274338b61e9SRichard Henderson if (arg_slot_reg_p(l->arg_slot)) {
527539004a71SRichard Henderson TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
527639004a71SRichard Henderson load_arg_reg(s, reg, ts, *allocated_regs);
527739004a71SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg);
527839004a71SRichard Henderson } else {
5279d78e4a4fSRichard Henderson load_arg_stk(s, l->arg_slot, ts, *allocated_regs);
5280c896fe29Sbellard }
528139cf05d3Sbellard }
5282c896fe29Sbellard
load_arg_ref(TCGContext * s,unsigned arg_slot,TCGReg ref_base,intptr_t ref_off,TCGRegSet * allocated_regs)5283d78e4a4fSRichard Henderson static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base,
5284313bdea8SRichard Henderson intptr_t ref_off, TCGRegSet *allocated_regs)
5285313bdea8SRichard Henderson {
5286313bdea8SRichard Henderson TCGReg reg;
5287313bdea8SRichard Henderson
5288d78e4a4fSRichard Henderson if (arg_slot_reg_p(arg_slot)) {
5289313bdea8SRichard Henderson reg = tcg_target_call_iarg_regs[arg_slot];
5290313bdea8SRichard Henderson tcg_reg_free(s, reg, *allocated_regs);
5291313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5292313bdea8SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg);
5293313bdea8SRichard Henderson } else {
5294313bdea8SRichard Henderson reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
5295313bdea8SRichard Henderson *allocated_regs, 0, false);
5296313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5297313bdea8SRichard Henderson tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
5298d78e4a4fSRichard Henderson arg_slot_stk_ofs(arg_slot));
5299313bdea8SRichard Henderson }
5300313bdea8SRichard Henderson }
5301313bdea8SRichard Henderson
tcg_reg_alloc_call(TCGContext * s,TCGOp * op)530239004a71SRichard Henderson static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
530339004a71SRichard Henderson {
530439004a71SRichard Henderson const int nb_oargs = TCGOP_CALLO(op);
530539004a71SRichard Henderson const int nb_iargs = TCGOP_CALLI(op);
530639004a71SRichard Henderson const TCGLifeData arg_life = op->life;
530739004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op);
530839004a71SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs;
530939004a71SRichard Henderson int i;
531039004a71SRichard Henderson
531139004a71SRichard Henderson /*
531239004a71SRichard Henderson * Move inputs into place in reverse order,
531339004a71SRichard Henderson * so that we place stacked arguments first.
531439004a71SRichard Henderson */
531539004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; --i) {
531639004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i];
531739004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
531839004a71SRichard Henderson
531939004a71SRichard Henderson switch (loc->kind) {
532039004a71SRichard Henderson case TCG_CALL_ARG_NORMAL:
532139004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U:
532239004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S:
532339004a71SRichard Henderson load_arg_normal(s, loc, ts, &allocated_regs);
532439004a71SRichard Henderson break;
5325313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF:
5326313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5327313bdea8SRichard Henderson load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
5328d78e4a4fSRichard Henderson arg_slot_stk_ofs(loc->ref_slot),
5329313bdea8SRichard Henderson &allocated_regs);
5330313bdea8SRichard Henderson break;
5331313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N:
5332313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5333313bdea8SRichard Henderson break;
533439004a71SRichard Henderson default:
533539004a71SRichard Henderson g_assert_not_reached();
533639004a71SRichard Henderson }
533739004a71SRichard Henderson }
533839004a71SRichard Henderson
533939004a71SRichard Henderson /* Mark dead temporaries and free the associated registers. */
5340866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
5341866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) {
534243439139SRichard Henderson temp_dead(s, arg_temp(op->args[i]));
5343c896fe29Sbellard }
5344c896fe29Sbellard }
5345c896fe29Sbellard
534639004a71SRichard Henderson /* Clobber call registers. */
5347c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5348c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
5349b3915dbbSRichard Henderson tcg_reg_free(s, i, allocated_regs);
5350c896fe29Sbellard }
5351c896fe29Sbellard }
5352c896fe29Sbellard
535339004a71SRichard Henderson /*
535439004a71SRichard Henderson * Save globals if they might be written by the helper,
535539004a71SRichard Henderson * sync them if they might be read.
535639004a71SRichard Henderson */
535739004a71SRichard Henderson if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
535878505279SAurelien Jarno /* Nothing to do */
535939004a71SRichard Henderson } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
536078505279SAurelien Jarno sync_globals(s, allocated_regs);
536178505279SAurelien Jarno } else {
5362e8996ee0Sbellard save_globals(s, allocated_regs);
5363b9c18f56Saurel32 }
5364c896fe29Sbellard
5365313bdea8SRichard Henderson /*
5366313bdea8SRichard Henderson * If the ABI passes a pointer to the returned struct as the first
5367313bdea8SRichard Henderson * argument, load that now. Pass a pointer to the output home slot.
5368313bdea8SRichard Henderson */
5369313bdea8SRichard Henderson if (info->out_kind == TCG_CALL_RET_BY_REF) {
5370313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]);
5371313bdea8SRichard Henderson
5372313bdea8SRichard Henderson if (!ts->mem_allocated) {
5373313bdea8SRichard Henderson temp_allocate_frame(s, ts);
5374313bdea8SRichard Henderson }
5375313bdea8SRichard Henderson load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
5376313bdea8SRichard Henderson }
5377313bdea8SRichard Henderson
5378cee44b03SRichard Henderson tcg_out_call(s, tcg_call_func(op), info);
5379c896fe29Sbellard
538039004a71SRichard Henderson /* Assign output registers and emit moves if needed. */
538139004a71SRichard Henderson switch (info->out_kind) {
538239004a71SRichard Henderson case TCG_CALL_RET_NORMAL:
5383c896fe29Sbellard for (i = 0; i < nb_oargs; i++) {
538439004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]);
53855e3d0c19SRichard Henderson TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
5386d63e3b6eSRichard Henderson
5387d63e3b6eSRichard Henderson /* ENV should not be modified. */
5388e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts));
5389d63e3b6eSRichard Henderson
5390098859f1SRichard Henderson set_temp_val_reg(s, ts, reg);
5391c896fe29Sbellard ts->mem_coherent = 0;
539239004a71SRichard Henderson }
539339004a71SRichard Henderson break;
5394313bdea8SRichard Henderson
5395c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC:
5396c6556aa0SRichard Henderson {
5397c6556aa0SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]);
5398c6556aa0SRichard Henderson
5399c6556aa0SRichard Henderson tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
5400c6556aa0SRichard Henderson tcg_debug_assert(ts->temp_subindex == 0);
5401c6556aa0SRichard Henderson if (!ts->mem_allocated) {
5402c6556aa0SRichard Henderson temp_allocate_frame(s, ts);
5403c6556aa0SRichard Henderson }
5404c6556aa0SRichard Henderson tcg_out_st(s, TCG_TYPE_V128,
5405c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
5406c6556aa0SRichard Henderson ts->mem_base->reg, ts->mem_offset);
5407c6556aa0SRichard Henderson }
5408c6556aa0SRichard Henderson /* fall through to mark all parts in memory */
5409c6556aa0SRichard Henderson
5410313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF:
5411313bdea8SRichard Henderson /* The callee has performed a write through the reference. */
5412313bdea8SRichard Henderson for (i = 0; i < nb_oargs; i++) {
5413313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]);
5414313bdea8SRichard Henderson ts->val_type = TEMP_VAL_MEM;
5415313bdea8SRichard Henderson }
5416313bdea8SRichard Henderson break;
5417313bdea8SRichard Henderson
541839004a71SRichard Henderson default:
541939004a71SRichard Henderson g_assert_not_reached();
542039004a71SRichard Henderson }
542139004a71SRichard Henderson
542239004a71SRichard Henderson /* Flush or discard output registers as needed. */
542339004a71SRichard Henderson for (i = 0; i < nb_oargs; i++) {
542439004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]);
5425ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) {
542639004a71SRichard Henderson temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
542759d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) {
5428f8bf00f1SRichard Henderson temp_dead(s, ts);
5429c896fe29Sbellard }
5430c896fe29Sbellard }
54318c11ad25SAurelien Jarno }
5432c896fe29Sbellard
5433e63b8a29SRichard Henderson /**
5434e63b8a29SRichard Henderson * atom_and_align_for_opc:
5435e63b8a29SRichard Henderson * @s: tcg context
5436e63b8a29SRichard Henderson * @opc: memory operation code
5437e63b8a29SRichard Henderson * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations
5438e63b8a29SRichard Henderson * @allow_two_ops: true if we are prepared to issue two operations
5439e63b8a29SRichard Henderson *
5440e63b8a29SRichard Henderson * Return the alignment and atomicity to use for the inline fast path
5441e63b8a29SRichard Henderson * for the given memory operation. The alignment may be larger than
5442e63b8a29SRichard Henderson * that specified in @opc, and the correct alignment will be diagnosed
5443e63b8a29SRichard Henderson * by the slow path helper.
5444e63b8a29SRichard Henderson *
5445e63b8a29SRichard Henderson * If @allow_two_ops, the host is prepared to test for 2x alignment,
5446e63b8a29SRichard Henderson * and issue two loads or stores for subalignment.
5447e63b8a29SRichard Henderson */
atom_and_align_for_opc(TCGContext * s,MemOp opc,MemOp host_atom,bool allow_two_ops)5448e63b8a29SRichard Henderson static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
5449e63b8a29SRichard Henderson MemOp host_atom, bool allow_two_ops)
5450e63b8a29SRichard Henderson {
5451e63b8a29SRichard Henderson MemOp align = get_alignment_bits(opc);
5452e63b8a29SRichard Henderson MemOp size = opc & MO_SIZE;
5453e63b8a29SRichard Henderson MemOp half = size ? size - 1 : 0;
5454cbb14556SRichard Henderson MemOp atom = opc & MO_ATOM_MASK;
5455e63b8a29SRichard Henderson MemOp atmax;
5456e63b8a29SRichard Henderson
5457e63b8a29SRichard Henderson switch (atom) {
5458e63b8a29SRichard Henderson case MO_ATOM_NONE:
5459e63b8a29SRichard Henderson /* The operation requires no specific atomicity. */
5460e63b8a29SRichard Henderson atmax = MO_8;
5461e63b8a29SRichard Henderson break;
5462e63b8a29SRichard Henderson
5463e63b8a29SRichard Henderson case MO_ATOM_IFALIGN:
5464e63b8a29SRichard Henderson atmax = size;
5465e63b8a29SRichard Henderson break;
5466e63b8a29SRichard Henderson
5467e63b8a29SRichard Henderson case MO_ATOM_IFALIGN_PAIR:
5468e63b8a29SRichard Henderson atmax = half;
5469e63b8a29SRichard Henderson break;
5470e63b8a29SRichard Henderson
5471e63b8a29SRichard Henderson case MO_ATOM_WITHIN16:
5472e63b8a29SRichard Henderson atmax = size;
5473e63b8a29SRichard Henderson if (size == MO_128) {
5474e63b8a29SRichard Henderson /* Misalignment implies !within16, and therefore no atomicity. */
5475e63b8a29SRichard Henderson } else if (host_atom != MO_ATOM_WITHIN16) {
5476e63b8a29SRichard Henderson /* The host does not implement within16, so require alignment. */
5477e63b8a29SRichard Henderson align = MAX(align, size);
5478e63b8a29SRichard Henderson }
5479e63b8a29SRichard Henderson break;
5480e63b8a29SRichard Henderson
5481e63b8a29SRichard Henderson case MO_ATOM_WITHIN16_PAIR:
5482e63b8a29SRichard Henderson atmax = size;
5483e63b8a29SRichard Henderson /*
5484e63b8a29SRichard Henderson * Misalignment implies !within16, and therefore half atomicity.
5485e63b8a29SRichard Henderson * Any host prepared for two operations can implement this with
5486e63b8a29SRichard Henderson * half alignment.
5487e63b8a29SRichard Henderson */
5488e63b8a29SRichard Henderson if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) {
5489e63b8a29SRichard Henderson align = MAX(align, half);
5490e63b8a29SRichard Henderson }
5491e63b8a29SRichard Henderson break;
5492e63b8a29SRichard Henderson
5493e63b8a29SRichard Henderson case MO_ATOM_SUBALIGN:
5494e63b8a29SRichard Henderson atmax = size;
5495e63b8a29SRichard Henderson if (host_atom != MO_ATOM_SUBALIGN) {
5496e63b8a29SRichard Henderson /* If unaligned but not odd, there are subobjects up to half. */
5497e63b8a29SRichard Henderson if (allow_two_ops) {
5498e63b8a29SRichard Henderson align = MAX(align, half);
5499e63b8a29SRichard Henderson } else {
5500e63b8a29SRichard Henderson align = MAX(align, size);
5501e63b8a29SRichard Henderson }
5502e63b8a29SRichard Henderson }
5503e63b8a29SRichard Henderson break;
5504e63b8a29SRichard Henderson
5505e63b8a29SRichard Henderson default:
5506e63b8a29SRichard Henderson g_assert_not_reached();
5507e63b8a29SRichard Henderson }
5508e63b8a29SRichard Henderson
5509e63b8a29SRichard Henderson return (TCGAtomAlign){ .atom = atmax, .align = align };
5510e63b8a29SRichard Henderson }
5511e63b8a29SRichard Henderson
55128429a1caSRichard Henderson /*
55138429a1caSRichard Henderson * Similarly for qemu_ld/st slow path helpers.
55148429a1caSRichard Henderson * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
55158429a1caSRichard Henderson * using only the provided backend tcg_out_* functions.
55168429a1caSRichard Henderson */
55178429a1caSRichard Henderson
tcg_out_helper_stk_ofs(TCGType type,unsigned slot)55188429a1caSRichard Henderson static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot)
55198429a1caSRichard Henderson {
55208429a1caSRichard Henderson int ofs = arg_slot_stk_ofs(slot);
55218429a1caSRichard Henderson
55228429a1caSRichard Henderson /*
55238429a1caSRichard Henderson * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not
55248429a1caSRichard Henderson * require extension to uint64_t, adjust the address for uint32_t.
55258429a1caSRichard Henderson */
55268429a1caSRichard Henderson if (HOST_BIG_ENDIAN &&
55278429a1caSRichard Henderson TCG_TARGET_REG_BITS == 64 &&
55288429a1caSRichard Henderson type == TCG_TYPE_I32) {
55298429a1caSRichard Henderson ofs += 4;
55308429a1caSRichard Henderson }
55318429a1caSRichard Henderson return ofs;
55328429a1caSRichard Henderson }
55338429a1caSRichard Henderson
tcg_out_helper_load_slots(TCGContext * s,unsigned nmov,TCGMovExtend * mov,const TCGLdstHelperParam * parm)55348d314041SRichard Henderson static void tcg_out_helper_load_slots(TCGContext *s,
55358429a1caSRichard Henderson unsigned nmov, TCGMovExtend *mov,
55362462e30eSRichard Henderson const TCGLdstHelperParam *parm)
55378429a1caSRichard Henderson {
55388d314041SRichard Henderson unsigned i;
55392462e30eSRichard Henderson TCGReg dst3;
55402462e30eSRichard Henderson
55418d314041SRichard Henderson /*
55428d314041SRichard Henderson * Start from the end, storing to the stack first.
55438d314041SRichard Henderson * This frees those registers, so we need not consider overlap.
55448d314041SRichard Henderson */
55458d314041SRichard Henderson for (i = nmov; i-- > 0; ) {
55468d314041SRichard Henderson unsigned slot = mov[i].dst;
55478d314041SRichard Henderson
55488d314041SRichard Henderson if (arg_slot_reg_p(slot)) {
55498d314041SRichard Henderson goto found_reg;
55508d314041SRichard Henderson }
55518d314041SRichard Henderson
55528d314041SRichard Henderson TCGReg src = mov[i].src;
55538d314041SRichard Henderson TCGType dst_type = mov[i].dst_type;
55548d314041SRichard Henderson MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64;
55558d314041SRichard Henderson
55568d314041SRichard Henderson /* The argument is going onto the stack; extend into scratch. */
55578d314041SRichard Henderson if ((mov[i].src_ext & MO_SIZE) != dst_mo) {
55588d314041SRichard Henderson tcg_debug_assert(parm->ntmp != 0);
55598d314041SRichard Henderson mov[i].dst = src = parm->tmp[0];
55608d314041SRichard Henderson tcg_out_movext1(s, &mov[i]);
55618d314041SRichard Henderson }
55628d314041SRichard Henderson
55638d314041SRichard Henderson tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK,
55648d314041SRichard Henderson tcg_out_helper_stk_ofs(dst_type, slot));
55658d314041SRichard Henderson }
55668d314041SRichard Henderson return;
55678d314041SRichard Henderson
55688d314041SRichard Henderson found_reg:
55698d314041SRichard Henderson /*
55708d314041SRichard Henderson * The remaining arguments are in registers.
55718d314041SRichard Henderson * Convert slot numbers to argument registers.
55728d314041SRichard Henderson */
55738d314041SRichard Henderson nmov = i + 1;
55748d314041SRichard Henderson for (i = 0; i < nmov; ++i) {
55758d314041SRichard Henderson mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst];
55768d314041SRichard Henderson }
55778d314041SRichard Henderson
55788429a1caSRichard Henderson switch (nmov) {
55792462e30eSRichard Henderson case 4:
55808429a1caSRichard Henderson /* The backend must have provided enough temps for the worst case. */
55812462e30eSRichard Henderson tcg_debug_assert(parm->ntmp >= 2);
55828429a1caSRichard Henderson
55832462e30eSRichard Henderson dst3 = mov[3].dst;
55842462e30eSRichard Henderson for (unsigned j = 0; j < 3; ++j) {
55852462e30eSRichard Henderson if (dst3 == mov[j].src) {
55868429a1caSRichard Henderson /*
55872462e30eSRichard Henderson * Conflict. Copy the source to a temporary, perform the
55882462e30eSRichard Henderson * remaining moves, then the extension from our scratch
55892462e30eSRichard Henderson * on the way out.
55908429a1caSRichard Henderson */
55912462e30eSRichard Henderson TCGReg scratch = parm->tmp[1];
55928429a1caSRichard Henderson
55932462e30eSRichard Henderson tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src);
55942462e30eSRichard Henderson tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]);
55952462e30eSRichard Henderson tcg_out_movext1_new_src(s, &mov[3], scratch);
55962462e30eSRichard Henderson break;
55978429a1caSRichard Henderson }
55988429a1caSRichard Henderson }
55998429a1caSRichard Henderson
56008429a1caSRichard Henderson /* No conflicts: perform this move and continue. */
56012462e30eSRichard Henderson tcg_out_movext1(s, &mov[3]);
56022462e30eSRichard Henderson /* fall through */
56038429a1caSRichard Henderson
56042462e30eSRichard Henderson case 3:
56052462e30eSRichard Henderson tcg_out_movext3(s, mov, mov + 1, mov + 2,
56062462e30eSRichard Henderson parm->ntmp ? parm->tmp[0] : -1);
56072462e30eSRichard Henderson break;
56088429a1caSRichard Henderson case 2:
56092462e30eSRichard Henderson tcg_out_movext2(s, mov, mov + 1,
56102462e30eSRichard Henderson parm->ntmp ? parm->tmp[0] : -1);
56112462e30eSRichard Henderson break;
56128429a1caSRichard Henderson case 1:
56138429a1caSRichard Henderson tcg_out_movext1(s, mov);
56142462e30eSRichard Henderson break;
56152462e30eSRichard Henderson default:
56168429a1caSRichard Henderson g_assert_not_reached();
56178429a1caSRichard Henderson }
56188429a1caSRichard Henderson }
56198429a1caSRichard Henderson
tcg_out_helper_load_imm(TCGContext * s,unsigned slot,TCGType type,tcg_target_long imm,const TCGLdstHelperParam * parm)56208429a1caSRichard Henderson static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot,
56218429a1caSRichard Henderson TCGType type, tcg_target_long imm,
56228429a1caSRichard Henderson const TCGLdstHelperParam *parm)
56238429a1caSRichard Henderson {
56248429a1caSRichard Henderson if (arg_slot_reg_p(slot)) {
56258429a1caSRichard Henderson tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm);
56268429a1caSRichard Henderson } else {
56278429a1caSRichard Henderson int ofs = tcg_out_helper_stk_ofs(type, slot);
56288429a1caSRichard Henderson if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) {
56298429a1caSRichard Henderson tcg_debug_assert(parm->ntmp != 0);
56308429a1caSRichard Henderson tcg_out_movi(s, type, parm->tmp[0], imm);
56318429a1caSRichard Henderson tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs);
56328429a1caSRichard Henderson }
56338429a1caSRichard Henderson }
56348429a1caSRichard Henderson }
56358429a1caSRichard Henderson
tcg_out_helper_load_common_args(TCGContext * s,const TCGLabelQemuLdst * ldst,const TCGLdstHelperParam * parm,const TCGHelperInfo * info,unsigned next_arg)56368429a1caSRichard Henderson static void tcg_out_helper_load_common_args(TCGContext *s,
56378429a1caSRichard Henderson const TCGLabelQemuLdst *ldst,
56388429a1caSRichard Henderson const TCGLdstHelperParam *parm,
56398429a1caSRichard Henderson const TCGHelperInfo *info,
56408429a1caSRichard Henderson unsigned next_arg)
56418429a1caSRichard Henderson {
56428429a1caSRichard Henderson TCGMovExtend ptr_mov = {
56438429a1caSRichard Henderson .dst_type = TCG_TYPE_PTR,
56448429a1caSRichard Henderson .src_type = TCG_TYPE_PTR,
56458429a1caSRichard Henderson .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64
56468429a1caSRichard Henderson };
56478429a1caSRichard Henderson const TCGCallArgumentLoc *loc = &info->in[0];
56488429a1caSRichard Henderson TCGType type;
56498429a1caSRichard Henderson unsigned slot;
56508429a1caSRichard Henderson tcg_target_ulong imm;
56518429a1caSRichard Henderson
56528429a1caSRichard Henderson /*
56538429a1caSRichard Henderson * Handle env, which is always first.
56548429a1caSRichard Henderson */
56558429a1caSRichard Henderson ptr_mov.dst = loc->arg_slot;
56568429a1caSRichard Henderson ptr_mov.src = TCG_AREG0;
56578429a1caSRichard Henderson tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
56588429a1caSRichard Henderson
56598429a1caSRichard Henderson /*
56608429a1caSRichard Henderson * Handle oi.
56618429a1caSRichard Henderson */
56628429a1caSRichard Henderson imm = ldst->oi;
56638429a1caSRichard Henderson loc = &info->in[next_arg];
56648429a1caSRichard Henderson type = TCG_TYPE_I32;
56658429a1caSRichard Henderson switch (loc->kind) {
56668429a1caSRichard Henderson case TCG_CALL_ARG_NORMAL:
56678429a1caSRichard Henderson break;
56688429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_U:
56698429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_S:
56708429a1caSRichard Henderson /* No extension required for MemOpIdx. */
56718429a1caSRichard Henderson tcg_debug_assert(imm <= INT32_MAX);
56728429a1caSRichard Henderson type = TCG_TYPE_REG;
56738429a1caSRichard Henderson break;
56748429a1caSRichard Henderson default:
56758429a1caSRichard Henderson g_assert_not_reached();
56768429a1caSRichard Henderson }
56778429a1caSRichard Henderson tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm);
56788429a1caSRichard Henderson next_arg++;
56798429a1caSRichard Henderson
56808429a1caSRichard Henderson /*
56818429a1caSRichard Henderson * Handle ra.
56828429a1caSRichard Henderson */
56838429a1caSRichard Henderson loc = &info->in[next_arg];
56848429a1caSRichard Henderson slot = loc->arg_slot;
56858429a1caSRichard Henderson if (parm->ra_gen) {
56868429a1caSRichard Henderson int arg_reg = -1;
56878429a1caSRichard Henderson TCGReg ra_reg;
56888429a1caSRichard Henderson
56898429a1caSRichard Henderson if (arg_slot_reg_p(slot)) {
56908429a1caSRichard Henderson arg_reg = tcg_target_call_iarg_regs[slot];
56918429a1caSRichard Henderson }
56928429a1caSRichard Henderson ra_reg = parm->ra_gen(s, ldst, arg_reg);
56938429a1caSRichard Henderson
56948429a1caSRichard Henderson ptr_mov.dst = slot;
56958429a1caSRichard Henderson ptr_mov.src = ra_reg;
56968429a1caSRichard Henderson tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
56978429a1caSRichard Henderson } else {
56988429a1caSRichard Henderson imm = (uintptr_t)ldst->raddr;
56998429a1caSRichard Henderson tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm);
57008429a1caSRichard Henderson }
57018429a1caSRichard Henderson }
57028429a1caSRichard Henderson
tcg_out_helper_add_mov(TCGMovExtend * mov,const TCGCallArgumentLoc * loc,TCGType dst_type,TCGType src_type,TCGReg lo,TCGReg hi)57038429a1caSRichard Henderson static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov,
57048429a1caSRichard Henderson const TCGCallArgumentLoc *loc,
57058429a1caSRichard Henderson TCGType dst_type, TCGType src_type,
57068429a1caSRichard Henderson TCGReg lo, TCGReg hi)
57078429a1caSRichard Henderson {
5708ebebea53SRichard Henderson MemOp reg_mo;
5709ebebea53SRichard Henderson
57108429a1caSRichard Henderson if (dst_type <= TCG_TYPE_REG) {
57118429a1caSRichard Henderson MemOp src_ext;
57128429a1caSRichard Henderson
57138429a1caSRichard Henderson switch (loc->kind) {
57148429a1caSRichard Henderson case TCG_CALL_ARG_NORMAL:
57158429a1caSRichard Henderson src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64;
57168429a1caSRichard Henderson break;
57178429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_U:
57188429a1caSRichard Henderson dst_type = TCG_TYPE_REG;
57198429a1caSRichard Henderson src_ext = MO_UL;
57208429a1caSRichard Henderson break;
57218429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_S:
57228429a1caSRichard Henderson dst_type = TCG_TYPE_REG;
57238429a1caSRichard Henderson src_ext = MO_SL;
57248429a1caSRichard Henderson break;
57258429a1caSRichard Henderson default:
57268429a1caSRichard Henderson g_assert_not_reached();
57278429a1caSRichard Henderson }
57288429a1caSRichard Henderson
57298429a1caSRichard Henderson mov[0].dst = loc->arg_slot;
57308429a1caSRichard Henderson mov[0].dst_type = dst_type;
57318429a1caSRichard Henderson mov[0].src = lo;
57328429a1caSRichard Henderson mov[0].src_type = src_type;
57338429a1caSRichard Henderson mov[0].src_ext = src_ext;
57348429a1caSRichard Henderson return 1;
57358429a1caSRichard Henderson }
57368429a1caSRichard Henderson
5737ebebea53SRichard Henderson if (TCG_TARGET_REG_BITS == 32) {
5738ebebea53SRichard Henderson assert(dst_type == TCG_TYPE_I64);
5739ebebea53SRichard Henderson reg_mo = MO_32;
5740ebebea53SRichard Henderson } else {
5741ebebea53SRichard Henderson assert(dst_type == TCG_TYPE_I128);
5742ebebea53SRichard Henderson reg_mo = MO_64;
5743ebebea53SRichard Henderson }
57448429a1caSRichard Henderson
57458429a1caSRichard Henderson mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot;
57468429a1caSRichard Henderson mov[0].src = lo;
5747ebebea53SRichard Henderson mov[0].dst_type = TCG_TYPE_REG;
5748ebebea53SRichard Henderson mov[0].src_type = TCG_TYPE_REG;
5749ebebea53SRichard Henderson mov[0].src_ext = reg_mo;
57508429a1caSRichard Henderson
57518429a1caSRichard Henderson mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot;
57528429a1caSRichard Henderson mov[1].src = hi;
5753ebebea53SRichard Henderson mov[1].dst_type = TCG_TYPE_REG;
5754ebebea53SRichard Henderson mov[1].src_type = TCG_TYPE_REG;
5755ebebea53SRichard Henderson mov[1].src_ext = reg_mo;
57568429a1caSRichard Henderson
57578429a1caSRichard Henderson return 2;
57588429a1caSRichard Henderson }
57598429a1caSRichard Henderson
tcg_out_ld_helper_args(TCGContext * s,const TCGLabelQemuLdst * ldst,const TCGLdstHelperParam * parm)57608429a1caSRichard Henderson static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
57618429a1caSRichard Henderson const TCGLdstHelperParam *parm)
57628429a1caSRichard Henderson {
57638429a1caSRichard Henderson const TCGHelperInfo *info;
57648429a1caSRichard Henderson const TCGCallArgumentLoc *loc;
57658429a1caSRichard Henderson TCGMovExtend mov[2];
57668429a1caSRichard Henderson unsigned next_arg, nmov;
57678429a1caSRichard Henderson MemOp mop = get_memop(ldst->oi);
57688429a1caSRichard Henderson
57698429a1caSRichard Henderson switch (mop & MO_SIZE) {
57708429a1caSRichard Henderson case MO_8:
57718429a1caSRichard Henderson case MO_16:
57728429a1caSRichard Henderson case MO_32:
57738429a1caSRichard Henderson info = &info_helper_ld32_mmu;
57748429a1caSRichard Henderson break;
57758429a1caSRichard Henderson case MO_64:
57768429a1caSRichard Henderson info = &info_helper_ld64_mmu;
57778429a1caSRichard Henderson break;
5778ebebea53SRichard Henderson case MO_128:
5779ebebea53SRichard Henderson info = &info_helper_ld128_mmu;
5780ebebea53SRichard Henderson break;
57818429a1caSRichard Henderson default:
57828429a1caSRichard Henderson g_assert_not_reached();
57838429a1caSRichard Henderson }
57848429a1caSRichard Henderson
57858429a1caSRichard Henderson /* Defer env argument. */
57868429a1caSRichard Henderson next_arg = 1;
57878429a1caSRichard Henderson
57888429a1caSRichard Henderson loc = &info->in[next_arg];
5789c31e5fa4SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
579024e46e6cSRichard Henderson /*
579124e46e6cSRichard Henderson * 32-bit host with 32-bit guest: zero-extend the guest address
579224e46e6cSRichard Henderson * to 64-bits for the helper by storing the low part, then
579324e46e6cSRichard Henderson * load a zero for the high part.
579424e46e6cSRichard Henderson */
579524e46e6cSRichard Henderson tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
579624e46e6cSRichard Henderson TCG_TYPE_I32, TCG_TYPE_I32,
579724e46e6cSRichard Henderson ldst->addrlo_reg, -1);
579824e46e6cSRichard Henderson tcg_out_helper_load_slots(s, 1, mov, parm);
579924e46e6cSRichard Henderson
580024e46e6cSRichard Henderson tcg_out_helper_load_imm(s, loc[!HOST_BIG_ENDIAN].arg_slot,
580124e46e6cSRichard Henderson TCG_TYPE_I32, 0, parm);
580224e46e6cSRichard Henderson next_arg += 2;
5803c31e5fa4SRichard Henderson } else {
5804c31e5fa4SRichard Henderson nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
5805c31e5fa4SRichard Henderson ldst->addrlo_reg, ldst->addrhi_reg);
5806c31e5fa4SRichard Henderson tcg_out_helper_load_slots(s, nmov, mov, parm);
5807c31e5fa4SRichard Henderson next_arg += nmov;
580824e46e6cSRichard Henderson }
58098429a1caSRichard Henderson
5810ebebea53SRichard Henderson switch (info->out_kind) {
5811ebebea53SRichard Henderson case TCG_CALL_RET_NORMAL:
5812ebebea53SRichard Henderson case TCG_CALL_RET_BY_VEC:
5813ebebea53SRichard Henderson break;
5814ebebea53SRichard Henderson case TCG_CALL_RET_BY_REF:
5815ebebea53SRichard Henderson /*
5816ebebea53SRichard Henderson * The return reference is in the first argument slot.
5817ebebea53SRichard Henderson * We need memory in which to return: re-use the top of stack.
5818ebebea53SRichard Henderson */
5819ebebea53SRichard Henderson {
5820ebebea53SRichard Henderson int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
5821ebebea53SRichard Henderson
5822ebebea53SRichard Henderson if (arg_slot_reg_p(0)) {
5823ebebea53SRichard Henderson tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0],
5824ebebea53SRichard Henderson TCG_REG_CALL_STACK, ofs_slot0);
5825ebebea53SRichard Henderson } else {
5826ebebea53SRichard Henderson tcg_debug_assert(parm->ntmp != 0);
5827ebebea53SRichard Henderson tcg_out_addi_ptr(s, parm->tmp[0],
5828ebebea53SRichard Henderson TCG_REG_CALL_STACK, ofs_slot0);
5829ebebea53SRichard Henderson tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
5830ebebea53SRichard Henderson TCG_REG_CALL_STACK, ofs_slot0);
5831ebebea53SRichard Henderson }
5832ebebea53SRichard Henderson }
5833ebebea53SRichard Henderson break;
5834ebebea53SRichard Henderson default:
5835ebebea53SRichard Henderson g_assert_not_reached();
5836ebebea53SRichard Henderson }
58378429a1caSRichard Henderson
58388429a1caSRichard Henderson tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
58398429a1caSRichard Henderson }
58408429a1caSRichard Henderson
tcg_out_ld_helper_ret(TCGContext * s,const TCGLabelQemuLdst * ldst,bool load_sign,const TCGLdstHelperParam * parm)58418429a1caSRichard Henderson static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst,
58428429a1caSRichard Henderson bool load_sign,
58438429a1caSRichard Henderson const TCGLdstHelperParam *parm)
58448429a1caSRichard Henderson {
58458429a1caSRichard Henderson MemOp mop = get_memop(ldst->oi);
5846ebebea53SRichard Henderson TCGMovExtend mov[2];
5847ebebea53SRichard Henderson int ofs_slot0;
58488429a1caSRichard Henderson
5849ebebea53SRichard Henderson switch (ldst->type) {
5850ebebea53SRichard Henderson case TCG_TYPE_I64:
5851ebebea53SRichard Henderson if (TCG_TARGET_REG_BITS == 32) {
5852ebebea53SRichard Henderson break;
5853ebebea53SRichard Henderson }
5854ebebea53SRichard Henderson /* fall through */
5855ebebea53SRichard Henderson
5856ebebea53SRichard Henderson case TCG_TYPE_I32:
58578429a1caSRichard Henderson mov[0].dst = ldst->datalo_reg;
58588429a1caSRichard Henderson mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0);
58598429a1caSRichard Henderson mov[0].dst_type = ldst->type;
58608429a1caSRichard Henderson mov[0].src_type = TCG_TYPE_REG;
58618429a1caSRichard Henderson
58628429a1caSRichard Henderson /*
58638429a1caSRichard Henderson * If load_sign, then we allowed the helper to perform the
58648429a1caSRichard Henderson * appropriate sign extension to tcg_target_ulong, and all
58658429a1caSRichard Henderson * we need now is a plain move.
58668429a1caSRichard Henderson *
58678429a1caSRichard Henderson * If they do not, then we expect the relevant extension
58688429a1caSRichard Henderson * instruction to be no more expensive than a move, and
58698429a1caSRichard Henderson * we thus save the icache etc by only using one of two
58708429a1caSRichard Henderson * helper functions.
58718429a1caSRichard Henderson */
58728429a1caSRichard Henderson if (load_sign || !(mop & MO_SIGN)) {
58738429a1caSRichard Henderson if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) {
58748429a1caSRichard Henderson mov[0].src_ext = MO_32;
58758429a1caSRichard Henderson } else {
58768429a1caSRichard Henderson mov[0].src_ext = MO_64;
58778429a1caSRichard Henderson }
58788429a1caSRichard Henderson } else {
58798429a1caSRichard Henderson mov[0].src_ext = mop & MO_SSIZE;
58808429a1caSRichard Henderson }
58818429a1caSRichard Henderson tcg_out_movext1(s, mov);
5882ebebea53SRichard Henderson return;
5883ebebea53SRichard Henderson
5884ebebea53SRichard Henderson case TCG_TYPE_I128:
5885ebebea53SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
5886ebebea53SRichard Henderson ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
5887ebebea53SRichard Henderson switch (TCG_TARGET_CALL_RET_I128) {
5888ebebea53SRichard Henderson case TCG_CALL_RET_NORMAL:
5889ebebea53SRichard Henderson break;
5890ebebea53SRichard Henderson case TCG_CALL_RET_BY_VEC:
5891ebebea53SRichard Henderson tcg_out_st(s, TCG_TYPE_V128,
5892ebebea53SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
5893ebebea53SRichard Henderson TCG_REG_CALL_STACK, ofs_slot0);
5894ebebea53SRichard Henderson /* fall through */
5895ebebea53SRichard Henderson case TCG_CALL_RET_BY_REF:
5896ebebea53SRichard Henderson tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg,
5897ebebea53SRichard Henderson TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN);
5898ebebea53SRichard Henderson tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg,
5899ebebea53SRichard Henderson TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN);
5900ebebea53SRichard Henderson return;
5901ebebea53SRichard Henderson default:
5902ebebea53SRichard Henderson g_assert_not_reached();
5903ebebea53SRichard Henderson }
5904ebebea53SRichard Henderson break;
5905ebebea53SRichard Henderson
5906ebebea53SRichard Henderson default:
5907ebebea53SRichard Henderson g_assert_not_reached();
5908ebebea53SRichard Henderson }
59098429a1caSRichard Henderson
59108429a1caSRichard Henderson mov[0].dst = ldst->datalo_reg;
59118429a1caSRichard Henderson mov[0].src =
59128429a1caSRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN);
5913723d3a27SRichard Henderson mov[0].dst_type = TCG_TYPE_REG;
5914723d3a27SRichard Henderson mov[0].src_type = TCG_TYPE_REG;
5915ebebea53SRichard Henderson mov[0].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
59168429a1caSRichard Henderson
59178429a1caSRichard Henderson mov[1].dst = ldst->datahi_reg;
59188429a1caSRichard Henderson mov[1].src =
59198429a1caSRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN);
59208429a1caSRichard Henderson mov[1].dst_type = TCG_TYPE_REG;
59218429a1caSRichard Henderson mov[1].src_type = TCG_TYPE_REG;
5922ebebea53SRichard Henderson mov[1].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
59238429a1caSRichard Henderson
59248429a1caSRichard Henderson tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1);
59258429a1caSRichard Henderson }
59268429a1caSRichard Henderson
tcg_out_st_helper_args(TCGContext * s,const TCGLabelQemuLdst * ldst,const TCGLdstHelperParam * parm)59278429a1caSRichard Henderson static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
59288429a1caSRichard Henderson const TCGLdstHelperParam *parm)
59298429a1caSRichard Henderson {
59308429a1caSRichard Henderson const TCGHelperInfo *info;
59318429a1caSRichard Henderson const TCGCallArgumentLoc *loc;
59328429a1caSRichard Henderson TCGMovExtend mov[4];
59338429a1caSRichard Henderson TCGType data_type;
59348429a1caSRichard Henderson unsigned next_arg, nmov, n;
59358429a1caSRichard Henderson MemOp mop = get_memop(ldst->oi);
59368429a1caSRichard Henderson
59378429a1caSRichard Henderson switch (mop & MO_SIZE) {
59388429a1caSRichard Henderson case MO_8:
59398429a1caSRichard Henderson case MO_16:
59408429a1caSRichard Henderson case MO_32:
59418429a1caSRichard Henderson info = &info_helper_st32_mmu;
59428429a1caSRichard Henderson data_type = TCG_TYPE_I32;
59438429a1caSRichard Henderson break;
59448429a1caSRichard Henderson case MO_64:
59458429a1caSRichard Henderson info = &info_helper_st64_mmu;
59468429a1caSRichard Henderson data_type = TCG_TYPE_I64;
59478429a1caSRichard Henderson break;
5948ebebea53SRichard Henderson case MO_128:
5949ebebea53SRichard Henderson info = &info_helper_st128_mmu;
5950ebebea53SRichard Henderson data_type = TCG_TYPE_I128;
5951ebebea53SRichard Henderson break;
59528429a1caSRichard Henderson default:
59538429a1caSRichard Henderson g_assert_not_reached();
59548429a1caSRichard Henderson }
59558429a1caSRichard Henderson
59568429a1caSRichard Henderson /* Defer env argument. */
59578429a1caSRichard Henderson next_arg = 1;
59588429a1caSRichard Henderson nmov = 0;
59598429a1caSRichard Henderson
59608429a1caSRichard Henderson /* Handle addr argument. */
59618429a1caSRichard Henderson loc = &info->in[next_arg];
5962c31e5fa4SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
596324e46e6cSRichard Henderson /*
596424e46e6cSRichard Henderson * 32-bit host with 32-bit guest: zero-extend the guest address
596524e46e6cSRichard Henderson * to 64-bits for the helper by storing the low part. Later,
596624e46e6cSRichard Henderson * after we have processed the register inputs, we will load a
596724e46e6cSRichard Henderson * zero for the high part.
596824e46e6cSRichard Henderson */
596924e46e6cSRichard Henderson tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
597024e46e6cSRichard Henderson TCG_TYPE_I32, TCG_TYPE_I32,
597124e46e6cSRichard Henderson ldst->addrlo_reg, -1);
597224e46e6cSRichard Henderson next_arg += 2;
597324e46e6cSRichard Henderson nmov += 1;
5974c31e5fa4SRichard Henderson } else {
5975c31e5fa4SRichard Henderson n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
5976c31e5fa4SRichard Henderson ldst->addrlo_reg, ldst->addrhi_reg);
5977c31e5fa4SRichard Henderson next_arg += n;
5978c31e5fa4SRichard Henderson nmov += n;
597924e46e6cSRichard Henderson }
59808429a1caSRichard Henderson
59818429a1caSRichard Henderson /* Handle data argument. */
59828429a1caSRichard Henderson loc = &info->in[next_arg];
5983ebebea53SRichard Henderson switch (loc->kind) {
5984ebebea53SRichard Henderson case TCG_CALL_ARG_NORMAL:
5985ebebea53SRichard Henderson case TCG_CALL_ARG_EXTEND_U:
5986ebebea53SRichard Henderson case TCG_CALL_ARG_EXTEND_S:
59878429a1caSRichard Henderson n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type,
59888429a1caSRichard Henderson ldst->datalo_reg, ldst->datahi_reg);
59898429a1caSRichard Henderson next_arg += n;
59908429a1caSRichard Henderson nmov += n;
5991ebebea53SRichard Henderson tcg_out_helper_load_slots(s, nmov, mov, parm);
5992ebebea53SRichard Henderson break;
5993ebebea53SRichard Henderson
5994ebebea53SRichard Henderson case TCG_CALL_ARG_BY_REF:
5995ebebea53SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
5996ebebea53SRichard Henderson tcg_debug_assert(data_type == TCG_TYPE_I128);
5997ebebea53SRichard Henderson tcg_out_st(s, TCG_TYPE_I64,
5998ebebea53SRichard Henderson HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg,
5999ebebea53SRichard Henderson TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot));
6000ebebea53SRichard Henderson tcg_out_st(s, TCG_TYPE_I64,
6001ebebea53SRichard Henderson HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg,
6002ebebea53SRichard Henderson TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot));
60038429a1caSRichard Henderson
60048429a1caSRichard Henderson tcg_out_helper_load_slots(s, nmov, mov, parm);
6005ebebea53SRichard Henderson
6006ebebea53SRichard Henderson if (arg_slot_reg_p(loc->arg_slot)) {
6007ebebea53SRichard Henderson tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot],
6008ebebea53SRichard Henderson TCG_REG_CALL_STACK,
6009ebebea53SRichard Henderson arg_slot_stk_ofs(loc->ref_slot));
6010ebebea53SRichard Henderson } else {
6011ebebea53SRichard Henderson tcg_debug_assert(parm->ntmp != 0);
6012ebebea53SRichard Henderson tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK,
6013ebebea53SRichard Henderson arg_slot_stk_ofs(loc->ref_slot));
6014ebebea53SRichard Henderson tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6015ebebea53SRichard Henderson TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot));
6016ebebea53SRichard Henderson }
6017ebebea53SRichard Henderson next_arg += 2;
6018ebebea53SRichard Henderson break;
6019ebebea53SRichard Henderson
6020ebebea53SRichard Henderson default:
6021ebebea53SRichard Henderson g_assert_not_reached();
6022ebebea53SRichard Henderson }
6023ebebea53SRichard Henderson
6024c31e5fa4SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
6025c31e5fa4SRichard Henderson /* Zero extend the address by loading a zero for the high part. */
602624e46e6cSRichard Henderson loc = &info->in[1 + !HOST_BIG_ENDIAN];
602724e46e6cSRichard Henderson tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm);
602824e46e6cSRichard Henderson }
602924e46e6cSRichard Henderson
60308429a1caSRichard Henderson tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
60318429a1caSRichard Henderson }
60328429a1caSRichard Henderson
tcg_gen_code(TCGContext * s,TranslationBlock * tb,uint64_t pc_start)603376cef4b2SRichard Henderson int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
6034c896fe29Sbellard {
6035747bd69dSRichard Henderson int i, start_words, num_insns;
603615fa08f8SRichard Henderson TCGOp *op;
6037c896fe29Sbellard
6038d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
6039fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) {
6040c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock();
604178b54858SRichard Henderson if (logfile) {
604278b54858SRichard Henderson fprintf(logfile, "OP:\n");
6043b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false);
604478b54858SRichard Henderson fprintf(logfile, "\n");
6045fc59d2d8SRobert Foley qemu_log_unlock(logfile);
6046c896fe29Sbellard }
604778b54858SRichard Henderson }
6048c896fe29Sbellard
6049bef16ab4SRichard Henderson #ifdef CONFIG_DEBUG_TCG
6050bef16ab4SRichard Henderson /* Ensure all labels referenced have been emitted. */
6051bef16ab4SRichard Henderson {
6052bef16ab4SRichard Henderson TCGLabel *l;
6053bef16ab4SRichard Henderson bool error = false;
6054bef16ab4SRichard Henderson
6055bef16ab4SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) {
6056f85b1fc4SRichard Henderson if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
6057bef16ab4SRichard Henderson qemu_log_mask(CPU_LOG_TB_OP,
6058bef16ab4SRichard Henderson "$L%d referenced but not present.\n", l->id);
6059bef16ab4SRichard Henderson error = true;
6060bef16ab4SRichard Henderson }
6061bef16ab4SRichard Henderson }
6062bef16ab4SRichard Henderson assert(!error);
6063bef16ab4SRichard Henderson }
6064bef16ab4SRichard Henderson #endif
6065bef16ab4SRichard Henderson
6066c45cb8bbSRichard Henderson tcg_optimize(s);
60678f2e8c07SKirill Batuzov
6068b4fc67c7SRichard Henderson reachable_code_pass(s);
6069874b8574SRichard Henderson liveness_pass_0(s);
6070b83eabeaSRichard Henderson liveness_pass_1(s);
60715a18407fSRichard Henderson
60725a18407fSRichard Henderson if (s->nb_indirects > 0) {
60735a18407fSRichard Henderson if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
6074fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) {
6075c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock();
607678b54858SRichard Henderson if (logfile) {
607778b54858SRichard Henderson fprintf(logfile, "OP before indirect lowering:\n");
6078b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false);
607978b54858SRichard Henderson fprintf(logfile, "\n");
6080fc59d2d8SRobert Foley qemu_log_unlock(logfile);
60815a18407fSRichard Henderson }
608278b54858SRichard Henderson }
6083645e3a81SRichard Henderson
60845a18407fSRichard Henderson /* Replace indirect temps with direct temps. */
6085b83eabeaSRichard Henderson if (liveness_pass_2(s)) {
60865a18407fSRichard Henderson /* If changes were made, re-run liveness. */
6087b83eabeaSRichard Henderson liveness_pass_1(s);
60885a18407fSRichard Henderson }
60895a18407fSRichard Henderson }
6090c5cc28ffSAurelien Jarno
6091d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
6092fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) {
6093c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock();
609478b54858SRichard Henderson if (logfile) {
609578b54858SRichard Henderson fprintf(logfile, "OP after optimization and liveness analysis:\n");
6096b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, true);
609778b54858SRichard Henderson fprintf(logfile, "\n");
6098fc59d2d8SRobert Foley qemu_log_unlock(logfile);
6099c896fe29Sbellard }
610078b54858SRichard Henderson }
6101c896fe29Sbellard
610235abb009SRichard Henderson /* Initialize goto_tb jump offsets. */
61033a50f424SRichard Henderson tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
61043a50f424SRichard Henderson tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
61059da6079bSRichard Henderson tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
61069da6079bSRichard Henderson tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
610735abb009SRichard Henderson
6108c896fe29Sbellard tcg_reg_alloc_start(s);
6109c896fe29Sbellard
6110db0c51a3SRichard Henderson /*
6111db0c51a3SRichard Henderson * Reset the buffer pointers when restarting after overflow.
6112db0c51a3SRichard Henderson * TODO: Move this into translate-all.c with the rest of the
6113db0c51a3SRichard Henderson * buffer management. Having only this done here is confusing.
6114db0c51a3SRichard Henderson */
6115db0c51a3SRichard Henderson s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
6116db0c51a3SRichard Henderson s->code_ptr = s->code_buf;
6117c896fe29Sbellard
6118659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS
61196001f772SLaurent Vivier QSIMPLEQ_INIT(&s->ldst_labels);
6120659ef5cbSRichard Henderson #endif
612157a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS
612257a26946SRichard Henderson s->pool_labels = NULL;
612357a26946SRichard Henderson #endif
61249ecefc84SRichard Henderson
6125747bd69dSRichard Henderson start_words = s->insn_start_words;
6126747bd69dSRichard Henderson s->gen_insn_data =
6127747bd69dSRichard Henderson tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words);
6128747bd69dSRichard Henderson
61299358fbbfSRichard Henderson tcg_out_tb_start(s);
61309358fbbfSRichard Henderson
6131fca8a500SRichard Henderson num_insns = -1;
613215fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) {
6133c45cb8bbSRichard Henderson TCGOpcode opc = op->opc;
6134b3db8758Sblueswir1
6135c896fe29Sbellard switch (opc) {
6136c896fe29Sbellard case INDEX_op_mov_i32:
6137c896fe29Sbellard case INDEX_op_mov_i64:
6138d2fd745fSRichard Henderson case INDEX_op_mov_vec:
6139dd186292SRichard Henderson tcg_reg_alloc_mov(s, op);
6140c896fe29Sbellard break;
6141bab1671fSRichard Henderson case INDEX_op_dup_vec:
6142bab1671fSRichard Henderson tcg_reg_alloc_dup(s, op);
6143bab1671fSRichard Henderson break;
6144765b842aSRichard Henderson case INDEX_op_insn_start:
6145fca8a500SRichard Henderson if (num_insns >= 0) {
61469f754620SRichard Henderson size_t off = tcg_current_code_size(s);
61479f754620SRichard Henderson s->gen_insn_end_off[num_insns] = off;
61489f754620SRichard Henderson /* Assert that we do not overflow our stored offset. */
61499f754620SRichard Henderson assert(s->gen_insn_end_off[num_insns] == off);
6150fca8a500SRichard Henderson }
6151fca8a500SRichard Henderson num_insns++;
6152747bd69dSRichard Henderson for (i = 0; i < start_words; ++i) {
6153747bd69dSRichard Henderson s->gen_insn_data[num_insns * start_words + i] =
6154c9ad8d27SRichard Henderson tcg_get_insn_start_param(op, i);
6155bad729e2SRichard Henderson }
6156c896fe29Sbellard break;
61575ff9d6a4Sbellard case INDEX_op_discard:
615843439139SRichard Henderson temp_dead(s, arg_temp(op->args[0]));
61595ff9d6a4Sbellard break;
6160c896fe29Sbellard case INDEX_op_set_label:
6161e8996ee0Sbellard tcg_reg_alloc_bb_end(s, s->reserved_regs);
616292ab8e7dSRichard Henderson tcg_out_label(s, arg_label(op->args[0]));
6163c896fe29Sbellard break;
6164c896fe29Sbellard case INDEX_op_call:
6165dd186292SRichard Henderson tcg_reg_alloc_call(s, op);
6166c45cb8bbSRichard Henderson break;
6167b55a8d9dSRichard Henderson case INDEX_op_exit_tb:
6168b55a8d9dSRichard Henderson tcg_out_exit_tb(s, op->args[0]);
6169b55a8d9dSRichard Henderson break;
6170cf7d6b8eSRichard Henderson case INDEX_op_goto_tb:
6171cf7d6b8eSRichard Henderson tcg_out_goto_tb(s, op->args[0]);
6172cf7d6b8eSRichard Henderson break;
6173efe86b21SRichard Henderson case INDEX_op_dup2_vec:
6174efe86b21SRichard Henderson if (tcg_reg_alloc_dup2(s, op)) {
6175efe86b21SRichard Henderson break;
6176efe86b21SRichard Henderson }
6177efe86b21SRichard Henderson /* fall through */
6178c896fe29Sbellard default:
617925c4d9ccSRichard Henderson /* Sanity check that we've not introduced any unhandled opcodes. */
6180be0f34b5SRichard Henderson tcg_debug_assert(tcg_op_supported(opc));
6181c896fe29Sbellard /* Note: in order to speed up the code, it would be much
6182c896fe29Sbellard faster to have specialized register allocator functions for
6183c896fe29Sbellard some common argument patterns */
6184dd186292SRichard Henderson tcg_reg_alloc_op(s, op);
6185c896fe29Sbellard break;
6186c896fe29Sbellard }
6187b125f9dcSRichard Henderson /* Test for (pending) buffer overflow. The assumption is that any
6188b125f9dcSRichard Henderson one operation beginning below the high water mark cannot overrun
6189b125f9dcSRichard Henderson the buffer completely. Thus we can test for overflow after
6190b125f9dcSRichard Henderson generating code without having to check during generation. */
6191644da9b3SJohn Clarke if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
6192b125f9dcSRichard Henderson return -1;
6193b125f9dcSRichard Henderson }
61946e6c4efeSRichard Henderson /* Test for TB overflow, as seen by gen_insn_end_off. */
61956e6c4efeSRichard Henderson if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
61966e6c4efeSRichard Henderson return -2;
61976e6c4efeSRichard Henderson }
6198c896fe29Sbellard }
6199747bd69dSRichard Henderson tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
6200fca8a500SRichard Henderson s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
6201c45cb8bbSRichard Henderson
6202b76f0d8cSYeongkyoon Lee /* Generate TB finalization at the end of block */
6203659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS
6204aeee05f5SRichard Henderson i = tcg_out_ldst_finalize(s);
6205aeee05f5SRichard Henderson if (i < 0) {
6206aeee05f5SRichard Henderson return i;
620723dceda6SRichard Henderson }
6208659ef5cbSRichard Henderson #endif
620957a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS
62101768987bSRichard Henderson i = tcg_out_pool_finalize(s);
62111768987bSRichard Henderson if (i < 0) {
62121768987bSRichard Henderson return i;
621357a26946SRichard Henderson }
621457a26946SRichard Henderson #endif
62157ecd02a0SRichard Henderson if (!tcg_resolve_relocs(s)) {
62167ecd02a0SRichard Henderson return -2;
62177ecd02a0SRichard Henderson }
6218c896fe29Sbellard
6219df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER
6220c896fe29Sbellard /* flush instruction cache */
6221db0c51a3SRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
6222db0c51a3SRichard Henderson (uintptr_t)s->code_buf,
62231da8de39SRichard Henderson tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
6224df5d2b16SRichard Henderson #endif
62252aeabc08SStefan Weil
62261813e175SRichard Henderson return tcg_current_code_size(s);
6227c896fe29Sbellard }
6228c896fe29Sbellard
6229813da627SRichard Henderson #ifdef ELF_HOST_MACHINE
62305872bbf2SRichard Henderson /* In order to use this feature, the backend needs to do three things:
62315872bbf2SRichard Henderson
62325872bbf2SRichard Henderson (1) Define ELF_HOST_MACHINE to indicate both what value to
62335872bbf2SRichard Henderson put into the ELF image and to indicate support for the feature.
62345872bbf2SRichard Henderson
62355872bbf2SRichard Henderson (2) Define tcg_register_jit. This should create a buffer containing
62365872bbf2SRichard Henderson the contents of a .debug_frame section that describes the post-
62375872bbf2SRichard Henderson prologue unwind info for the tcg machine.
62385872bbf2SRichard Henderson
62395872bbf2SRichard Henderson (3) Call tcg_register_jit_int, with the constructed .debug_frame.
62405872bbf2SRichard Henderson */
6241813da627SRichard Henderson
6242813da627SRichard Henderson /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
6243813da627SRichard Henderson typedef enum {
6244813da627SRichard Henderson JIT_NOACTION = 0,
6245813da627SRichard Henderson JIT_REGISTER_FN,
6246813da627SRichard Henderson JIT_UNREGISTER_FN
6247813da627SRichard Henderson } jit_actions_t;
6248813da627SRichard Henderson
6249813da627SRichard Henderson struct jit_code_entry {
6250813da627SRichard Henderson struct jit_code_entry *next_entry;
6251813da627SRichard Henderson struct jit_code_entry *prev_entry;
6252813da627SRichard Henderson const void *symfile_addr;
6253813da627SRichard Henderson uint64_t symfile_size;
6254813da627SRichard Henderson };
6255813da627SRichard Henderson
6256813da627SRichard Henderson struct jit_descriptor {
6257813da627SRichard Henderson uint32_t version;
6258813da627SRichard Henderson uint32_t action_flag;
6259813da627SRichard Henderson struct jit_code_entry *relevant_entry;
6260813da627SRichard Henderson struct jit_code_entry *first_entry;
6261813da627SRichard Henderson };
6262813da627SRichard Henderson
6263813da627SRichard Henderson void __jit_debug_register_code(void) __attribute__((noinline));
__jit_debug_register_code(void)6264813da627SRichard Henderson void __jit_debug_register_code(void)
6265813da627SRichard Henderson {
6266813da627SRichard Henderson asm("");
6267813da627SRichard Henderson }
6268813da627SRichard Henderson
6269813da627SRichard Henderson /* Must statically initialize the version, because GDB may check
6270813da627SRichard Henderson the version before we can set it. */
6271813da627SRichard Henderson struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
6272813da627SRichard Henderson
6273813da627SRichard Henderson /* End GDB interface. */
6274813da627SRichard Henderson
find_string(const char * strtab,const char * str)6275813da627SRichard Henderson static int find_string(const char *strtab, const char *str)
6276813da627SRichard Henderson {
6277813da627SRichard Henderson const char *p = strtab + 1;
6278813da627SRichard Henderson
6279813da627SRichard Henderson while (1) {
6280813da627SRichard Henderson if (strcmp(p, str) == 0) {
6281813da627SRichard Henderson return p - strtab;
6282813da627SRichard Henderson }
6283813da627SRichard Henderson p += strlen(p) + 1;
6284813da627SRichard Henderson }
6285813da627SRichard Henderson }
6286813da627SRichard Henderson
tcg_register_jit_int(const void * buf_ptr,size_t buf_size,const void * debug_frame,size_t debug_frame_size)6287755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
62882c90784aSRichard Henderson const void *debug_frame,
62892c90784aSRichard Henderson size_t debug_frame_size)
6290813da627SRichard Henderson {
62915872bbf2SRichard Henderson struct __attribute__((packed)) DebugInfo {
62925872bbf2SRichard Henderson uint32_t len;
62935872bbf2SRichard Henderson uint16_t version;
62945872bbf2SRichard Henderson uint32_t abbrev;
62955872bbf2SRichard Henderson uint8_t ptr_size;
62965872bbf2SRichard Henderson uint8_t cu_die;
62975872bbf2SRichard Henderson uint16_t cu_lang;
62985872bbf2SRichard Henderson uintptr_t cu_low_pc;
62995872bbf2SRichard Henderson uintptr_t cu_high_pc;
63005872bbf2SRichard Henderson uint8_t fn_die;
63015872bbf2SRichard Henderson char fn_name[16];
63025872bbf2SRichard Henderson uintptr_t fn_low_pc;
63035872bbf2SRichard Henderson uintptr_t fn_high_pc;
63045872bbf2SRichard Henderson uint8_t cu_eoc;
63055872bbf2SRichard Henderson };
6306813da627SRichard Henderson
6307813da627SRichard Henderson struct ElfImage {
6308813da627SRichard Henderson ElfW(Ehdr) ehdr;
6309813da627SRichard Henderson ElfW(Phdr) phdr;
63105872bbf2SRichard Henderson ElfW(Shdr) shdr[7];
63115872bbf2SRichard Henderson ElfW(Sym) sym[2];
63125872bbf2SRichard Henderson struct DebugInfo di;
63135872bbf2SRichard Henderson uint8_t da[24];
63145872bbf2SRichard Henderson char str[80];
63155872bbf2SRichard Henderson };
63165872bbf2SRichard Henderson
63175872bbf2SRichard Henderson struct ElfImage *img;
63185872bbf2SRichard Henderson
63195872bbf2SRichard Henderson static const struct ElfImage img_template = {
63205872bbf2SRichard Henderson .ehdr = {
63215872bbf2SRichard Henderson .e_ident[EI_MAG0] = ELFMAG0,
63225872bbf2SRichard Henderson .e_ident[EI_MAG1] = ELFMAG1,
63235872bbf2SRichard Henderson .e_ident[EI_MAG2] = ELFMAG2,
63245872bbf2SRichard Henderson .e_ident[EI_MAG3] = ELFMAG3,
63255872bbf2SRichard Henderson .e_ident[EI_CLASS] = ELF_CLASS,
63265872bbf2SRichard Henderson .e_ident[EI_DATA] = ELF_DATA,
63275872bbf2SRichard Henderson .e_ident[EI_VERSION] = EV_CURRENT,
63285872bbf2SRichard Henderson .e_type = ET_EXEC,
63295872bbf2SRichard Henderson .e_machine = ELF_HOST_MACHINE,
63305872bbf2SRichard Henderson .e_version = EV_CURRENT,
63315872bbf2SRichard Henderson .e_phoff = offsetof(struct ElfImage, phdr),
63325872bbf2SRichard Henderson .e_shoff = offsetof(struct ElfImage, shdr),
63335872bbf2SRichard Henderson .e_ehsize = sizeof(ElfW(Shdr)),
63345872bbf2SRichard Henderson .e_phentsize = sizeof(ElfW(Phdr)),
63355872bbf2SRichard Henderson .e_phnum = 1,
63365872bbf2SRichard Henderson .e_shentsize = sizeof(ElfW(Shdr)),
63375872bbf2SRichard Henderson .e_shnum = ARRAY_SIZE(img->shdr),
63385872bbf2SRichard Henderson .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
6339abbb3eaeSRichard Henderson #ifdef ELF_HOST_FLAGS
6340abbb3eaeSRichard Henderson .e_flags = ELF_HOST_FLAGS,
6341abbb3eaeSRichard Henderson #endif
6342abbb3eaeSRichard Henderson #ifdef ELF_OSABI
6343abbb3eaeSRichard Henderson .e_ident[EI_OSABI] = ELF_OSABI,
6344abbb3eaeSRichard Henderson #endif
63455872bbf2SRichard Henderson },
63465872bbf2SRichard Henderson .phdr = {
63475872bbf2SRichard Henderson .p_type = PT_LOAD,
63485872bbf2SRichard Henderson .p_flags = PF_X,
63495872bbf2SRichard Henderson },
63505872bbf2SRichard Henderson .shdr = {
63515872bbf2SRichard Henderson [0] = { .sh_type = SHT_NULL },
63525872bbf2SRichard Henderson /* Trick: The contents of code_gen_buffer are not present in
63535872bbf2SRichard Henderson this fake ELF file; that got allocated elsewhere. Therefore
63545872bbf2SRichard Henderson we mark .text as SHT_NOBITS (similar to .bss) so that readers
63555872bbf2SRichard Henderson will not look for contents. We can record any address. */
63565872bbf2SRichard Henderson [1] = { /* .text */
63575872bbf2SRichard Henderson .sh_type = SHT_NOBITS,
63585872bbf2SRichard Henderson .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
63595872bbf2SRichard Henderson },
63605872bbf2SRichard Henderson [2] = { /* .debug_info */
63615872bbf2SRichard Henderson .sh_type = SHT_PROGBITS,
63625872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, di),
63635872bbf2SRichard Henderson .sh_size = sizeof(struct DebugInfo),
63645872bbf2SRichard Henderson },
63655872bbf2SRichard Henderson [3] = { /* .debug_abbrev */
63665872bbf2SRichard Henderson .sh_type = SHT_PROGBITS,
63675872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, da),
63685872bbf2SRichard Henderson .sh_size = sizeof(img->da),
63695872bbf2SRichard Henderson },
63705872bbf2SRichard Henderson [4] = { /* .debug_frame */
63715872bbf2SRichard Henderson .sh_type = SHT_PROGBITS,
63725872bbf2SRichard Henderson .sh_offset = sizeof(struct ElfImage),
63735872bbf2SRichard Henderson },
63745872bbf2SRichard Henderson [5] = { /* .symtab */
63755872bbf2SRichard Henderson .sh_type = SHT_SYMTAB,
63765872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, sym),
63775872bbf2SRichard Henderson .sh_size = sizeof(img->sym),
63785872bbf2SRichard Henderson .sh_info = 1,
63795872bbf2SRichard Henderson .sh_link = ARRAY_SIZE(img->shdr) - 1,
63805872bbf2SRichard Henderson .sh_entsize = sizeof(ElfW(Sym)),
63815872bbf2SRichard Henderson },
63825872bbf2SRichard Henderson [6] = { /* .strtab */
63835872bbf2SRichard Henderson .sh_type = SHT_STRTAB,
63845872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, str),
63855872bbf2SRichard Henderson .sh_size = sizeof(img->str),
63865872bbf2SRichard Henderson }
63875872bbf2SRichard Henderson },
63885872bbf2SRichard Henderson .sym = {
63895872bbf2SRichard Henderson [1] = { /* code_gen_buffer */
63905872bbf2SRichard Henderson .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
63915872bbf2SRichard Henderson .st_shndx = 1,
63925872bbf2SRichard Henderson }
63935872bbf2SRichard Henderson },
63945872bbf2SRichard Henderson .di = {
63955872bbf2SRichard Henderson .len = sizeof(struct DebugInfo) - 4,
63965872bbf2SRichard Henderson .version = 2,
63975872bbf2SRichard Henderson .ptr_size = sizeof(void *),
63985872bbf2SRichard Henderson .cu_die = 1,
63995872bbf2SRichard Henderson .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
64005872bbf2SRichard Henderson .fn_die = 2,
64015872bbf2SRichard Henderson .fn_name = "code_gen_buffer"
64025872bbf2SRichard Henderson },
64035872bbf2SRichard Henderson .da = {
64045872bbf2SRichard Henderson 1, /* abbrev number (the cu) */
64055872bbf2SRichard Henderson 0x11, 1, /* DW_TAG_compile_unit, has children */
64065872bbf2SRichard Henderson 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
64075872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
64085872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
64095872bbf2SRichard Henderson 0, 0, /* end of abbrev */
64105872bbf2SRichard Henderson 2, /* abbrev number (the fn) */
64115872bbf2SRichard Henderson 0x2e, 0, /* DW_TAG_subprogram, no children */
64125872bbf2SRichard Henderson 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
64135872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
64145872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
64155872bbf2SRichard Henderson 0, 0, /* end of abbrev */
64165872bbf2SRichard Henderson 0 /* no more abbrev */
64175872bbf2SRichard Henderson },
64185872bbf2SRichard Henderson .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
64195872bbf2SRichard Henderson ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
6420813da627SRichard Henderson };
6421813da627SRichard Henderson
6422813da627SRichard Henderson /* We only need a single jit entry; statically allocate it. */
6423813da627SRichard Henderson static struct jit_code_entry one_entry;
6424813da627SRichard Henderson
64255872bbf2SRichard Henderson uintptr_t buf = (uintptr_t)buf_ptr;
6426813da627SRichard Henderson size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
64272c90784aSRichard Henderson DebugFrameHeader *dfh;
6428813da627SRichard Henderson
64295872bbf2SRichard Henderson img = g_malloc(img_size);
64305872bbf2SRichard Henderson *img = img_template;
6431813da627SRichard Henderson
64325872bbf2SRichard Henderson img->phdr.p_vaddr = buf;
64335872bbf2SRichard Henderson img->phdr.p_paddr = buf;
64345872bbf2SRichard Henderson img->phdr.p_memsz = buf_size;
6435813da627SRichard Henderson
64365872bbf2SRichard Henderson img->shdr[1].sh_name = find_string(img->str, ".text");
64375872bbf2SRichard Henderson img->shdr[1].sh_addr = buf;
64385872bbf2SRichard Henderson img->shdr[1].sh_size = buf_size;
6439813da627SRichard Henderson
64405872bbf2SRichard Henderson img->shdr[2].sh_name = find_string(img->str, ".debug_info");
64415872bbf2SRichard Henderson img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
64425872bbf2SRichard Henderson
64435872bbf2SRichard Henderson img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
64445872bbf2SRichard Henderson img->shdr[4].sh_size = debug_frame_size;
64455872bbf2SRichard Henderson
64465872bbf2SRichard Henderson img->shdr[5].sh_name = find_string(img->str, ".symtab");
64475872bbf2SRichard Henderson img->shdr[6].sh_name = find_string(img->str, ".strtab");
64485872bbf2SRichard Henderson
64495872bbf2SRichard Henderson img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
64505872bbf2SRichard Henderson img->sym[1].st_value = buf;
64515872bbf2SRichard Henderson img->sym[1].st_size = buf_size;
64525872bbf2SRichard Henderson
64535872bbf2SRichard Henderson img->di.cu_low_pc = buf;
645445aba097SRichard Henderson img->di.cu_high_pc = buf + buf_size;
64555872bbf2SRichard Henderson img->di.fn_low_pc = buf;
645645aba097SRichard Henderson img->di.fn_high_pc = buf + buf_size;
6457813da627SRichard Henderson
64582c90784aSRichard Henderson dfh = (DebugFrameHeader *)(img + 1);
64592c90784aSRichard Henderson memcpy(dfh, debug_frame, debug_frame_size);
64602c90784aSRichard Henderson dfh->fde.func_start = buf;
64612c90784aSRichard Henderson dfh->fde.func_len = buf_size;
64622c90784aSRichard Henderson
6463813da627SRichard Henderson #ifdef DEBUG_JIT
6464813da627SRichard Henderson /* Enable this block to be able to debug the ELF image file creation.
6465813da627SRichard Henderson One can use readelf, objdump, or other inspection utilities. */
6466813da627SRichard Henderson {
6467eb6b2edfSBin Meng g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
6468eb6b2edfSBin Meng FILE *f = fopen(jit, "w+b");
6469813da627SRichard Henderson if (f) {
64705872bbf2SRichard Henderson if (fwrite(img, img_size, 1, f) != img_size) {
6471813da627SRichard Henderson /* Avoid stupid unused return value warning for fwrite. */
6472813da627SRichard Henderson }
6473813da627SRichard Henderson fclose(f);
6474813da627SRichard Henderson }
6475813da627SRichard Henderson }
6476813da627SRichard Henderson #endif
6477813da627SRichard Henderson
6478813da627SRichard Henderson one_entry.symfile_addr = img;
6479813da627SRichard Henderson one_entry.symfile_size = img_size;
6480813da627SRichard Henderson
6481813da627SRichard Henderson __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
6482813da627SRichard Henderson __jit_debug_descriptor.relevant_entry = &one_entry;
6483813da627SRichard Henderson __jit_debug_descriptor.first_entry = &one_entry;
6484813da627SRichard Henderson __jit_debug_register_code();
6485813da627SRichard Henderson }
6486813da627SRichard Henderson #else
64875872bbf2SRichard Henderson /* No support for the feature. Provide the entry point expected by exec.c,
64885872bbf2SRichard Henderson and implement the internal function we declared earlier. */
6489813da627SRichard Henderson
tcg_register_jit_int(const void * buf,size_t size,const void * debug_frame,size_t debug_frame_size)6490755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size,
64912c90784aSRichard Henderson const void *debug_frame,
64922c90784aSRichard Henderson size_t debug_frame_size)
6493813da627SRichard Henderson {
6494813da627SRichard Henderson }
6495813da627SRichard Henderson
tcg_register_jit(const void * buf,size_t buf_size)6496755bf9e5SRichard Henderson void tcg_register_jit(const void *buf, size_t buf_size)
6497813da627SRichard Henderson {
6498813da627SRichard Henderson }
6499813da627SRichard Henderson #endif /* ELF_HOST_MACHINE */
6500db432672SRichard Henderson
6501db432672SRichard Henderson #if !TCG_TARGET_MAYBE_vec
tcg_expand_vec_op(TCGOpcode o,TCGType t,unsigned e,TCGArg a0,...)6502db432672SRichard Henderson void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
6503db432672SRichard Henderson {
6504db432672SRichard Henderson g_assert_not_reached();
6505db432672SRichard Henderson }
6506db432672SRichard Henderson #endif
6507