1c896fe29Sbellard /* 2c896fe29Sbellard * Tiny Code Generator for QEMU 3c896fe29Sbellard * 4c896fe29Sbellard * Copyright (c) 2008 Fabrice Bellard 5c896fe29Sbellard * 6c896fe29Sbellard * Permission is hereby granted, free of charge, to any person obtaining a copy 7c896fe29Sbellard * of this software and associated documentation files (the "Software"), to deal 8c896fe29Sbellard * in the Software without restriction, including without limitation the rights 9c896fe29Sbellard * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10c896fe29Sbellard * copies of the Software, and to permit persons to whom the Software is 11c896fe29Sbellard * furnished to do so, subject to the following conditions: 12c896fe29Sbellard * 13c896fe29Sbellard * The above copyright notice and this permission notice shall be included in 14c896fe29Sbellard * all copies or substantial portions of the Software. 15c896fe29Sbellard * 16c896fe29Sbellard * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17c896fe29Sbellard * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18c896fe29Sbellard * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19c896fe29Sbellard * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20c896fe29Sbellard * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21c896fe29Sbellard * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22c896fe29Sbellard * THE SOFTWARE. 23c896fe29Sbellard */ 24c896fe29Sbellard 25c896fe29Sbellard /* define it to use liveness analysis (better code) */ 268f2e8c07SKirill Batuzov #define USE_TCG_OPTIMIZATIONS 27c896fe29Sbellard 28757e725bSPeter Maydell #include "qemu/osdep.h" 29cca82982Saurel32 30813da627SRichard Henderson /* Define to jump the ELF file used to communicate with GDB. */ 31813da627SRichard Henderson #undef DEBUG_JIT 32813da627SRichard Henderson 3372fd2efbSEmilio G. Cota #include "qemu/error-report.h" 34f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 351de7afc9SPaolo Bonzini #include "qemu/host-utils.h" 36d4c51a0aSMarkus Armbruster #include "qemu/qemu-print.h" 37084cfca1SRichard Henderson #include "qemu/cacheflush.h" 38ad768e6fSPeter Maydell #include "qemu/cacheinfo.h" 39533206f0SRichard W.M. Jones #include "qemu/timer.h" 40c896fe29Sbellard 41c5d3c498SStefan Weil /* Note: the long term plan is to reduce the dependencies on the QEMU 42c896fe29Sbellard CPU definitions. Currently they are used for qemu_ld/st 43c896fe29Sbellard instructions */ 44c896fe29Sbellard #define NO_CPU_IO_DEFS 45c896fe29Sbellard 4663c91552SPaolo Bonzini #include "exec/exec-all.h" 47dcb32f1dSPhilippe Mathieu-Daudé #include "tcg/tcg-op.h" 48813da627SRichard Henderson 49edee2579SRichard Henderson #if UINTPTR_MAX == UINT32_MAX 50813da627SRichard Henderson # define ELF_CLASS ELFCLASS32 51edee2579SRichard Henderson #else 52edee2579SRichard Henderson # define ELF_CLASS ELFCLASS64 53813da627SRichard Henderson #endif 54e03b5686SMarc-André Lureau #if HOST_BIG_ENDIAN 55813da627SRichard Henderson # define ELF_DATA ELFDATA2MSB 56813da627SRichard Henderson #else 57813da627SRichard Henderson # define ELF_DATA ELFDATA2LSB 58813da627SRichard Henderson #endif 59813da627SRichard Henderson 60c896fe29Sbellard #include "elf.h" 61508127e2SPaolo Bonzini #include "exec/log.h" 62d2ba8026SRichard Henderson #include "tcg/tcg-ldst.h" 6347f7313dSRichard Henderson #include "tcg/tcg-temp-internal.h" 645ff7258cSRichard Henderson #include "tcg-internal.h" 655584e2dbSIlya Leoshkevich #include "accel/tcg/perf.h" 66c896fe29Sbellard 67139c1837SPaolo Bonzini /* Forward declarations for functions declared in tcg-target.c.inc and 68ce151109SPeter Maydell used here. */ 69e4d58b41SRichard Henderson static void tcg_target_init(TCGContext *s); 70e4d58b41SRichard Henderson static void tcg_target_qemu_prologue(TCGContext *s); 716ac17786SRichard Henderson static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 722ba7fae2SRichard Henderson intptr_t value, intptr_t addend); 73c896fe29Sbellard 74497a22ebSRichard Henderson /* The CIE and FDE header definitions will be common to all hosts. */ 75497a22ebSRichard Henderson typedef struct { 76497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 77497a22ebSRichard Henderson uint32_t id; 78497a22ebSRichard Henderson uint8_t version; 79497a22ebSRichard Henderson char augmentation[1]; 80497a22ebSRichard Henderson uint8_t code_align; 81497a22ebSRichard Henderson uint8_t data_align; 82497a22ebSRichard Henderson uint8_t return_column; 83497a22ebSRichard Henderson } DebugFrameCIE; 84497a22ebSRichard Henderson 85497a22ebSRichard Henderson typedef struct QEMU_PACKED { 86497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 87497a22ebSRichard Henderson uint32_t cie_offset; 88edee2579SRichard Henderson uintptr_t func_start; 89edee2579SRichard Henderson uintptr_t func_len; 90497a22ebSRichard Henderson } DebugFrameFDEHeader; 91497a22ebSRichard Henderson 922c90784aSRichard Henderson typedef struct QEMU_PACKED { 932c90784aSRichard Henderson DebugFrameCIE cie; 942c90784aSRichard Henderson DebugFrameFDEHeader fde; 952c90784aSRichard Henderson } DebugFrameHeader; 962c90784aSRichard Henderson 97755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 982c90784aSRichard Henderson const void *debug_frame, 992c90784aSRichard Henderson size_t debug_frame_size) 100813da627SRichard Henderson __attribute__((unused)); 101813da627SRichard Henderson 102139c1837SPaolo Bonzini /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 1032a534affSRichard Henderson static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 104a05b5b9bSRichard Henderson intptr_t arg2); 10578113e83SRichard Henderson static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 106c0ad3001SStefan Weil static void tcg_out_movi(TCGContext *s, TCGType type, 1072a534affSRichard Henderson TCGReg ret, tcg_target_long arg); 108*678155b2SRichard Henderson static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 109313bdea8SRichard Henderson static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); 110b55a8d9dSRichard Henderson static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); 111cf7d6b8eSRichard Henderson static void tcg_out_goto_tb(TCGContext *s, int which); 1125e8892dbSMiroslav Rezanina static void tcg_out_op(TCGContext *s, TCGOpcode opc, 1135e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1145e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 115d2fd745fSRichard Henderson #if TCG_TARGET_MAYBE_vec 116e7632cfaSRichard Henderson static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 117e7632cfaSRichard Henderson TCGReg dst, TCGReg src); 118d6ecb4a9SRichard Henderson static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 119d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset); 1204e186175SRichard Henderson static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1214e186175SRichard Henderson TCGReg dst, int64_t arg); 1225e8892dbSMiroslav Rezanina static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1235e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1245e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1255e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 126d2fd745fSRichard Henderson #else 127e7632cfaSRichard Henderson static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 128e7632cfaSRichard Henderson TCGReg dst, TCGReg src) 129e7632cfaSRichard Henderson { 130e7632cfaSRichard Henderson g_assert_not_reached(); 131e7632cfaSRichard Henderson } 132d6ecb4a9SRichard Henderson static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 133d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset) 134d6ecb4a9SRichard Henderson { 135d6ecb4a9SRichard Henderson g_assert_not_reached(); 136d6ecb4a9SRichard Henderson } 1374e186175SRichard Henderson static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1384e186175SRichard Henderson TCGReg dst, int64_t arg) 139e7632cfaSRichard Henderson { 140e7632cfaSRichard Henderson g_assert_not_reached(); 141e7632cfaSRichard Henderson } 1425e8892dbSMiroslav Rezanina static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1435e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1445e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1455e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]) 146d2fd745fSRichard Henderson { 147d2fd745fSRichard Henderson g_assert_not_reached(); 148d2fd745fSRichard Henderson } 149d2fd745fSRichard Henderson #endif 1502a534affSRichard Henderson static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 151a05b5b9bSRichard Henderson intptr_t arg2); 15259d7c14eSRichard Henderson static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 15359d7c14eSRichard Henderson TCGReg base, intptr_t ofs); 1547b7d8b2dSRichard Henderson static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 155cee44b03SRichard Henderson const TCGHelperInfo *info); 1565e3d0c19SRichard Henderson static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); 157a4fbbd77SRichard Henderson static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 158659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 159aeee05f5SRichard Henderson static int tcg_out_ldst_finalize(TCGContext *s); 160659ef5cbSRichard Henderson #endif 161c896fe29Sbellard 16242eb6dfcSRichard Henderson TCGContext tcg_init_ctx; 16342eb6dfcSRichard Henderson __thread TCGContext *tcg_ctx; 16442eb6dfcSRichard Henderson 1655ff7258cSRichard Henderson TCGContext **tcg_ctxs; 1660e2d61cfSRichard Henderson unsigned int tcg_cur_ctxs; 1670e2d61cfSRichard Henderson unsigned int tcg_max_ctxs; 1681c2adb95SRichard Henderson TCGv_env cpu_env = 0; 169c8bc1168SRichard Henderson const void *tcg_code_gen_epilogue; 170db0c51a3SRichard Henderson uintptr_t tcg_splitwx_diff; 171df2cce29SEmilio G. Cota 172b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 173b91ccb31SRichard Henderson tcg_prologue_fn *tcg_qemu_tb_exec; 174b91ccb31SRichard Henderson #endif 175b91ccb31SRichard Henderson 176d2fd745fSRichard Henderson static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 177b1d8e52eSblueswir1 static TCGRegSet tcg_target_call_clobber_regs; 178c896fe29Sbellard 1791813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE == 1 1804196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 181c896fe29Sbellard { 182c896fe29Sbellard *s->code_ptr++ = v; 183c896fe29Sbellard } 184c896fe29Sbellard 1854196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 1864196dca6SPeter Maydell uint8_t v) 1875c53bb81SPeter Maydell { 1881813e175SRichard Henderson *p = v; 1895c53bb81SPeter Maydell } 1901813e175SRichard Henderson #endif 1915c53bb81SPeter Maydell 1921813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 2 1934196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 194c896fe29Sbellard { 1951813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 1961813e175SRichard Henderson *s->code_ptr++ = v; 1971813e175SRichard Henderson } else { 1981813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 1994387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2001813e175SRichard Henderson s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 2011813e175SRichard Henderson } 202c896fe29Sbellard } 203c896fe29Sbellard 2044196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 2054196dca6SPeter Maydell uint16_t v) 2065c53bb81SPeter Maydell { 2071813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 2081813e175SRichard Henderson *p = v; 2091813e175SRichard Henderson } else { 2105c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2115c53bb81SPeter Maydell } 2121813e175SRichard Henderson } 2131813e175SRichard Henderson #endif 2145c53bb81SPeter Maydell 2151813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 4 2164196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 217c896fe29Sbellard { 2181813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2191813e175SRichard Henderson *s->code_ptr++ = v; 2201813e175SRichard Henderson } else { 2211813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2224387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2231813e175SRichard Henderson s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 2241813e175SRichard Henderson } 225c896fe29Sbellard } 226c896fe29Sbellard 2274196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 2284196dca6SPeter Maydell uint32_t v) 2295c53bb81SPeter Maydell { 2301813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2311813e175SRichard Henderson *p = v; 2321813e175SRichard Henderson } else { 2335c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2345c53bb81SPeter Maydell } 2351813e175SRichard Henderson } 2361813e175SRichard Henderson #endif 2375c53bb81SPeter Maydell 2381813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 8 2394196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 240ac26eb69SRichard Henderson { 2411813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2421813e175SRichard Henderson *s->code_ptr++ = v; 2431813e175SRichard Henderson } else { 2441813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2454387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2461813e175SRichard Henderson s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 2471813e175SRichard Henderson } 248ac26eb69SRichard Henderson } 249ac26eb69SRichard Henderson 2504196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 2514196dca6SPeter Maydell uint64_t v) 2525c53bb81SPeter Maydell { 2531813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2541813e175SRichard Henderson *p = v; 2551813e175SRichard Henderson } else { 2565c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2575c53bb81SPeter Maydell } 2581813e175SRichard Henderson } 2591813e175SRichard Henderson #endif 2605c53bb81SPeter Maydell 261c896fe29Sbellard /* label relocation processing */ 262c896fe29Sbellard 2631813e175SRichard Henderson static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 264bec16311SRichard Henderson TCGLabel *l, intptr_t addend) 265c896fe29Sbellard { 2667ecd02a0SRichard Henderson TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 267c896fe29Sbellard 268c896fe29Sbellard r->type = type; 269c896fe29Sbellard r->ptr = code_ptr; 270c896fe29Sbellard r->addend = addend; 2717ecd02a0SRichard Henderson QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 272c896fe29Sbellard } 273c896fe29Sbellard 27492ab8e7dSRichard Henderson static void tcg_out_label(TCGContext *s, TCGLabel *l) 275c896fe29Sbellard { 276eabb7b91SAurelien Jarno tcg_debug_assert(!l->has_value); 277c896fe29Sbellard l->has_value = 1; 27892ab8e7dSRichard Henderson l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 279c896fe29Sbellard } 280c896fe29Sbellard 28142a268c2SRichard Henderson TCGLabel *gen_new_label(void) 282c896fe29Sbellard { 283b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 28451e3972cSRichard Henderson TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 285c896fe29Sbellard 2867ecd02a0SRichard Henderson memset(l, 0, sizeof(TCGLabel)); 2877ecd02a0SRichard Henderson l->id = s->nb_labels++; 288f85b1fc4SRichard Henderson QSIMPLEQ_INIT(&l->branches); 2897ecd02a0SRichard Henderson QSIMPLEQ_INIT(&l->relocs); 2907ecd02a0SRichard Henderson 291bef16ab4SRichard Henderson QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 29242a268c2SRichard Henderson 29342a268c2SRichard Henderson return l; 294c896fe29Sbellard } 295c896fe29Sbellard 2967ecd02a0SRichard Henderson static bool tcg_resolve_relocs(TCGContext *s) 2977ecd02a0SRichard Henderson { 2987ecd02a0SRichard Henderson TCGLabel *l; 2997ecd02a0SRichard Henderson 3007ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 3017ecd02a0SRichard Henderson TCGRelocation *r; 3027ecd02a0SRichard Henderson uintptr_t value = l->u.value; 3037ecd02a0SRichard Henderson 3047ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(r, &l->relocs, next) { 3057ecd02a0SRichard Henderson if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 3067ecd02a0SRichard Henderson return false; 3077ecd02a0SRichard Henderson } 3087ecd02a0SRichard Henderson } 3097ecd02a0SRichard Henderson } 3107ecd02a0SRichard Henderson return true; 3117ecd02a0SRichard Henderson } 3127ecd02a0SRichard Henderson 3139f754620SRichard Henderson static void set_jmp_reset_offset(TCGContext *s, int which) 3149f754620SRichard Henderson { 315f14bed3fSRichard Henderson /* 316f14bed3fSRichard Henderson * We will check for overflow at the end of the opcode loop in 317f14bed3fSRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 318f14bed3fSRichard Henderson */ 319b7e4afbdSRichard Henderson s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); 3209f754620SRichard Henderson } 3219f754620SRichard Henderson 322b52a2c03SRichard Henderson static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) 323b52a2c03SRichard Henderson { 324b52a2c03SRichard Henderson /* 325b52a2c03SRichard Henderson * We will check for overflow at the end of the opcode loop in 326b52a2c03SRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 327b52a2c03SRichard Henderson */ 3289da6079bSRichard Henderson s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); 329b52a2c03SRichard Henderson } 330b52a2c03SRichard Henderson 331becc452aSRichard Henderson static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) 332becc452aSRichard Henderson { 333becc452aSRichard Henderson /* 334becc452aSRichard Henderson * Return the read-execute version of the pointer, for the benefit 335becc452aSRichard Henderson * of any pc-relative addressing mode. 336becc452aSRichard Henderson */ 3379da6079bSRichard Henderson return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); 338becc452aSRichard Henderson } 339becc452aSRichard Henderson 340db6b7d0cSRichard Henderson /* Signal overflow, starting over with fewer guest insns. */ 3418905770bSMarc-André Lureau static G_NORETURN 3428905770bSMarc-André Lureau void tcg_raise_tb_overflow(TCGContext *s) 343db6b7d0cSRichard Henderson { 344db6b7d0cSRichard Henderson siglongjmp(s->jmp_trans, -2); 345db6b7d0cSRichard Henderson } 346db6b7d0cSRichard Henderson 3474c22e840SRichard Henderson #define C_PFX1(P, A) P##A 3484c22e840SRichard Henderson #define C_PFX2(P, A, B) P##A##_##B 3494c22e840SRichard Henderson #define C_PFX3(P, A, B, C) P##A##_##B##_##C 3504c22e840SRichard Henderson #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 3514c22e840SRichard Henderson #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 3524c22e840SRichard Henderson #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 3534c22e840SRichard Henderson 3544c22e840SRichard Henderson /* Define an enumeration for the various combinations. */ 3554c22e840SRichard Henderson 3564c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 3574c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 3584c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 3594c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 3604c22e840SRichard Henderson 3614c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 3624c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 3634c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 3644c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 3654c22e840SRichard Henderson 3664c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 3674c22e840SRichard Henderson 3684c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 3694c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 3704c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 3714c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 3724c22e840SRichard Henderson 3734c22e840SRichard Henderson typedef enum { 3744c22e840SRichard Henderson #include "tcg-target-con-set.h" 3754c22e840SRichard Henderson } TCGConstraintSetIndex; 3764c22e840SRichard Henderson 3774c22e840SRichard Henderson static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 3784c22e840SRichard Henderson 3794c22e840SRichard Henderson #undef C_O0_I1 3804c22e840SRichard Henderson #undef C_O0_I2 3814c22e840SRichard Henderson #undef C_O0_I3 3824c22e840SRichard Henderson #undef C_O0_I4 3834c22e840SRichard Henderson #undef C_O1_I1 3844c22e840SRichard Henderson #undef C_O1_I2 3854c22e840SRichard Henderson #undef C_O1_I3 3864c22e840SRichard Henderson #undef C_O1_I4 3874c22e840SRichard Henderson #undef C_N1_I2 3884c22e840SRichard Henderson #undef C_O2_I1 3894c22e840SRichard Henderson #undef C_O2_I2 3904c22e840SRichard Henderson #undef C_O2_I3 3914c22e840SRichard Henderson #undef C_O2_I4 3924c22e840SRichard Henderson 3934c22e840SRichard Henderson /* Put all of the constraint sets into an array, indexed by the enum. */ 3944c22e840SRichard Henderson 3954c22e840SRichard Henderson #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 3964c22e840SRichard Henderson #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 3974c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 3984c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 3994c22e840SRichard Henderson 4004c22e840SRichard Henderson #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 4014c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 4024c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 4034c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 4044c22e840SRichard Henderson 4054c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 4064c22e840SRichard Henderson 4074c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 4084c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 4094c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 4104c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 4114c22e840SRichard Henderson 4124c22e840SRichard Henderson static const TCGTargetOpDef constraint_sets[] = { 4134c22e840SRichard Henderson #include "tcg-target-con-set.h" 4144c22e840SRichard Henderson }; 4154c22e840SRichard Henderson 4164c22e840SRichard Henderson 4174c22e840SRichard Henderson #undef C_O0_I1 4184c22e840SRichard Henderson #undef C_O0_I2 4194c22e840SRichard Henderson #undef C_O0_I3 4204c22e840SRichard Henderson #undef C_O0_I4 4214c22e840SRichard Henderson #undef C_O1_I1 4224c22e840SRichard Henderson #undef C_O1_I2 4234c22e840SRichard Henderson #undef C_O1_I3 4244c22e840SRichard Henderson #undef C_O1_I4 4254c22e840SRichard Henderson #undef C_N1_I2 4264c22e840SRichard Henderson #undef C_O2_I1 4274c22e840SRichard Henderson #undef C_O2_I2 4284c22e840SRichard Henderson #undef C_O2_I3 4294c22e840SRichard Henderson #undef C_O2_I4 4304c22e840SRichard Henderson 4314c22e840SRichard Henderson /* Expand the enumerator to be returned from tcg_target_op_def(). */ 4324c22e840SRichard Henderson 4334c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 4344c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 4354c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 4364c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 4374c22e840SRichard Henderson 4384c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 4394c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 4404c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 4414c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 4424c22e840SRichard Henderson 4434c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 4444c22e840SRichard Henderson 4454c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 4464c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 4474c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 4484c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 4494c22e840SRichard Henderson 450139c1837SPaolo Bonzini #include "tcg-target.c.inc" 451c896fe29Sbellard 45238b47b19SEmilio G. Cota static void alloc_tcg_plugin_context(TCGContext *s) 45338b47b19SEmilio G. Cota { 45438b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 45538b47b19SEmilio G. Cota s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 45638b47b19SEmilio G. Cota s->plugin_tb->insns = 45738b47b19SEmilio G. Cota g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 45838b47b19SEmilio G. Cota #endif 45938b47b19SEmilio G. Cota } 46038b47b19SEmilio G. Cota 461e8feb96fSEmilio G. Cota /* 4623468b59eSEmilio G. Cota * All TCG threads except the parent (i.e. the one that called tcg_context_init 4633468b59eSEmilio G. Cota * and registered the target's TCG globals) must register with this function 4643468b59eSEmilio G. Cota * before initiating translation. 4653468b59eSEmilio G. Cota * 4663468b59eSEmilio G. Cota * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 4673468b59eSEmilio G. Cota * of tcg_region_init() for the reasoning behind this. 4683468b59eSEmilio G. Cota * 4693468b59eSEmilio G. Cota * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 4703468b59eSEmilio G. Cota * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 4713468b59eSEmilio G. Cota * is not used anymore for translation once this function is called. 4723468b59eSEmilio G. Cota * 4733468b59eSEmilio G. Cota * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 4743468b59eSEmilio G. Cota * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 4753468b59eSEmilio G. Cota */ 4763468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 4773468b59eSEmilio G. Cota void tcg_register_thread(void) 4783468b59eSEmilio G. Cota { 4793468b59eSEmilio G. Cota tcg_ctx = &tcg_init_ctx; 4803468b59eSEmilio G. Cota } 4813468b59eSEmilio G. Cota #else 4823468b59eSEmilio G. Cota void tcg_register_thread(void) 4833468b59eSEmilio G. Cota { 4843468b59eSEmilio G. Cota TCGContext *s = g_malloc(sizeof(*s)); 4853468b59eSEmilio G. Cota unsigned int i, n; 4863468b59eSEmilio G. Cota 4873468b59eSEmilio G. Cota *s = tcg_init_ctx; 4883468b59eSEmilio G. Cota 4893468b59eSEmilio G. Cota /* Relink mem_base. */ 4903468b59eSEmilio G. Cota for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 4913468b59eSEmilio G. Cota if (tcg_init_ctx.temps[i].mem_base) { 4923468b59eSEmilio G. Cota ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 4933468b59eSEmilio G. Cota tcg_debug_assert(b >= 0 && b < n); 4943468b59eSEmilio G. Cota s->temps[i].mem_base = &s->temps[b]; 4953468b59eSEmilio G. Cota } 4963468b59eSEmilio G. Cota } 4973468b59eSEmilio G. Cota 4983468b59eSEmilio G. Cota /* Claim an entry in tcg_ctxs */ 4990e2d61cfSRichard Henderson n = qatomic_fetch_inc(&tcg_cur_ctxs); 5000e2d61cfSRichard Henderson g_assert(n < tcg_max_ctxs); 501d73415a3SStefan Hajnoczi qatomic_set(&tcg_ctxs[n], s); 5023468b59eSEmilio G. Cota 50338b47b19SEmilio G. Cota if (n > 0) { 50438b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 505bf042e8eSRichard Henderson tcg_region_initial_alloc(s); 50638b47b19SEmilio G. Cota } 50738b47b19SEmilio G. Cota 5083468b59eSEmilio G. Cota tcg_ctx = s; 5093468b59eSEmilio G. Cota } 5103468b59eSEmilio G. Cota #endif /* !CONFIG_USER_ONLY */ 5113468b59eSEmilio G. Cota 512c896fe29Sbellard /* pool based memory allocation */ 513c896fe29Sbellard void *tcg_malloc_internal(TCGContext *s, int size) 514c896fe29Sbellard { 515c896fe29Sbellard TCGPool *p; 516c896fe29Sbellard int pool_size; 517c896fe29Sbellard 518c896fe29Sbellard if (size > TCG_POOL_CHUNK_SIZE) { 519c896fe29Sbellard /* big malloc: insert a new pool (XXX: could optimize) */ 5207267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + size); 521c896fe29Sbellard p->size = size; 5224055299eSKirill Batuzov p->next = s->pool_first_large; 5234055299eSKirill Batuzov s->pool_first_large = p; 5244055299eSKirill Batuzov return p->data; 525c896fe29Sbellard } else { 526c896fe29Sbellard p = s->pool_current; 527c896fe29Sbellard if (!p) { 528c896fe29Sbellard p = s->pool_first; 529c896fe29Sbellard if (!p) 530c896fe29Sbellard goto new_pool; 531c896fe29Sbellard } else { 532c896fe29Sbellard if (!p->next) { 533c896fe29Sbellard new_pool: 534c896fe29Sbellard pool_size = TCG_POOL_CHUNK_SIZE; 5357267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + pool_size); 536c896fe29Sbellard p->size = pool_size; 537c896fe29Sbellard p->next = NULL; 538a813e36fSRichard Henderson if (s->pool_current) { 539c896fe29Sbellard s->pool_current->next = p; 540a813e36fSRichard Henderson } else { 541c896fe29Sbellard s->pool_first = p; 542a813e36fSRichard Henderson } 543c896fe29Sbellard } else { 544c896fe29Sbellard p = p->next; 545c896fe29Sbellard } 546c896fe29Sbellard } 547c896fe29Sbellard } 548c896fe29Sbellard s->pool_current = p; 549c896fe29Sbellard s->pool_cur = p->data + size; 550c896fe29Sbellard s->pool_end = p->data + p->size; 551c896fe29Sbellard return p->data; 552c896fe29Sbellard } 553c896fe29Sbellard 554c896fe29Sbellard void tcg_pool_reset(TCGContext *s) 555c896fe29Sbellard { 5564055299eSKirill Batuzov TCGPool *p, *t; 5574055299eSKirill Batuzov for (p = s->pool_first_large; p; p = t) { 5584055299eSKirill Batuzov t = p->next; 5594055299eSKirill Batuzov g_free(p); 5604055299eSKirill Batuzov } 5614055299eSKirill Batuzov s->pool_first_large = NULL; 562c896fe29Sbellard s->pool_cur = s->pool_end = NULL; 563c896fe29Sbellard s->pool_current = NULL; 564c896fe29Sbellard } 565c896fe29Sbellard 5662ef6175aSRichard Henderson #include "exec/helper-proto.h" 5672ef6175aSRichard Henderson 56839004a71SRichard Henderson static TCGHelperInfo all_helpers[] = { 5692ef6175aSRichard Henderson #include "exec/helper-tcg.h" 570100b5e01SRichard Henderson }; 571619205fdSEmilio G. Cota static GHashTable *helper_table; 572100b5e01SRichard Henderson 57322f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 574c6ef8c7bSPhilippe Mathieu-Daudé static ffi_type *typecode_to_ffi(int argmask) 575c6ef8c7bSPhilippe Mathieu-Daudé { 576e9709e17SRichard Henderson /* 577e9709e17SRichard Henderson * libffi does not support __int128_t, so we have forced Int128 578e9709e17SRichard Henderson * to use the structure definition instead of the builtin type. 579e9709e17SRichard Henderson */ 580e9709e17SRichard Henderson static ffi_type *ffi_type_i128_elements[3] = { 581e9709e17SRichard Henderson &ffi_type_uint64, 582e9709e17SRichard Henderson &ffi_type_uint64, 583e9709e17SRichard Henderson NULL 584e9709e17SRichard Henderson }; 585e9709e17SRichard Henderson static ffi_type ffi_type_i128 = { 586e9709e17SRichard Henderson .size = 16, 587e9709e17SRichard Henderson .alignment = __alignof__(Int128), 588e9709e17SRichard Henderson .type = FFI_TYPE_STRUCT, 589e9709e17SRichard Henderson .elements = ffi_type_i128_elements, 590e9709e17SRichard Henderson }; 591e9709e17SRichard Henderson 592c6ef8c7bSPhilippe Mathieu-Daudé switch (argmask) { 593c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_void: 594c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_void; 595c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i32: 596c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint32; 597c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s32: 598c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint32; 599c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i64: 600c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint64; 601c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s64: 602c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint64; 603c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_ptr: 604c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_pointer; 605e9709e17SRichard Henderson case dh_typecode_i128: 606e9709e17SRichard Henderson return &ffi_type_i128; 607c6ef8c7bSPhilippe Mathieu-Daudé } 608c6ef8c7bSPhilippe Mathieu-Daudé g_assert_not_reached(); 609c6ef8c7bSPhilippe Mathieu-Daudé } 6100c22e176SPhilippe Mathieu-Daudé 6110c22e176SPhilippe Mathieu-Daudé static void init_ffi_layouts(void) 6120c22e176SPhilippe Mathieu-Daudé { 6130c22e176SPhilippe Mathieu-Daudé /* g_direct_hash/equal for direct comparisons on uint32_t. */ 614f9c4bb80SRichard Henderson GHashTable *ffi_table = g_hash_table_new(NULL, NULL); 615f9c4bb80SRichard Henderson 6160c22e176SPhilippe Mathieu-Daudé for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 617f9c4bb80SRichard Henderson TCGHelperInfo *info = &all_helpers[i]; 618f9c4bb80SRichard Henderson unsigned typemask = info->typemask; 6190c22e176SPhilippe Mathieu-Daudé gpointer hash = (gpointer)(uintptr_t)typemask; 6200c22e176SPhilippe Mathieu-Daudé struct { 6210c22e176SPhilippe Mathieu-Daudé ffi_cif cif; 6220c22e176SPhilippe Mathieu-Daudé ffi_type *args[]; 6230c22e176SPhilippe Mathieu-Daudé } *ca; 6240c22e176SPhilippe Mathieu-Daudé ffi_status status; 6250c22e176SPhilippe Mathieu-Daudé int nargs; 626f9c4bb80SRichard Henderson ffi_cif *cif; 6270c22e176SPhilippe Mathieu-Daudé 628f9c4bb80SRichard Henderson cif = g_hash_table_lookup(ffi_table, hash); 629f9c4bb80SRichard Henderson if (cif) { 630f9c4bb80SRichard Henderson info->cif = cif; 6310c22e176SPhilippe Mathieu-Daudé continue; 6320c22e176SPhilippe Mathieu-Daudé } 6330c22e176SPhilippe Mathieu-Daudé 6340c22e176SPhilippe Mathieu-Daudé /* Ignoring the return type, find the last non-zero field. */ 6350c22e176SPhilippe Mathieu-Daudé nargs = 32 - clz32(typemask >> 3); 6360c22e176SPhilippe Mathieu-Daudé nargs = DIV_ROUND_UP(nargs, 3); 637e9709e17SRichard Henderson assert(nargs <= MAX_CALL_IARGS); 6380c22e176SPhilippe Mathieu-Daudé 6390c22e176SPhilippe Mathieu-Daudé ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 6400c22e176SPhilippe Mathieu-Daudé ca->cif.rtype = typecode_to_ffi(typemask & 7); 6410c22e176SPhilippe Mathieu-Daudé ca->cif.nargs = nargs; 6420c22e176SPhilippe Mathieu-Daudé 6430c22e176SPhilippe Mathieu-Daudé if (nargs != 0) { 6440c22e176SPhilippe Mathieu-Daudé ca->cif.arg_types = ca->args; 6450c22e176SPhilippe Mathieu-Daudé for (int j = 0; j < nargs; ++j) { 6460c22e176SPhilippe Mathieu-Daudé int typecode = extract32(typemask, (j + 1) * 3, 3); 6470c22e176SPhilippe Mathieu-Daudé ca->args[j] = typecode_to_ffi(typecode); 6480c22e176SPhilippe Mathieu-Daudé } 6490c22e176SPhilippe Mathieu-Daudé } 6500c22e176SPhilippe Mathieu-Daudé 6510c22e176SPhilippe Mathieu-Daudé status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 6520c22e176SPhilippe Mathieu-Daudé ca->cif.rtype, ca->cif.arg_types); 6530c22e176SPhilippe Mathieu-Daudé assert(status == FFI_OK); 6540c22e176SPhilippe Mathieu-Daudé 655f9c4bb80SRichard Henderson cif = &ca->cif; 656f9c4bb80SRichard Henderson info->cif = cif; 657f9c4bb80SRichard Henderson g_hash_table_insert(ffi_table, hash, (gpointer)cif); 6580c22e176SPhilippe Mathieu-Daudé } 659f9c4bb80SRichard Henderson 660f9c4bb80SRichard Henderson g_hash_table_destroy(ffi_table); 6610c22e176SPhilippe Mathieu-Daudé } 6620c22e176SPhilippe Mathieu-Daudé #endif /* CONFIG_TCG_INTERPRETER */ 66322f15579SRichard Henderson 66439004a71SRichard Henderson typedef struct TCGCumulativeArgs { 66539004a71SRichard Henderson int arg_idx; /* tcg_gen_callN args[] */ 66639004a71SRichard Henderson int info_in_idx; /* TCGHelperInfo in[] */ 66739004a71SRichard Henderson int arg_slot; /* regs+stack slot */ 66839004a71SRichard Henderson int ref_slot; /* stack slots for references */ 66939004a71SRichard Henderson } TCGCumulativeArgs; 67039004a71SRichard Henderson 67139004a71SRichard Henderson static void layout_arg_even(TCGCumulativeArgs *cum) 67239004a71SRichard Henderson { 67339004a71SRichard Henderson cum->arg_slot += cum->arg_slot & 1; 67439004a71SRichard Henderson } 67539004a71SRichard Henderson 67639004a71SRichard Henderson static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, 67739004a71SRichard Henderson TCGCallArgumentKind kind) 67839004a71SRichard Henderson { 67939004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 68039004a71SRichard Henderson 68139004a71SRichard Henderson *loc = (TCGCallArgumentLoc){ 68239004a71SRichard Henderson .kind = kind, 68339004a71SRichard Henderson .arg_idx = cum->arg_idx, 68439004a71SRichard Henderson .arg_slot = cum->arg_slot, 68539004a71SRichard Henderson }; 68639004a71SRichard Henderson cum->info_in_idx++; 68739004a71SRichard Henderson cum->arg_slot++; 68839004a71SRichard Henderson } 68939004a71SRichard Henderson 69039004a71SRichard Henderson static void layout_arg_normal_n(TCGCumulativeArgs *cum, 69139004a71SRichard Henderson TCGHelperInfo *info, int n) 69239004a71SRichard Henderson { 69339004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 69439004a71SRichard Henderson 69539004a71SRichard Henderson for (int i = 0; i < n; ++i) { 69639004a71SRichard Henderson /* Layout all using the same arg_idx, adjusting the subindex. */ 69739004a71SRichard Henderson loc[i] = (TCGCallArgumentLoc){ 69839004a71SRichard Henderson .kind = TCG_CALL_ARG_NORMAL, 69939004a71SRichard Henderson .arg_idx = cum->arg_idx, 70039004a71SRichard Henderson .tmp_subindex = i, 70139004a71SRichard Henderson .arg_slot = cum->arg_slot + i, 70239004a71SRichard Henderson }; 70339004a71SRichard Henderson } 70439004a71SRichard Henderson cum->info_in_idx += n; 70539004a71SRichard Henderson cum->arg_slot += n; 70639004a71SRichard Henderson } 70739004a71SRichard Henderson 708313bdea8SRichard Henderson static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info) 709313bdea8SRichard Henderson { 710313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 711313bdea8SRichard Henderson int n = 128 / TCG_TARGET_REG_BITS; 712313bdea8SRichard Henderson 713313bdea8SRichard Henderson /* The first subindex carries the pointer. */ 714313bdea8SRichard Henderson layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF); 715313bdea8SRichard Henderson 716313bdea8SRichard Henderson /* 717313bdea8SRichard Henderson * The callee is allowed to clobber memory associated with 718313bdea8SRichard Henderson * structure pass by-reference. Therefore we must make copies. 719313bdea8SRichard Henderson * Allocate space from "ref_slot", which will be adjusted to 720313bdea8SRichard Henderson * follow the parameters on the stack. 721313bdea8SRichard Henderson */ 722313bdea8SRichard Henderson loc[0].ref_slot = cum->ref_slot; 723313bdea8SRichard Henderson 724313bdea8SRichard Henderson /* 725313bdea8SRichard Henderson * Subsequent words also go into the reference slot, but 726313bdea8SRichard Henderson * do not accumulate into the regular arguments. 727313bdea8SRichard Henderson */ 728313bdea8SRichard Henderson for (int i = 1; i < n; ++i) { 729313bdea8SRichard Henderson loc[i] = (TCGCallArgumentLoc){ 730313bdea8SRichard Henderson .kind = TCG_CALL_ARG_BY_REF_N, 731313bdea8SRichard Henderson .arg_idx = cum->arg_idx, 732313bdea8SRichard Henderson .tmp_subindex = i, 733313bdea8SRichard Henderson .ref_slot = cum->ref_slot + i, 734313bdea8SRichard Henderson }; 735313bdea8SRichard Henderson } 736313bdea8SRichard Henderson cum->info_in_idx += n; 737313bdea8SRichard Henderson cum->ref_slot += n; 738313bdea8SRichard Henderson } 739313bdea8SRichard Henderson 74039004a71SRichard Henderson static void init_call_layout(TCGHelperInfo *info) 74139004a71SRichard Henderson { 74239004a71SRichard Henderson int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); 74339004a71SRichard Henderson int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 74439004a71SRichard Henderson unsigned typemask = info->typemask; 74539004a71SRichard Henderson unsigned typecode; 74639004a71SRichard Henderson TCGCumulativeArgs cum = { }; 74739004a71SRichard Henderson 74839004a71SRichard Henderson /* 74939004a71SRichard Henderson * Parse and place any function return value. 75039004a71SRichard Henderson */ 75139004a71SRichard Henderson typecode = typemask & 7; 75239004a71SRichard Henderson switch (typecode) { 75339004a71SRichard Henderson case dh_typecode_void: 75439004a71SRichard Henderson info->nr_out = 0; 75539004a71SRichard Henderson break; 75639004a71SRichard Henderson case dh_typecode_i32: 75739004a71SRichard Henderson case dh_typecode_s32: 75839004a71SRichard Henderson case dh_typecode_ptr: 75939004a71SRichard Henderson info->nr_out = 1; 76039004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL; 76139004a71SRichard Henderson break; 76239004a71SRichard Henderson case dh_typecode_i64: 76339004a71SRichard Henderson case dh_typecode_s64: 76439004a71SRichard Henderson info->nr_out = 64 / TCG_TARGET_REG_BITS; 76539004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL; 7665e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */ 7675e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 768466d3759SRichard Henderson break; 769466d3759SRichard Henderson case dh_typecode_i128: 770466d3759SRichard Henderson info->nr_out = 128 / TCG_TARGET_REG_BITS; 7715427a9a7SRichard Henderson info->out_kind = TCG_TARGET_CALL_RET_I128; 7725427a9a7SRichard Henderson switch (TCG_TARGET_CALL_RET_I128) { 773466d3759SRichard Henderson case TCG_CALL_RET_NORMAL: 7745e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */ 7755e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 776466d3759SRichard Henderson break; 777c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC: 778c6556aa0SRichard Henderson /* Query the single register now to trigger any assert early. */ 779c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0); 780c6556aa0SRichard Henderson break; 781313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF: 782313bdea8SRichard Henderson /* 783313bdea8SRichard Henderson * Allocate the first argument to the output. 784313bdea8SRichard Henderson * We don't need to store this anywhere, just make it 785313bdea8SRichard Henderson * unavailable for use in the input loop below. 786313bdea8SRichard Henderson */ 787313bdea8SRichard Henderson cum.arg_slot = 1; 788313bdea8SRichard Henderson break; 789466d3759SRichard Henderson default: 790466d3759SRichard Henderson qemu_build_not_reached(); 791466d3759SRichard Henderson } 79239004a71SRichard Henderson break; 79339004a71SRichard Henderson default: 79439004a71SRichard Henderson g_assert_not_reached(); 79539004a71SRichard Henderson } 79639004a71SRichard Henderson 79739004a71SRichard Henderson /* 79839004a71SRichard Henderson * Parse and place function arguments. 79939004a71SRichard Henderson */ 80039004a71SRichard Henderson for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { 80139004a71SRichard Henderson TCGCallArgumentKind kind; 80239004a71SRichard Henderson TCGType type; 80339004a71SRichard Henderson 80439004a71SRichard Henderson typecode = typemask & 7; 80539004a71SRichard Henderson switch (typecode) { 80639004a71SRichard Henderson case dh_typecode_i32: 80739004a71SRichard Henderson case dh_typecode_s32: 80839004a71SRichard Henderson type = TCG_TYPE_I32; 80939004a71SRichard Henderson break; 81039004a71SRichard Henderson case dh_typecode_i64: 81139004a71SRichard Henderson case dh_typecode_s64: 81239004a71SRichard Henderson type = TCG_TYPE_I64; 81339004a71SRichard Henderson break; 81439004a71SRichard Henderson case dh_typecode_ptr: 81539004a71SRichard Henderson type = TCG_TYPE_PTR; 81639004a71SRichard Henderson break; 817466d3759SRichard Henderson case dh_typecode_i128: 818466d3759SRichard Henderson type = TCG_TYPE_I128; 819466d3759SRichard Henderson break; 82039004a71SRichard Henderson default: 82139004a71SRichard Henderson g_assert_not_reached(); 82239004a71SRichard Henderson } 82339004a71SRichard Henderson 82439004a71SRichard Henderson switch (type) { 82539004a71SRichard Henderson case TCG_TYPE_I32: 82639004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I32) { 82739004a71SRichard Henderson case TCG_CALL_ARG_EVEN: 82839004a71SRichard Henderson layout_arg_even(&cum); 82939004a71SRichard Henderson /* fall through */ 83039004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 83139004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 83239004a71SRichard Henderson break; 83339004a71SRichard Henderson case TCG_CALL_ARG_EXTEND: 83439004a71SRichard Henderson kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); 83539004a71SRichard Henderson layout_arg_1(&cum, info, kind); 83639004a71SRichard Henderson break; 83739004a71SRichard Henderson default: 83839004a71SRichard Henderson qemu_build_not_reached(); 83939004a71SRichard Henderson } 84039004a71SRichard Henderson break; 84139004a71SRichard Henderson 84239004a71SRichard Henderson case TCG_TYPE_I64: 84339004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I64) { 84439004a71SRichard Henderson case TCG_CALL_ARG_EVEN: 84539004a71SRichard Henderson layout_arg_even(&cum); 84639004a71SRichard Henderson /* fall through */ 84739004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 84839004a71SRichard Henderson if (TCG_TARGET_REG_BITS == 32) { 84939004a71SRichard Henderson layout_arg_normal_n(&cum, info, 2); 85039004a71SRichard Henderson } else { 85139004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 85239004a71SRichard Henderson } 85339004a71SRichard Henderson break; 85439004a71SRichard Henderson default: 85539004a71SRichard Henderson qemu_build_not_reached(); 85639004a71SRichard Henderson } 85739004a71SRichard Henderson break; 85839004a71SRichard Henderson 859466d3759SRichard Henderson case TCG_TYPE_I128: 8605427a9a7SRichard Henderson switch (TCG_TARGET_CALL_ARG_I128) { 861466d3759SRichard Henderson case TCG_CALL_ARG_EVEN: 862466d3759SRichard Henderson layout_arg_even(&cum); 863466d3759SRichard Henderson /* fall through */ 864466d3759SRichard Henderson case TCG_CALL_ARG_NORMAL: 865466d3759SRichard Henderson layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS); 866466d3759SRichard Henderson break; 867313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 868313bdea8SRichard Henderson layout_arg_by_ref(&cum, info); 869313bdea8SRichard Henderson break; 870466d3759SRichard Henderson default: 871466d3759SRichard Henderson qemu_build_not_reached(); 872466d3759SRichard Henderson } 873466d3759SRichard Henderson break; 874466d3759SRichard Henderson 87539004a71SRichard Henderson default: 87639004a71SRichard Henderson g_assert_not_reached(); 87739004a71SRichard Henderson } 87839004a71SRichard Henderson } 87939004a71SRichard Henderson info->nr_in = cum.info_in_idx; 88039004a71SRichard Henderson 88139004a71SRichard Henderson /* Validate that we didn't overrun the input array. */ 88239004a71SRichard Henderson assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); 88339004a71SRichard Henderson /* Validate the backend has enough argument space. */ 88439004a71SRichard Henderson assert(cum.arg_slot <= max_reg_slots + max_stk_slots); 885313bdea8SRichard Henderson 886313bdea8SRichard Henderson /* 887313bdea8SRichard Henderson * Relocate the "ref_slot" area to the end of the parameters. 888313bdea8SRichard Henderson * Minimizing this stack offset helps code size for x86, 889313bdea8SRichard Henderson * which has a signed 8-bit offset encoding. 890313bdea8SRichard Henderson */ 891313bdea8SRichard Henderson if (cum.ref_slot != 0) { 892313bdea8SRichard Henderson int ref_base = 0; 893313bdea8SRichard Henderson 894313bdea8SRichard Henderson if (cum.arg_slot > max_reg_slots) { 895313bdea8SRichard Henderson int align = __alignof(Int128) / sizeof(tcg_target_long); 896313bdea8SRichard Henderson 897313bdea8SRichard Henderson ref_base = cum.arg_slot - max_reg_slots; 898313bdea8SRichard Henderson if (align > 1) { 899313bdea8SRichard Henderson ref_base = ROUND_UP(ref_base, align); 900313bdea8SRichard Henderson } 901313bdea8SRichard Henderson } 902313bdea8SRichard Henderson assert(ref_base + cum.ref_slot <= max_stk_slots); 903313bdea8SRichard Henderson 904313bdea8SRichard Henderson if (ref_base != 0) { 905313bdea8SRichard Henderson for (int i = cum.info_in_idx - 1; i >= 0; --i) { 906313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[i]; 907313bdea8SRichard Henderson switch (loc->kind) { 908313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 909313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 910313bdea8SRichard Henderson loc->ref_slot += ref_base; 911313bdea8SRichard Henderson break; 912313bdea8SRichard Henderson default: 913313bdea8SRichard Henderson break; 914313bdea8SRichard Henderson } 915313bdea8SRichard Henderson } 916313bdea8SRichard Henderson } 917313bdea8SRichard Henderson } 91839004a71SRichard Henderson } 91939004a71SRichard Henderson 92091478cefSRichard Henderson static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 921f69d277eSRichard Henderson static void process_op_defs(TCGContext *s); 9221c2adb95SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 9231c2adb95SRichard Henderson TCGReg reg, const char *name); 92491478cefSRichard Henderson 92543b972b7SRichard Henderson static void tcg_context_init(unsigned max_cpus) 926c896fe29Sbellard { 927a76aabd3SRichard Henderson TCGContext *s = &tcg_init_ctx; 928100b5e01SRichard Henderson int op, total_args, n, i; 929c896fe29Sbellard TCGOpDef *def; 930c896fe29Sbellard TCGArgConstraint *args_ct; 9311c2adb95SRichard Henderson TCGTemp *ts; 932c896fe29Sbellard 933c896fe29Sbellard memset(s, 0, sizeof(*s)); 934c896fe29Sbellard s->nb_globals = 0; 935c896fe29Sbellard 936c896fe29Sbellard /* Count total number of arguments and allocate the corresponding 937c896fe29Sbellard space */ 938c896fe29Sbellard total_args = 0; 939c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 940c896fe29Sbellard def = &tcg_op_defs[op]; 941c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 942c896fe29Sbellard total_args += n; 943c896fe29Sbellard } 944c896fe29Sbellard 945bc2b17e6SRichard Henderson args_ct = g_new0(TCGArgConstraint, total_args); 946c896fe29Sbellard 947c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 948c896fe29Sbellard def = &tcg_op_defs[op]; 949c896fe29Sbellard def->args_ct = args_ct; 950c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 951c896fe29Sbellard args_ct += n; 952c896fe29Sbellard } 953c896fe29Sbellard 9545cd8f621SRichard Henderson /* Register helpers. */ 95584fd9dd3SRichard Henderson /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 956619205fdSEmilio G. Cota helper_table = g_hash_table_new(NULL, NULL); 95784fd9dd3SRichard Henderson 958100b5e01SRichard Henderson for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 95939004a71SRichard Henderson init_call_layout(&all_helpers[i]); 96084fd9dd3SRichard Henderson g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 96172866e82SRichard Henderson (gpointer)&all_helpers[i]); 962100b5e01SRichard Henderson } 9635cd8f621SRichard Henderson 96422f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 9650c22e176SPhilippe Mathieu-Daudé init_ffi_layouts(); 96622f15579SRichard Henderson #endif 96722f15579SRichard Henderson 968c896fe29Sbellard tcg_target_init(s); 969f69d277eSRichard Henderson process_op_defs(s); 97091478cefSRichard Henderson 97191478cefSRichard Henderson /* Reverse the order of the saved registers, assuming they're all at 97291478cefSRichard Henderson the start of tcg_target_reg_alloc_order. */ 97391478cefSRichard Henderson for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 97491478cefSRichard Henderson int r = tcg_target_reg_alloc_order[n]; 97591478cefSRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 97691478cefSRichard Henderson break; 97791478cefSRichard Henderson } 97891478cefSRichard Henderson } 97991478cefSRichard Henderson for (i = 0; i < n; ++i) { 98091478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 98191478cefSRichard Henderson } 98291478cefSRichard Henderson for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 98391478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 98491478cefSRichard Henderson } 985b1311c4aSEmilio G. Cota 98638b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 98738b47b19SEmilio G. Cota 988b1311c4aSEmilio G. Cota tcg_ctx = s; 9893468b59eSEmilio G. Cota /* 9903468b59eSEmilio G. Cota * In user-mode we simply share the init context among threads, since we 9913468b59eSEmilio G. Cota * use a single region. See the documentation tcg_region_init() for the 9923468b59eSEmilio G. Cota * reasoning behind this. 9933468b59eSEmilio G. Cota * In softmmu we will have at most max_cpus TCG threads. 9943468b59eSEmilio G. Cota */ 9953468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 996df2cce29SEmilio G. Cota tcg_ctxs = &tcg_ctx; 9970e2d61cfSRichard Henderson tcg_cur_ctxs = 1; 9980e2d61cfSRichard Henderson tcg_max_ctxs = 1; 9993468b59eSEmilio G. Cota #else 10000e2d61cfSRichard Henderson tcg_max_ctxs = max_cpus; 10010e2d61cfSRichard Henderson tcg_ctxs = g_new0(TCGContext *, max_cpus); 10023468b59eSEmilio G. Cota #endif 10031c2adb95SRichard Henderson 10041c2adb95SRichard Henderson tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 10051c2adb95SRichard Henderson ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 10061c2adb95SRichard Henderson cpu_env = temp_tcgv_ptr(ts); 10079002ec79SRichard Henderson } 1008b03cce8eSbellard 100943b972b7SRichard Henderson void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 1010a76aabd3SRichard Henderson { 101143b972b7SRichard Henderson tcg_context_init(max_cpus); 101243b972b7SRichard Henderson tcg_region_init(tb_size, splitwx, max_cpus); 1013a76aabd3SRichard Henderson } 1014a76aabd3SRichard Henderson 10156e3b2bfdSEmilio G. Cota /* 10166e3b2bfdSEmilio G. Cota * Allocate TBs right before their corresponding translated code, making 10176e3b2bfdSEmilio G. Cota * sure that TBs and code are on different cache lines. 10186e3b2bfdSEmilio G. Cota */ 10196e3b2bfdSEmilio G. Cota TranslationBlock *tcg_tb_alloc(TCGContext *s) 10206e3b2bfdSEmilio G. Cota { 10216e3b2bfdSEmilio G. Cota uintptr_t align = qemu_icache_linesize; 10226e3b2bfdSEmilio G. Cota TranslationBlock *tb; 10236e3b2bfdSEmilio G. Cota void *next; 10246e3b2bfdSEmilio G. Cota 1025e8feb96fSEmilio G. Cota retry: 10266e3b2bfdSEmilio G. Cota tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 10276e3b2bfdSEmilio G. Cota next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 10286e3b2bfdSEmilio G. Cota 10296e3b2bfdSEmilio G. Cota if (unlikely(next > s->code_gen_highwater)) { 1030e8feb96fSEmilio G. Cota if (tcg_region_alloc(s)) { 10316e3b2bfdSEmilio G. Cota return NULL; 10326e3b2bfdSEmilio G. Cota } 1033e8feb96fSEmilio G. Cota goto retry; 1034e8feb96fSEmilio G. Cota } 1035d73415a3SStefan Hajnoczi qatomic_set(&s->code_gen_ptr, next); 103657a26946SRichard Henderson s->data_gen_ptr = NULL; 10376e3b2bfdSEmilio G. Cota return tb; 10386e3b2bfdSEmilio G. Cota } 10396e3b2bfdSEmilio G. Cota 10409002ec79SRichard Henderson void tcg_prologue_init(TCGContext *s) 10419002ec79SRichard Henderson { 1042b0a0794aSRichard Henderson size_t prologue_size; 10438163b749SRichard Henderson 1044b0a0794aSRichard Henderson s->code_ptr = s->code_gen_ptr; 1045b0a0794aSRichard Henderson s->code_buf = s->code_gen_ptr; 10465b38ee31SRichard Henderson s->data_gen_ptr = NULL; 1047b91ccb31SRichard Henderson 1048b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 1049b0a0794aSRichard Henderson tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 1050b91ccb31SRichard Henderson #endif 10518163b749SRichard Henderson 10525b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 10535b38ee31SRichard Henderson s->pool_labels = NULL; 10545b38ee31SRichard Henderson #endif 10555b38ee31SRichard Henderson 1056653b87ebSRoman Bolshakov qemu_thread_jit_write(); 10578163b749SRichard Henderson /* Generate the prologue. */ 1058b03cce8eSbellard tcg_target_qemu_prologue(s); 10595b38ee31SRichard Henderson 10605b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 10615b38ee31SRichard Henderson /* Allow the prologue to put e.g. guest_base into a pool entry. */ 10625b38ee31SRichard Henderson { 10631768987bSRichard Henderson int result = tcg_out_pool_finalize(s); 10641768987bSRichard Henderson tcg_debug_assert(result == 0); 10655b38ee31SRichard Henderson } 10665b38ee31SRichard Henderson #endif 10675b38ee31SRichard Henderson 1068b0a0794aSRichard Henderson prologue_size = tcg_current_code_size(s); 10695584e2dbSIlya Leoshkevich perf_report_prologue(s->code_gen_ptr, prologue_size); 1070b0a0794aSRichard Henderson 1071df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 1072b0a0794aSRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 1073b0a0794aSRichard Henderson (uintptr_t)s->code_buf, prologue_size); 1074df5d2b16SRichard Henderson #endif 10758163b749SRichard Henderson 1076d6b64b2bSRichard Henderson #ifdef DEBUG_DISAS 1077d6b64b2bSRichard Henderson if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 1078c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 107978b54858SRichard Henderson if (logfile) { 108078b54858SRichard Henderson fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 10815b38ee31SRichard Henderson if (s->data_gen_ptr) { 1082b0a0794aSRichard Henderson size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 10835b38ee31SRichard Henderson size_t data_size = prologue_size - code_size; 10845b38ee31SRichard Henderson size_t i; 10855b38ee31SRichard Henderson 108678b54858SRichard Henderson disas(logfile, s->code_gen_ptr, code_size); 10875b38ee31SRichard Henderson 10885b38ee31SRichard Henderson for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 10895b38ee31SRichard Henderson if (sizeof(tcg_target_ulong) == 8) { 109078b54858SRichard Henderson fprintf(logfile, 109178b54858SRichard Henderson "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 10925b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 10935b38ee31SRichard Henderson *(uint64_t *)(s->data_gen_ptr + i)); 10945b38ee31SRichard Henderson } else { 109578b54858SRichard Henderson fprintf(logfile, 109678b54858SRichard Henderson "0x%08" PRIxPTR ": .long 0x%08x\n", 10975b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 10985b38ee31SRichard Henderson *(uint32_t *)(s->data_gen_ptr + i)); 10995b38ee31SRichard Henderson } 11005b38ee31SRichard Henderson } 11015b38ee31SRichard Henderson } else { 110278b54858SRichard Henderson disas(logfile, s->code_gen_ptr, prologue_size); 11035b38ee31SRichard Henderson } 110478b54858SRichard Henderson fprintf(logfile, "\n"); 1105fc59d2d8SRobert Foley qemu_log_unlock(logfile); 1106d6b64b2bSRichard Henderson } 110778b54858SRichard Henderson } 1108d6b64b2bSRichard Henderson #endif 1109cedbcb01SEmilio G. Cota 11106eea0434SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 11116eea0434SRichard Henderson /* 11126eea0434SRichard Henderson * Assert that goto_ptr is implemented completely, setting an epilogue. 11136eea0434SRichard Henderson * For tci, we use NULL as the signal to return from the interpreter, 11146eea0434SRichard Henderson * so skip this check. 11156eea0434SRichard Henderson */ 11168b5c2b62SRichard Henderson tcg_debug_assert(tcg_code_gen_epilogue != NULL); 11176eea0434SRichard Henderson #endif 1118d1c74ab3SRichard Henderson 1119d1c74ab3SRichard Henderson tcg_region_prologue_set(s); 1120c896fe29Sbellard } 1121c896fe29Sbellard 1122c896fe29Sbellard void tcg_func_start(TCGContext *s) 1123c896fe29Sbellard { 1124c896fe29Sbellard tcg_pool_reset(s); 1125c896fe29Sbellard s->nb_temps = s->nb_globals; 11260ec9eabcSRichard Henderson 11270ec9eabcSRichard Henderson /* No temps have been previously allocated for size or locality. */ 11280ec9eabcSRichard Henderson memset(s->free_temps, 0, sizeof(s->free_temps)); 11290ec9eabcSRichard Henderson 1130c0522136SRichard Henderson /* No constant temps have been previously allocated. */ 1131c0522136SRichard Henderson for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 1132c0522136SRichard Henderson if (s->const_table[i]) { 1133c0522136SRichard Henderson g_hash_table_remove_all(s->const_table[i]); 1134c0522136SRichard Henderson } 1135c0522136SRichard Henderson } 1136c0522136SRichard Henderson 1137abebf925SRichard Henderson s->nb_ops = 0; 1138c896fe29Sbellard s->nb_labels = 0; 1139c896fe29Sbellard s->current_frame_offset = s->frame_start; 1140c896fe29Sbellard 11410a209d4bSRichard Henderson #ifdef CONFIG_DEBUG_TCG 11420a209d4bSRichard Henderson s->goto_tb_issue_mask = 0; 11430a209d4bSRichard Henderson #endif 11440a209d4bSRichard Henderson 114515fa08f8SRichard Henderson QTAILQ_INIT(&s->ops); 114615fa08f8SRichard Henderson QTAILQ_INIT(&s->free_ops); 1147bef16ab4SRichard Henderson QSIMPLEQ_INIT(&s->labels); 1148c896fe29Sbellard } 1149c896fe29Sbellard 1150ae30e866SRichard Henderson static TCGTemp *tcg_temp_alloc(TCGContext *s) 11517ca4b752SRichard Henderson { 11527ca4b752SRichard Henderson int n = s->nb_temps++; 1153ae30e866SRichard Henderson 1154ae30e866SRichard Henderson if (n >= TCG_MAX_TEMPS) { 1155db6b7d0cSRichard Henderson tcg_raise_tb_overflow(s); 1156ae30e866SRichard Henderson } 11577ca4b752SRichard Henderson return memset(&s->temps[n], 0, sizeof(TCGTemp)); 11587ca4b752SRichard Henderson } 11597ca4b752SRichard Henderson 1160ae30e866SRichard Henderson static TCGTemp *tcg_global_alloc(TCGContext *s) 11617ca4b752SRichard Henderson { 1162fa477d25SRichard Henderson TCGTemp *ts; 1163fa477d25SRichard Henderson 11647ca4b752SRichard Henderson tcg_debug_assert(s->nb_globals == s->nb_temps); 1165ae30e866SRichard Henderson tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 11667ca4b752SRichard Henderson s->nb_globals++; 1167fa477d25SRichard Henderson ts = tcg_temp_alloc(s); 1168ee17db83SRichard Henderson ts->kind = TEMP_GLOBAL; 1169fa477d25SRichard Henderson 1170fa477d25SRichard Henderson return ts; 1171c896fe29Sbellard } 1172c896fe29Sbellard 1173085272b3SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1174b6638662SRichard Henderson TCGReg reg, const char *name) 1175c896fe29Sbellard { 1176c896fe29Sbellard TCGTemp *ts; 1177c896fe29Sbellard 11781a057554SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); 11797ca4b752SRichard Henderson 11807ca4b752SRichard Henderson ts = tcg_global_alloc(s); 1181c896fe29Sbellard ts->base_type = type; 1182c896fe29Sbellard ts->type = type; 1183ee17db83SRichard Henderson ts->kind = TEMP_FIXED; 1184c896fe29Sbellard ts->reg = reg; 1185c896fe29Sbellard ts->name = name; 1186c896fe29Sbellard tcg_regset_set_reg(s->reserved_regs, reg); 11877ca4b752SRichard Henderson 1188085272b3SRichard Henderson return ts; 1189a7812ae4Spbrook } 1190a7812ae4Spbrook 1191b6638662SRichard Henderson void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 1192a7812ae4Spbrook { 1193b3a62939SRichard Henderson s->frame_start = start; 1194b3a62939SRichard Henderson s->frame_end = start + size; 1195085272b3SRichard Henderson s->frame_temp 1196085272b3SRichard Henderson = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 1197b3a62939SRichard Henderson } 1198a7812ae4Spbrook 1199085272b3SRichard Henderson TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 1200e1ccc054SRichard Henderson intptr_t offset, const char *name) 1201c896fe29Sbellard { 1202b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1203dc41aa7dSRichard Henderson TCGTemp *base_ts = tcgv_ptr_temp(base); 12047ca4b752SRichard Henderson TCGTemp *ts = tcg_global_alloc(s); 1205aef85402SRichard Henderson int indirect_reg = 0; 1206c896fe29Sbellard 1207c0522136SRichard Henderson switch (base_ts->kind) { 1208c0522136SRichard Henderson case TEMP_FIXED: 1209c0522136SRichard Henderson break; 1210c0522136SRichard Henderson case TEMP_GLOBAL: 12115a18407fSRichard Henderson /* We do not support double-indirect registers. */ 12125a18407fSRichard Henderson tcg_debug_assert(!base_ts->indirect_reg); 1213b3915dbbSRichard Henderson base_ts->indirect_base = 1; 12145a18407fSRichard Henderson s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 12155a18407fSRichard Henderson ? 2 : 1); 12165a18407fSRichard Henderson indirect_reg = 1; 1217c0522136SRichard Henderson break; 1218c0522136SRichard Henderson default: 1219c0522136SRichard Henderson g_assert_not_reached(); 1220b3915dbbSRichard Henderson } 1221b3915dbbSRichard Henderson 12227ca4b752SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 12237ca4b752SRichard Henderson TCGTemp *ts2 = tcg_global_alloc(s); 1224c896fe29Sbellard char buf[64]; 12257ca4b752SRichard Henderson 12267ca4b752SRichard Henderson ts->base_type = TCG_TYPE_I64; 1227c896fe29Sbellard ts->type = TCG_TYPE_I32; 1228b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 1229c896fe29Sbellard ts->mem_allocated = 1; 1230b3a62939SRichard Henderson ts->mem_base = base_ts; 1231aef85402SRichard Henderson ts->mem_offset = offset; 1232c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 1233c896fe29Sbellard pstrcat(buf, sizeof(buf), "_0"); 1234c896fe29Sbellard ts->name = strdup(buf); 1235c896fe29Sbellard 12367ca4b752SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 12377ca4b752SRichard Henderson ts2->base_type = TCG_TYPE_I64; 12387ca4b752SRichard Henderson ts2->type = TCG_TYPE_I32; 1239b3915dbbSRichard Henderson ts2->indirect_reg = indirect_reg; 12407ca4b752SRichard Henderson ts2->mem_allocated = 1; 12417ca4b752SRichard Henderson ts2->mem_base = base_ts; 1242aef85402SRichard Henderson ts2->mem_offset = offset + 4; 1243fac87bd2SRichard Henderson ts2->temp_subindex = 1; 1244c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 1245c896fe29Sbellard pstrcat(buf, sizeof(buf), "_1"); 1246120c1084SRichard Henderson ts2->name = strdup(buf); 12477ca4b752SRichard Henderson } else { 1248c896fe29Sbellard ts->base_type = type; 1249c896fe29Sbellard ts->type = type; 1250b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 1251c896fe29Sbellard ts->mem_allocated = 1; 1252b3a62939SRichard Henderson ts->mem_base = base_ts; 1253c896fe29Sbellard ts->mem_offset = offset; 1254c896fe29Sbellard ts->name = name; 1255c896fe29Sbellard } 1256085272b3SRichard Henderson return ts; 1257c896fe29Sbellard } 1258c896fe29Sbellard 1259bbf989bfSRichard Henderson TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) 1260c896fe29Sbellard { 1261b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1262c896fe29Sbellard TCGTemp *ts; 1263e1c08b00SRichard Henderson int n; 1264c896fe29Sbellard 1265e1c08b00SRichard Henderson if (kind == TEMP_EBB) { 1266e1c08b00SRichard Henderson int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); 1267e1c08b00SRichard Henderson 12680ec9eabcSRichard Henderson if (idx < TCG_MAX_TEMPS) { 12690ec9eabcSRichard Henderson /* There is already an available temp with the right type. */ 1270e1c08b00SRichard Henderson clear_bit(idx, s->free_temps[type].l); 12710ec9eabcSRichard Henderson 1272e8996ee0Sbellard ts = &s->temps[idx]; 1273e8996ee0Sbellard ts->temp_allocated = 1; 12747ca4b752SRichard Henderson tcg_debug_assert(ts->base_type == type); 1275ee17db83SRichard Henderson tcg_debug_assert(ts->kind == kind); 12762f2e911dSRichard Henderson return ts; 1277e1c08b00SRichard Henderson } 1278e8996ee0Sbellard } else { 1279e1c08b00SRichard Henderson tcg_debug_assert(kind == TEMP_TB); 1280e1c08b00SRichard Henderson } 128143eef72fSRichard Henderson 128243eef72fSRichard Henderson switch (type) { 128343eef72fSRichard Henderson case TCG_TYPE_I32: 128443eef72fSRichard Henderson case TCG_TYPE_V64: 128543eef72fSRichard Henderson case TCG_TYPE_V128: 128643eef72fSRichard Henderson case TCG_TYPE_V256: 128743eef72fSRichard Henderson n = 1; 128843eef72fSRichard Henderson break; 128943eef72fSRichard Henderson case TCG_TYPE_I64: 129043eef72fSRichard Henderson n = 64 / TCG_TARGET_REG_BITS; 129143eef72fSRichard Henderson break; 129243eef72fSRichard Henderson case TCG_TYPE_I128: 129343eef72fSRichard Henderson n = 128 / TCG_TARGET_REG_BITS; 129443eef72fSRichard Henderson break; 129543eef72fSRichard Henderson default: 129643eef72fSRichard Henderson g_assert_not_reached(); 129743eef72fSRichard Henderson } 129843eef72fSRichard Henderson 12997ca4b752SRichard Henderson ts = tcg_temp_alloc(s); 130043eef72fSRichard Henderson ts->base_type = type; 130143eef72fSRichard Henderson ts->temp_allocated = 1; 130243eef72fSRichard Henderson ts->kind = kind; 130343eef72fSRichard Henderson 130443eef72fSRichard Henderson if (n == 1) { 130543eef72fSRichard Henderson ts->type = type; 130643eef72fSRichard Henderson } else { 130743eef72fSRichard Henderson ts->type = TCG_TYPE_REG; 130843eef72fSRichard Henderson 1309e1c08b00SRichard Henderson for (int i = 1; i < n; ++i) { 13107ca4b752SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 13117ca4b752SRichard Henderson 131243eef72fSRichard Henderson tcg_debug_assert(ts2 == ts + i); 131343eef72fSRichard Henderson ts2->base_type = type; 131443eef72fSRichard Henderson ts2->type = TCG_TYPE_REG; 13157ca4b752SRichard Henderson ts2->temp_allocated = 1; 131643eef72fSRichard Henderson ts2->temp_subindex = i; 1317ee17db83SRichard Henderson ts2->kind = kind; 131843eef72fSRichard Henderson } 1319c896fe29Sbellard } 1320085272b3SRichard Henderson return ts; 1321c896fe29Sbellard } 1322c896fe29Sbellard 1323d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec(TCGType type) 1324d2fd745fSRichard Henderson { 1325d2fd745fSRichard Henderson TCGTemp *t; 1326d2fd745fSRichard Henderson 1327d2fd745fSRichard Henderson #ifdef CONFIG_DEBUG_TCG 1328d2fd745fSRichard Henderson switch (type) { 1329d2fd745fSRichard Henderson case TCG_TYPE_V64: 1330d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v64); 1331d2fd745fSRichard Henderson break; 1332d2fd745fSRichard Henderson case TCG_TYPE_V128: 1333d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v128); 1334d2fd745fSRichard Henderson break; 1335d2fd745fSRichard Henderson case TCG_TYPE_V256: 1336d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v256); 1337d2fd745fSRichard Henderson break; 1338d2fd745fSRichard Henderson default: 1339d2fd745fSRichard Henderson g_assert_not_reached(); 1340d2fd745fSRichard Henderson } 1341d2fd745fSRichard Henderson #endif 1342d2fd745fSRichard Henderson 1343bbf989bfSRichard Henderson t = tcg_temp_new_internal(type, TEMP_EBB); 1344d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1345d2fd745fSRichard Henderson } 1346d2fd745fSRichard Henderson 1347d2fd745fSRichard Henderson /* Create a new temp of the same type as an existing temp. */ 1348d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1349d2fd745fSRichard Henderson { 1350d2fd745fSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 1351d2fd745fSRichard Henderson 1352d2fd745fSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 1353d2fd745fSRichard Henderson 1354bbf989bfSRichard Henderson t = tcg_temp_new_internal(t->base_type, TEMP_EBB); 1355d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1356d2fd745fSRichard Henderson } 1357d2fd745fSRichard Henderson 13585bfa8034SRichard Henderson void tcg_temp_free_internal(TCGTemp *ts) 1359c896fe29Sbellard { 1360b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1361c896fe29Sbellard 1362c7482438SRichard Henderson switch (ts->kind) { 1363c7482438SRichard Henderson case TEMP_CONST: 1364f57c6915SRichard Henderson case TEMP_TB: 13652f2e911dSRichard Henderson /* Silently ignore free. */ 1366c7482438SRichard Henderson break; 13672f2e911dSRichard Henderson case TEMP_EBB: 1368eabb7b91SAurelien Jarno tcg_debug_assert(ts->temp_allocated != 0); 1369e8996ee0Sbellard ts->temp_allocated = 0; 13702f2e911dSRichard Henderson set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); 13712f2e911dSRichard Henderson break; 13722f2e911dSRichard Henderson default: 13732f2e911dSRichard Henderson /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */ 13742f2e911dSRichard Henderson g_assert_not_reached(); 1375e1c08b00SRichard Henderson } 1376e8996ee0Sbellard } 1377e8996ee0Sbellard 1378c0522136SRichard Henderson TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1379c0522136SRichard Henderson { 1380c0522136SRichard Henderson TCGContext *s = tcg_ctx; 1381c0522136SRichard Henderson GHashTable *h = s->const_table[type]; 1382c0522136SRichard Henderson TCGTemp *ts; 1383c0522136SRichard Henderson 1384c0522136SRichard Henderson if (h == NULL) { 1385c0522136SRichard Henderson h = g_hash_table_new(g_int64_hash, g_int64_equal); 1386c0522136SRichard Henderson s->const_table[type] = h; 1387c0522136SRichard Henderson } 1388c0522136SRichard Henderson 1389c0522136SRichard Henderson ts = g_hash_table_lookup(h, &val); 1390c0522136SRichard Henderson if (ts == NULL) { 1391aef85402SRichard Henderson int64_t *val_ptr; 1392aef85402SRichard Henderson 1393c0522136SRichard Henderson ts = tcg_temp_alloc(s); 1394c0522136SRichard Henderson 1395c0522136SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1396c0522136SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 1397c0522136SRichard Henderson 1398aef85402SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 1399aef85402SRichard Henderson 1400c0522136SRichard Henderson ts->base_type = TCG_TYPE_I64; 1401c0522136SRichard Henderson ts->type = TCG_TYPE_I32; 1402c0522136SRichard Henderson ts->kind = TEMP_CONST; 1403c0522136SRichard Henderson ts->temp_allocated = 1; 1404c0522136SRichard Henderson 1405c0522136SRichard Henderson ts2->base_type = TCG_TYPE_I64; 1406c0522136SRichard Henderson ts2->type = TCG_TYPE_I32; 1407c0522136SRichard Henderson ts2->kind = TEMP_CONST; 1408c0522136SRichard Henderson ts2->temp_allocated = 1; 1409fac87bd2SRichard Henderson ts2->temp_subindex = 1; 1410aef85402SRichard Henderson 1411aef85402SRichard Henderson /* 1412aef85402SRichard Henderson * Retain the full value of the 64-bit constant in the low 1413aef85402SRichard Henderson * part, so that the hash table works. Actual uses will 1414aef85402SRichard Henderson * truncate the value to the low part. 1415aef85402SRichard Henderson */ 1416aef85402SRichard Henderson ts[HOST_BIG_ENDIAN].val = val; 1417aef85402SRichard Henderson ts[!HOST_BIG_ENDIAN].val = val >> 32; 1418aef85402SRichard Henderson val_ptr = &ts[HOST_BIG_ENDIAN].val; 1419c0522136SRichard Henderson } else { 1420c0522136SRichard Henderson ts->base_type = type; 1421c0522136SRichard Henderson ts->type = type; 1422c0522136SRichard Henderson ts->kind = TEMP_CONST; 1423c0522136SRichard Henderson ts->temp_allocated = 1; 1424c0522136SRichard Henderson ts->val = val; 1425aef85402SRichard Henderson val_ptr = &ts->val; 1426c0522136SRichard Henderson } 1427aef85402SRichard Henderson g_hash_table_insert(h, val_ptr, ts); 1428c0522136SRichard Henderson } 1429c0522136SRichard Henderson 1430c0522136SRichard Henderson return ts; 1431c0522136SRichard Henderson } 1432c0522136SRichard Henderson 1433c0522136SRichard Henderson TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1434c0522136SRichard Henderson { 1435c0522136SRichard Henderson val = dup_const(vece, val); 1436c0522136SRichard Henderson return temp_tcgv_vec(tcg_constant_internal(type, val)); 1437c0522136SRichard Henderson } 1438c0522136SRichard Henderson 143988d4005bSRichard Henderson TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 144088d4005bSRichard Henderson { 144188d4005bSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 144288d4005bSRichard Henderson 144388d4005bSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 144488d4005bSRichard Henderson return tcg_constant_vec(t->base_type, vece, val); 144588d4005bSRichard Henderson } 144688d4005bSRichard Henderson 1447be0f34b5SRichard Henderson /* Return true if OP may appear in the opcode stream. 1448be0f34b5SRichard Henderson Test the runtime variable that controls each opcode. */ 1449be0f34b5SRichard Henderson bool tcg_op_supported(TCGOpcode op) 1450be0f34b5SRichard Henderson { 1451d2fd745fSRichard Henderson const bool have_vec 1452d2fd745fSRichard Henderson = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1453d2fd745fSRichard Henderson 1454be0f34b5SRichard Henderson switch (op) { 1455be0f34b5SRichard Henderson case INDEX_op_discard: 1456be0f34b5SRichard Henderson case INDEX_op_set_label: 1457be0f34b5SRichard Henderson case INDEX_op_call: 1458be0f34b5SRichard Henderson case INDEX_op_br: 1459be0f34b5SRichard Henderson case INDEX_op_mb: 1460be0f34b5SRichard Henderson case INDEX_op_insn_start: 1461be0f34b5SRichard Henderson case INDEX_op_exit_tb: 1462be0f34b5SRichard Henderson case INDEX_op_goto_tb: 1463f4e01e30SRichard Henderson case INDEX_op_goto_ptr: 1464be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i32: 1465be0f34b5SRichard Henderson case INDEX_op_qemu_st_i32: 1466be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i64: 1467be0f34b5SRichard Henderson case INDEX_op_qemu_st_i64: 1468be0f34b5SRichard Henderson return true; 1469be0f34b5SRichard Henderson 147007ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 147107ce0b05SRichard Henderson return TCG_TARGET_HAS_qemu_st8_i32; 147207ce0b05SRichard Henderson 1473be0f34b5SRichard Henderson case INDEX_op_mov_i32: 1474be0f34b5SRichard Henderson case INDEX_op_setcond_i32: 1475be0f34b5SRichard Henderson case INDEX_op_brcond_i32: 1476be0f34b5SRichard Henderson case INDEX_op_ld8u_i32: 1477be0f34b5SRichard Henderson case INDEX_op_ld8s_i32: 1478be0f34b5SRichard Henderson case INDEX_op_ld16u_i32: 1479be0f34b5SRichard Henderson case INDEX_op_ld16s_i32: 1480be0f34b5SRichard Henderson case INDEX_op_ld_i32: 1481be0f34b5SRichard Henderson case INDEX_op_st8_i32: 1482be0f34b5SRichard Henderson case INDEX_op_st16_i32: 1483be0f34b5SRichard Henderson case INDEX_op_st_i32: 1484be0f34b5SRichard Henderson case INDEX_op_add_i32: 1485be0f34b5SRichard Henderson case INDEX_op_sub_i32: 1486be0f34b5SRichard Henderson case INDEX_op_mul_i32: 1487be0f34b5SRichard Henderson case INDEX_op_and_i32: 1488be0f34b5SRichard Henderson case INDEX_op_or_i32: 1489be0f34b5SRichard Henderson case INDEX_op_xor_i32: 1490be0f34b5SRichard Henderson case INDEX_op_shl_i32: 1491be0f34b5SRichard Henderson case INDEX_op_shr_i32: 1492be0f34b5SRichard Henderson case INDEX_op_sar_i32: 1493be0f34b5SRichard Henderson return true; 1494be0f34b5SRichard Henderson 1495be0f34b5SRichard Henderson case INDEX_op_movcond_i32: 1496be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i32; 1497be0f34b5SRichard Henderson case INDEX_op_div_i32: 1498be0f34b5SRichard Henderson case INDEX_op_divu_i32: 1499be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i32; 1500be0f34b5SRichard Henderson case INDEX_op_rem_i32: 1501be0f34b5SRichard Henderson case INDEX_op_remu_i32: 1502be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i32; 1503be0f34b5SRichard Henderson case INDEX_op_div2_i32: 1504be0f34b5SRichard Henderson case INDEX_op_divu2_i32: 1505be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i32; 1506be0f34b5SRichard Henderson case INDEX_op_rotl_i32: 1507be0f34b5SRichard Henderson case INDEX_op_rotr_i32: 1508be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i32; 1509be0f34b5SRichard Henderson case INDEX_op_deposit_i32: 1510be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i32; 1511be0f34b5SRichard Henderson case INDEX_op_extract_i32: 1512be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i32; 1513be0f34b5SRichard Henderson case INDEX_op_sextract_i32: 1514be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i32; 1515fce1296fSRichard Henderson case INDEX_op_extract2_i32: 1516fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i32; 1517be0f34b5SRichard Henderson case INDEX_op_add2_i32: 1518be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i32; 1519be0f34b5SRichard Henderson case INDEX_op_sub2_i32: 1520be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i32; 1521be0f34b5SRichard Henderson case INDEX_op_mulu2_i32: 1522be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i32; 1523be0f34b5SRichard Henderson case INDEX_op_muls2_i32: 1524be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i32; 1525be0f34b5SRichard Henderson case INDEX_op_muluh_i32: 1526be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i32; 1527be0f34b5SRichard Henderson case INDEX_op_mulsh_i32: 1528be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i32; 1529be0f34b5SRichard Henderson case INDEX_op_ext8s_i32: 1530be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i32; 1531be0f34b5SRichard Henderson case INDEX_op_ext16s_i32: 1532be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i32; 1533be0f34b5SRichard Henderson case INDEX_op_ext8u_i32: 1534be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i32; 1535be0f34b5SRichard Henderson case INDEX_op_ext16u_i32: 1536be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i32; 1537be0f34b5SRichard Henderson case INDEX_op_bswap16_i32: 1538be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i32; 1539be0f34b5SRichard Henderson case INDEX_op_bswap32_i32: 1540be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i32; 1541be0f34b5SRichard Henderson case INDEX_op_not_i32: 1542be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i32; 1543be0f34b5SRichard Henderson case INDEX_op_neg_i32: 1544be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i32; 1545be0f34b5SRichard Henderson case INDEX_op_andc_i32: 1546be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i32; 1547be0f34b5SRichard Henderson case INDEX_op_orc_i32: 1548be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i32; 1549be0f34b5SRichard Henderson case INDEX_op_eqv_i32: 1550be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i32; 1551be0f34b5SRichard Henderson case INDEX_op_nand_i32: 1552be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i32; 1553be0f34b5SRichard Henderson case INDEX_op_nor_i32: 1554be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i32; 1555be0f34b5SRichard Henderson case INDEX_op_clz_i32: 1556be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i32; 1557be0f34b5SRichard Henderson case INDEX_op_ctz_i32: 1558be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i32; 1559be0f34b5SRichard Henderson case INDEX_op_ctpop_i32: 1560be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i32; 1561be0f34b5SRichard Henderson 1562be0f34b5SRichard Henderson case INDEX_op_brcond2_i32: 1563be0f34b5SRichard Henderson case INDEX_op_setcond2_i32: 1564be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 32; 1565be0f34b5SRichard Henderson 1566be0f34b5SRichard Henderson case INDEX_op_mov_i64: 1567be0f34b5SRichard Henderson case INDEX_op_setcond_i64: 1568be0f34b5SRichard Henderson case INDEX_op_brcond_i64: 1569be0f34b5SRichard Henderson case INDEX_op_ld8u_i64: 1570be0f34b5SRichard Henderson case INDEX_op_ld8s_i64: 1571be0f34b5SRichard Henderson case INDEX_op_ld16u_i64: 1572be0f34b5SRichard Henderson case INDEX_op_ld16s_i64: 1573be0f34b5SRichard Henderson case INDEX_op_ld32u_i64: 1574be0f34b5SRichard Henderson case INDEX_op_ld32s_i64: 1575be0f34b5SRichard Henderson case INDEX_op_ld_i64: 1576be0f34b5SRichard Henderson case INDEX_op_st8_i64: 1577be0f34b5SRichard Henderson case INDEX_op_st16_i64: 1578be0f34b5SRichard Henderson case INDEX_op_st32_i64: 1579be0f34b5SRichard Henderson case INDEX_op_st_i64: 1580be0f34b5SRichard Henderson case INDEX_op_add_i64: 1581be0f34b5SRichard Henderson case INDEX_op_sub_i64: 1582be0f34b5SRichard Henderson case INDEX_op_mul_i64: 1583be0f34b5SRichard Henderson case INDEX_op_and_i64: 1584be0f34b5SRichard Henderson case INDEX_op_or_i64: 1585be0f34b5SRichard Henderson case INDEX_op_xor_i64: 1586be0f34b5SRichard Henderson case INDEX_op_shl_i64: 1587be0f34b5SRichard Henderson case INDEX_op_shr_i64: 1588be0f34b5SRichard Henderson case INDEX_op_sar_i64: 1589be0f34b5SRichard Henderson case INDEX_op_ext_i32_i64: 1590be0f34b5SRichard Henderson case INDEX_op_extu_i32_i64: 1591be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 64; 1592be0f34b5SRichard Henderson 1593be0f34b5SRichard Henderson case INDEX_op_movcond_i64: 1594be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i64; 1595be0f34b5SRichard Henderson case INDEX_op_div_i64: 1596be0f34b5SRichard Henderson case INDEX_op_divu_i64: 1597be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i64; 1598be0f34b5SRichard Henderson case INDEX_op_rem_i64: 1599be0f34b5SRichard Henderson case INDEX_op_remu_i64: 1600be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i64; 1601be0f34b5SRichard Henderson case INDEX_op_div2_i64: 1602be0f34b5SRichard Henderson case INDEX_op_divu2_i64: 1603be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i64; 1604be0f34b5SRichard Henderson case INDEX_op_rotl_i64: 1605be0f34b5SRichard Henderson case INDEX_op_rotr_i64: 1606be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i64; 1607be0f34b5SRichard Henderson case INDEX_op_deposit_i64: 1608be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i64; 1609be0f34b5SRichard Henderson case INDEX_op_extract_i64: 1610be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i64; 1611be0f34b5SRichard Henderson case INDEX_op_sextract_i64: 1612be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i64; 1613fce1296fSRichard Henderson case INDEX_op_extract2_i64: 1614fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i64; 1615be0f34b5SRichard Henderson case INDEX_op_extrl_i64_i32: 1616be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrl_i64_i32; 1617be0f34b5SRichard Henderson case INDEX_op_extrh_i64_i32: 1618be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrh_i64_i32; 1619be0f34b5SRichard Henderson case INDEX_op_ext8s_i64: 1620be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i64; 1621be0f34b5SRichard Henderson case INDEX_op_ext16s_i64: 1622be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i64; 1623be0f34b5SRichard Henderson case INDEX_op_ext32s_i64: 1624be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32s_i64; 1625be0f34b5SRichard Henderson case INDEX_op_ext8u_i64: 1626be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i64; 1627be0f34b5SRichard Henderson case INDEX_op_ext16u_i64: 1628be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i64; 1629be0f34b5SRichard Henderson case INDEX_op_ext32u_i64: 1630be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32u_i64; 1631be0f34b5SRichard Henderson case INDEX_op_bswap16_i64: 1632be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i64; 1633be0f34b5SRichard Henderson case INDEX_op_bswap32_i64: 1634be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i64; 1635be0f34b5SRichard Henderson case INDEX_op_bswap64_i64: 1636be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap64_i64; 1637be0f34b5SRichard Henderson case INDEX_op_not_i64: 1638be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i64; 1639be0f34b5SRichard Henderson case INDEX_op_neg_i64: 1640be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i64; 1641be0f34b5SRichard Henderson case INDEX_op_andc_i64: 1642be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i64; 1643be0f34b5SRichard Henderson case INDEX_op_orc_i64: 1644be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i64; 1645be0f34b5SRichard Henderson case INDEX_op_eqv_i64: 1646be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i64; 1647be0f34b5SRichard Henderson case INDEX_op_nand_i64: 1648be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i64; 1649be0f34b5SRichard Henderson case INDEX_op_nor_i64: 1650be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i64; 1651be0f34b5SRichard Henderson case INDEX_op_clz_i64: 1652be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i64; 1653be0f34b5SRichard Henderson case INDEX_op_ctz_i64: 1654be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i64; 1655be0f34b5SRichard Henderson case INDEX_op_ctpop_i64: 1656be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i64; 1657be0f34b5SRichard Henderson case INDEX_op_add2_i64: 1658be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i64; 1659be0f34b5SRichard Henderson case INDEX_op_sub2_i64: 1660be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i64; 1661be0f34b5SRichard Henderson case INDEX_op_mulu2_i64: 1662be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i64; 1663be0f34b5SRichard Henderson case INDEX_op_muls2_i64: 1664be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i64; 1665be0f34b5SRichard Henderson case INDEX_op_muluh_i64: 1666be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i64; 1667be0f34b5SRichard Henderson case INDEX_op_mulsh_i64: 1668be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i64; 1669be0f34b5SRichard Henderson 1670d2fd745fSRichard Henderson case INDEX_op_mov_vec: 1671d2fd745fSRichard Henderson case INDEX_op_dup_vec: 167237ee55a0SRichard Henderson case INDEX_op_dupm_vec: 1673d2fd745fSRichard Henderson case INDEX_op_ld_vec: 1674d2fd745fSRichard Henderson case INDEX_op_st_vec: 1675d2fd745fSRichard Henderson case INDEX_op_add_vec: 1676d2fd745fSRichard Henderson case INDEX_op_sub_vec: 1677d2fd745fSRichard Henderson case INDEX_op_and_vec: 1678d2fd745fSRichard Henderson case INDEX_op_or_vec: 1679d2fd745fSRichard Henderson case INDEX_op_xor_vec: 1680212be173SRichard Henderson case INDEX_op_cmp_vec: 1681d2fd745fSRichard Henderson return have_vec; 1682d2fd745fSRichard Henderson case INDEX_op_dup2_vec: 1683d2fd745fSRichard Henderson return have_vec && TCG_TARGET_REG_BITS == 32; 1684d2fd745fSRichard Henderson case INDEX_op_not_vec: 1685d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_not_vec; 1686d2fd745fSRichard Henderson case INDEX_op_neg_vec: 1687d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_neg_vec; 1688bcefc902SRichard Henderson case INDEX_op_abs_vec: 1689bcefc902SRichard Henderson return have_vec && TCG_TARGET_HAS_abs_vec; 1690d2fd745fSRichard Henderson case INDEX_op_andc_vec: 1691d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_andc_vec; 1692d2fd745fSRichard Henderson case INDEX_op_orc_vec: 1693d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_orc_vec; 1694ed523473SRichard Henderson case INDEX_op_nand_vec: 1695ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nand_vec; 1696ed523473SRichard Henderson case INDEX_op_nor_vec: 1697ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nor_vec; 1698ed523473SRichard Henderson case INDEX_op_eqv_vec: 1699ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_eqv_vec; 17003774030aSRichard Henderson case INDEX_op_mul_vec: 17013774030aSRichard Henderson return have_vec && TCG_TARGET_HAS_mul_vec; 1702d0ec9796SRichard Henderson case INDEX_op_shli_vec: 1703d0ec9796SRichard Henderson case INDEX_op_shri_vec: 1704d0ec9796SRichard Henderson case INDEX_op_sari_vec: 1705d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shi_vec; 1706d0ec9796SRichard Henderson case INDEX_op_shls_vec: 1707d0ec9796SRichard Henderson case INDEX_op_shrs_vec: 1708d0ec9796SRichard Henderson case INDEX_op_sars_vec: 1709d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shs_vec; 1710d0ec9796SRichard Henderson case INDEX_op_shlv_vec: 1711d0ec9796SRichard Henderson case INDEX_op_shrv_vec: 1712d0ec9796SRichard Henderson case INDEX_op_sarv_vec: 1713d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shv_vec; 1714b0f7e744SRichard Henderson case INDEX_op_rotli_vec: 1715b0f7e744SRichard Henderson return have_vec && TCG_TARGET_HAS_roti_vec; 171623850a74SRichard Henderson case INDEX_op_rotls_vec: 171723850a74SRichard Henderson return have_vec && TCG_TARGET_HAS_rots_vec; 17185d0ceda9SRichard Henderson case INDEX_op_rotlv_vec: 17195d0ceda9SRichard Henderson case INDEX_op_rotrv_vec: 17205d0ceda9SRichard Henderson return have_vec && TCG_TARGET_HAS_rotv_vec; 17218afaf050SRichard Henderson case INDEX_op_ssadd_vec: 17228afaf050SRichard Henderson case INDEX_op_usadd_vec: 17238afaf050SRichard Henderson case INDEX_op_sssub_vec: 17248afaf050SRichard Henderson case INDEX_op_ussub_vec: 17258afaf050SRichard Henderson return have_vec && TCG_TARGET_HAS_sat_vec; 1726dd0a0fcdSRichard Henderson case INDEX_op_smin_vec: 1727dd0a0fcdSRichard Henderson case INDEX_op_umin_vec: 1728dd0a0fcdSRichard Henderson case INDEX_op_smax_vec: 1729dd0a0fcdSRichard Henderson case INDEX_op_umax_vec: 1730dd0a0fcdSRichard Henderson return have_vec && TCG_TARGET_HAS_minmax_vec; 173138dc1294SRichard Henderson case INDEX_op_bitsel_vec: 173238dc1294SRichard Henderson return have_vec && TCG_TARGET_HAS_bitsel_vec; 1733f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 1734f75da298SRichard Henderson return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1735d2fd745fSRichard Henderson 1736db432672SRichard Henderson default: 1737db432672SRichard Henderson tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1738db432672SRichard Henderson return true; 1739be0f34b5SRichard Henderson } 1740be0f34b5SRichard Henderson } 1741be0f34b5SRichard Henderson 174239004a71SRichard Henderson static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); 174339004a71SRichard Henderson 1744ae8b75dcSRichard Henderson void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1745c896fe29Sbellard { 17463e92aa34SRichard Henderson const TCGHelperInfo *info; 174739004a71SRichard Henderson TCGv_i64 extend_free[MAX_CALL_IARGS]; 174839004a71SRichard Henderson int n_extend = 0; 174975e8b9b7SRichard Henderson TCGOp *op; 175039004a71SRichard Henderson int i, n, pi = 0, total_args; 1751afb49896SRichard Henderson 1752619205fdSEmilio G. Cota info = g_hash_table_lookup(helper_table, (gpointer)func); 175339004a71SRichard Henderson total_args = info->nr_out + info->nr_in + 2; 175439004a71SRichard Henderson op = tcg_op_alloc(INDEX_op_call, total_args); 17552bece2c8SRichard Henderson 175638b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 175717083f6fSEmilio Cota /* Flag helpers that may affect guest state */ 175817083f6fSEmilio Cota if (tcg_ctx->plugin_insn && 175917083f6fSEmilio Cota !(info->flags & TCG_CALL_PLUGIN) && 176017083f6fSEmilio Cota !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { 176138b47b19SEmilio G. Cota tcg_ctx->plugin_insn->calls_helpers = true; 176238b47b19SEmilio G. Cota } 176338b47b19SEmilio G. Cota #endif 176438b47b19SEmilio G. Cota 176539004a71SRichard Henderson TCGOP_CALLO(op) = n = info->nr_out; 176639004a71SRichard Henderson switch (n) { 176739004a71SRichard Henderson case 0: 176839004a71SRichard Henderson tcg_debug_assert(ret == NULL); 176939004a71SRichard Henderson break; 177039004a71SRichard Henderson case 1: 177139004a71SRichard Henderson tcg_debug_assert(ret != NULL); 177239004a71SRichard Henderson op->args[pi++] = temp_arg(ret); 177339004a71SRichard Henderson break; 177439004a71SRichard Henderson case 2: 1775466d3759SRichard Henderson case 4: 177639004a71SRichard Henderson tcg_debug_assert(ret != NULL); 1777466d3759SRichard Henderson tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); 177839004a71SRichard Henderson tcg_debug_assert(ret->temp_subindex == 0); 1779466d3759SRichard Henderson for (i = 0; i < n; ++i) { 1780466d3759SRichard Henderson op->args[pi++] = temp_arg(ret + i); 1781466d3759SRichard Henderson } 178239004a71SRichard Henderson break; 178339004a71SRichard Henderson default: 178439004a71SRichard Henderson g_assert_not_reached(); 178539004a71SRichard Henderson } 17867319d83aSRichard Henderson 178739004a71SRichard Henderson TCGOP_CALLI(op) = n = info->nr_in; 178839004a71SRichard Henderson for (i = 0; i < n; i++) { 178939004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 179039004a71SRichard Henderson TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; 179139004a71SRichard Henderson 179239004a71SRichard Henderson switch (loc->kind) { 179339004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 1794313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 1795313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 179639004a71SRichard Henderson op->args[pi++] = temp_arg(ts); 179739004a71SRichard Henderson break; 179839004a71SRichard Henderson 179939004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 180039004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 180139004a71SRichard Henderson { 18025dd48602SRichard Henderson TCGv_i64 temp = tcg_temp_ebb_new_i64(); 180339004a71SRichard Henderson TCGv_i32 orig = temp_tcgv_i32(ts); 180439004a71SRichard Henderson 180539004a71SRichard Henderson if (loc->kind == TCG_CALL_ARG_EXTEND_S) { 180618cf3d07SRichard Henderson tcg_gen_ext_i32_i64(temp, orig); 18072bece2c8SRichard Henderson } else { 180818cf3d07SRichard Henderson tcg_gen_extu_i32_i64(temp, orig); 18092bece2c8SRichard Henderson } 181039004a71SRichard Henderson op->args[pi++] = tcgv_i64_arg(temp); 181139004a71SRichard Henderson extend_free[n_extend++] = temp; 18122bece2c8SRichard Henderson } 181339004a71SRichard Henderson break; 18142bece2c8SRichard Henderson 1815e2a9dd6bSRichard Henderson default: 1816e2a9dd6bSRichard Henderson g_assert_not_reached(); 1817e2a9dd6bSRichard Henderson } 1818c896fe29Sbellard } 181975e8b9b7SRichard Henderson op->args[pi++] = (uintptr_t)func; 18203e92aa34SRichard Henderson op->args[pi++] = (uintptr_t)info; 182139004a71SRichard Henderson tcg_debug_assert(pi == total_args); 1822a7812ae4Spbrook 182339004a71SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 18242bece2c8SRichard Henderson 182539004a71SRichard Henderson tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); 182639004a71SRichard Henderson for (i = 0; i < n_extend; ++i) { 182739004a71SRichard Henderson tcg_temp_free_i64(extend_free[i]); 1828eb8b0224SRichard Henderson } 1829a7812ae4Spbrook } 1830c896fe29Sbellard 18318fcd3692Sblueswir1 static void tcg_reg_alloc_start(TCGContext *s) 1832c896fe29Sbellard { 1833ac3b8891SRichard Henderson int i, n; 1834ac3b8891SRichard Henderson 1835ee17db83SRichard Henderson for (i = 0, n = s->nb_temps; i < n; i++) { 1836ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 1837ee17db83SRichard Henderson TCGTempVal val = TEMP_VAL_MEM; 1838ee17db83SRichard Henderson 1839ee17db83SRichard Henderson switch (ts->kind) { 1840c0522136SRichard Henderson case TEMP_CONST: 1841c0522136SRichard Henderson val = TEMP_VAL_CONST; 1842c0522136SRichard Henderson break; 1843ee17db83SRichard Henderson case TEMP_FIXED: 1844ee17db83SRichard Henderson val = TEMP_VAL_REG; 1845ee17db83SRichard Henderson break; 1846ee17db83SRichard Henderson case TEMP_GLOBAL: 1847ee17db83SRichard Henderson break; 1848c7482438SRichard Henderson case TEMP_EBB: 1849ee17db83SRichard Henderson val = TEMP_VAL_DEAD; 1850ee17db83SRichard Henderson /* fall through */ 1851f57c6915SRichard Henderson case TEMP_TB: 1852e8996ee0Sbellard ts->mem_allocated = 0; 1853ee17db83SRichard Henderson break; 1854ee17db83SRichard Henderson default: 1855ee17db83SRichard Henderson g_assert_not_reached(); 1856ee17db83SRichard Henderson } 1857ee17db83SRichard Henderson ts->val_type = val; 1858e8996ee0Sbellard } 1859f8b2f202SRichard Henderson 1860f8b2f202SRichard Henderson memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1861c896fe29Sbellard } 1862c896fe29Sbellard 1863f8b2f202SRichard Henderson static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1864f8b2f202SRichard Henderson TCGTemp *ts) 1865c896fe29Sbellard { 18661807f4c4SRichard Henderson int idx = temp_idx(ts); 1867ac56dd48Spbrook 1868ee17db83SRichard Henderson switch (ts->kind) { 1869ee17db83SRichard Henderson case TEMP_FIXED: 1870ee17db83SRichard Henderson case TEMP_GLOBAL: 1871ac56dd48Spbrook pstrcpy(buf, buf_size, ts->name); 1872ee17db83SRichard Henderson break; 1873f57c6915SRichard Henderson case TEMP_TB: 1874641d5fbeSbellard snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1875ee17db83SRichard Henderson break; 1876c7482438SRichard Henderson case TEMP_EBB: 1877ac56dd48Spbrook snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1878ee17db83SRichard Henderson break; 1879c0522136SRichard Henderson case TEMP_CONST: 1880c0522136SRichard Henderson switch (ts->type) { 1881c0522136SRichard Henderson case TCG_TYPE_I32: 1882c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1883c0522136SRichard Henderson break; 1884c0522136SRichard Henderson #if TCG_TARGET_REG_BITS > 32 1885c0522136SRichard Henderson case TCG_TYPE_I64: 1886c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1887c0522136SRichard Henderson break; 1888c0522136SRichard Henderson #endif 1889c0522136SRichard Henderson case TCG_TYPE_V64: 1890c0522136SRichard Henderson case TCG_TYPE_V128: 1891c0522136SRichard Henderson case TCG_TYPE_V256: 1892c0522136SRichard Henderson snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1893c0522136SRichard Henderson 64 << (ts->type - TCG_TYPE_V64), ts->val); 1894c0522136SRichard Henderson break; 1895c0522136SRichard Henderson default: 1896c0522136SRichard Henderson g_assert_not_reached(); 1897c0522136SRichard Henderson } 1898c0522136SRichard Henderson break; 1899c896fe29Sbellard } 1900c896fe29Sbellard return buf; 1901c896fe29Sbellard } 1902c896fe29Sbellard 190343439139SRichard Henderson static char *tcg_get_arg_str(TCGContext *s, char *buf, 190443439139SRichard Henderson int buf_size, TCGArg arg) 1905f8b2f202SRichard Henderson { 190643439139SRichard Henderson return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1907f8b2f202SRichard Henderson } 1908f8b2f202SRichard Henderson 1909f48f3edeSblueswir1 static const char * const cond_name[] = 1910f48f3edeSblueswir1 { 19110aed257fSRichard Henderson [TCG_COND_NEVER] = "never", 19120aed257fSRichard Henderson [TCG_COND_ALWAYS] = "always", 1913f48f3edeSblueswir1 [TCG_COND_EQ] = "eq", 1914f48f3edeSblueswir1 [TCG_COND_NE] = "ne", 1915f48f3edeSblueswir1 [TCG_COND_LT] = "lt", 1916f48f3edeSblueswir1 [TCG_COND_GE] = "ge", 1917f48f3edeSblueswir1 [TCG_COND_LE] = "le", 1918f48f3edeSblueswir1 [TCG_COND_GT] = "gt", 1919f48f3edeSblueswir1 [TCG_COND_LTU] = "ltu", 1920f48f3edeSblueswir1 [TCG_COND_GEU] = "geu", 1921f48f3edeSblueswir1 [TCG_COND_LEU] = "leu", 1922f48f3edeSblueswir1 [TCG_COND_GTU] = "gtu" 1923f48f3edeSblueswir1 }; 1924f48f3edeSblueswir1 1925f713d6adSRichard Henderson static const char * const ldst_name[] = 1926f713d6adSRichard Henderson { 1927f713d6adSRichard Henderson [MO_UB] = "ub", 1928f713d6adSRichard Henderson [MO_SB] = "sb", 1929f713d6adSRichard Henderson [MO_LEUW] = "leuw", 1930f713d6adSRichard Henderson [MO_LESW] = "lesw", 1931f713d6adSRichard Henderson [MO_LEUL] = "leul", 1932f713d6adSRichard Henderson [MO_LESL] = "lesl", 1933fc313c64SFrédéric Pétrot [MO_LEUQ] = "leq", 1934f713d6adSRichard Henderson [MO_BEUW] = "beuw", 1935f713d6adSRichard Henderson [MO_BESW] = "besw", 1936f713d6adSRichard Henderson [MO_BEUL] = "beul", 1937f713d6adSRichard Henderson [MO_BESL] = "besl", 1938fc313c64SFrédéric Pétrot [MO_BEUQ] = "beq", 1939f713d6adSRichard Henderson }; 1940f713d6adSRichard Henderson 19411f00b27fSSergey Sorokin static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 194252bf9771Stony.nguyen@bt.com #ifdef TARGET_ALIGNED_ONLY 19431f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "un+", 19441f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "", 19451f00b27fSSergey Sorokin #else 19461f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "", 19471f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "al+", 19481f00b27fSSergey Sorokin #endif 19491f00b27fSSergey Sorokin [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 19501f00b27fSSergey Sorokin [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 19511f00b27fSSergey Sorokin [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 19521f00b27fSSergey Sorokin [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 19531f00b27fSSergey Sorokin [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 19541f00b27fSSergey Sorokin [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 19551f00b27fSSergey Sorokin }; 19561f00b27fSSergey Sorokin 1957587195bdSRichard Henderson static const char bswap_flag_name[][6] = { 1958587195bdSRichard Henderson [TCG_BSWAP_IZ] = "iz", 1959587195bdSRichard Henderson [TCG_BSWAP_OZ] = "oz", 1960587195bdSRichard Henderson [TCG_BSWAP_OS] = "os", 1961587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1962587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1963587195bdSRichard Henderson }; 1964587195bdSRichard Henderson 1965b016486eSRichard Henderson static inline bool tcg_regset_single(TCGRegSet d) 1966b016486eSRichard Henderson { 1967b016486eSRichard Henderson return (d & (d - 1)) == 0; 1968b016486eSRichard Henderson } 1969b016486eSRichard Henderson 1970b016486eSRichard Henderson static inline TCGReg tcg_regset_first(TCGRegSet d) 1971b016486eSRichard Henderson { 1972b016486eSRichard Henderson if (TCG_TARGET_NB_REGS <= 32) { 1973b016486eSRichard Henderson return ctz32(d); 1974b016486eSRichard Henderson } else { 1975b016486eSRichard Henderson return ctz64(d); 1976b016486eSRichard Henderson } 1977b016486eSRichard Henderson } 1978b016486eSRichard Henderson 1979b7a83ff8SRichard Henderson /* Return only the number of characters output -- no error return. */ 1980b7a83ff8SRichard Henderson #define ne_fprintf(...) \ 1981b7a83ff8SRichard Henderson ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 1982b7a83ff8SRichard Henderson 1983b7a83ff8SRichard Henderson static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 1984c896fe29Sbellard { 1985c896fe29Sbellard char buf[128]; 1986c45cb8bbSRichard Henderson TCGOp *op; 1987c896fe29Sbellard 198815fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 1989c45cb8bbSRichard Henderson int i, k, nb_oargs, nb_iargs, nb_cargs; 1990c45cb8bbSRichard Henderson const TCGOpDef *def; 1991c45cb8bbSRichard Henderson TCGOpcode c; 1992bdfb460eSRichard Henderson int col = 0; 1993c45cb8bbSRichard Henderson 1994c45cb8bbSRichard Henderson c = op->opc; 1995c896fe29Sbellard def = &tcg_op_defs[c]; 1996c45cb8bbSRichard Henderson 1997765b842aSRichard Henderson if (c == INDEX_op_insn_start) { 1998b016486eSRichard Henderson nb_oargs = 0; 1999b7a83ff8SRichard Henderson col += ne_fprintf(f, "\n ----"); 20009aef40edSRichard Henderson 20019aef40edSRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 20029aef40edSRichard Henderson target_ulong a; 20037e4597d7Sbellard #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 2004efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 20057e4597d7Sbellard #else 2006efee3746SRichard Henderson a = op->args[i]; 20077e4597d7Sbellard #endif 2008b7a83ff8SRichard Henderson col += ne_fprintf(f, " " TARGET_FMT_lx, a); 2009eeacee4dSBlue Swirl } 20107e4597d7Sbellard } else if (c == INDEX_op_call) { 20113e92aa34SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 2012fa52e660SRichard Henderson void *func = tcg_call_func(op); 20133e92aa34SRichard Henderson 2014c896fe29Sbellard /* variable number of arguments */ 2015cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 2016cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 2017c896fe29Sbellard nb_cargs = def->nb_cargs; 2018b03cce8eSbellard 2019b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name); 20203e92aa34SRichard Henderson 20213e92aa34SRichard Henderson /* 20223e92aa34SRichard Henderson * Print the function name from TCGHelperInfo, if available. 20233e92aa34SRichard Henderson * Note that plugins have a template function for the info, 20243e92aa34SRichard Henderson * but the actual function pointer comes from the plugin. 20253e92aa34SRichard Henderson */ 20263e92aa34SRichard Henderson if (func == info->func) { 2027b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s", info->name); 20283e92aa34SRichard Henderson } else { 2029b7a83ff8SRichard Henderson col += ne_fprintf(f, "plugin(%p)", func); 20303e92aa34SRichard Henderson } 20313e92aa34SRichard Henderson 2032b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 2033b03cce8eSbellard for (i = 0; i < nb_oargs; i++) { 2034b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 2035efee3746SRichard Henderson op->args[i])); 2036b03cce8eSbellard } 2037cf066674SRichard Henderson for (i = 0; i < nb_iargs; i++) { 2038efee3746SRichard Henderson TCGArg arg = op->args[nb_oargs + i]; 203939004a71SRichard Henderson const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 2040b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", t); 2041e8996ee0Sbellard } 2042b03cce8eSbellard } else { 2043b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name); 2044c45cb8bbSRichard Henderson 2045c896fe29Sbellard nb_oargs = def->nb_oargs; 2046c896fe29Sbellard nb_iargs = def->nb_iargs; 2047c896fe29Sbellard nb_cargs = def->nb_cargs; 2048c896fe29Sbellard 2049d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 2050b7a83ff8SRichard Henderson col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op), 2051d2fd745fSRichard Henderson 8 << TCGOP_VECE(op)); 2052d2fd745fSRichard Henderson } 2053d2fd745fSRichard Henderson 2054c896fe29Sbellard k = 0; 2055c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 2056b7a83ff8SRichard Henderson const char *sep = k ? "," : ""; 2057b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep, 2058b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf), 2059efee3746SRichard Henderson op->args[k++])); 2060c896fe29Sbellard } 2061c896fe29Sbellard for (i = 0; i < nb_iargs; i++) { 2062b7a83ff8SRichard Henderson const char *sep = k ? "," : ""; 2063b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep, 2064b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf), 2065efee3746SRichard Henderson op->args[k++])); 2066c896fe29Sbellard } 2067be210acbSRichard Henderson switch (c) { 2068be210acbSRichard Henderson case INDEX_op_brcond_i32: 2069ffc5ea09SRichard Henderson case INDEX_op_setcond_i32: 2070ffc5ea09SRichard Henderson case INDEX_op_movcond_i32: 2071be210acbSRichard Henderson case INDEX_op_brcond2_i32: 2072be210acbSRichard Henderson case INDEX_op_setcond2_i32: 2073ffc5ea09SRichard Henderson case INDEX_op_brcond_i64: 2074be210acbSRichard Henderson case INDEX_op_setcond_i64: 2075ffc5ea09SRichard Henderson case INDEX_op_movcond_i64: 2076212be173SRichard Henderson case INDEX_op_cmp_vec: 2077f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 2078efee3746SRichard Henderson if (op->args[k] < ARRAY_SIZE(cond_name) 2079efee3746SRichard Henderson && cond_name[op->args[k]]) { 2080b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 2081eeacee4dSBlue Swirl } else { 2082b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 2083eeacee4dSBlue Swirl } 2084f48f3edeSblueswir1 i = 1; 2085be210acbSRichard Henderson break; 2086f713d6adSRichard Henderson case INDEX_op_qemu_ld_i32: 2087f713d6adSRichard Henderson case INDEX_op_qemu_st_i32: 208807ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 2089f713d6adSRichard Henderson case INDEX_op_qemu_ld_i64: 2090f713d6adSRichard Henderson case INDEX_op_qemu_st_i64: 209159227d5dSRichard Henderson { 20929002ffcbSRichard Henderson MemOpIdx oi = op->args[k++]; 209314776ab5STony Nguyen MemOp op = get_memop(oi); 209459227d5dSRichard Henderson unsigned ix = get_mmuidx(oi); 209559227d5dSRichard Henderson 209659c4b7e8SRichard Henderson if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 2097b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%x,%u", op, ix); 209859c4b7e8SRichard Henderson } else { 20991f00b27fSSergey Sorokin const char *s_al, *s_op; 21001f00b27fSSergey Sorokin s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 210159c4b7e8SRichard Henderson s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 2102b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix); 2103f713d6adSRichard Henderson } 2104f713d6adSRichard Henderson i = 1; 210559227d5dSRichard Henderson } 2106f713d6adSRichard Henderson break; 2107587195bdSRichard Henderson case INDEX_op_bswap16_i32: 2108587195bdSRichard Henderson case INDEX_op_bswap16_i64: 2109587195bdSRichard Henderson case INDEX_op_bswap32_i32: 2110587195bdSRichard Henderson case INDEX_op_bswap32_i64: 2111587195bdSRichard Henderson case INDEX_op_bswap64_i64: 2112587195bdSRichard Henderson { 2113587195bdSRichard Henderson TCGArg flags = op->args[k]; 2114587195bdSRichard Henderson const char *name = NULL; 2115587195bdSRichard Henderson 2116587195bdSRichard Henderson if (flags < ARRAY_SIZE(bswap_flag_name)) { 2117587195bdSRichard Henderson name = bswap_flag_name[flags]; 2118587195bdSRichard Henderson } 2119587195bdSRichard Henderson if (name) { 2120b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", name); 2121587195bdSRichard Henderson } else { 2122b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 2123587195bdSRichard Henderson } 2124587195bdSRichard Henderson i = k = 1; 2125587195bdSRichard Henderson } 2126587195bdSRichard Henderson break; 2127be210acbSRichard Henderson default: 2128f48f3edeSblueswir1 i = 0; 2129be210acbSRichard Henderson break; 2130be210acbSRichard Henderson } 213151e3972cSRichard Henderson switch (c) { 213251e3972cSRichard Henderson case INDEX_op_set_label: 213351e3972cSRichard Henderson case INDEX_op_br: 213451e3972cSRichard Henderson case INDEX_op_brcond_i32: 213551e3972cSRichard Henderson case INDEX_op_brcond_i64: 213651e3972cSRichard Henderson case INDEX_op_brcond2_i32: 2137b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$L%d", k ? "," : "", 2138efee3746SRichard Henderson arg_label(op->args[k])->id); 213951e3972cSRichard Henderson i++, k++; 214051e3972cSRichard Henderson break; 21413470867bSRichard Henderson case INDEX_op_mb: 21423470867bSRichard Henderson { 21433470867bSRichard Henderson TCGBar membar = op->args[k]; 21443470867bSRichard Henderson const char *b_op, *m_op; 21453470867bSRichard Henderson 21463470867bSRichard Henderson switch (membar & TCG_BAR_SC) { 21473470867bSRichard Henderson case 0: 21483470867bSRichard Henderson b_op = "none"; 21493470867bSRichard Henderson break; 21503470867bSRichard Henderson case TCG_BAR_LDAQ: 21513470867bSRichard Henderson b_op = "acq"; 21523470867bSRichard Henderson break; 21533470867bSRichard Henderson case TCG_BAR_STRL: 21543470867bSRichard Henderson b_op = "rel"; 21553470867bSRichard Henderson break; 21563470867bSRichard Henderson case TCG_BAR_SC: 21573470867bSRichard Henderson b_op = "seq"; 21583470867bSRichard Henderson break; 21593470867bSRichard Henderson default: 21603470867bSRichard Henderson g_assert_not_reached(); 21613470867bSRichard Henderson } 21623470867bSRichard Henderson 21633470867bSRichard Henderson switch (membar & TCG_MO_ALL) { 21643470867bSRichard Henderson case 0: 21653470867bSRichard Henderson m_op = "none"; 21663470867bSRichard Henderson break; 21673470867bSRichard Henderson case TCG_MO_LD_LD: 21683470867bSRichard Henderson m_op = "rr"; 21693470867bSRichard Henderson break; 21703470867bSRichard Henderson case TCG_MO_LD_ST: 21713470867bSRichard Henderson m_op = "rw"; 21723470867bSRichard Henderson break; 21733470867bSRichard Henderson case TCG_MO_ST_LD: 21743470867bSRichard Henderson m_op = "wr"; 21753470867bSRichard Henderson break; 21763470867bSRichard Henderson case TCG_MO_ST_ST: 21773470867bSRichard Henderson m_op = "ww"; 21783470867bSRichard Henderson break; 21793470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST: 21803470867bSRichard Henderson m_op = "rr+rw"; 21813470867bSRichard Henderson break; 21823470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD: 21833470867bSRichard Henderson m_op = "rr+wr"; 21843470867bSRichard Henderson break; 21853470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_ST: 21863470867bSRichard Henderson m_op = "rr+ww"; 21873470867bSRichard Henderson break; 21883470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD: 21893470867bSRichard Henderson m_op = "rw+wr"; 21903470867bSRichard Henderson break; 21913470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_ST: 21923470867bSRichard Henderson m_op = "rw+ww"; 21933470867bSRichard Henderson break; 21943470867bSRichard Henderson case TCG_MO_ST_LD | TCG_MO_ST_ST: 21953470867bSRichard Henderson m_op = "wr+ww"; 21963470867bSRichard Henderson break; 21973470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD: 21983470867bSRichard Henderson m_op = "rr+rw+wr"; 21993470867bSRichard Henderson break; 22003470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST: 22013470867bSRichard Henderson m_op = "rr+rw+ww"; 22023470867bSRichard Henderson break; 22033470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST: 22043470867bSRichard Henderson m_op = "rr+wr+ww"; 22053470867bSRichard Henderson break; 22063470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST: 22073470867bSRichard Henderson m_op = "rw+wr+ww"; 22083470867bSRichard Henderson break; 22093470867bSRichard Henderson case TCG_MO_ALL: 22103470867bSRichard Henderson m_op = "all"; 22113470867bSRichard Henderson break; 22123470867bSRichard Henderson default: 22133470867bSRichard Henderson g_assert_not_reached(); 22143470867bSRichard Henderson } 22153470867bSRichard Henderson 22163470867bSRichard Henderson col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op); 22173470867bSRichard Henderson i++, k++; 22183470867bSRichard Henderson } 22193470867bSRichard Henderson break; 222051e3972cSRichard Henderson default: 222151e3972cSRichard Henderson break; 2222eeacee4dSBlue Swirl } 222351e3972cSRichard Henderson for (; i < nb_cargs; i++, k++) { 2224b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 2225b7a83ff8SRichard Henderson op->args[k]); 2226bdfb460eSRichard Henderson } 2227bdfb460eSRichard Henderson } 2228bdfb460eSRichard Henderson 22291894f69aSRichard Henderson if (have_prefs || op->life) { 22301894f69aSRichard Henderson for (; col < 40; ++col) { 2231b7a83ff8SRichard Henderson putc(' ', f); 2232bdfb460eSRichard Henderson } 22331894f69aSRichard Henderson } 22341894f69aSRichard Henderson 22351894f69aSRichard Henderson if (op->life) { 22361894f69aSRichard Henderson unsigned life = op->life; 2237bdfb460eSRichard Henderson 2238bdfb460eSRichard Henderson if (life & (SYNC_ARG * 3)) { 2239b7a83ff8SRichard Henderson ne_fprintf(f, " sync:"); 2240bdfb460eSRichard Henderson for (i = 0; i < 2; ++i) { 2241bdfb460eSRichard Henderson if (life & (SYNC_ARG << i)) { 2242b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i); 2243bdfb460eSRichard Henderson } 2244bdfb460eSRichard Henderson } 2245bdfb460eSRichard Henderson } 2246bdfb460eSRichard Henderson life /= DEAD_ARG; 2247bdfb460eSRichard Henderson if (life) { 2248b7a83ff8SRichard Henderson ne_fprintf(f, " dead:"); 2249bdfb460eSRichard Henderson for (i = 0; life; ++i, life >>= 1) { 2250bdfb460eSRichard Henderson if (life & 1) { 2251b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i); 2252bdfb460eSRichard Henderson } 2253bdfb460eSRichard Henderson } 2254c896fe29Sbellard } 2255b03cce8eSbellard } 22561894f69aSRichard Henderson 22571894f69aSRichard Henderson if (have_prefs) { 22581894f69aSRichard Henderson for (i = 0; i < nb_oargs; ++i) { 225931fd884bSRichard Henderson TCGRegSet set = output_pref(op, i); 22601894f69aSRichard Henderson 22611894f69aSRichard Henderson if (i == 0) { 2262b7a83ff8SRichard Henderson ne_fprintf(f, " pref="); 22631894f69aSRichard Henderson } else { 2264b7a83ff8SRichard Henderson ne_fprintf(f, ","); 22651894f69aSRichard Henderson } 22661894f69aSRichard Henderson if (set == 0) { 2267b7a83ff8SRichard Henderson ne_fprintf(f, "none"); 22681894f69aSRichard Henderson } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 2269b7a83ff8SRichard Henderson ne_fprintf(f, "all"); 22701894f69aSRichard Henderson #ifdef CONFIG_DEBUG_TCG 22711894f69aSRichard Henderson } else if (tcg_regset_single(set)) { 22721894f69aSRichard Henderson TCGReg reg = tcg_regset_first(set); 2273b7a83ff8SRichard Henderson ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 22741894f69aSRichard Henderson #endif 22751894f69aSRichard Henderson } else if (TCG_TARGET_NB_REGS <= 32) { 2276b7a83ff8SRichard Henderson ne_fprintf(f, "0x%x", (uint32_t)set); 22771894f69aSRichard Henderson } else { 2278b7a83ff8SRichard Henderson ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 22791894f69aSRichard Henderson } 22801894f69aSRichard Henderson } 22811894f69aSRichard Henderson } 22821894f69aSRichard Henderson 2283b7a83ff8SRichard Henderson putc('\n', f); 2284c896fe29Sbellard } 2285c896fe29Sbellard } 2286c896fe29Sbellard 2287c896fe29Sbellard /* we give more priority to constraints with less registers */ 2288c896fe29Sbellard static int get_constraint_priority(const TCGOpDef *def, int k) 2289c896fe29Sbellard { 229074a11790SRichard Henderson const TCGArgConstraint *arg_ct = &def->args_ct[k]; 229129f5e925SRichard Henderson int n = ctpop64(arg_ct->regs); 2292c896fe29Sbellard 229329f5e925SRichard Henderson /* 229429f5e925SRichard Henderson * Sort constraints of a single register first, which includes output 229529f5e925SRichard Henderson * aliases (which must exactly match the input already allocated). 229629f5e925SRichard Henderson */ 229729f5e925SRichard Henderson if (n == 1 || arg_ct->oalias) { 229829f5e925SRichard Henderson return INT_MAX; 2299c896fe29Sbellard } 230029f5e925SRichard Henderson 230129f5e925SRichard Henderson /* 230229f5e925SRichard Henderson * Sort register pairs next, first then second immediately after. 230329f5e925SRichard Henderson * Arbitrarily sort multiple pairs by the index of the first reg; 230429f5e925SRichard Henderson * there shouldn't be many pairs. 230529f5e925SRichard Henderson */ 230629f5e925SRichard Henderson switch (arg_ct->pair) { 230729f5e925SRichard Henderson case 1: 230829f5e925SRichard Henderson case 3: 230929f5e925SRichard Henderson return (k + 1) * 2; 231029f5e925SRichard Henderson case 2: 231129f5e925SRichard Henderson return (arg_ct->pair_index + 1) * 2 - 1; 231229f5e925SRichard Henderson } 231329f5e925SRichard Henderson 231429f5e925SRichard Henderson /* Finally, sort by decreasing register count. */ 231529f5e925SRichard Henderson assert(n > 1); 231629f5e925SRichard Henderson return -n; 2317c896fe29Sbellard } 2318c896fe29Sbellard 2319c896fe29Sbellard /* sort from highest priority to lowest */ 2320c896fe29Sbellard static void sort_constraints(TCGOpDef *def, int start, int n) 2321c896fe29Sbellard { 232266792f90SRichard Henderson int i, j; 232366792f90SRichard Henderson TCGArgConstraint *a = def->args_ct; 2324c896fe29Sbellard 232566792f90SRichard Henderson for (i = 0; i < n; i++) { 232666792f90SRichard Henderson a[start + i].sort_index = start + i; 232766792f90SRichard Henderson } 232866792f90SRichard Henderson if (n <= 1) { 2329c896fe29Sbellard return; 233066792f90SRichard Henderson } 2331c896fe29Sbellard for (i = 0; i < n - 1; i++) { 2332c896fe29Sbellard for (j = i + 1; j < n; j++) { 233366792f90SRichard Henderson int p1 = get_constraint_priority(def, a[start + i].sort_index); 233466792f90SRichard Henderson int p2 = get_constraint_priority(def, a[start + j].sort_index); 2335c896fe29Sbellard if (p1 < p2) { 233666792f90SRichard Henderson int tmp = a[start + i].sort_index; 233766792f90SRichard Henderson a[start + i].sort_index = a[start + j].sort_index; 233866792f90SRichard Henderson a[start + j].sort_index = tmp; 2339c896fe29Sbellard } 2340c896fe29Sbellard } 2341c896fe29Sbellard } 2342c896fe29Sbellard } 2343c896fe29Sbellard 2344f69d277eSRichard Henderson static void process_op_defs(TCGContext *s) 2345c896fe29Sbellard { 2346a9751609SRichard Henderson TCGOpcode op; 2347c896fe29Sbellard 2348f69d277eSRichard Henderson for (op = 0; op < NB_OPS; op++) { 2349f69d277eSRichard Henderson TCGOpDef *def = &tcg_op_defs[op]; 2350f69d277eSRichard Henderson const TCGTargetOpDef *tdefs; 235129f5e925SRichard Henderson bool saw_alias_pair = false; 235229f5e925SRichard Henderson int i, o, i2, o2, nb_args; 2353f69d277eSRichard Henderson 2354f69d277eSRichard Henderson if (def->flags & TCG_OPF_NOT_PRESENT) { 2355f69d277eSRichard Henderson continue; 2356f69d277eSRichard Henderson } 2357f69d277eSRichard Henderson 2358c896fe29Sbellard nb_args = def->nb_iargs + def->nb_oargs; 2359f69d277eSRichard Henderson if (nb_args == 0) { 2360f69d277eSRichard Henderson continue; 2361f69d277eSRichard Henderson } 2362f69d277eSRichard Henderson 23634c22e840SRichard Henderson /* 23644c22e840SRichard Henderson * Macro magic should make it impossible, but double-check that 23654c22e840SRichard Henderson * the array index is in range. Since the signness of an enum 23664c22e840SRichard Henderson * is implementation defined, force the result to unsigned. 23674c22e840SRichard Henderson */ 23684c22e840SRichard Henderson unsigned con_set = tcg_target_op_def(op); 23694c22e840SRichard Henderson tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 23704c22e840SRichard Henderson tdefs = &constraint_sets[con_set]; 2371f69d277eSRichard Henderson 2372c896fe29Sbellard for (i = 0; i < nb_args; i++) { 2373f69d277eSRichard Henderson const char *ct_str = tdefs->args_ct_str[i]; 23748940ea0dSPhilippe Mathieu-Daudé bool input_p = i >= def->nb_oargs; 23758940ea0dSPhilippe Mathieu-Daudé 2376f69d277eSRichard Henderson /* Incomplete TCGTargetOpDef entry. */ 2377eabb7b91SAurelien Jarno tcg_debug_assert(ct_str != NULL); 2378f69d277eSRichard Henderson 237917280ff4SRichard Henderson switch (*ct_str) { 238017280ff4SRichard Henderson case '0' ... '9': 23818940ea0dSPhilippe Mathieu-Daudé o = *ct_str - '0'; 23828940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(input_p); 23838940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(o < def->nb_oargs); 23848940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(def->args_ct[o].regs != 0); 23858940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!def->args_ct[o].oalias); 23868940ea0dSPhilippe Mathieu-Daudé def->args_ct[i] = def->args_ct[o]; 2387bc2b17e6SRichard Henderson /* The output sets oalias. */ 23888940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].oalias = 1; 23898940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].alias_index = i; 2390bc2b17e6SRichard Henderson /* The input sets ialias. */ 23918940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].ialias = 1; 23928940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].alias_index = o; 239329f5e925SRichard Henderson if (def->args_ct[i].pair) { 239429f5e925SRichard Henderson saw_alias_pair = true; 239529f5e925SRichard Henderson } 23968940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(ct_str[1] == '\0'); 23978940ea0dSPhilippe Mathieu-Daudé continue; 23988940ea0dSPhilippe Mathieu-Daudé 239982790a87SRichard Henderson case '&': 24008940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!input_p); 2401bc2b17e6SRichard Henderson def->args_ct[i].newreg = true; 240282790a87SRichard Henderson ct_str++; 240382790a87SRichard Henderson break; 240429f5e925SRichard Henderson 240529f5e925SRichard Henderson case 'p': /* plus */ 240629f5e925SRichard Henderson /* Allocate to the register after the previous. */ 240729f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 240829f5e925SRichard Henderson o = i - 1; 240929f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair); 241029f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct); 241129f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){ 241229f5e925SRichard Henderson .pair = 2, 241329f5e925SRichard Henderson .pair_index = o, 241429f5e925SRichard Henderson .regs = def->args_ct[o].regs << 1, 241529f5e925SRichard Henderson }; 241629f5e925SRichard Henderson def->args_ct[o].pair = 1; 241729f5e925SRichard Henderson def->args_ct[o].pair_index = i; 241829f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0'); 241929f5e925SRichard Henderson continue; 242029f5e925SRichard Henderson 242129f5e925SRichard Henderson case 'm': /* minus */ 242229f5e925SRichard Henderson /* Allocate to the register before the previous. */ 242329f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 242429f5e925SRichard Henderson o = i - 1; 242529f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair); 242629f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct); 242729f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){ 242829f5e925SRichard Henderson .pair = 1, 242929f5e925SRichard Henderson .pair_index = o, 243029f5e925SRichard Henderson .regs = def->args_ct[o].regs >> 1, 243129f5e925SRichard Henderson }; 243229f5e925SRichard Henderson def->args_ct[o].pair = 2; 243329f5e925SRichard Henderson def->args_ct[o].pair_index = i; 243429f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0'); 243529f5e925SRichard Henderson continue; 24368940ea0dSPhilippe Mathieu-Daudé } 24378940ea0dSPhilippe Mathieu-Daudé 24388940ea0dSPhilippe Mathieu-Daudé do { 24398940ea0dSPhilippe Mathieu-Daudé switch (*ct_str) { 2440c896fe29Sbellard case 'i': 2441c896fe29Sbellard def->args_ct[i].ct |= TCG_CT_CONST; 2442c896fe29Sbellard break; 2443358b4923SRichard Henderson 2444358b4923SRichard Henderson /* Include all of the target-specific constraints. */ 2445358b4923SRichard Henderson 2446358b4923SRichard Henderson #undef CONST 2447358b4923SRichard Henderson #define CONST(CASE, MASK) \ 24488940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].ct |= MASK; break; 2449358b4923SRichard Henderson #define REGS(CASE, MASK) \ 24508940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].regs |= MASK; break; 2451358b4923SRichard Henderson 2452358b4923SRichard Henderson #include "tcg-target-con-str.h" 2453358b4923SRichard Henderson 2454358b4923SRichard Henderson #undef REGS 2455358b4923SRichard Henderson #undef CONST 2456c896fe29Sbellard default: 24578940ea0dSPhilippe Mathieu-Daudé case '0' ... '9': 24588940ea0dSPhilippe Mathieu-Daudé case '&': 245929f5e925SRichard Henderson case 'p': 246029f5e925SRichard Henderson case 'm': 2461358b4923SRichard Henderson /* Typo in TCGTargetOpDef constraint. */ 2462358b4923SRichard Henderson g_assert_not_reached(); 2463358b4923SRichard Henderson } 24648940ea0dSPhilippe Mathieu-Daudé } while (*++ct_str != '\0'); 2465c896fe29Sbellard } 2466c896fe29Sbellard 2467c68aaa18SStefan Weil /* TCGTargetOpDef entry with too much information? */ 2468eabb7b91SAurelien Jarno tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2469c68aaa18SStefan Weil 247029f5e925SRichard Henderson /* 247129f5e925SRichard Henderson * Fix up output pairs that are aliased with inputs. 247229f5e925SRichard Henderson * When we created the alias, we copied pair from the output. 247329f5e925SRichard Henderson * There are three cases: 247429f5e925SRichard Henderson * (1a) Pairs of inputs alias pairs of outputs. 247529f5e925SRichard Henderson * (1b) One input aliases the first of a pair of outputs. 247629f5e925SRichard Henderson * (2) One input aliases the second of a pair of outputs. 247729f5e925SRichard Henderson * 247829f5e925SRichard Henderson * Case 1a is handled by making sure that the pair_index'es are 247929f5e925SRichard Henderson * properly updated so that they appear the same as a pair of inputs. 248029f5e925SRichard Henderson * 248129f5e925SRichard Henderson * Case 1b is handled by setting the pair_index of the input to 248229f5e925SRichard Henderson * itself, simply so it doesn't point to an unrelated argument. 248329f5e925SRichard Henderson * Since we don't encounter the "second" during the input allocation 248429f5e925SRichard Henderson * phase, nothing happens with the second half of the input pair. 248529f5e925SRichard Henderson * 248629f5e925SRichard Henderson * Case 2 is handled by setting the second input to pair=3, the 248729f5e925SRichard Henderson * first output to pair=3, and the pair_index'es to match. 248829f5e925SRichard Henderson */ 248929f5e925SRichard Henderson if (saw_alias_pair) { 249029f5e925SRichard Henderson for (i = def->nb_oargs; i < nb_args; i++) { 249129f5e925SRichard Henderson /* 249229f5e925SRichard Henderson * Since [0-9pm] must be alone in the constraint string, 249329f5e925SRichard Henderson * the only way they can both be set is if the pair comes 249429f5e925SRichard Henderson * from the output alias. 249529f5e925SRichard Henderson */ 249629f5e925SRichard Henderson if (!def->args_ct[i].ialias) { 249729f5e925SRichard Henderson continue; 249829f5e925SRichard Henderson } 249929f5e925SRichard Henderson switch (def->args_ct[i].pair) { 250029f5e925SRichard Henderson case 0: 250129f5e925SRichard Henderson break; 250229f5e925SRichard Henderson case 1: 250329f5e925SRichard Henderson o = def->args_ct[i].alias_index; 250429f5e925SRichard Henderson o2 = def->args_ct[o].pair_index; 250529f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 1); 250629f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 2); 250729f5e925SRichard Henderson if (def->args_ct[o2].oalias) { 250829f5e925SRichard Henderson /* Case 1a */ 250929f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index; 251029f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 2); 251129f5e925SRichard Henderson def->args_ct[i2].pair_index = i; 251229f5e925SRichard Henderson def->args_ct[i].pair_index = i2; 251329f5e925SRichard Henderson } else { 251429f5e925SRichard Henderson /* Case 1b */ 251529f5e925SRichard Henderson def->args_ct[i].pair_index = i; 251629f5e925SRichard Henderson } 251729f5e925SRichard Henderson break; 251829f5e925SRichard Henderson case 2: 251929f5e925SRichard Henderson o = def->args_ct[i].alias_index; 252029f5e925SRichard Henderson o2 = def->args_ct[o].pair_index; 252129f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 2); 252229f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 1); 252329f5e925SRichard Henderson if (def->args_ct[o2].oalias) { 252429f5e925SRichard Henderson /* Case 1a */ 252529f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index; 252629f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 1); 252729f5e925SRichard Henderson def->args_ct[i2].pair_index = i; 252829f5e925SRichard Henderson def->args_ct[i].pair_index = i2; 252929f5e925SRichard Henderson } else { 253029f5e925SRichard Henderson /* Case 2 */ 253129f5e925SRichard Henderson def->args_ct[i].pair = 3; 253229f5e925SRichard Henderson def->args_ct[o2].pair = 3; 253329f5e925SRichard Henderson def->args_ct[i].pair_index = o2; 253429f5e925SRichard Henderson def->args_ct[o2].pair_index = i; 253529f5e925SRichard Henderson } 253629f5e925SRichard Henderson break; 253729f5e925SRichard Henderson default: 253829f5e925SRichard Henderson g_assert_not_reached(); 253929f5e925SRichard Henderson } 254029f5e925SRichard Henderson } 254129f5e925SRichard Henderson } 254229f5e925SRichard Henderson 2543c896fe29Sbellard /* sort the constraints (XXX: this is just an heuristic) */ 2544c896fe29Sbellard sort_constraints(def, 0, def->nb_oargs); 2545c896fe29Sbellard sort_constraints(def, def->nb_oargs, def->nb_iargs); 2546c896fe29Sbellard } 2547c896fe29Sbellard } 2548c896fe29Sbellard 2549f85b1fc4SRichard Henderson static void remove_label_use(TCGOp *op, int idx) 2550f85b1fc4SRichard Henderson { 2551f85b1fc4SRichard Henderson TCGLabel *label = arg_label(op->args[idx]); 2552f85b1fc4SRichard Henderson TCGLabelUse *use; 2553f85b1fc4SRichard Henderson 2554f85b1fc4SRichard Henderson QSIMPLEQ_FOREACH(use, &label->branches, next) { 2555f85b1fc4SRichard Henderson if (use->op == op) { 2556f85b1fc4SRichard Henderson QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); 2557f85b1fc4SRichard Henderson return; 2558f85b1fc4SRichard Henderson } 2559f85b1fc4SRichard Henderson } 2560f85b1fc4SRichard Henderson g_assert_not_reached(); 2561f85b1fc4SRichard Henderson } 2562f85b1fc4SRichard Henderson 25630c627cdcSRichard Henderson void tcg_op_remove(TCGContext *s, TCGOp *op) 25640c627cdcSRichard Henderson { 2565d88a117eSRichard Henderson switch (op->opc) { 2566d88a117eSRichard Henderson case INDEX_op_br: 2567f85b1fc4SRichard Henderson remove_label_use(op, 0); 2568d88a117eSRichard Henderson break; 2569d88a117eSRichard Henderson case INDEX_op_brcond_i32: 2570d88a117eSRichard Henderson case INDEX_op_brcond_i64: 2571f85b1fc4SRichard Henderson remove_label_use(op, 3); 2572d88a117eSRichard Henderson break; 2573d88a117eSRichard Henderson case INDEX_op_brcond2_i32: 2574f85b1fc4SRichard Henderson remove_label_use(op, 5); 2575d88a117eSRichard Henderson break; 2576d88a117eSRichard Henderson default: 2577d88a117eSRichard Henderson break; 2578d88a117eSRichard Henderson } 2579d88a117eSRichard Henderson 258015fa08f8SRichard Henderson QTAILQ_REMOVE(&s->ops, op, link); 258115fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2582abebf925SRichard Henderson s->nb_ops--; 25830c627cdcSRichard Henderson 25840c627cdcSRichard Henderson #ifdef CONFIG_PROFILER 2585d73415a3SStefan Hajnoczi qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 25860c627cdcSRichard Henderson #endif 25870c627cdcSRichard Henderson } 25880c627cdcSRichard Henderson 2589a80cdd31SRichard Henderson void tcg_remove_ops_after(TCGOp *op) 2590a80cdd31SRichard Henderson { 2591a80cdd31SRichard Henderson TCGContext *s = tcg_ctx; 2592a80cdd31SRichard Henderson 2593a80cdd31SRichard Henderson while (true) { 2594a80cdd31SRichard Henderson TCGOp *last = tcg_last_op(); 2595a80cdd31SRichard Henderson if (last == op) { 2596a80cdd31SRichard Henderson return; 2597a80cdd31SRichard Henderson } 2598a80cdd31SRichard Henderson tcg_op_remove(s, last); 2599a80cdd31SRichard Henderson } 2600a80cdd31SRichard Henderson } 2601a80cdd31SRichard Henderson 2602d4478943SPhilippe Mathieu-Daudé static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) 260315fa08f8SRichard Henderson { 260415fa08f8SRichard Henderson TCGContext *s = tcg_ctx; 2605cb10bc63SRichard Henderson TCGOp *op = NULL; 260615fa08f8SRichard Henderson 2607cb10bc63SRichard Henderson if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { 2608cb10bc63SRichard Henderson QTAILQ_FOREACH(op, &s->free_ops, link) { 2609cb10bc63SRichard Henderson if (nargs <= op->nargs) { 261015fa08f8SRichard Henderson QTAILQ_REMOVE(&s->free_ops, op, link); 2611cb10bc63SRichard Henderson nargs = op->nargs; 2612cb10bc63SRichard Henderson goto found; 261315fa08f8SRichard Henderson } 2614cb10bc63SRichard Henderson } 2615cb10bc63SRichard Henderson } 2616cb10bc63SRichard Henderson 2617cb10bc63SRichard Henderson /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ 2618cb10bc63SRichard Henderson nargs = MAX(4, nargs); 2619cb10bc63SRichard Henderson op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); 2620cb10bc63SRichard Henderson 2621cb10bc63SRichard Henderson found: 262215fa08f8SRichard Henderson memset(op, 0, offsetof(TCGOp, link)); 262315fa08f8SRichard Henderson op->opc = opc; 2624cb10bc63SRichard Henderson op->nargs = nargs; 262515fa08f8SRichard Henderson 2626cb10bc63SRichard Henderson /* Check for bitfield overflow. */ 2627cb10bc63SRichard Henderson tcg_debug_assert(op->nargs == nargs); 2628cb10bc63SRichard Henderson 2629cb10bc63SRichard Henderson s->nb_ops++; 263015fa08f8SRichard Henderson return op; 263115fa08f8SRichard Henderson } 263215fa08f8SRichard Henderson 2633d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) 263415fa08f8SRichard Henderson { 2635d4478943SPhilippe Mathieu-Daudé TCGOp *op = tcg_op_alloc(opc, nargs); 263615fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 263715fa08f8SRichard Henderson return op; 263815fa08f8SRichard Henderson } 263915fa08f8SRichard Henderson 2640d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 2641d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs) 26425a18407fSRichard Henderson { 2643d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs); 264415fa08f8SRichard Henderson QTAILQ_INSERT_BEFORE(old_op, new_op, link); 26455a18407fSRichard Henderson return new_op; 26465a18407fSRichard Henderson } 26475a18407fSRichard Henderson 2648d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 2649d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs) 26505a18407fSRichard Henderson { 2651d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs); 265215fa08f8SRichard Henderson QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 26535a18407fSRichard Henderson return new_op; 26545a18407fSRichard Henderson } 26555a18407fSRichard Henderson 2656968f305eSRichard Henderson static void move_label_uses(TCGLabel *to, TCGLabel *from) 2657968f305eSRichard Henderson { 2658968f305eSRichard Henderson TCGLabelUse *u; 2659968f305eSRichard Henderson 2660968f305eSRichard Henderson QSIMPLEQ_FOREACH(u, &from->branches, next) { 2661968f305eSRichard Henderson TCGOp *op = u->op; 2662968f305eSRichard Henderson switch (op->opc) { 2663968f305eSRichard Henderson case INDEX_op_br: 2664968f305eSRichard Henderson op->args[0] = label_arg(to); 2665968f305eSRichard Henderson break; 2666968f305eSRichard Henderson case INDEX_op_brcond_i32: 2667968f305eSRichard Henderson case INDEX_op_brcond_i64: 2668968f305eSRichard Henderson op->args[3] = label_arg(to); 2669968f305eSRichard Henderson break; 2670968f305eSRichard Henderson case INDEX_op_brcond2_i32: 2671968f305eSRichard Henderson op->args[5] = label_arg(to); 2672968f305eSRichard Henderson break; 2673968f305eSRichard Henderson default: 2674968f305eSRichard Henderson g_assert_not_reached(); 2675968f305eSRichard Henderson } 2676968f305eSRichard Henderson } 2677968f305eSRichard Henderson 2678968f305eSRichard Henderson QSIMPLEQ_CONCAT(&to->branches, &from->branches); 2679968f305eSRichard Henderson } 2680968f305eSRichard Henderson 2681b4fc67c7SRichard Henderson /* Reachable analysis : remove unreachable code. */ 26829bbee4c0SRichard Henderson static void __attribute__((noinline)) 26839bbee4c0SRichard Henderson reachable_code_pass(TCGContext *s) 2684b4fc67c7SRichard Henderson { 26854d89d0bbSRichard Henderson TCGOp *op, *op_next, *op_prev; 2686b4fc67c7SRichard Henderson bool dead = false; 2687b4fc67c7SRichard Henderson 2688b4fc67c7SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2689b4fc67c7SRichard Henderson bool remove = dead; 2690b4fc67c7SRichard Henderson TCGLabel *label; 2691b4fc67c7SRichard Henderson 2692b4fc67c7SRichard Henderson switch (op->opc) { 2693b4fc67c7SRichard Henderson case INDEX_op_set_label: 2694b4fc67c7SRichard Henderson label = arg_label(op->args[0]); 26954d89d0bbSRichard Henderson 26964d89d0bbSRichard Henderson /* 2697968f305eSRichard Henderson * Note that the first op in the TB is always a load, 2698968f305eSRichard Henderson * so there is always something before a label. 2699968f305eSRichard Henderson */ 2700968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link); 2701968f305eSRichard Henderson 2702968f305eSRichard Henderson /* 2703968f305eSRichard Henderson * If we find two sequential labels, move all branches to 2704968f305eSRichard Henderson * reference the second label and remove the first label. 2705968f305eSRichard Henderson * Do this before branch to next optimization, so that the 2706968f305eSRichard Henderson * middle label is out of the way. 2707968f305eSRichard Henderson */ 2708968f305eSRichard Henderson if (op_prev->opc == INDEX_op_set_label) { 2709968f305eSRichard Henderson move_label_uses(label, arg_label(op_prev->args[0])); 2710968f305eSRichard Henderson tcg_op_remove(s, op_prev); 2711968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link); 2712968f305eSRichard Henderson } 2713968f305eSRichard Henderson 2714968f305eSRichard Henderson /* 27154d89d0bbSRichard Henderson * Optimization can fold conditional branches to unconditional. 27164d89d0bbSRichard Henderson * If we find a label which is preceded by an unconditional 27174d89d0bbSRichard Henderson * branch to next, remove the branch. We couldn't do this when 27184d89d0bbSRichard Henderson * processing the branch because any dead code between the branch 27194d89d0bbSRichard Henderson * and label had not yet been removed. 27204d89d0bbSRichard Henderson */ 27214d89d0bbSRichard Henderson if (op_prev->opc == INDEX_op_br && 27224d89d0bbSRichard Henderson label == arg_label(op_prev->args[0])) { 27234d89d0bbSRichard Henderson tcg_op_remove(s, op_prev); 27244d89d0bbSRichard Henderson /* Fall through means insns become live again. */ 27254d89d0bbSRichard Henderson dead = false; 27264d89d0bbSRichard Henderson } 27274d89d0bbSRichard Henderson 2728f85b1fc4SRichard Henderson if (QSIMPLEQ_EMPTY(&label->branches)) { 2729b4fc67c7SRichard Henderson /* 2730b4fc67c7SRichard Henderson * While there is an occasional backward branch, virtually 2731b4fc67c7SRichard Henderson * all branches generated by the translators are forward. 2732b4fc67c7SRichard Henderson * Which means that generally we will have already removed 2733b4fc67c7SRichard Henderson * all references to the label that will be, and there is 2734b4fc67c7SRichard Henderson * little to be gained by iterating. 2735b4fc67c7SRichard Henderson */ 2736b4fc67c7SRichard Henderson remove = true; 2737b4fc67c7SRichard Henderson } else { 2738b4fc67c7SRichard Henderson /* Once we see a label, insns become live again. */ 2739b4fc67c7SRichard Henderson dead = false; 2740b4fc67c7SRichard Henderson remove = false; 2741b4fc67c7SRichard Henderson } 2742b4fc67c7SRichard Henderson break; 2743b4fc67c7SRichard Henderson 2744b4fc67c7SRichard Henderson case INDEX_op_br: 2745b4fc67c7SRichard Henderson case INDEX_op_exit_tb: 2746b4fc67c7SRichard Henderson case INDEX_op_goto_ptr: 2747b4fc67c7SRichard Henderson /* Unconditional branches; everything following is dead. */ 2748b4fc67c7SRichard Henderson dead = true; 2749b4fc67c7SRichard Henderson break; 2750b4fc67c7SRichard Henderson 2751b4fc67c7SRichard Henderson case INDEX_op_call: 2752b4fc67c7SRichard Henderson /* Notice noreturn helper calls, raising exceptions. */ 275390163900SRichard Henderson if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2754b4fc67c7SRichard Henderson dead = true; 2755b4fc67c7SRichard Henderson } 2756b4fc67c7SRichard Henderson break; 2757b4fc67c7SRichard Henderson 2758b4fc67c7SRichard Henderson case INDEX_op_insn_start: 2759b4fc67c7SRichard Henderson /* Never remove -- we need to keep these for unwind. */ 2760b4fc67c7SRichard Henderson remove = false; 2761b4fc67c7SRichard Henderson break; 2762b4fc67c7SRichard Henderson 2763b4fc67c7SRichard Henderson default: 2764b4fc67c7SRichard Henderson break; 2765b4fc67c7SRichard Henderson } 2766b4fc67c7SRichard Henderson 2767b4fc67c7SRichard Henderson if (remove) { 2768b4fc67c7SRichard Henderson tcg_op_remove(s, op); 2769b4fc67c7SRichard Henderson } 2770b4fc67c7SRichard Henderson } 2771b4fc67c7SRichard Henderson } 2772b4fc67c7SRichard Henderson 2773c70fbf0aSRichard Henderson #define TS_DEAD 1 2774c70fbf0aSRichard Henderson #define TS_MEM 2 2775c70fbf0aSRichard Henderson 27765a18407fSRichard Henderson #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 27775a18407fSRichard Henderson #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 27785a18407fSRichard Henderson 277925f49c5fSRichard Henderson /* For liveness_pass_1, the register preferences for a given temp. */ 278025f49c5fSRichard Henderson static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 278125f49c5fSRichard Henderson { 278225f49c5fSRichard Henderson return ts->state_ptr; 278325f49c5fSRichard Henderson } 278425f49c5fSRichard Henderson 278525f49c5fSRichard Henderson /* For liveness_pass_1, reset the preferences for a given temp to the 278625f49c5fSRichard Henderson * maximal regset for its type. 278725f49c5fSRichard Henderson */ 278825f49c5fSRichard Henderson static inline void la_reset_pref(TCGTemp *ts) 278925f49c5fSRichard Henderson { 279025f49c5fSRichard Henderson *la_temp_pref(ts) 279125f49c5fSRichard Henderson = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 279225f49c5fSRichard Henderson } 279325f49c5fSRichard Henderson 27949c43b68dSAurelien Jarno /* liveness analysis: end of function: all temps are dead, and globals 27959c43b68dSAurelien Jarno should be in memory. */ 27962616c808SRichard Henderson static void la_func_end(TCGContext *s, int ng, int nt) 2797c896fe29Sbellard { 2798b83eabeaSRichard Henderson int i; 2799b83eabeaSRichard Henderson 2800b83eabeaSRichard Henderson for (i = 0; i < ng; ++i) { 2801b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 280225f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 2803b83eabeaSRichard Henderson } 2804b83eabeaSRichard Henderson for (i = ng; i < nt; ++i) { 2805b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD; 280625f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 2807b83eabeaSRichard Henderson } 2808c896fe29Sbellard } 2809c896fe29Sbellard 28109c43b68dSAurelien Jarno /* liveness analysis: end of basic block: all temps are dead, globals 28119c43b68dSAurelien Jarno and local temps should be in memory. */ 28122616c808SRichard Henderson static void la_bb_end(TCGContext *s, int ng, int nt) 2813641d5fbeSbellard { 2814b83eabeaSRichard Henderson int i; 2815641d5fbeSbellard 2816ee17db83SRichard Henderson for (i = 0; i < nt; ++i) { 2817ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 2818ee17db83SRichard Henderson int state; 2819ee17db83SRichard Henderson 2820ee17db83SRichard Henderson switch (ts->kind) { 2821ee17db83SRichard Henderson case TEMP_FIXED: 2822ee17db83SRichard Henderson case TEMP_GLOBAL: 2823f57c6915SRichard Henderson case TEMP_TB: 2824ee17db83SRichard Henderson state = TS_DEAD | TS_MEM; 2825ee17db83SRichard Henderson break; 2826c7482438SRichard Henderson case TEMP_EBB: 2827c0522136SRichard Henderson case TEMP_CONST: 2828ee17db83SRichard Henderson state = TS_DEAD; 2829ee17db83SRichard Henderson break; 2830ee17db83SRichard Henderson default: 2831ee17db83SRichard Henderson g_assert_not_reached(); 2832c70fbf0aSRichard Henderson } 2833ee17db83SRichard Henderson ts->state = state; 2834ee17db83SRichard Henderson la_reset_pref(ts); 2835641d5fbeSbellard } 2836641d5fbeSbellard } 2837641d5fbeSbellard 2838f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory. */ 2839f65a061cSRichard Henderson static void la_global_sync(TCGContext *s, int ng) 2840f65a061cSRichard Henderson { 2841f65a061cSRichard Henderson int i; 2842f65a061cSRichard Henderson 2843f65a061cSRichard Henderson for (i = 0; i < ng; ++i) { 284425f49c5fSRichard Henderson int state = s->temps[i].state; 284525f49c5fSRichard Henderson s->temps[i].state = state | TS_MEM; 284625f49c5fSRichard Henderson if (state == TS_DEAD) { 284725f49c5fSRichard Henderson /* If the global was previously dead, reset prefs. */ 284825f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 284925f49c5fSRichard Henderson } 2850f65a061cSRichard Henderson } 2851f65a061cSRichard Henderson } 2852f65a061cSRichard Henderson 2853b4cb76e6SRichard Henderson /* 2854c7482438SRichard Henderson * liveness analysis: conditional branch: all temps are dead unless 2855c7482438SRichard Henderson * explicitly live-across-conditional-branch, globals and local temps 2856c7482438SRichard Henderson * should be synced. 2857b4cb76e6SRichard Henderson */ 2858b4cb76e6SRichard Henderson static void la_bb_sync(TCGContext *s, int ng, int nt) 2859b4cb76e6SRichard Henderson { 2860b4cb76e6SRichard Henderson la_global_sync(s, ng); 2861b4cb76e6SRichard Henderson 2862b4cb76e6SRichard Henderson for (int i = ng; i < nt; ++i) { 2863c0522136SRichard Henderson TCGTemp *ts = &s->temps[i]; 2864c0522136SRichard Henderson int state; 2865c0522136SRichard Henderson 2866c0522136SRichard Henderson switch (ts->kind) { 2867f57c6915SRichard Henderson case TEMP_TB: 2868c0522136SRichard Henderson state = ts->state; 2869c0522136SRichard Henderson ts->state = state | TS_MEM; 2870b4cb76e6SRichard Henderson if (state != TS_DEAD) { 2871b4cb76e6SRichard Henderson continue; 2872b4cb76e6SRichard Henderson } 2873c0522136SRichard Henderson break; 2874c7482438SRichard Henderson case TEMP_EBB: 2875c0522136SRichard Henderson case TEMP_CONST: 2876c0522136SRichard Henderson continue; 2877c0522136SRichard Henderson default: 2878c0522136SRichard Henderson g_assert_not_reached(); 2879b4cb76e6SRichard Henderson } 2880b4cb76e6SRichard Henderson la_reset_pref(&s->temps[i]); 2881b4cb76e6SRichard Henderson } 2882b4cb76e6SRichard Henderson } 2883b4cb76e6SRichard Henderson 2884f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory and kill. */ 2885f65a061cSRichard Henderson static void la_global_kill(TCGContext *s, int ng) 2886f65a061cSRichard Henderson { 2887f65a061cSRichard Henderson int i; 2888f65a061cSRichard Henderson 2889f65a061cSRichard Henderson for (i = 0; i < ng; i++) { 2890f65a061cSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 289125f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 289225f49c5fSRichard Henderson } 289325f49c5fSRichard Henderson } 289425f49c5fSRichard Henderson 289525f49c5fSRichard Henderson /* liveness analysis: note live globals crossing calls. */ 289625f49c5fSRichard Henderson static void la_cross_call(TCGContext *s, int nt) 289725f49c5fSRichard Henderson { 289825f49c5fSRichard Henderson TCGRegSet mask = ~tcg_target_call_clobber_regs; 289925f49c5fSRichard Henderson int i; 290025f49c5fSRichard Henderson 290125f49c5fSRichard Henderson for (i = 0; i < nt; i++) { 290225f49c5fSRichard Henderson TCGTemp *ts = &s->temps[i]; 290325f49c5fSRichard Henderson if (!(ts->state & TS_DEAD)) { 290425f49c5fSRichard Henderson TCGRegSet *pset = la_temp_pref(ts); 290525f49c5fSRichard Henderson TCGRegSet set = *pset; 290625f49c5fSRichard Henderson 290725f49c5fSRichard Henderson set &= mask; 290825f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 290925f49c5fSRichard Henderson if (set == 0) { 291025f49c5fSRichard Henderson set = tcg_target_available_regs[ts->type] & mask; 291125f49c5fSRichard Henderson } 291225f49c5fSRichard Henderson *pset = set; 291325f49c5fSRichard Henderson } 2914f65a061cSRichard Henderson } 2915f65a061cSRichard Henderson } 2916f65a061cSRichard Henderson 2917874b8574SRichard Henderson /* 2918874b8574SRichard Henderson * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce 2919874b8574SRichard Henderson * to TEMP_EBB, if possible. 2920874b8574SRichard Henderson */ 2921874b8574SRichard Henderson static void __attribute__((noinline)) 2922874b8574SRichard Henderson liveness_pass_0(TCGContext *s) 2923874b8574SRichard Henderson { 2924874b8574SRichard Henderson void * const multiple_ebb = (void *)(uintptr_t)-1; 2925874b8574SRichard Henderson int nb_temps = s->nb_temps; 2926874b8574SRichard Henderson TCGOp *op, *ebb; 2927874b8574SRichard Henderson 2928874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) { 2929874b8574SRichard Henderson s->temps[i].state_ptr = NULL; 2930874b8574SRichard Henderson } 2931874b8574SRichard Henderson 2932874b8574SRichard Henderson /* 2933874b8574SRichard Henderson * Represent each EBB by the op at which it begins. In the case of 2934874b8574SRichard Henderson * the first EBB, this is the first op, otherwise it is a label. 2935874b8574SRichard Henderson * Collect the uses of each TEMP_TB: NULL for unused, EBB for use 2936874b8574SRichard Henderson * within a single EBB, else MULTIPLE_EBB. 2937874b8574SRichard Henderson */ 2938874b8574SRichard Henderson ebb = QTAILQ_FIRST(&s->ops); 2939874b8574SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 2940874b8574SRichard Henderson const TCGOpDef *def; 2941874b8574SRichard Henderson int nb_oargs, nb_iargs; 2942874b8574SRichard Henderson 2943874b8574SRichard Henderson switch (op->opc) { 2944874b8574SRichard Henderson case INDEX_op_set_label: 2945874b8574SRichard Henderson ebb = op; 2946874b8574SRichard Henderson continue; 2947874b8574SRichard Henderson case INDEX_op_discard: 2948874b8574SRichard Henderson continue; 2949874b8574SRichard Henderson case INDEX_op_call: 2950874b8574SRichard Henderson nb_oargs = TCGOP_CALLO(op); 2951874b8574SRichard Henderson nb_iargs = TCGOP_CALLI(op); 2952874b8574SRichard Henderson break; 2953874b8574SRichard Henderson default: 2954874b8574SRichard Henderson def = &tcg_op_defs[op->opc]; 2955874b8574SRichard Henderson nb_oargs = def->nb_oargs; 2956874b8574SRichard Henderson nb_iargs = def->nb_iargs; 2957874b8574SRichard Henderson break; 2958874b8574SRichard Henderson } 2959874b8574SRichard Henderson 2960874b8574SRichard Henderson for (int i = 0; i < nb_oargs + nb_iargs; ++i) { 2961874b8574SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 2962874b8574SRichard Henderson 2963874b8574SRichard Henderson if (ts->kind != TEMP_TB) { 2964874b8574SRichard Henderson continue; 2965874b8574SRichard Henderson } 2966874b8574SRichard Henderson if (ts->state_ptr == NULL) { 2967874b8574SRichard Henderson ts->state_ptr = ebb; 2968874b8574SRichard Henderson } else if (ts->state_ptr != ebb) { 2969874b8574SRichard Henderson ts->state_ptr = multiple_ebb; 2970874b8574SRichard Henderson } 2971874b8574SRichard Henderson } 2972874b8574SRichard Henderson } 2973874b8574SRichard Henderson 2974874b8574SRichard Henderson /* 2975874b8574SRichard Henderson * For TEMP_TB that turned out not to be used beyond one EBB, 2976874b8574SRichard Henderson * reduce the liveness to TEMP_EBB. 2977874b8574SRichard Henderson */ 2978874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) { 2979874b8574SRichard Henderson TCGTemp *ts = &s->temps[i]; 2980874b8574SRichard Henderson if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { 2981874b8574SRichard Henderson ts->kind = TEMP_EBB; 2982874b8574SRichard Henderson } 2983874b8574SRichard Henderson } 2984874b8574SRichard Henderson } 2985874b8574SRichard Henderson 2986a1b3c48dSRichard Henderson /* Liveness analysis : update the opc_arg_life array to tell if a 2987c896fe29Sbellard given input arguments is dead. Instructions updating dead 2988c896fe29Sbellard temporaries are removed. */ 29899bbee4c0SRichard Henderson static void __attribute__((noinline)) 29909bbee4c0SRichard Henderson liveness_pass_1(TCGContext *s) 2991c896fe29Sbellard { 2992c70fbf0aSRichard Henderson int nb_globals = s->nb_globals; 29932616c808SRichard Henderson int nb_temps = s->nb_temps; 299415fa08f8SRichard Henderson TCGOp *op, *op_prev; 299525f49c5fSRichard Henderson TCGRegSet *prefs; 299625f49c5fSRichard Henderson int i; 299725f49c5fSRichard Henderson 299825f49c5fSRichard Henderson prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 299925f49c5fSRichard Henderson for (i = 0; i < nb_temps; ++i) { 300025f49c5fSRichard Henderson s->temps[i].state_ptr = prefs + i; 300125f49c5fSRichard Henderson } 3002c896fe29Sbellard 3003ae36a246SRichard Henderson /* ??? Should be redundant with the exit_tb that ends the TB. */ 30042616c808SRichard Henderson la_func_end(s, nb_globals, nb_temps); 3005c896fe29Sbellard 3006eae3eb3eSPaolo Bonzini QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 300725f49c5fSRichard Henderson int nb_iargs, nb_oargs; 3008c45cb8bbSRichard Henderson TCGOpcode opc_new, opc_new2; 3009c45cb8bbSRichard Henderson bool have_opc_new2; 3010a1b3c48dSRichard Henderson TCGLifeData arg_life = 0; 301125f49c5fSRichard Henderson TCGTemp *ts; 3012c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 3013c45cb8bbSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 3014c45cb8bbSRichard Henderson 3015c45cb8bbSRichard Henderson switch (opc) { 3016c896fe29Sbellard case INDEX_op_call: 3017c6e113f5Sbellard { 301839004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 301939004a71SRichard Henderson int call_flags = tcg_call_flags(op); 3020c6e113f5Sbellard 3021cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 3022cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 3023c6e113f5Sbellard 3024c45cb8bbSRichard Henderson /* pure functions can be removed if their result is unused */ 302578505279SAurelien Jarno if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 3026c6e113f5Sbellard for (i = 0; i < nb_oargs; i++) { 302725f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 302825f49c5fSRichard Henderson if (ts->state != TS_DEAD) { 3029c6e113f5Sbellard goto do_not_remove_call; 3030c6e113f5Sbellard } 30319c43b68dSAurelien Jarno } 3032c45cb8bbSRichard Henderson goto do_remove; 3033152c35aaSRichard Henderson } 3034c6e113f5Sbellard do_not_remove_call: 3035c896fe29Sbellard 303625f49c5fSRichard Henderson /* Output args are dead. */ 3037c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 303825f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 303925f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3040a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 30416b64b624SAurelien Jarno } 304225f49c5fSRichard Henderson if (ts->state & TS_MEM) { 3043a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 30449c43b68dSAurelien Jarno } 304525f49c5fSRichard Henderson ts->state = TS_DEAD; 304625f49c5fSRichard Henderson la_reset_pref(ts); 3047c896fe29Sbellard } 3048c896fe29Sbellard 304931fd884bSRichard Henderson /* Not used -- it will be tcg_target_call_oarg_reg(). */ 305031fd884bSRichard Henderson memset(op->output_pref, 0, sizeof(op->output_pref)); 305131fd884bSRichard Henderson 305278505279SAurelien Jarno if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 305378505279SAurelien Jarno TCG_CALL_NO_READ_GLOBALS))) { 3054f65a061cSRichard Henderson la_global_kill(s, nb_globals); 3055c70fbf0aSRichard Henderson } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 3056f65a061cSRichard Henderson la_global_sync(s, nb_globals); 3057b9c18f56Saurel32 } 3058c896fe29Sbellard 305925f49c5fSRichard Henderson /* Record arguments that die in this helper. */ 3060866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 306125f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 306239004a71SRichard Henderson if (ts->state & TS_DEAD) { 3063a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 3064c896fe29Sbellard } 3065c896fe29Sbellard } 306625f49c5fSRichard Henderson 306725f49c5fSRichard Henderson /* For all live registers, remove call-clobbered prefs. */ 306825f49c5fSRichard Henderson la_cross_call(s, nb_temps); 306925f49c5fSRichard Henderson 307039004a71SRichard Henderson /* 307139004a71SRichard Henderson * Input arguments are live for preceding opcodes. 307239004a71SRichard Henderson * 307339004a71SRichard Henderson * For those arguments that die, and will be allocated in 307439004a71SRichard Henderson * registers, clear the register set for that arg, to be 307539004a71SRichard Henderson * filled in below. For args that will be on the stack, 307639004a71SRichard Henderson * reset to any available reg. Process arguments in reverse 307739004a71SRichard Henderson * order so that if a temp is used more than once, the stack 307839004a71SRichard Henderson * reset to max happens before the register reset to 0. 307925f49c5fSRichard Henderson */ 308039004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; i--) { 308139004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 308239004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]); 308339004a71SRichard Henderson 308439004a71SRichard Henderson if (ts->state & TS_DEAD) { 308539004a71SRichard Henderson switch (loc->kind) { 308639004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 308739004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 308839004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 308939004a71SRichard Henderson if (REG_P(loc)) { 309039004a71SRichard Henderson *la_temp_pref(ts) = 0; 309139004a71SRichard Henderson break; 309239004a71SRichard Henderson } 309339004a71SRichard Henderson /* fall through */ 309439004a71SRichard Henderson default: 309539004a71SRichard Henderson *la_temp_pref(ts) = 309639004a71SRichard Henderson tcg_target_available_regs[ts->type]; 309739004a71SRichard Henderson break; 309839004a71SRichard Henderson } 309925f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 310025f49c5fSRichard Henderson } 310125f49c5fSRichard Henderson } 310225f49c5fSRichard Henderson 310339004a71SRichard Henderson /* 310439004a71SRichard Henderson * For each input argument, add its input register to prefs. 310539004a71SRichard Henderson * If a temp is used once, this produces a single set bit; 310639004a71SRichard Henderson * if a temp is used multiple times, this produces a set. 310739004a71SRichard Henderson */ 310839004a71SRichard Henderson for (i = 0; i < nb_iargs; i++) { 310939004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 311039004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]); 311139004a71SRichard Henderson 311239004a71SRichard Henderson switch (loc->kind) { 311339004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 311439004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 311539004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 311639004a71SRichard Henderson if (REG_P(loc)) { 311725f49c5fSRichard Henderson tcg_regset_set_reg(*la_temp_pref(ts), 311839004a71SRichard Henderson tcg_target_call_iarg_regs[loc->arg_slot]); 311939004a71SRichard Henderson } 312039004a71SRichard Henderson break; 312139004a71SRichard Henderson default: 312239004a71SRichard Henderson break; 3123c70fbf0aSRichard Henderson } 3124c19f47bfSAurelien Jarno } 3125c6e113f5Sbellard } 3126c896fe29Sbellard break; 3127765b842aSRichard Henderson case INDEX_op_insn_start: 3128c896fe29Sbellard break; 31295ff9d6a4Sbellard case INDEX_op_discard: 31305ff9d6a4Sbellard /* mark the temporary as dead */ 313125f49c5fSRichard Henderson ts = arg_temp(op->args[0]); 313225f49c5fSRichard Henderson ts->state = TS_DEAD; 313325f49c5fSRichard Henderson la_reset_pref(ts); 31345ff9d6a4Sbellard break; 31351305c451SRichard Henderson 31361305c451SRichard Henderson case INDEX_op_add2_i32: 3137c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i32; 3138f1fae40cSRichard Henderson goto do_addsub2; 31391305c451SRichard Henderson case INDEX_op_sub2_i32: 3140c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i32; 3141f1fae40cSRichard Henderson goto do_addsub2; 3142f1fae40cSRichard Henderson case INDEX_op_add2_i64: 3143c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i64; 3144f1fae40cSRichard Henderson goto do_addsub2; 3145f1fae40cSRichard Henderson case INDEX_op_sub2_i64: 3146c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i64; 3147f1fae40cSRichard Henderson do_addsub2: 31481305c451SRichard Henderson nb_iargs = 4; 31491305c451SRichard Henderson nb_oargs = 2; 31501305c451SRichard Henderson /* Test if the high part of the operation is dead, but not 31511305c451SRichard Henderson the low part. The result can be optimized to a simple 31521305c451SRichard Henderson add or sub. This happens often for x86_64 guest when the 31531305c451SRichard Henderson cpu mode is set to 32 bit. */ 3154b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 3155b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 31561305c451SRichard Henderson goto do_remove; 31571305c451SRichard Henderson } 3158c45cb8bbSRichard Henderson /* Replace the opcode and adjust the args in place, 3159c45cb8bbSRichard Henderson leaving 3 unused args at the end. */ 3160c45cb8bbSRichard Henderson op->opc = opc = opc_new; 3161efee3746SRichard Henderson op->args[1] = op->args[2]; 3162efee3746SRichard Henderson op->args[2] = op->args[4]; 31631305c451SRichard Henderson /* Fall through and mark the single-word operation live. */ 31641305c451SRichard Henderson nb_iargs = 2; 31651305c451SRichard Henderson nb_oargs = 1; 31661305c451SRichard Henderson } 31671305c451SRichard Henderson goto do_not_remove; 31681305c451SRichard Henderson 31691414968aSRichard Henderson case INDEX_op_mulu2_i32: 3170c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 3171c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i32; 3172c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 317303271524SRichard Henderson goto do_mul2; 3174f1fae40cSRichard Henderson case INDEX_op_muls2_i32: 3175c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 3176c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i32; 3177c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 3178f1fae40cSRichard Henderson goto do_mul2; 3179f1fae40cSRichard Henderson case INDEX_op_mulu2_i64: 3180c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 3181c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i64; 3182c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 318303271524SRichard Henderson goto do_mul2; 3184f1fae40cSRichard Henderson case INDEX_op_muls2_i64: 3185c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 3186c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i64; 3187c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 318803271524SRichard Henderson goto do_mul2; 3189f1fae40cSRichard Henderson do_mul2: 31901414968aSRichard Henderson nb_iargs = 2; 31911414968aSRichard Henderson nb_oargs = 2; 3192b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 3193b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 319403271524SRichard Henderson /* Both parts of the operation are dead. */ 31951414968aSRichard Henderson goto do_remove; 31961414968aSRichard Henderson } 319703271524SRichard Henderson /* The high part of the operation is dead; generate the low. */ 3198c45cb8bbSRichard Henderson op->opc = opc = opc_new; 3199efee3746SRichard Henderson op->args[1] = op->args[2]; 3200efee3746SRichard Henderson op->args[2] = op->args[3]; 3201b83eabeaSRichard Henderson } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 320203271524SRichard Henderson /* The low part of the operation is dead; generate the high. */ 3203c45cb8bbSRichard Henderson op->opc = opc = opc_new2; 3204efee3746SRichard Henderson op->args[0] = op->args[1]; 3205efee3746SRichard Henderson op->args[1] = op->args[2]; 3206efee3746SRichard Henderson op->args[2] = op->args[3]; 320703271524SRichard Henderson } else { 320803271524SRichard Henderson goto do_not_remove; 320903271524SRichard Henderson } 321003271524SRichard Henderson /* Mark the single-word operation live. */ 32111414968aSRichard Henderson nb_oargs = 1; 32121414968aSRichard Henderson goto do_not_remove; 32131414968aSRichard Henderson 3214c896fe29Sbellard default: 32151305c451SRichard Henderson /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 3216c896fe29Sbellard nb_iargs = def->nb_iargs; 3217c896fe29Sbellard nb_oargs = def->nb_oargs; 3218c896fe29Sbellard 3219c896fe29Sbellard /* Test if the operation can be removed because all 32205ff9d6a4Sbellard its outputs are dead. We assume that nb_oargs == 0 32215ff9d6a4Sbellard implies side effects */ 32225ff9d6a4Sbellard if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 3223c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 3224b83eabeaSRichard Henderson if (arg_temp(op->args[i])->state != TS_DEAD) { 3225c896fe29Sbellard goto do_not_remove; 3226c896fe29Sbellard } 32279c43b68dSAurelien Jarno } 3228152c35aaSRichard Henderson goto do_remove; 3229152c35aaSRichard Henderson } 3230152c35aaSRichard Henderson goto do_not_remove; 3231152c35aaSRichard Henderson 32321305c451SRichard Henderson do_remove: 32330c627cdcSRichard Henderson tcg_op_remove(s, op); 3234152c35aaSRichard Henderson break; 3235152c35aaSRichard Henderson 3236c896fe29Sbellard do_not_remove: 3237c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 323825f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 323925f49c5fSRichard Henderson 324025f49c5fSRichard Henderson /* Remember the preference of the uses that followed. */ 324131fd884bSRichard Henderson if (i < ARRAY_SIZE(op->output_pref)) { 324225f49c5fSRichard Henderson op->output_pref[i] = *la_temp_pref(ts); 324331fd884bSRichard Henderson } 324425f49c5fSRichard Henderson 324525f49c5fSRichard Henderson /* Output args are dead. */ 324625f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3247a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 32486b64b624SAurelien Jarno } 324925f49c5fSRichard Henderson if (ts->state & TS_MEM) { 3250a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 32519c43b68dSAurelien Jarno } 325225f49c5fSRichard Henderson ts->state = TS_DEAD; 325325f49c5fSRichard Henderson la_reset_pref(ts); 3254c896fe29Sbellard } 3255c896fe29Sbellard 325625f49c5fSRichard Henderson /* If end of basic block, update. */ 3257ae36a246SRichard Henderson if (def->flags & TCG_OPF_BB_EXIT) { 3258ae36a246SRichard Henderson la_func_end(s, nb_globals, nb_temps); 3259b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_COND_BRANCH) { 3260b4cb76e6SRichard Henderson la_bb_sync(s, nb_globals, nb_temps); 3261ae36a246SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 32622616c808SRichard Henderson la_bb_end(s, nb_globals, nb_temps); 32633d5c5f87SAurelien Jarno } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3264f65a061cSRichard Henderson la_global_sync(s, nb_globals); 326525f49c5fSRichard Henderson if (def->flags & TCG_OPF_CALL_CLOBBER) { 326625f49c5fSRichard Henderson la_cross_call(s, nb_temps); 326725f49c5fSRichard Henderson } 3268c896fe29Sbellard } 3269c896fe29Sbellard 327025f49c5fSRichard Henderson /* Record arguments that die in this opcode. */ 3271866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 327225f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 327325f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3274a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 3275c896fe29Sbellard } 3276c19f47bfSAurelien Jarno } 327725f49c5fSRichard Henderson 327825f49c5fSRichard Henderson /* Input arguments are live for preceding opcodes. */ 3279c19f47bfSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 328025f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 328125f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 328225f49c5fSRichard Henderson /* For operands that were dead, initially allow 328325f49c5fSRichard Henderson all regs for the type. */ 328425f49c5fSRichard Henderson *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 328525f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 328625f49c5fSRichard Henderson } 328725f49c5fSRichard Henderson } 328825f49c5fSRichard Henderson 328925f49c5fSRichard Henderson /* Incorporate constraints for this operand. */ 329025f49c5fSRichard Henderson switch (opc) { 329125f49c5fSRichard Henderson case INDEX_op_mov_i32: 329225f49c5fSRichard Henderson case INDEX_op_mov_i64: 329325f49c5fSRichard Henderson /* Note that these are TCG_OPF_NOT_PRESENT and do not 329425f49c5fSRichard Henderson have proper constraints. That said, special case 329525f49c5fSRichard Henderson moves to propagate preferences backward. */ 329625f49c5fSRichard Henderson if (IS_DEAD_ARG(1)) { 329725f49c5fSRichard Henderson *la_temp_pref(arg_temp(op->args[0])) 329825f49c5fSRichard Henderson = *la_temp_pref(arg_temp(op->args[1])); 329925f49c5fSRichard Henderson } 330025f49c5fSRichard Henderson break; 330125f49c5fSRichard Henderson 330225f49c5fSRichard Henderson default: 330325f49c5fSRichard Henderson for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 330425f49c5fSRichard Henderson const TCGArgConstraint *ct = &def->args_ct[i]; 330525f49c5fSRichard Henderson TCGRegSet set, *pset; 330625f49c5fSRichard Henderson 330725f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 330825f49c5fSRichard Henderson pset = la_temp_pref(ts); 330925f49c5fSRichard Henderson set = *pset; 331025f49c5fSRichard Henderson 33119be0d080SRichard Henderson set &= ct->regs; 3312bc2b17e6SRichard Henderson if (ct->ialias) { 331331fd884bSRichard Henderson set &= output_pref(op, ct->alias_index); 331425f49c5fSRichard Henderson } 331525f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 331625f49c5fSRichard Henderson if (set == 0) { 33179be0d080SRichard Henderson set = ct->regs; 331825f49c5fSRichard Henderson } 331925f49c5fSRichard Henderson *pset = set; 332025f49c5fSRichard Henderson } 332125f49c5fSRichard Henderson break; 3322c896fe29Sbellard } 3323c896fe29Sbellard break; 3324c896fe29Sbellard } 3325bee158cbSRichard Henderson op->life = arg_life; 3326c896fe29Sbellard } 33271ff0a2c5SEvgeny Voevodin } 3328c896fe29Sbellard 33295a18407fSRichard Henderson /* Liveness analysis: Convert indirect regs to direct temporaries. */ 33309bbee4c0SRichard Henderson static bool __attribute__((noinline)) 33319bbee4c0SRichard Henderson liveness_pass_2(TCGContext *s) 33325a18407fSRichard Henderson { 33335a18407fSRichard Henderson int nb_globals = s->nb_globals; 333415fa08f8SRichard Henderson int nb_temps, i; 33355a18407fSRichard Henderson bool changes = false; 333615fa08f8SRichard Henderson TCGOp *op, *op_next; 33375a18407fSRichard Henderson 33385a18407fSRichard Henderson /* Create a temporary for each indirect global. */ 33395a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 33405a18407fSRichard Henderson TCGTemp *its = &s->temps[i]; 33415a18407fSRichard Henderson if (its->indirect_reg) { 33425a18407fSRichard Henderson TCGTemp *dts = tcg_temp_alloc(s); 33435a18407fSRichard Henderson dts->type = its->type; 33445a18407fSRichard Henderson dts->base_type = its->base_type; 3345e1e64652SRichard Henderson dts->temp_subindex = its->temp_subindex; 3346c7482438SRichard Henderson dts->kind = TEMP_EBB; 3347b83eabeaSRichard Henderson its->state_ptr = dts; 3348b83eabeaSRichard Henderson } else { 3349b83eabeaSRichard Henderson its->state_ptr = NULL; 33505a18407fSRichard Henderson } 3351b83eabeaSRichard Henderson /* All globals begin dead. */ 3352b83eabeaSRichard Henderson its->state = TS_DEAD; 33535a18407fSRichard Henderson } 3354b83eabeaSRichard Henderson for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 3355b83eabeaSRichard Henderson TCGTemp *its = &s->temps[i]; 3356b83eabeaSRichard Henderson its->state_ptr = NULL; 3357b83eabeaSRichard Henderson its->state = TS_DEAD; 3358b83eabeaSRichard Henderson } 33595a18407fSRichard Henderson 336015fa08f8SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 33615a18407fSRichard Henderson TCGOpcode opc = op->opc; 33625a18407fSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 33635a18407fSRichard Henderson TCGLifeData arg_life = op->life; 33645a18407fSRichard Henderson int nb_iargs, nb_oargs, call_flags; 3365b83eabeaSRichard Henderson TCGTemp *arg_ts, *dir_ts; 33665a18407fSRichard Henderson 33675a18407fSRichard Henderson if (opc == INDEX_op_call) { 3368cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 3369cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 337090163900SRichard Henderson call_flags = tcg_call_flags(op); 33715a18407fSRichard Henderson } else { 33725a18407fSRichard Henderson nb_iargs = def->nb_iargs; 33735a18407fSRichard Henderson nb_oargs = def->nb_oargs; 33745a18407fSRichard Henderson 33755a18407fSRichard Henderson /* Set flags similar to how calls require. */ 3376b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 3377b4cb76e6SRichard Henderson /* Like reading globals: sync_globals */ 3378b4cb76e6SRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 3379b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 33805a18407fSRichard Henderson /* Like writing globals: save_globals */ 33815a18407fSRichard Henderson call_flags = 0; 33825a18407fSRichard Henderson } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 33835a18407fSRichard Henderson /* Like reading globals: sync_globals */ 33845a18407fSRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 33855a18407fSRichard Henderson } else { 33865a18407fSRichard Henderson /* No effect on globals. */ 33875a18407fSRichard Henderson call_flags = (TCG_CALL_NO_READ_GLOBALS | 33885a18407fSRichard Henderson TCG_CALL_NO_WRITE_GLOBALS); 33895a18407fSRichard Henderson } 33905a18407fSRichard Henderson } 33915a18407fSRichard Henderson 33925a18407fSRichard Henderson /* Make sure that input arguments are available. */ 33935a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3394b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3395b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3396b83eabeaSRichard Henderson if (dir_ts && arg_ts->state == TS_DEAD) { 3397b83eabeaSRichard Henderson TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 33985a18407fSRichard Henderson ? INDEX_op_ld_i32 33995a18407fSRichard Henderson : INDEX_op_ld_i64); 3400d4478943SPhilippe Mathieu-Daudé TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 34015a18407fSRichard Henderson 3402b83eabeaSRichard Henderson lop->args[0] = temp_arg(dir_ts); 3403b83eabeaSRichard Henderson lop->args[1] = temp_arg(arg_ts->mem_base); 3404b83eabeaSRichard Henderson lop->args[2] = arg_ts->mem_offset; 34055a18407fSRichard Henderson 34065a18407fSRichard Henderson /* Loaded, but synced with memory. */ 3407b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 34085a18407fSRichard Henderson } 34095a18407fSRichard Henderson } 34105a18407fSRichard Henderson 34115a18407fSRichard Henderson /* Perform input replacement, and mark inputs that became dead. 34125a18407fSRichard Henderson No action is required except keeping temp_state up to date 34135a18407fSRichard Henderson so that we reload when needed. */ 34145a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3415b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3416b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3417b83eabeaSRichard Henderson if (dir_ts) { 3418b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 34195a18407fSRichard Henderson changes = true; 34205a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 3421b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 34225a18407fSRichard Henderson } 34235a18407fSRichard Henderson } 34245a18407fSRichard Henderson } 34255a18407fSRichard Henderson 34265a18407fSRichard Henderson /* Liveness analysis should ensure that the following are 34275a18407fSRichard Henderson all correct, for call sites and basic block end points. */ 34285a18407fSRichard Henderson if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 34295a18407fSRichard Henderson /* Nothing to do */ 34305a18407fSRichard Henderson } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 34315a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 34325a18407fSRichard Henderson /* Liveness should see that globals are synced back, 34335a18407fSRichard Henderson that is, either TS_DEAD or TS_MEM. */ 3434b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 3435b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 3436b83eabeaSRichard Henderson || arg_ts->state != 0); 34375a18407fSRichard Henderson } 34385a18407fSRichard Henderson } else { 34395a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 34405a18407fSRichard Henderson /* Liveness should see that globals are saved back, 34415a18407fSRichard Henderson that is, TS_DEAD, waiting to be reloaded. */ 3442b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 3443b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 3444b83eabeaSRichard Henderson || arg_ts->state == TS_DEAD); 34455a18407fSRichard Henderson } 34465a18407fSRichard Henderson } 34475a18407fSRichard Henderson 34485a18407fSRichard Henderson /* Outputs become available. */ 344961f15c48SRichard Henderson if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 345061f15c48SRichard Henderson arg_ts = arg_temp(op->args[0]); 345161f15c48SRichard Henderson dir_ts = arg_ts->state_ptr; 345261f15c48SRichard Henderson if (dir_ts) { 345361f15c48SRichard Henderson op->args[0] = temp_arg(dir_ts); 345461f15c48SRichard Henderson changes = true; 345561f15c48SRichard Henderson 345661f15c48SRichard Henderson /* The output is now live and modified. */ 345761f15c48SRichard Henderson arg_ts->state = 0; 345861f15c48SRichard Henderson 345961f15c48SRichard Henderson if (NEED_SYNC_ARG(0)) { 346061f15c48SRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 346161f15c48SRichard Henderson ? INDEX_op_st_i32 346261f15c48SRichard Henderson : INDEX_op_st_i64); 3463d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 346461f15c48SRichard Henderson TCGTemp *out_ts = dir_ts; 346561f15c48SRichard Henderson 346661f15c48SRichard Henderson if (IS_DEAD_ARG(0)) { 346761f15c48SRichard Henderson out_ts = arg_temp(op->args[1]); 346861f15c48SRichard Henderson arg_ts->state = TS_DEAD; 346961f15c48SRichard Henderson tcg_op_remove(s, op); 347061f15c48SRichard Henderson } else { 347161f15c48SRichard Henderson arg_ts->state = TS_MEM; 347261f15c48SRichard Henderson } 347361f15c48SRichard Henderson 347461f15c48SRichard Henderson sop->args[0] = temp_arg(out_ts); 347561f15c48SRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 347661f15c48SRichard Henderson sop->args[2] = arg_ts->mem_offset; 347761f15c48SRichard Henderson } else { 347861f15c48SRichard Henderson tcg_debug_assert(!IS_DEAD_ARG(0)); 347961f15c48SRichard Henderson } 348061f15c48SRichard Henderson } 348161f15c48SRichard Henderson } else { 34825a18407fSRichard Henderson for (i = 0; i < nb_oargs; i++) { 3483b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3484b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3485b83eabeaSRichard Henderson if (!dir_ts) { 34865a18407fSRichard Henderson continue; 34875a18407fSRichard Henderson } 3488b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 34895a18407fSRichard Henderson changes = true; 34905a18407fSRichard Henderson 34915a18407fSRichard Henderson /* The output is now live and modified. */ 3492b83eabeaSRichard Henderson arg_ts->state = 0; 34935a18407fSRichard Henderson 34945a18407fSRichard Henderson /* Sync outputs upon their last write. */ 34955a18407fSRichard Henderson if (NEED_SYNC_ARG(i)) { 3496b83eabeaSRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 34975a18407fSRichard Henderson ? INDEX_op_st_i32 34985a18407fSRichard Henderson : INDEX_op_st_i64); 3499d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 35005a18407fSRichard Henderson 3501b83eabeaSRichard Henderson sop->args[0] = temp_arg(dir_ts); 3502b83eabeaSRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 3503b83eabeaSRichard Henderson sop->args[2] = arg_ts->mem_offset; 35045a18407fSRichard Henderson 3505b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 35065a18407fSRichard Henderson } 35075a18407fSRichard Henderson /* Drop outputs that are dead. */ 35085a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 3509b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 35105a18407fSRichard Henderson } 35115a18407fSRichard Henderson } 35125a18407fSRichard Henderson } 351361f15c48SRichard Henderson } 35145a18407fSRichard Henderson 35155a18407fSRichard Henderson return changes; 35165a18407fSRichard Henderson } 35175a18407fSRichard Henderson 35182272e4a7SRichard Henderson static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3519c896fe29Sbellard { 352031c96417SRichard Henderson intptr_t off; 3521273eb50cSRichard Henderson int size, align; 3522c1c09194SRichard Henderson 3523273eb50cSRichard Henderson /* When allocating an object, look at the full type. */ 3524273eb50cSRichard Henderson size = tcg_type_size(ts->base_type); 3525273eb50cSRichard Henderson switch (ts->base_type) { 3526c1c09194SRichard Henderson case TCG_TYPE_I32: 352731c96417SRichard Henderson align = 4; 3528c1c09194SRichard Henderson break; 3529c1c09194SRichard Henderson case TCG_TYPE_I64: 3530c1c09194SRichard Henderson case TCG_TYPE_V64: 353131c96417SRichard Henderson align = 8; 3532c1c09194SRichard Henderson break; 353343eef72fSRichard Henderson case TCG_TYPE_I128: 3534c1c09194SRichard Henderson case TCG_TYPE_V128: 3535c1c09194SRichard Henderson case TCG_TYPE_V256: 353643eef72fSRichard Henderson /* 353743eef72fSRichard Henderson * Note that we do not require aligned storage for V256, 353843eef72fSRichard Henderson * and that we provide alignment for I128 to match V128, 353943eef72fSRichard Henderson * even if that's above what the host ABI requires. 354043eef72fSRichard Henderson */ 354131c96417SRichard Henderson align = 16; 3542c1c09194SRichard Henderson break; 3543c1c09194SRichard Henderson default: 3544c1c09194SRichard Henderson g_assert_not_reached(); 3545b591dc59SBlue Swirl } 3546c1c09194SRichard Henderson 3547b9537d59SRichard Henderson /* 3548b9537d59SRichard Henderson * Assume the stack is sufficiently aligned. 3549b9537d59SRichard Henderson * This affects e.g. ARM NEON, where we have 8 byte stack alignment 3550b9537d59SRichard Henderson * and do not require 16 byte vector alignment. This seems slightly 3551b9537d59SRichard Henderson * easier than fully parameterizing the above switch statement. 3552b9537d59SRichard Henderson */ 3553b9537d59SRichard Henderson align = MIN(TCG_TARGET_STACK_ALIGN, align); 3554c1c09194SRichard Henderson off = ROUND_UP(s->current_frame_offset, align); 3555732d5897SRichard Henderson 3556732d5897SRichard Henderson /* If we've exhausted the stack frame, restart with a smaller TB. */ 3557732d5897SRichard Henderson if (off + size > s->frame_end) { 3558732d5897SRichard Henderson tcg_raise_tb_overflow(s); 3559732d5897SRichard Henderson } 3560c1c09194SRichard Henderson s->current_frame_offset = off + size; 35619defd1bdSRichard Henderson #if defined(__sparc__) 3562273eb50cSRichard Henderson off += TCG_TARGET_STACK_BIAS; 35639defd1bdSRichard Henderson #endif 3564273eb50cSRichard Henderson 3565273eb50cSRichard Henderson /* If the object was subdivided, assign memory to all the parts. */ 3566273eb50cSRichard Henderson if (ts->base_type != ts->type) { 3567273eb50cSRichard Henderson int part_size = tcg_type_size(ts->type); 3568273eb50cSRichard Henderson int part_count = size / part_size; 3569273eb50cSRichard Henderson 3570273eb50cSRichard Henderson /* 3571273eb50cSRichard Henderson * Each part is allocated sequentially in tcg_temp_new_internal. 3572273eb50cSRichard Henderson * Jump back to the first part by subtracting the current index. 3573273eb50cSRichard Henderson */ 3574273eb50cSRichard Henderson ts -= ts->temp_subindex; 3575273eb50cSRichard Henderson for (int i = 0; i < part_count; ++i) { 3576273eb50cSRichard Henderson ts[i].mem_offset = off + i * part_size; 3577273eb50cSRichard Henderson ts[i].mem_base = s->frame_temp; 3578273eb50cSRichard Henderson ts[i].mem_allocated = 1; 3579273eb50cSRichard Henderson } 3580273eb50cSRichard Henderson } else { 3581273eb50cSRichard Henderson ts->mem_offset = off; 3582b3a62939SRichard Henderson ts->mem_base = s->frame_temp; 3583c896fe29Sbellard ts->mem_allocated = 1; 3584c896fe29Sbellard } 3585273eb50cSRichard Henderson } 3586c896fe29Sbellard 3587098859f1SRichard Henderson /* Assign @reg to @ts, and update reg_to_temp[]. */ 3588098859f1SRichard Henderson static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 3589098859f1SRichard Henderson { 3590098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 3591098859f1SRichard Henderson TCGReg old = ts->reg; 3592098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[old] == ts); 3593098859f1SRichard Henderson if (old == reg) { 3594098859f1SRichard Henderson return; 3595098859f1SRichard Henderson } 3596098859f1SRichard Henderson s->reg_to_temp[old] = NULL; 3597098859f1SRichard Henderson } 3598098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3599098859f1SRichard Henderson s->reg_to_temp[reg] = ts; 3600098859f1SRichard Henderson ts->val_type = TEMP_VAL_REG; 3601098859f1SRichard Henderson ts->reg = reg; 3602098859f1SRichard Henderson } 3603098859f1SRichard Henderson 3604098859f1SRichard Henderson /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 3605098859f1SRichard Henderson static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 3606098859f1SRichard Henderson { 3607098859f1SRichard Henderson tcg_debug_assert(type != TEMP_VAL_REG); 3608098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 3609098859f1SRichard Henderson TCGReg reg = ts->reg; 3610098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == ts); 3611098859f1SRichard Henderson s->reg_to_temp[reg] = NULL; 3612098859f1SRichard Henderson } 3613098859f1SRichard Henderson ts->val_type = type; 3614098859f1SRichard Henderson } 3615098859f1SRichard Henderson 3616b722452aSRichard Henderson static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3617b3915dbbSRichard Henderson 361859d7c14eSRichard Henderson /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 361959d7c14eSRichard Henderson mark it free; otherwise mark it dead. */ 362059d7c14eSRichard Henderson static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3621c896fe29Sbellard { 3622c0522136SRichard Henderson TCGTempVal new_type; 3623c0522136SRichard Henderson 3624c0522136SRichard Henderson switch (ts->kind) { 3625c0522136SRichard Henderson case TEMP_FIXED: 362659d7c14eSRichard Henderson return; 3627c0522136SRichard Henderson case TEMP_GLOBAL: 3628f57c6915SRichard Henderson case TEMP_TB: 3629c0522136SRichard Henderson new_type = TEMP_VAL_MEM; 3630c0522136SRichard Henderson break; 3631c7482438SRichard Henderson case TEMP_EBB: 3632c0522136SRichard Henderson new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3633c0522136SRichard Henderson break; 3634c0522136SRichard Henderson case TEMP_CONST: 3635c0522136SRichard Henderson new_type = TEMP_VAL_CONST; 3636c0522136SRichard Henderson break; 3637c0522136SRichard Henderson default: 3638c0522136SRichard Henderson g_assert_not_reached(); 363959d7c14eSRichard Henderson } 3640098859f1SRichard Henderson set_temp_val_nonreg(s, ts, new_type); 364159d7c14eSRichard Henderson } 3642c896fe29Sbellard 364359d7c14eSRichard Henderson /* Mark a temporary as dead. */ 364459d7c14eSRichard Henderson static inline void temp_dead(TCGContext *s, TCGTemp *ts) 364559d7c14eSRichard Henderson { 364659d7c14eSRichard Henderson temp_free_or_dead(s, ts, 1); 364759d7c14eSRichard Henderson } 364859d7c14eSRichard Henderson 364959d7c14eSRichard Henderson /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 365059d7c14eSRichard Henderson registers needs to be allocated to store a constant. If 'free_or_dead' 365159d7c14eSRichard Henderson is non-zero, subsequently release the temporary; if it is positive, the 365259d7c14eSRichard Henderson temp is dead; if it is negative, the temp is free. */ 365398b4e186SRichard Henderson static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 365498b4e186SRichard Henderson TCGRegSet preferred_regs, int free_or_dead) 365559d7c14eSRichard Henderson { 3656c0522136SRichard Henderson if (!temp_readonly(ts) && !ts->mem_coherent) { 36577f6ceedfSAurelien Jarno if (!ts->mem_allocated) { 36582272e4a7SRichard Henderson temp_allocate_frame(s, ts); 365959d7c14eSRichard Henderson } 366059d7c14eSRichard Henderson switch (ts->val_type) { 366159d7c14eSRichard Henderson case TEMP_VAL_CONST: 366259d7c14eSRichard Henderson /* If we're going to free the temp immediately, then we won't 366359d7c14eSRichard Henderson require it later in a register, so attempt to store the 366459d7c14eSRichard Henderson constant to memory directly. */ 366559d7c14eSRichard Henderson if (free_or_dead 366659d7c14eSRichard Henderson && tcg_out_sti(s, ts->type, ts->val, 366759d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset)) { 366859d7c14eSRichard Henderson break; 366959d7c14eSRichard Henderson } 367059d7c14eSRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 367198b4e186SRichard Henderson allocated_regs, preferred_regs); 367259d7c14eSRichard Henderson /* fallthrough */ 367359d7c14eSRichard Henderson 367459d7c14eSRichard Henderson case TEMP_VAL_REG: 367559d7c14eSRichard Henderson tcg_out_st(s, ts->type, ts->reg, 367659d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset); 367759d7c14eSRichard Henderson break; 367859d7c14eSRichard Henderson 367959d7c14eSRichard Henderson case TEMP_VAL_MEM: 368059d7c14eSRichard Henderson break; 368159d7c14eSRichard Henderson 368259d7c14eSRichard Henderson case TEMP_VAL_DEAD: 368359d7c14eSRichard Henderson default: 3684732e89f4SRichard Henderson g_assert_not_reached(); 3685c896fe29Sbellard } 36867f6ceedfSAurelien Jarno ts->mem_coherent = 1; 36877f6ceedfSAurelien Jarno } 368859d7c14eSRichard Henderson if (free_or_dead) { 368959d7c14eSRichard Henderson temp_free_or_dead(s, ts, free_or_dead); 369059d7c14eSRichard Henderson } 369159d7c14eSRichard Henderson } 36927f6ceedfSAurelien Jarno 36937f6ceedfSAurelien Jarno /* free register 'reg' by spilling the corresponding temporary if necessary */ 3694b3915dbbSRichard Henderson static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 36957f6ceedfSAurelien Jarno { 3696f8b2f202SRichard Henderson TCGTemp *ts = s->reg_to_temp[reg]; 3697f8b2f202SRichard Henderson if (ts != NULL) { 369898b4e186SRichard Henderson temp_sync(s, ts, allocated_regs, 0, -1); 3699c896fe29Sbellard } 3700c896fe29Sbellard } 3701c896fe29Sbellard 3702b016486eSRichard Henderson /** 3703b016486eSRichard Henderson * tcg_reg_alloc: 3704b016486eSRichard Henderson * @required_regs: Set of registers in which we must allocate. 3705b016486eSRichard Henderson * @allocated_regs: Set of registers which must be avoided. 3706b016486eSRichard Henderson * @preferred_regs: Set of registers we should prefer. 3707b016486eSRichard Henderson * @rev: True if we search the registers in "indirect" order. 3708b016486eSRichard Henderson * 3709b016486eSRichard Henderson * The allocated register must be in @required_regs & ~@allocated_regs, 3710b016486eSRichard Henderson * but if we can put it in @preferred_regs we may save a move later. 3711b016486eSRichard Henderson */ 3712b016486eSRichard Henderson static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3713b016486eSRichard Henderson TCGRegSet allocated_regs, 3714b016486eSRichard Henderson TCGRegSet preferred_regs, bool rev) 3715c896fe29Sbellard { 3716b016486eSRichard Henderson int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3717b016486eSRichard Henderson TCGRegSet reg_ct[2]; 371891478cefSRichard Henderson const int *order; 3719c896fe29Sbellard 3720b016486eSRichard Henderson reg_ct[1] = required_regs & ~allocated_regs; 3721b016486eSRichard Henderson tcg_debug_assert(reg_ct[1] != 0); 3722b016486eSRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs; 3723b016486eSRichard Henderson 3724b016486eSRichard Henderson /* Skip the preferred_regs option if it cannot be satisfied, 3725b016486eSRichard Henderson or if the preference made no difference. */ 3726b016486eSRichard Henderson f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3727b016486eSRichard Henderson 372891478cefSRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3729c896fe29Sbellard 3730b016486eSRichard Henderson /* Try free registers, preferences first. */ 3731b016486eSRichard Henderson for (j = f; j < 2; j++) { 3732b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 3733b016486eSRichard Henderson 3734b016486eSRichard Henderson if (tcg_regset_single(set)) { 3735b016486eSRichard Henderson /* One register in the set. */ 3736b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 3737b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL) { 3738c896fe29Sbellard return reg; 3739c896fe29Sbellard } 3740b016486eSRichard Henderson } else { 374191478cefSRichard Henderson for (i = 0; i < n; i++) { 3742b016486eSRichard Henderson TCGReg reg = order[i]; 3743b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL && 3744b016486eSRichard Henderson tcg_regset_test_reg(set, reg)) { 3745b016486eSRichard Henderson return reg; 3746b016486eSRichard Henderson } 3747b016486eSRichard Henderson } 3748b016486eSRichard Henderson } 3749b016486eSRichard Henderson } 3750b016486eSRichard Henderson 3751b016486eSRichard Henderson /* We must spill something. */ 3752b016486eSRichard Henderson for (j = f; j < 2; j++) { 3753b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 3754b016486eSRichard Henderson 3755b016486eSRichard Henderson if (tcg_regset_single(set)) { 3756b016486eSRichard Henderson /* One register in the set. */ 3757b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 3758b3915dbbSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 3759c896fe29Sbellard return reg; 3760b016486eSRichard Henderson } else { 3761b016486eSRichard Henderson for (i = 0; i < n; i++) { 3762b016486eSRichard Henderson TCGReg reg = order[i]; 3763b016486eSRichard Henderson if (tcg_regset_test_reg(set, reg)) { 3764b016486eSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 3765b016486eSRichard Henderson return reg; 3766b016486eSRichard Henderson } 3767b016486eSRichard Henderson } 3768c896fe29Sbellard } 3769c896fe29Sbellard } 3770c896fe29Sbellard 3771732e89f4SRichard Henderson g_assert_not_reached(); 3772c896fe29Sbellard } 3773c896fe29Sbellard 377429f5e925SRichard Henderson static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 377529f5e925SRichard Henderson TCGRegSet allocated_regs, 377629f5e925SRichard Henderson TCGRegSet preferred_regs, bool rev) 377729f5e925SRichard Henderson { 377829f5e925SRichard Henderson int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 377929f5e925SRichard Henderson TCGRegSet reg_ct[2]; 378029f5e925SRichard Henderson const int *order; 378129f5e925SRichard Henderson 378229f5e925SRichard Henderson /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 378329f5e925SRichard Henderson reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 378429f5e925SRichard Henderson tcg_debug_assert(reg_ct[1] != 0); 378529f5e925SRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs; 378629f5e925SRichard Henderson 378729f5e925SRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 378829f5e925SRichard Henderson 378929f5e925SRichard Henderson /* 379029f5e925SRichard Henderson * Skip the preferred_regs option if it cannot be satisfied, 379129f5e925SRichard Henderson * or if the preference made no difference. 379229f5e925SRichard Henderson */ 379329f5e925SRichard Henderson k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 379429f5e925SRichard Henderson 379529f5e925SRichard Henderson /* 379629f5e925SRichard Henderson * Minimize the number of flushes by looking for 2 free registers first, 379729f5e925SRichard Henderson * then a single flush, then two flushes. 379829f5e925SRichard Henderson */ 379929f5e925SRichard Henderson for (fmin = 2; fmin >= 0; fmin--) { 380029f5e925SRichard Henderson for (j = k; j < 2; j++) { 380129f5e925SRichard Henderson TCGRegSet set = reg_ct[j]; 380229f5e925SRichard Henderson 380329f5e925SRichard Henderson for (i = 0; i < n; i++) { 380429f5e925SRichard Henderson TCGReg reg = order[i]; 380529f5e925SRichard Henderson 380629f5e925SRichard Henderson if (tcg_regset_test_reg(set, reg)) { 380729f5e925SRichard Henderson int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 380829f5e925SRichard Henderson if (f >= fmin) { 380929f5e925SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 381029f5e925SRichard Henderson tcg_reg_free(s, reg + 1, allocated_regs); 381129f5e925SRichard Henderson return reg; 381229f5e925SRichard Henderson } 381329f5e925SRichard Henderson } 381429f5e925SRichard Henderson } 381529f5e925SRichard Henderson } 381629f5e925SRichard Henderson } 3817732e89f4SRichard Henderson g_assert_not_reached(); 381829f5e925SRichard Henderson } 381929f5e925SRichard Henderson 382040ae5c62SRichard Henderson /* Make sure the temporary is in a register. If needed, allocate the register 382140ae5c62SRichard Henderson from DESIRED while avoiding ALLOCATED. */ 382240ae5c62SRichard Henderson static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3823b722452aSRichard Henderson TCGRegSet allocated_regs, TCGRegSet preferred_regs) 382440ae5c62SRichard Henderson { 382540ae5c62SRichard Henderson TCGReg reg; 382640ae5c62SRichard Henderson 382740ae5c62SRichard Henderson switch (ts->val_type) { 382840ae5c62SRichard Henderson case TEMP_VAL_REG: 382940ae5c62SRichard Henderson return; 383040ae5c62SRichard Henderson case TEMP_VAL_CONST: 3831b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3832b722452aSRichard Henderson preferred_regs, ts->indirect_base); 38330a6a8bc8SRichard Henderson if (ts->type <= TCG_TYPE_I64) { 383440ae5c62SRichard Henderson tcg_out_movi(s, ts->type, reg, ts->val); 38350a6a8bc8SRichard Henderson } else { 38364e186175SRichard Henderson uint64_t val = ts->val; 38374e186175SRichard Henderson MemOp vece = MO_64; 38384e186175SRichard Henderson 38394e186175SRichard Henderson /* 38404e186175SRichard Henderson * Find the minimal vector element that matches the constant. 38414e186175SRichard Henderson * The targets will, in general, have to do this search anyway, 38424e186175SRichard Henderson * do this generically. 38434e186175SRichard Henderson */ 38444e186175SRichard Henderson if (val == dup_const(MO_8, val)) { 38454e186175SRichard Henderson vece = MO_8; 38464e186175SRichard Henderson } else if (val == dup_const(MO_16, val)) { 38474e186175SRichard Henderson vece = MO_16; 38480b4286ddSRichard Henderson } else if (val == dup_const(MO_32, val)) { 38494e186175SRichard Henderson vece = MO_32; 38504e186175SRichard Henderson } 38514e186175SRichard Henderson 38524e186175SRichard Henderson tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 38530a6a8bc8SRichard Henderson } 385440ae5c62SRichard Henderson ts->mem_coherent = 0; 385540ae5c62SRichard Henderson break; 385640ae5c62SRichard Henderson case TEMP_VAL_MEM: 3857b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3858b722452aSRichard Henderson preferred_regs, ts->indirect_base); 385940ae5c62SRichard Henderson tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 386040ae5c62SRichard Henderson ts->mem_coherent = 1; 386140ae5c62SRichard Henderson break; 386240ae5c62SRichard Henderson case TEMP_VAL_DEAD: 386340ae5c62SRichard Henderson default: 3864732e89f4SRichard Henderson g_assert_not_reached(); 386540ae5c62SRichard Henderson } 3866098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 386740ae5c62SRichard Henderson } 386840ae5c62SRichard Henderson 386959d7c14eSRichard Henderson /* Save a temporary to memory. 'allocated_regs' is used in case a 3870e8996ee0Sbellard temporary registers needs to be allocated to store a constant. */ 387159d7c14eSRichard Henderson static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 38721ad80729SAurelien Jarno { 38732c0366f0SAurelien Jarno /* The liveness analysis already ensures that globals are back 3874eabb7b91SAurelien Jarno in memory. Keep an tcg_debug_assert for safety. */ 3875e01fa97dSRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 38761ad80729SAurelien Jarno } 38771ad80729SAurelien Jarno 38789814dd27SDong Xu Wang /* save globals to their canonical location and assume they can be 3879641d5fbeSbellard modified be the following code. 'allocated_regs' is used in case a 3880641d5fbeSbellard temporary registers needs to be allocated to store a constant. */ 3881641d5fbeSbellard static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3882641d5fbeSbellard { 3883ac3b8891SRichard Henderson int i, n; 3884641d5fbeSbellard 3885ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 3886b13eb728SRichard Henderson temp_save(s, &s->temps[i], allocated_regs); 3887641d5fbeSbellard } 3888e5097dc8Sbellard } 3889e5097dc8Sbellard 38903d5c5f87SAurelien Jarno /* sync globals to their canonical location and assume they can be 38913d5c5f87SAurelien Jarno read by the following code. 'allocated_regs' is used in case a 38923d5c5f87SAurelien Jarno temporary registers needs to be allocated to store a constant. */ 38933d5c5f87SAurelien Jarno static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 38943d5c5f87SAurelien Jarno { 3895ac3b8891SRichard Henderson int i, n; 38963d5c5f87SAurelien Jarno 3897ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 389812b9b11aSRichard Henderson TCGTemp *ts = &s->temps[i]; 389912b9b11aSRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3900ee17db83SRichard Henderson || ts->kind == TEMP_FIXED 390112b9b11aSRichard Henderson || ts->mem_coherent); 39023d5c5f87SAurelien Jarno } 39033d5c5f87SAurelien Jarno } 39043d5c5f87SAurelien Jarno 3905e5097dc8Sbellard /* at the end of a basic block, we assume all temporaries are dead and 3906e8996ee0Sbellard all globals are stored at their canonical location. */ 3907e8996ee0Sbellard static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3908e5097dc8Sbellard { 3909e5097dc8Sbellard int i; 3910e5097dc8Sbellard 3911c896fe29Sbellard for (i = s->nb_globals; i < s->nb_temps; i++) { 3912b13eb728SRichard Henderson TCGTemp *ts = &s->temps[i]; 3913c0522136SRichard Henderson 3914c0522136SRichard Henderson switch (ts->kind) { 3915f57c6915SRichard Henderson case TEMP_TB: 3916b13eb728SRichard Henderson temp_save(s, ts, allocated_regs); 3917c0522136SRichard Henderson break; 3918c7482438SRichard Henderson case TEMP_EBB: 39192c0366f0SAurelien Jarno /* The liveness analysis already ensures that temps are dead. 3920eabb7b91SAurelien Jarno Keep an tcg_debug_assert for safety. */ 3921eabb7b91SAurelien Jarno tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3922c0522136SRichard Henderson break; 3923c0522136SRichard Henderson case TEMP_CONST: 3924c0522136SRichard Henderson /* Similarly, we should have freed any allocated register. */ 3925c0522136SRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3926c0522136SRichard Henderson break; 3927c0522136SRichard Henderson default: 3928c0522136SRichard Henderson g_assert_not_reached(); 3929c896fe29Sbellard } 3930641d5fbeSbellard } 3931e8996ee0Sbellard 3932e8996ee0Sbellard save_globals(s, allocated_regs); 3933c896fe29Sbellard } 3934c896fe29Sbellard 3935bab1671fSRichard Henderson /* 3936c7482438SRichard Henderson * At a conditional branch, we assume all temporaries are dead unless 3937c7482438SRichard Henderson * explicitly live-across-conditional-branch; all globals and local 3938c7482438SRichard Henderson * temps are synced to their location. 3939b4cb76e6SRichard Henderson */ 3940b4cb76e6SRichard Henderson static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3941b4cb76e6SRichard Henderson { 3942b4cb76e6SRichard Henderson sync_globals(s, allocated_regs); 3943b4cb76e6SRichard Henderson 3944b4cb76e6SRichard Henderson for (int i = s->nb_globals; i < s->nb_temps; i++) { 3945b4cb76e6SRichard Henderson TCGTemp *ts = &s->temps[i]; 3946b4cb76e6SRichard Henderson /* 3947b4cb76e6SRichard Henderson * The liveness analysis already ensures that temps are dead. 3948b4cb76e6SRichard Henderson * Keep tcg_debug_asserts for safety. 3949b4cb76e6SRichard Henderson */ 3950c0522136SRichard Henderson switch (ts->kind) { 3951f57c6915SRichard Henderson case TEMP_TB: 3952b4cb76e6SRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3953c0522136SRichard Henderson break; 3954c7482438SRichard Henderson case TEMP_EBB: 3955c0522136SRichard Henderson case TEMP_CONST: 3956c0522136SRichard Henderson break; 3957c0522136SRichard Henderson default: 3958c0522136SRichard Henderson g_assert_not_reached(); 3959b4cb76e6SRichard Henderson } 3960b4cb76e6SRichard Henderson } 3961b4cb76e6SRichard Henderson } 3962b4cb76e6SRichard Henderson 3963b4cb76e6SRichard Henderson /* 3964c58f4c97SRichard Henderson * Specialized code generation for INDEX_op_mov_* with a constant. 3965bab1671fSRichard Henderson */ 39660fe4fca4SPaolo Bonzini static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3967ba87719cSRichard Henderson tcg_target_ulong val, TCGLifeData arg_life, 3968ba87719cSRichard Henderson TCGRegSet preferred_regs) 3969e8996ee0Sbellard { 3970d63e3b6eSRichard Henderson /* ENV should not be modified. */ 3971e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 397259d7c14eSRichard Henderson 397359d7c14eSRichard Henderson /* The movi is not explicitly generated here. */ 3974098859f1SRichard Henderson set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 3975e8996ee0Sbellard ots->val = val; 397659d7c14eSRichard Henderson ots->mem_coherent = 0; 3977ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 3978ba87719cSRichard Henderson temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 397959d7c14eSRichard Henderson } else if (IS_DEAD_ARG(0)) { 3980f8bf00f1SRichard Henderson temp_dead(s, ots); 39814c4e1ab2SAurelien Jarno } 3982e8996ee0Sbellard } 3983e8996ee0Sbellard 3984bab1671fSRichard Henderson /* 3985bab1671fSRichard Henderson * Specialized code generation for INDEX_op_mov_*. 3986bab1671fSRichard Henderson */ 3987dd186292SRichard Henderson static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 3988c896fe29Sbellard { 3989dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 399069e3706dSRichard Henderson TCGRegSet allocated_regs, preferred_regs; 3991c896fe29Sbellard TCGTemp *ts, *ots; 3992450445d5SRichard Henderson TCGType otype, itype; 3993098859f1SRichard Henderson TCGReg oreg, ireg; 3994c896fe29Sbellard 3995d21369f5SRichard Henderson allocated_regs = s->reserved_regs; 399631fd884bSRichard Henderson preferred_regs = output_pref(op, 0); 399743439139SRichard Henderson ots = arg_temp(op->args[0]); 399843439139SRichard Henderson ts = arg_temp(op->args[1]); 3999450445d5SRichard Henderson 4000d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4001e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4002d63e3b6eSRichard Henderson 4003450445d5SRichard Henderson /* Note that otype != itype for no-op truncation. */ 4004450445d5SRichard Henderson otype = ots->type; 4005450445d5SRichard Henderson itype = ts->type; 4006c896fe29Sbellard 40070fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_CONST) { 40080fe4fca4SPaolo Bonzini /* propagate constant or generate sti */ 40090fe4fca4SPaolo Bonzini tcg_target_ulong val = ts->val; 40100fe4fca4SPaolo Bonzini if (IS_DEAD_ARG(1)) { 40110fe4fca4SPaolo Bonzini temp_dead(s, ts); 40120fe4fca4SPaolo Bonzini } 401369e3706dSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 40140fe4fca4SPaolo Bonzini return; 40150fe4fca4SPaolo Bonzini } 40160fe4fca4SPaolo Bonzini 40170fe4fca4SPaolo Bonzini /* If the source value is in memory we're going to be forced 40180fe4fca4SPaolo Bonzini to have it in a register in order to perform the copy. Copy 40190fe4fca4SPaolo Bonzini the SOURCE value into its own register first, that way we 40200fe4fca4SPaolo Bonzini don't have to reload SOURCE the next time it is used. */ 40210fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_MEM) { 402269e3706dSRichard Henderson temp_load(s, ts, tcg_target_available_regs[itype], 402369e3706dSRichard Henderson allocated_regs, preferred_regs); 4024c29c1d7eSAurelien Jarno } 40250fe4fca4SPaolo Bonzini tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 4026098859f1SRichard Henderson ireg = ts->reg; 4027098859f1SRichard Henderson 4028d63e3b6eSRichard Henderson if (IS_DEAD_ARG(0)) { 4029c29c1d7eSAurelien Jarno /* mov to a non-saved dead register makes no sense (even with 4030c29c1d7eSAurelien Jarno liveness analysis disabled). */ 4031eabb7b91SAurelien Jarno tcg_debug_assert(NEED_SYNC_ARG(0)); 4032c29c1d7eSAurelien Jarno if (!ots->mem_allocated) { 40332272e4a7SRichard Henderson temp_allocate_frame(s, ots); 4034c29c1d7eSAurelien Jarno } 4035098859f1SRichard Henderson tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 4036c29c1d7eSAurelien Jarno if (IS_DEAD_ARG(1)) { 4037f8bf00f1SRichard Henderson temp_dead(s, ts); 4038c29c1d7eSAurelien Jarno } 4039f8bf00f1SRichard Henderson temp_dead(s, ots); 4040098859f1SRichard Henderson return; 4041098859f1SRichard Henderson } 4042098859f1SRichard Henderson 4043ee17db83SRichard Henderson if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 4044098859f1SRichard Henderson /* 4045098859f1SRichard Henderson * The mov can be suppressed. Kill input first, so that it 4046098859f1SRichard Henderson * is unlinked from reg_to_temp, then set the output to the 4047098859f1SRichard Henderson * reg that we saved from the input. 4048098859f1SRichard Henderson */ 4049f8bf00f1SRichard Henderson temp_dead(s, ts); 4050098859f1SRichard Henderson oreg = ireg; 4051c29c1d7eSAurelien Jarno } else { 4052098859f1SRichard Henderson if (ots->val_type == TEMP_VAL_REG) { 4053098859f1SRichard Henderson oreg = ots->reg; 4054098859f1SRichard Henderson } else { 4055098859f1SRichard Henderson /* Make sure to not spill the input register during allocation. */ 4056098859f1SRichard Henderson oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 4057098859f1SRichard Henderson allocated_regs | ((TCGRegSet)1 << ireg), 4058098859f1SRichard Henderson preferred_regs, ots->indirect_base); 4059c29c1d7eSAurelien Jarno } 4060098859f1SRichard Henderson if (!tcg_out_mov(s, otype, oreg, ireg)) { 4061240c08d0SRichard Henderson /* 4062240c08d0SRichard Henderson * Cross register class move not supported. 4063240c08d0SRichard Henderson * Store the source register into the destination slot 4064240c08d0SRichard Henderson * and leave the destination temp as TEMP_VAL_MEM. 4065240c08d0SRichard Henderson */ 4066e01fa97dSRichard Henderson assert(!temp_readonly(ots)); 4067240c08d0SRichard Henderson if (!ts->mem_allocated) { 4068240c08d0SRichard Henderson temp_allocate_frame(s, ots); 4069240c08d0SRichard Henderson } 4070098859f1SRichard Henderson tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 4071098859f1SRichard Henderson set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 4072240c08d0SRichard Henderson ots->mem_coherent = 1; 4073240c08d0SRichard Henderson return; 407478113e83SRichard Henderson } 4075c29c1d7eSAurelien Jarno } 4076098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4077c896fe29Sbellard ots->mem_coherent = 0; 4078098859f1SRichard Henderson 4079ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 408098b4e186SRichard Henderson temp_sync(s, ots, allocated_regs, 0, 0); 4081c29c1d7eSAurelien Jarno } 4082ec7a869dSAurelien Jarno } 4083c896fe29Sbellard 4084bab1671fSRichard Henderson /* 4085bab1671fSRichard Henderson * Specialized code generation for INDEX_op_dup_vec. 4086bab1671fSRichard Henderson */ 4087bab1671fSRichard Henderson static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 4088bab1671fSRichard Henderson { 4089bab1671fSRichard Henderson const TCGLifeData arg_life = op->life; 4090bab1671fSRichard Henderson TCGRegSet dup_out_regs, dup_in_regs; 4091bab1671fSRichard Henderson TCGTemp *its, *ots; 4092bab1671fSRichard Henderson TCGType itype, vtype; 4093bab1671fSRichard Henderson unsigned vece; 409431c96417SRichard Henderson int lowpart_ofs; 4095bab1671fSRichard Henderson bool ok; 4096bab1671fSRichard Henderson 4097bab1671fSRichard Henderson ots = arg_temp(op->args[0]); 4098bab1671fSRichard Henderson its = arg_temp(op->args[1]); 4099bab1671fSRichard Henderson 4100bab1671fSRichard Henderson /* ENV should not be modified. */ 4101e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4102bab1671fSRichard Henderson 4103bab1671fSRichard Henderson itype = its->type; 4104bab1671fSRichard Henderson vece = TCGOP_VECE(op); 4105bab1671fSRichard Henderson vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4106bab1671fSRichard Henderson 4107bab1671fSRichard Henderson if (its->val_type == TEMP_VAL_CONST) { 4108bab1671fSRichard Henderson /* Propagate constant via movi -> dupi. */ 4109bab1671fSRichard Henderson tcg_target_ulong val = its->val; 4110bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 4111bab1671fSRichard Henderson temp_dead(s, its); 4112bab1671fSRichard Henderson } 411331fd884bSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); 4114bab1671fSRichard Henderson return; 4115bab1671fSRichard Henderson } 4116bab1671fSRichard Henderson 41179be0d080SRichard Henderson dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 41189be0d080SRichard Henderson dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 4119bab1671fSRichard Henderson 4120bab1671fSRichard Henderson /* Allocate the output register now. */ 4121bab1671fSRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 4122bab1671fSRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 4123098859f1SRichard Henderson TCGReg oreg; 4124bab1671fSRichard Henderson 4125bab1671fSRichard Henderson if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 4126bab1671fSRichard Henderson /* Make sure to not spill the input register. */ 4127bab1671fSRichard Henderson tcg_regset_set_reg(allocated_regs, its->reg); 4128bab1671fSRichard Henderson } 4129098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 413031fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base); 4131098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4132bab1671fSRichard Henderson } 4133bab1671fSRichard Henderson 4134bab1671fSRichard Henderson switch (its->val_type) { 4135bab1671fSRichard Henderson case TEMP_VAL_REG: 4136bab1671fSRichard Henderson /* 4137bab1671fSRichard Henderson * The dup constriaints must be broad, covering all possible VECE. 4138bab1671fSRichard Henderson * However, tcg_op_dup_vec() gets to see the VECE and we allow it 4139bab1671fSRichard Henderson * to fail, indicating that extra moves are required for that case. 4140bab1671fSRichard Henderson */ 4141bab1671fSRichard Henderson if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 4142bab1671fSRichard Henderson if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 4143bab1671fSRichard Henderson goto done; 4144bab1671fSRichard Henderson } 4145bab1671fSRichard Henderson /* Try again from memory or a vector input register. */ 4146bab1671fSRichard Henderson } 4147bab1671fSRichard Henderson if (!its->mem_coherent) { 4148bab1671fSRichard Henderson /* 4149bab1671fSRichard Henderson * The input register is not synced, and so an extra store 4150bab1671fSRichard Henderson * would be required to use memory. Attempt an integer-vector 4151bab1671fSRichard Henderson * register move first. We do not have a TCGRegSet for this. 4152bab1671fSRichard Henderson */ 4153bab1671fSRichard Henderson if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 4154bab1671fSRichard Henderson break; 4155bab1671fSRichard Henderson } 4156bab1671fSRichard Henderson /* Sync the temp back to its slot and load from there. */ 4157bab1671fSRichard Henderson temp_sync(s, its, s->reserved_regs, 0, 0); 4158bab1671fSRichard Henderson } 4159bab1671fSRichard Henderson /* fall through */ 4160bab1671fSRichard Henderson 4161bab1671fSRichard Henderson case TEMP_VAL_MEM: 416231c96417SRichard Henderson lowpart_ofs = 0; 416331c96417SRichard Henderson if (HOST_BIG_ENDIAN) { 416431c96417SRichard Henderson lowpart_ofs = tcg_type_size(itype) - (1 << vece); 416531c96417SRichard Henderson } 4166d6ecb4a9SRichard Henderson if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 416731c96417SRichard Henderson its->mem_offset + lowpart_ofs)) { 4168d6ecb4a9SRichard Henderson goto done; 4169d6ecb4a9SRichard Henderson } 4170098859f1SRichard Henderson /* Load the input into the destination vector register. */ 4171bab1671fSRichard Henderson tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 4172bab1671fSRichard Henderson break; 4173bab1671fSRichard Henderson 4174bab1671fSRichard Henderson default: 4175bab1671fSRichard Henderson g_assert_not_reached(); 4176bab1671fSRichard Henderson } 4177bab1671fSRichard Henderson 4178bab1671fSRichard Henderson /* We now have a vector input register, so dup must succeed. */ 4179bab1671fSRichard Henderson ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 4180bab1671fSRichard Henderson tcg_debug_assert(ok); 4181bab1671fSRichard Henderson 4182bab1671fSRichard Henderson done: 418336f5539cSRichard Henderson ots->mem_coherent = 0; 4184bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 4185bab1671fSRichard Henderson temp_dead(s, its); 4186bab1671fSRichard Henderson } 4187bab1671fSRichard Henderson if (NEED_SYNC_ARG(0)) { 4188bab1671fSRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, 0); 4189bab1671fSRichard Henderson } 4190bab1671fSRichard Henderson if (IS_DEAD_ARG(0)) { 4191bab1671fSRichard Henderson temp_dead(s, ots); 4192bab1671fSRichard Henderson } 4193bab1671fSRichard Henderson } 4194bab1671fSRichard Henderson 4195dd186292SRichard Henderson static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 4196c896fe29Sbellard { 4197dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 4198dd186292SRichard Henderson const TCGOpDef * const def = &tcg_op_defs[op->opc]; 419982790a87SRichard Henderson TCGRegSet i_allocated_regs; 420082790a87SRichard Henderson TCGRegSet o_allocated_regs; 4201b6638662SRichard Henderson int i, k, nb_iargs, nb_oargs; 4202b6638662SRichard Henderson TCGReg reg; 4203c896fe29Sbellard TCGArg arg; 4204c896fe29Sbellard const TCGArgConstraint *arg_ct; 4205c896fe29Sbellard TCGTemp *ts; 4206c896fe29Sbellard TCGArg new_args[TCG_MAX_OP_ARGS]; 4207c896fe29Sbellard int const_args[TCG_MAX_OP_ARGS]; 4208c896fe29Sbellard 4209c896fe29Sbellard nb_oargs = def->nb_oargs; 4210c896fe29Sbellard nb_iargs = def->nb_iargs; 4211c896fe29Sbellard 4212c896fe29Sbellard /* copy constants */ 4213c896fe29Sbellard memcpy(new_args + nb_oargs + nb_iargs, 4214dd186292SRichard Henderson op->args + nb_oargs + nb_iargs, 4215c896fe29Sbellard sizeof(TCGArg) * def->nb_cargs); 4216c896fe29Sbellard 4217d21369f5SRichard Henderson i_allocated_regs = s->reserved_regs; 4218d21369f5SRichard Henderson o_allocated_regs = s->reserved_regs; 421982790a87SRichard Henderson 4220c896fe29Sbellard /* satisfy input constraints */ 4221c896fe29Sbellard for (k = 0; k < nb_iargs; k++) { 422229f5e925SRichard Henderson TCGRegSet i_preferred_regs, i_required_regs; 422329f5e925SRichard Henderson bool allocate_new_reg, copyto_new_reg; 422429f5e925SRichard Henderson TCGTemp *ts2; 422529f5e925SRichard Henderson int i1, i2; 4226d62816f2SRichard Henderson 422766792f90SRichard Henderson i = def->args_ct[nb_oargs + k].sort_index; 4228dd186292SRichard Henderson arg = op->args[i]; 4229c896fe29Sbellard arg_ct = &def->args_ct[i]; 423043439139SRichard Henderson ts = arg_temp(arg); 423140ae5c62SRichard Henderson 423240ae5c62SRichard Henderson if (ts->val_type == TEMP_VAL_CONST 4233a4fbbd77SRichard Henderson && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 4234c896fe29Sbellard /* constant is OK for instruction */ 4235c896fe29Sbellard const_args[i] = 1; 4236c896fe29Sbellard new_args[i] = ts->val; 4237d62816f2SRichard Henderson continue; 4238c896fe29Sbellard } 423940ae5c62SRichard Henderson 42401c1824dcSRichard Henderson reg = ts->reg; 42411c1824dcSRichard Henderson i_preferred_regs = 0; 424229f5e925SRichard Henderson i_required_regs = arg_ct->regs; 42431c1824dcSRichard Henderson allocate_new_reg = false; 424429f5e925SRichard Henderson copyto_new_reg = false; 42451c1824dcSRichard Henderson 424629f5e925SRichard Henderson switch (arg_ct->pair) { 424729f5e925SRichard Henderson case 0: /* not paired */ 4248bc2b17e6SRichard Henderson if (arg_ct->ialias) { 424931fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 4250c0522136SRichard Henderson 4251c0522136SRichard Henderson /* 4252c0522136SRichard Henderson * If the input is readonly, then it cannot also be an 4253c0522136SRichard Henderson * output and aliased to itself. If the input is not 4254c0522136SRichard Henderson * dead after the instruction, we must allocate a new 4255c0522136SRichard Henderson * register and move it. 4256c0522136SRichard Henderson */ 4257c0522136SRichard Henderson if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 42581c1824dcSRichard Henderson allocate_new_reg = true; 42591c1824dcSRichard Henderson } else if (ts->val_type == TEMP_VAL_REG) { 4260c0522136SRichard Henderson /* 42611c1824dcSRichard Henderson * Check if the current register has already been 42621c1824dcSRichard Henderson * allocated for another input. 4263c0522136SRichard Henderson */ 426429f5e925SRichard Henderson allocate_new_reg = 426529f5e925SRichard Henderson tcg_regset_test_reg(i_allocated_regs, reg); 42667e1df267SAurelien Jarno } 42677e1df267SAurelien Jarno } 42681c1824dcSRichard Henderson if (!allocate_new_reg) { 426929f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs, 427029f5e925SRichard Henderson i_preferred_regs); 4271c896fe29Sbellard reg = ts->reg; 427229f5e925SRichard Henderson allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 42731c1824dcSRichard Henderson } 42741c1824dcSRichard Henderson if (allocate_new_reg) { 4275c0522136SRichard Henderson /* 4276c0522136SRichard Henderson * Allocate a new register matching the constraint 4277c0522136SRichard Henderson * and move the temporary register into it. 4278c0522136SRichard Henderson */ 4279d62816f2SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 4280d62816f2SRichard Henderson i_allocated_regs, 0); 428129f5e925SRichard Henderson reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 42821c1824dcSRichard Henderson i_preferred_regs, ts->indirect_base); 428329f5e925SRichard Henderson copyto_new_reg = true; 428429f5e925SRichard Henderson } 428529f5e925SRichard Henderson break; 428629f5e925SRichard Henderson 428729f5e925SRichard Henderson case 1: 428829f5e925SRichard Henderson /* First of an input pair; if i1 == i2, the second is an output. */ 428929f5e925SRichard Henderson i1 = i; 429029f5e925SRichard Henderson i2 = arg_ct->pair_index; 429129f5e925SRichard Henderson ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 429229f5e925SRichard Henderson 429329f5e925SRichard Henderson /* 429429f5e925SRichard Henderson * It is easier to default to allocating a new pair 429529f5e925SRichard Henderson * and to identify a few cases where it's not required. 429629f5e925SRichard Henderson */ 429729f5e925SRichard Henderson if (arg_ct->ialias) { 429831fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 429929f5e925SRichard Henderson if (IS_DEAD_ARG(i1) && 430029f5e925SRichard Henderson IS_DEAD_ARG(i2) && 430129f5e925SRichard Henderson !temp_readonly(ts) && 430229f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG && 430329f5e925SRichard Henderson ts->reg < TCG_TARGET_NB_REGS - 1 && 430429f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) && 430529f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) && 430629f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 430729f5e925SRichard Henderson (ts2 430829f5e925SRichard Henderson ? ts2->val_type == TEMP_VAL_REG && 430929f5e925SRichard Henderson ts2->reg == reg + 1 && 431029f5e925SRichard Henderson !temp_readonly(ts2) 431129f5e925SRichard Henderson : s->reg_to_temp[reg + 1] == NULL)) { 431229f5e925SRichard Henderson break; 431329f5e925SRichard Henderson } 431429f5e925SRichard Henderson } else { 431529f5e925SRichard Henderson /* Without aliasing, the pair must also be an input. */ 431629f5e925SRichard Henderson tcg_debug_assert(ts2); 431729f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG && 431829f5e925SRichard Henderson ts2->val_type == TEMP_VAL_REG && 431929f5e925SRichard Henderson ts2->reg == reg + 1 && 432029f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg)) { 432129f5e925SRichard Henderson break; 432229f5e925SRichard Henderson } 432329f5e925SRichard Henderson } 432429f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 432529f5e925SRichard Henderson 0, ts->indirect_base); 432629f5e925SRichard Henderson goto do_pair; 432729f5e925SRichard Henderson 432829f5e925SRichard Henderson case 2: /* pair second */ 432929f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1; 433029f5e925SRichard Henderson goto do_pair; 433129f5e925SRichard Henderson 433229f5e925SRichard Henderson case 3: /* ialias with second output, no first input */ 433329f5e925SRichard Henderson tcg_debug_assert(arg_ct->ialias); 433431fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 433529f5e925SRichard Henderson 433629f5e925SRichard Henderson if (IS_DEAD_ARG(i) && 433729f5e925SRichard Henderson !temp_readonly(ts) && 433829f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG && 433929f5e925SRichard Henderson reg > 0 && 434029f5e925SRichard Henderson s->reg_to_temp[reg - 1] == NULL && 434129f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) && 434229f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) && 434329f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 434429f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg - 1); 434529f5e925SRichard Henderson break; 434629f5e925SRichard Henderson } 434729f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 434829f5e925SRichard Henderson i_allocated_regs, 0, 434929f5e925SRichard Henderson ts->indirect_base); 435029f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg); 435129f5e925SRichard Henderson reg += 1; 435229f5e925SRichard Henderson goto do_pair; 435329f5e925SRichard Henderson 435429f5e925SRichard Henderson do_pair: 435529f5e925SRichard Henderson /* 435629f5e925SRichard Henderson * If an aliased input is not dead after the instruction, 435729f5e925SRichard Henderson * we must allocate a new register and move it. 435829f5e925SRichard Henderson */ 435929f5e925SRichard Henderson if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 436029f5e925SRichard Henderson TCGRegSet t_allocated_regs = i_allocated_regs; 436129f5e925SRichard Henderson 436229f5e925SRichard Henderson /* 436329f5e925SRichard Henderson * Because of the alias, and the continued life, make sure 436429f5e925SRichard Henderson * that the temp is somewhere *other* than the reg pair, 436529f5e925SRichard Henderson * and we get a copy in reg. 436629f5e925SRichard Henderson */ 436729f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg); 436829f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg + 1); 436929f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 437029f5e925SRichard Henderson /* If ts was already in reg, copy it somewhere else. */ 437129f5e925SRichard Henderson TCGReg nr; 437229f5e925SRichard Henderson bool ok; 437329f5e925SRichard Henderson 437429f5e925SRichard Henderson tcg_debug_assert(ts->kind != TEMP_FIXED); 437529f5e925SRichard Henderson nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 437629f5e925SRichard Henderson t_allocated_regs, 0, ts->indirect_base); 437729f5e925SRichard Henderson ok = tcg_out_mov(s, ts->type, nr, reg); 437829f5e925SRichard Henderson tcg_debug_assert(ok); 437929f5e925SRichard Henderson 438029f5e925SRichard Henderson set_temp_val_reg(s, ts, nr); 438129f5e925SRichard Henderson } else { 438229f5e925SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 438329f5e925SRichard Henderson t_allocated_regs, 0); 438429f5e925SRichard Henderson copyto_new_reg = true; 438529f5e925SRichard Henderson } 438629f5e925SRichard Henderson } else { 438729f5e925SRichard Henderson /* Preferably allocate to reg, otherwise copy. */ 438829f5e925SRichard Henderson i_required_regs = (TCGRegSet)1 << reg; 438929f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs, 439029f5e925SRichard Henderson i_preferred_regs); 439129f5e925SRichard Henderson copyto_new_reg = ts->reg != reg; 439229f5e925SRichard Henderson } 439329f5e925SRichard Henderson break; 439429f5e925SRichard Henderson 439529f5e925SRichard Henderson default: 439629f5e925SRichard Henderson g_assert_not_reached(); 439729f5e925SRichard Henderson } 439829f5e925SRichard Henderson 439929f5e925SRichard Henderson if (copyto_new_reg) { 440078113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4401240c08d0SRichard Henderson /* 4402240c08d0SRichard Henderson * Cross register class move not supported. Sync the 4403240c08d0SRichard Henderson * temp back to its slot and load from there. 4404240c08d0SRichard Henderson */ 4405240c08d0SRichard Henderson temp_sync(s, ts, i_allocated_regs, 0, 0); 4406240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 4407240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 440878113e83SRichard Henderson } 4409c896fe29Sbellard } 4410c896fe29Sbellard new_args[i] = reg; 4411c896fe29Sbellard const_args[i] = 0; 441282790a87SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg); 4413c896fe29Sbellard } 4414c896fe29Sbellard 4415c896fe29Sbellard /* mark dead temporaries and free the associated registers */ 4416866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4417866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 441843439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 4419c896fe29Sbellard } 4420c896fe29Sbellard } 4421c896fe29Sbellard 4422b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 4423b4cb76e6SRichard Henderson tcg_reg_alloc_cbranch(s, i_allocated_regs); 4424b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 442582790a87SRichard Henderson tcg_reg_alloc_bb_end(s, i_allocated_regs); 4426a52ad07eSAurelien Jarno } else { 4427c896fe29Sbellard if (def->flags & TCG_OPF_CALL_CLOBBER) { 4428b03cce8eSbellard /* XXX: permit generic clobber register list ? */ 4429c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4430c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 443182790a87SRichard Henderson tcg_reg_free(s, i, i_allocated_regs); 4432c896fe29Sbellard } 4433c896fe29Sbellard } 44343d5c5f87SAurelien Jarno } 44353d5c5f87SAurelien Jarno if (def->flags & TCG_OPF_SIDE_EFFECTS) { 44363d5c5f87SAurelien Jarno /* sync globals if the op has side effects and might trigger 44373d5c5f87SAurelien Jarno an exception. */ 443882790a87SRichard Henderson sync_globals(s, i_allocated_regs); 4439c896fe29Sbellard } 4440c896fe29Sbellard 4441c896fe29Sbellard /* satisfy the output constraints */ 4442c896fe29Sbellard for(k = 0; k < nb_oargs; k++) { 444366792f90SRichard Henderson i = def->args_ct[k].sort_index; 4444dd186292SRichard Henderson arg = op->args[i]; 4445c896fe29Sbellard arg_ct = &def->args_ct[i]; 444643439139SRichard Henderson ts = arg_temp(arg); 4447d63e3b6eSRichard Henderson 4448d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4449e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4450d63e3b6eSRichard Henderson 445129f5e925SRichard Henderson switch (arg_ct->pair) { 445229f5e925SRichard Henderson case 0: /* not paired */ 4453bc2b17e6SRichard Henderson if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 44545ff9d6a4Sbellard reg = new_args[arg_ct->alias_index]; 4455bc2b17e6SRichard Henderson } else if (arg_ct->newreg) { 44569be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, 445782790a87SRichard Henderson i_allocated_regs | o_allocated_regs, 445831fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 4459c896fe29Sbellard } else { 44609be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 446131fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 4462c896fe29Sbellard } 446329f5e925SRichard Henderson break; 446429f5e925SRichard Henderson 446529f5e925SRichard Henderson case 1: /* first of pair */ 446629f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 446729f5e925SRichard Henderson if (arg_ct->oalias) { 446829f5e925SRichard Henderson reg = new_args[arg_ct->alias_index]; 446929f5e925SRichard Henderson break; 447029f5e925SRichard Henderson } 447129f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 447231fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 447329f5e925SRichard Henderson break; 447429f5e925SRichard Henderson 447529f5e925SRichard Henderson case 2: /* second of pair */ 447629f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 447729f5e925SRichard Henderson if (arg_ct->oalias) { 447829f5e925SRichard Henderson reg = new_args[arg_ct->alias_index]; 447929f5e925SRichard Henderson } else { 448029f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1; 448129f5e925SRichard Henderson } 448229f5e925SRichard Henderson break; 448329f5e925SRichard Henderson 448429f5e925SRichard Henderson case 3: /* first of pair, aliasing with a second input */ 448529f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 448629f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] - 1; 448729f5e925SRichard Henderson break; 448829f5e925SRichard Henderson 448929f5e925SRichard Henderson default: 449029f5e925SRichard Henderson g_assert_not_reached(); 449129f5e925SRichard Henderson } 449282790a87SRichard Henderson tcg_regset_set_reg(o_allocated_regs, reg); 4493098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 4494c896fe29Sbellard ts->mem_coherent = 0; 4495c896fe29Sbellard new_args[i] = reg; 4496c896fe29Sbellard } 4497e8996ee0Sbellard } 4498c896fe29Sbellard 4499c896fe29Sbellard /* emit instruction */ 4500*678155b2SRichard Henderson switch (op->opc) { 4501*678155b2SRichard Henderson case INDEX_op_ext8s_i32: 4502*678155b2SRichard Henderson tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]); 4503*678155b2SRichard Henderson break; 4504*678155b2SRichard Henderson case INDEX_op_ext8s_i64: 4505*678155b2SRichard Henderson tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]); 4506*678155b2SRichard Henderson break; 4507*678155b2SRichard Henderson default: 4508d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 4509d2fd745fSRichard Henderson tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 4510d2fd745fSRichard Henderson new_args, const_args); 4511d2fd745fSRichard Henderson } else { 4512dd186292SRichard Henderson tcg_out_op(s, op->opc, new_args, const_args); 4513d2fd745fSRichard Henderson } 4514*678155b2SRichard Henderson break; 4515*678155b2SRichard Henderson } 4516c896fe29Sbellard 4517c896fe29Sbellard /* move the outputs in the correct register if needed */ 4518c896fe29Sbellard for(i = 0; i < nb_oargs; i++) { 451943439139SRichard Henderson ts = arg_temp(op->args[i]); 4520d63e3b6eSRichard Henderson 4521d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4522e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4523d63e3b6eSRichard Henderson 4524ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 452598b4e186SRichard Henderson temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 452659d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 4527f8bf00f1SRichard Henderson temp_dead(s, ts); 4528ec7a869dSAurelien Jarno } 4529c896fe29Sbellard } 4530c896fe29Sbellard } 4531c896fe29Sbellard 4532efe86b21SRichard Henderson static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 4533efe86b21SRichard Henderson { 4534efe86b21SRichard Henderson const TCGLifeData arg_life = op->life; 4535efe86b21SRichard Henderson TCGTemp *ots, *itsl, *itsh; 4536efe86b21SRichard Henderson TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4537efe86b21SRichard Henderson 4538efe86b21SRichard Henderson /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 4539efe86b21SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 4540efe86b21SRichard Henderson tcg_debug_assert(TCGOP_VECE(op) == MO_64); 4541efe86b21SRichard Henderson 4542efe86b21SRichard Henderson ots = arg_temp(op->args[0]); 4543efe86b21SRichard Henderson itsl = arg_temp(op->args[1]); 4544efe86b21SRichard Henderson itsh = arg_temp(op->args[2]); 4545efe86b21SRichard Henderson 4546efe86b21SRichard Henderson /* ENV should not be modified. */ 4547efe86b21SRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4548efe86b21SRichard Henderson 4549efe86b21SRichard Henderson /* Allocate the output register now. */ 4550efe86b21SRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 4551efe86b21SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 4552efe86b21SRichard Henderson TCGRegSet dup_out_regs = 4553efe86b21SRichard Henderson tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 4554098859f1SRichard Henderson TCGReg oreg; 4555efe86b21SRichard Henderson 4556efe86b21SRichard Henderson /* Make sure to not spill the input registers. */ 4557efe86b21SRichard Henderson if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 4558efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsl->reg); 4559efe86b21SRichard Henderson } 4560efe86b21SRichard Henderson if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 4561efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsh->reg); 4562efe86b21SRichard Henderson } 4563efe86b21SRichard Henderson 4564098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 456531fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base); 4566098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4567efe86b21SRichard Henderson } 4568efe86b21SRichard Henderson 4569efe86b21SRichard Henderson /* Promote dup2 of immediates to dupi_vec. */ 4570efe86b21SRichard Henderson if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 4571efe86b21SRichard Henderson uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 4572efe86b21SRichard Henderson MemOp vece = MO_64; 4573efe86b21SRichard Henderson 4574efe86b21SRichard Henderson if (val == dup_const(MO_8, val)) { 4575efe86b21SRichard Henderson vece = MO_8; 4576efe86b21SRichard Henderson } else if (val == dup_const(MO_16, val)) { 4577efe86b21SRichard Henderson vece = MO_16; 4578efe86b21SRichard Henderson } else if (val == dup_const(MO_32, val)) { 4579efe86b21SRichard Henderson vece = MO_32; 4580efe86b21SRichard Henderson } 4581efe86b21SRichard Henderson 4582efe86b21SRichard Henderson tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 4583efe86b21SRichard Henderson goto done; 4584efe86b21SRichard Henderson } 4585efe86b21SRichard Henderson 4586efe86b21SRichard Henderson /* If the two inputs form one 64-bit value, try dupm_vec. */ 4587aef85402SRichard Henderson if (itsl->temp_subindex == HOST_BIG_ENDIAN && 4588aef85402SRichard Henderson itsh->temp_subindex == !HOST_BIG_ENDIAN && 4589aef85402SRichard Henderson itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { 4590aef85402SRichard Henderson TCGTemp *its = itsl - HOST_BIG_ENDIAN; 4591aef85402SRichard Henderson 4592aef85402SRichard Henderson temp_sync(s, its + 0, s->reserved_regs, 0, 0); 4593aef85402SRichard Henderson temp_sync(s, its + 1, s->reserved_regs, 0, 0); 4594aef85402SRichard Henderson 4595efe86b21SRichard Henderson if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 4596efe86b21SRichard Henderson its->mem_base->reg, its->mem_offset)) { 4597efe86b21SRichard Henderson goto done; 4598efe86b21SRichard Henderson } 4599efe86b21SRichard Henderson } 4600efe86b21SRichard Henderson 4601efe86b21SRichard Henderson /* Fall back to generic expansion. */ 4602efe86b21SRichard Henderson return false; 4603efe86b21SRichard Henderson 4604efe86b21SRichard Henderson done: 460536f5539cSRichard Henderson ots->mem_coherent = 0; 4606efe86b21SRichard Henderson if (IS_DEAD_ARG(1)) { 4607efe86b21SRichard Henderson temp_dead(s, itsl); 4608efe86b21SRichard Henderson } 4609efe86b21SRichard Henderson if (IS_DEAD_ARG(2)) { 4610efe86b21SRichard Henderson temp_dead(s, itsh); 4611efe86b21SRichard Henderson } 4612efe86b21SRichard Henderson if (NEED_SYNC_ARG(0)) { 4613efe86b21SRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 4614efe86b21SRichard Henderson } else if (IS_DEAD_ARG(0)) { 4615efe86b21SRichard Henderson temp_dead(s, ots); 4616efe86b21SRichard Henderson } 4617efe86b21SRichard Henderson return true; 4618efe86b21SRichard Henderson } 4619efe86b21SRichard Henderson 462039004a71SRichard Henderson static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, 462139004a71SRichard Henderson TCGRegSet allocated_regs) 4622c896fe29Sbellard { 4623c896fe29Sbellard if (ts->val_type == TEMP_VAL_REG) { 4624c896fe29Sbellard if (ts->reg != reg) { 46254250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 462678113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4627240c08d0SRichard Henderson /* 4628240c08d0SRichard Henderson * Cross register class move not supported. Sync the 4629240c08d0SRichard Henderson * temp back to its slot and load from there. 4630240c08d0SRichard Henderson */ 4631240c08d0SRichard Henderson temp_sync(s, ts, allocated_regs, 0, 0); 4632240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 4633240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 463478113e83SRichard Henderson } 4635c896fe29Sbellard } 4636c896fe29Sbellard } else { 4637ccb1bb66SRichard Henderson TCGRegSet arg_set = 0; 463840ae5c62SRichard Henderson 46394250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 464040ae5c62SRichard Henderson tcg_regset_set_reg(arg_set, reg); 4641b722452aSRichard Henderson temp_load(s, ts, arg_set, allocated_regs, 0); 4642c896fe29Sbellard } 464339004a71SRichard Henderson } 464440ae5c62SRichard Henderson 464539004a71SRichard Henderson static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, 464639004a71SRichard Henderson TCGRegSet allocated_regs) 464739004a71SRichard Henderson { 464839004a71SRichard Henderson /* 464939004a71SRichard Henderson * When the destination is on the stack, load up the temp and store. 465039004a71SRichard Henderson * If there are many call-saved registers, the temp might live to 465139004a71SRichard Henderson * see another use; otherwise it'll be discarded. 465239004a71SRichard Henderson */ 465339004a71SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); 465439004a71SRichard Henderson tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, 465539004a71SRichard Henderson TCG_TARGET_CALL_STACK_OFFSET + 465639004a71SRichard Henderson stk_slot * sizeof(tcg_target_long)); 465739004a71SRichard Henderson } 465839004a71SRichard Henderson 465939004a71SRichard Henderson static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, 466039004a71SRichard Henderson TCGTemp *ts, TCGRegSet *allocated_regs) 466139004a71SRichard Henderson { 466239004a71SRichard Henderson if (REG_P(l)) { 466339004a71SRichard Henderson TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; 466439004a71SRichard Henderson load_arg_reg(s, reg, ts, *allocated_regs); 466539004a71SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg); 466639004a71SRichard Henderson } else { 466739004a71SRichard Henderson load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs), 466839004a71SRichard Henderson ts, *allocated_regs); 4669c896fe29Sbellard } 467039cf05d3Sbellard } 4671c896fe29Sbellard 4672313bdea8SRichard Henderson static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base, 4673313bdea8SRichard Henderson intptr_t ref_off, TCGRegSet *allocated_regs) 4674313bdea8SRichard Henderson { 4675313bdea8SRichard Henderson TCGReg reg; 4676313bdea8SRichard Henderson int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); 4677313bdea8SRichard Henderson 4678313bdea8SRichard Henderson if (stk_slot < 0) { 4679313bdea8SRichard Henderson reg = tcg_target_call_iarg_regs[arg_slot]; 4680313bdea8SRichard Henderson tcg_reg_free(s, reg, *allocated_regs); 4681313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4682313bdea8SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg); 4683313bdea8SRichard Henderson } else { 4684313bdea8SRichard Henderson reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR], 4685313bdea8SRichard Henderson *allocated_regs, 0, false); 4686313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4687313bdea8SRichard Henderson tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, 4688313bdea8SRichard Henderson TCG_TARGET_CALL_STACK_OFFSET 4689313bdea8SRichard Henderson + stk_slot * sizeof(tcg_target_long)); 4690313bdea8SRichard Henderson } 4691313bdea8SRichard Henderson } 4692313bdea8SRichard Henderson 469339004a71SRichard Henderson static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 469439004a71SRichard Henderson { 469539004a71SRichard Henderson const int nb_oargs = TCGOP_CALLO(op); 469639004a71SRichard Henderson const int nb_iargs = TCGOP_CALLI(op); 469739004a71SRichard Henderson const TCGLifeData arg_life = op->life; 469839004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 469939004a71SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 470039004a71SRichard Henderson int i; 470139004a71SRichard Henderson 470239004a71SRichard Henderson /* 470339004a71SRichard Henderson * Move inputs into place in reverse order, 470439004a71SRichard Henderson * so that we place stacked arguments first. 470539004a71SRichard Henderson */ 470639004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; --i) { 470739004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 470839004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); 470939004a71SRichard Henderson 471039004a71SRichard Henderson switch (loc->kind) { 471139004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 471239004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 471339004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 471439004a71SRichard Henderson load_arg_normal(s, loc, ts, &allocated_regs); 471539004a71SRichard Henderson break; 4716313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 4717313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4718313bdea8SRichard Henderson load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, 4719313bdea8SRichard Henderson TCG_TARGET_CALL_STACK_OFFSET 4720313bdea8SRichard Henderson + loc->ref_slot * sizeof(tcg_target_long), 4721313bdea8SRichard Henderson &allocated_regs); 4722313bdea8SRichard Henderson break; 4723313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 4724313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4725313bdea8SRichard Henderson break; 472639004a71SRichard Henderson default: 472739004a71SRichard Henderson g_assert_not_reached(); 472839004a71SRichard Henderson } 472939004a71SRichard Henderson } 473039004a71SRichard Henderson 473139004a71SRichard Henderson /* Mark dead temporaries and free the associated registers. */ 4732866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4733866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 473443439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 4735c896fe29Sbellard } 4736c896fe29Sbellard } 4737c896fe29Sbellard 473839004a71SRichard Henderson /* Clobber call registers. */ 4739c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4740c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4741b3915dbbSRichard Henderson tcg_reg_free(s, i, allocated_regs); 4742c896fe29Sbellard } 4743c896fe29Sbellard } 4744c896fe29Sbellard 474539004a71SRichard Henderson /* 474639004a71SRichard Henderson * Save globals if they might be written by the helper, 474739004a71SRichard Henderson * sync them if they might be read. 474839004a71SRichard Henderson */ 474939004a71SRichard Henderson if (info->flags & TCG_CALL_NO_READ_GLOBALS) { 475078505279SAurelien Jarno /* Nothing to do */ 475139004a71SRichard Henderson } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { 475278505279SAurelien Jarno sync_globals(s, allocated_regs); 475378505279SAurelien Jarno } else { 4754e8996ee0Sbellard save_globals(s, allocated_regs); 4755b9c18f56Saurel32 } 4756c896fe29Sbellard 4757313bdea8SRichard Henderson /* 4758313bdea8SRichard Henderson * If the ABI passes a pointer to the returned struct as the first 4759313bdea8SRichard Henderson * argument, load that now. Pass a pointer to the output home slot. 4760313bdea8SRichard Henderson */ 4761313bdea8SRichard Henderson if (info->out_kind == TCG_CALL_RET_BY_REF) { 4762313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]); 4763313bdea8SRichard Henderson 4764313bdea8SRichard Henderson if (!ts->mem_allocated) { 4765313bdea8SRichard Henderson temp_allocate_frame(s, ts); 4766313bdea8SRichard Henderson } 4767313bdea8SRichard Henderson load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); 4768313bdea8SRichard Henderson } 4769313bdea8SRichard Henderson 4770cee44b03SRichard Henderson tcg_out_call(s, tcg_call_func(op), info); 4771c896fe29Sbellard 477239004a71SRichard Henderson /* Assign output registers and emit moves if needed. */ 477339004a71SRichard Henderson switch (info->out_kind) { 477439004a71SRichard Henderson case TCG_CALL_RET_NORMAL: 4775c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 477639004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 47775e3d0c19SRichard Henderson TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i); 4778d63e3b6eSRichard Henderson 4779d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4780e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4781d63e3b6eSRichard Henderson 4782098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 4783c896fe29Sbellard ts->mem_coherent = 0; 478439004a71SRichard Henderson } 478539004a71SRichard Henderson break; 4786313bdea8SRichard Henderson 4787c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC: 4788c6556aa0SRichard Henderson { 4789c6556aa0SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]); 4790c6556aa0SRichard Henderson 4791c6556aa0SRichard Henderson tcg_debug_assert(ts->base_type == TCG_TYPE_I128); 4792c6556aa0SRichard Henderson tcg_debug_assert(ts->temp_subindex == 0); 4793c6556aa0SRichard Henderson if (!ts->mem_allocated) { 4794c6556aa0SRichard Henderson temp_allocate_frame(s, ts); 4795c6556aa0SRichard Henderson } 4796c6556aa0SRichard Henderson tcg_out_st(s, TCG_TYPE_V128, 4797c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 4798c6556aa0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 4799c6556aa0SRichard Henderson } 4800c6556aa0SRichard Henderson /* fall through to mark all parts in memory */ 4801c6556aa0SRichard Henderson 4802313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF: 4803313bdea8SRichard Henderson /* The callee has performed a write through the reference. */ 4804313bdea8SRichard Henderson for (i = 0; i < nb_oargs; i++) { 4805313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 4806313bdea8SRichard Henderson ts->val_type = TEMP_VAL_MEM; 4807313bdea8SRichard Henderson } 4808313bdea8SRichard Henderson break; 4809313bdea8SRichard Henderson 481039004a71SRichard Henderson default: 481139004a71SRichard Henderson g_assert_not_reached(); 481239004a71SRichard Henderson } 481339004a71SRichard Henderson 481439004a71SRichard Henderson /* Flush or discard output registers as needed. */ 481539004a71SRichard Henderson for (i = 0; i < nb_oargs; i++) { 481639004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 4817ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 481839004a71SRichard Henderson temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); 481959d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 4820f8bf00f1SRichard Henderson temp_dead(s, ts); 4821c896fe29Sbellard } 4822c896fe29Sbellard } 48238c11ad25SAurelien Jarno } 4824c896fe29Sbellard 4825c896fe29Sbellard #ifdef CONFIG_PROFILER 4826c896fe29Sbellard 4827c3fac113SEmilio G. Cota /* avoid copy/paste errors */ 4828c3fac113SEmilio G. Cota #define PROF_ADD(to, from, field) \ 4829c3fac113SEmilio G. Cota do { \ 4830d73415a3SStefan Hajnoczi (to)->field += qatomic_read(&((from)->field)); \ 4831c3fac113SEmilio G. Cota } while (0) 4832c896fe29Sbellard 4833c3fac113SEmilio G. Cota #define PROF_MAX(to, from, field) \ 4834c3fac113SEmilio G. Cota do { \ 4835d73415a3SStefan Hajnoczi typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4836c3fac113SEmilio G. Cota if (val__ > (to)->field) { \ 4837c3fac113SEmilio G. Cota (to)->field = val__; \ 4838c3fac113SEmilio G. Cota } \ 4839c3fac113SEmilio G. Cota } while (0) 4840c3fac113SEmilio G. Cota 4841c3fac113SEmilio G. Cota /* Pass in a zero'ed @prof */ 4842c3fac113SEmilio G. Cota static inline 4843c3fac113SEmilio G. Cota void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4844c896fe29Sbellard { 48450e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4846c3fac113SEmilio G. Cota unsigned int i; 4847c3fac113SEmilio G. Cota 48483468b59eSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 4849d73415a3SStefan Hajnoczi TCGContext *s = qatomic_read(&tcg_ctxs[i]); 48503468b59eSEmilio G. Cota const TCGProfile *orig = &s->prof; 4851c3fac113SEmilio G. Cota 4852c3fac113SEmilio G. Cota if (counters) { 485372fd2efbSEmilio G. Cota PROF_ADD(prof, orig, cpu_exec_time); 4854c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count1); 4855c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count); 4856c3fac113SEmilio G. Cota PROF_ADD(prof, orig, op_count); 4857c3fac113SEmilio G. Cota PROF_MAX(prof, orig, op_count_max); 4858c3fac113SEmilio G. Cota PROF_ADD(prof, orig, temp_count); 4859c3fac113SEmilio G. Cota PROF_MAX(prof, orig, temp_count_max); 4860c3fac113SEmilio G. Cota PROF_ADD(prof, orig, del_op_count); 4861c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_in_len); 4862c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_out_len); 4863c3fac113SEmilio G. Cota PROF_ADD(prof, orig, search_out_len); 4864c3fac113SEmilio G. Cota PROF_ADD(prof, orig, interm_time); 4865c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_time); 4866c3fac113SEmilio G. Cota PROF_ADD(prof, orig, la_time); 4867c3fac113SEmilio G. Cota PROF_ADD(prof, orig, opt_time); 4868c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_count); 4869c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_time); 4870c3fac113SEmilio G. Cota } 4871c3fac113SEmilio G. Cota if (table) { 4872c896fe29Sbellard int i; 4873d70724ceSzhanghailiang 487415fc7daaSRichard Henderson for (i = 0; i < NB_OPS; i++) { 4875c3fac113SEmilio G. Cota PROF_ADD(prof, orig, table_op_count[i]); 4876c3fac113SEmilio G. Cota } 4877c3fac113SEmilio G. Cota } 4878c3fac113SEmilio G. Cota } 4879c3fac113SEmilio G. Cota } 4880c3fac113SEmilio G. Cota 4881c3fac113SEmilio G. Cota #undef PROF_ADD 4882c3fac113SEmilio G. Cota #undef PROF_MAX 4883c3fac113SEmilio G. Cota 4884c3fac113SEmilio G. Cota static void tcg_profile_snapshot_counters(TCGProfile *prof) 4885c3fac113SEmilio G. Cota { 4886c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, true, false); 4887c3fac113SEmilio G. Cota } 4888c3fac113SEmilio G. Cota 4889c3fac113SEmilio G. Cota static void tcg_profile_snapshot_table(TCGProfile *prof) 4890c3fac113SEmilio G. Cota { 4891c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, false, true); 4892c3fac113SEmilio G. Cota } 4893c3fac113SEmilio G. Cota 4894b6a7f3e0SDaniel P. Berrangé void tcg_dump_op_count(GString *buf) 4895c3fac113SEmilio G. Cota { 4896c3fac113SEmilio G. Cota TCGProfile prof = {}; 4897c3fac113SEmilio G. Cota int i; 4898c3fac113SEmilio G. Cota 4899c3fac113SEmilio G. Cota tcg_profile_snapshot_table(&prof); 4900c3fac113SEmilio G. Cota for (i = 0; i < NB_OPS; i++) { 4901b6a7f3e0SDaniel P. Berrangé g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name, 4902c3fac113SEmilio G. Cota prof.table_op_count[i]); 4903c896fe29Sbellard } 4904c896fe29Sbellard } 490572fd2efbSEmilio G. Cota 490672fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 490772fd2efbSEmilio G. Cota { 49080e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 490972fd2efbSEmilio G. Cota unsigned int i; 491072fd2efbSEmilio G. Cota int64_t ret = 0; 491172fd2efbSEmilio G. Cota 491272fd2efbSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 4913d73415a3SStefan Hajnoczi const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 491472fd2efbSEmilio G. Cota const TCGProfile *prof = &s->prof; 491572fd2efbSEmilio G. Cota 4916d73415a3SStefan Hajnoczi ret += qatomic_read(&prof->cpu_exec_time); 491772fd2efbSEmilio G. Cota } 491872fd2efbSEmilio G. Cota return ret; 491972fd2efbSEmilio G. Cota } 4920246ae24dSMax Filippov #else 4921b6a7f3e0SDaniel P. Berrangé void tcg_dump_op_count(GString *buf) 4922246ae24dSMax Filippov { 4923b6a7f3e0SDaniel P. Berrangé g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 4924246ae24dSMax Filippov } 492572fd2efbSEmilio G. Cota 492672fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 492772fd2efbSEmilio G. Cota { 492872fd2efbSEmilio G. Cota error_report("%s: TCG profiler not compiled", __func__); 492972fd2efbSEmilio G. Cota exit(EXIT_FAILURE); 493072fd2efbSEmilio G. Cota } 4931c896fe29Sbellard #endif 4932c896fe29Sbellard 4933c896fe29Sbellard 4934fbf59aadSRichard Henderson int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) 4935c896fe29Sbellard { 4936c3fac113SEmilio G. Cota #ifdef CONFIG_PROFILER 4937c3fac113SEmilio G. Cota TCGProfile *prof = &s->prof; 4938c3fac113SEmilio G. Cota #endif 493915fa08f8SRichard Henderson int i, num_insns; 494015fa08f8SRichard Henderson TCGOp *op; 4941c896fe29Sbellard 494204fe6400SRichard Henderson #ifdef CONFIG_PROFILER 494304fe6400SRichard Henderson { 4944c1f543b7SEmilio G. Cota int n = 0; 494504fe6400SRichard Henderson 494615fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 494715fa08f8SRichard Henderson n++; 494815fa08f8SRichard Henderson } 4949d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count, prof->op_count + n); 4950c3fac113SEmilio G. Cota if (n > prof->op_count_max) { 4951d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count_max, n); 495204fe6400SRichard Henderson } 495304fe6400SRichard Henderson 495404fe6400SRichard Henderson n = s->nb_temps; 4955d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count, prof->temp_count + n); 4956c3fac113SEmilio G. Cota if (n > prof->temp_count_max) { 4957d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count_max, n); 495804fe6400SRichard Henderson } 495904fe6400SRichard Henderson } 496004fe6400SRichard Henderson #endif 496104fe6400SRichard Henderson 4962c896fe29Sbellard #ifdef DEBUG_DISAS 4963d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4964fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 4965c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 496678b54858SRichard Henderson if (logfile) { 496778b54858SRichard Henderson fprintf(logfile, "OP:\n"); 4968b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false); 496978b54858SRichard Henderson fprintf(logfile, "\n"); 4970fc59d2d8SRobert Foley qemu_log_unlock(logfile); 4971c896fe29Sbellard } 497278b54858SRichard Henderson } 4973c896fe29Sbellard #endif 4974c896fe29Sbellard 4975bef16ab4SRichard Henderson #ifdef CONFIG_DEBUG_TCG 4976bef16ab4SRichard Henderson /* Ensure all labels referenced have been emitted. */ 4977bef16ab4SRichard Henderson { 4978bef16ab4SRichard Henderson TCGLabel *l; 4979bef16ab4SRichard Henderson bool error = false; 4980bef16ab4SRichard Henderson 4981bef16ab4SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 4982f85b1fc4SRichard Henderson if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { 4983bef16ab4SRichard Henderson qemu_log_mask(CPU_LOG_TB_OP, 4984bef16ab4SRichard Henderson "$L%d referenced but not present.\n", l->id); 4985bef16ab4SRichard Henderson error = true; 4986bef16ab4SRichard Henderson } 4987bef16ab4SRichard Henderson } 4988bef16ab4SRichard Henderson assert(!error); 4989bef16ab4SRichard Henderson } 4990bef16ab4SRichard Henderson #endif 4991bef16ab4SRichard Henderson 4992c5cc28ffSAurelien Jarno #ifdef CONFIG_PROFILER 4993d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 4994c5cc28ffSAurelien Jarno #endif 4995c5cc28ffSAurelien Jarno 49968f2e8c07SKirill Batuzov #ifdef USE_TCG_OPTIMIZATIONS 4997c45cb8bbSRichard Henderson tcg_optimize(s); 49988f2e8c07SKirill Batuzov #endif 49998f2e8c07SKirill Batuzov 5000a23a9ec6Sbellard #ifdef CONFIG_PROFILER 5001d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 5002d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 5003a23a9ec6Sbellard #endif 5004c5cc28ffSAurelien Jarno 5005b4fc67c7SRichard Henderson reachable_code_pass(s); 5006874b8574SRichard Henderson liveness_pass_0(s); 5007b83eabeaSRichard Henderson liveness_pass_1(s); 50085a18407fSRichard Henderson 50095a18407fSRichard Henderson if (s->nb_indirects > 0) { 50105a18407fSRichard Henderson #ifdef DEBUG_DISAS 50115a18407fSRichard Henderson if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 5012fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 5013c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 501478b54858SRichard Henderson if (logfile) { 501578b54858SRichard Henderson fprintf(logfile, "OP before indirect lowering:\n"); 5016b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false); 501778b54858SRichard Henderson fprintf(logfile, "\n"); 5018fc59d2d8SRobert Foley qemu_log_unlock(logfile); 50195a18407fSRichard Henderson } 502078b54858SRichard Henderson } 50215a18407fSRichard Henderson #endif 50225a18407fSRichard Henderson /* Replace indirect temps with direct temps. */ 5023b83eabeaSRichard Henderson if (liveness_pass_2(s)) { 50245a18407fSRichard Henderson /* If changes were made, re-run liveness. */ 5025b83eabeaSRichard Henderson liveness_pass_1(s); 50265a18407fSRichard Henderson } 50275a18407fSRichard Henderson } 5028c5cc28ffSAurelien Jarno 5029a23a9ec6Sbellard #ifdef CONFIG_PROFILER 5030d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 5031a23a9ec6Sbellard #endif 5032c896fe29Sbellard 5033c896fe29Sbellard #ifdef DEBUG_DISAS 5034d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 5035fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 5036c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 503778b54858SRichard Henderson if (logfile) { 503878b54858SRichard Henderson fprintf(logfile, "OP after optimization and liveness analysis:\n"); 5039b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, true); 504078b54858SRichard Henderson fprintf(logfile, "\n"); 5041fc59d2d8SRobert Foley qemu_log_unlock(logfile); 5042c896fe29Sbellard } 504378b54858SRichard Henderson } 5044c896fe29Sbellard #endif 5045c896fe29Sbellard 504635abb009SRichard Henderson /* Initialize goto_tb jump offsets. */ 50473a50f424SRichard Henderson tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; 50483a50f424SRichard Henderson tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; 50499da6079bSRichard Henderson tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; 50509da6079bSRichard Henderson tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; 505135abb009SRichard Henderson 5052c896fe29Sbellard tcg_reg_alloc_start(s); 5053c896fe29Sbellard 5054db0c51a3SRichard Henderson /* 5055db0c51a3SRichard Henderson * Reset the buffer pointers when restarting after overflow. 5056db0c51a3SRichard Henderson * TODO: Move this into translate-all.c with the rest of the 5057db0c51a3SRichard Henderson * buffer management. Having only this done here is confusing. 5058db0c51a3SRichard Henderson */ 5059db0c51a3SRichard Henderson s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 5060db0c51a3SRichard Henderson s->code_ptr = s->code_buf; 5061c896fe29Sbellard 5062659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 50636001f772SLaurent Vivier QSIMPLEQ_INIT(&s->ldst_labels); 5064659ef5cbSRichard Henderson #endif 506557a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 506657a26946SRichard Henderson s->pool_labels = NULL; 506757a26946SRichard Henderson #endif 50689ecefc84SRichard Henderson 5069fca8a500SRichard Henderson num_insns = -1; 507015fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 5071c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 5072b3db8758Sblueswir1 5073c896fe29Sbellard #ifdef CONFIG_PROFILER 5074d73415a3SStefan Hajnoczi qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 5075c896fe29Sbellard #endif 5076c45cb8bbSRichard Henderson 5077c896fe29Sbellard switch (opc) { 5078c896fe29Sbellard case INDEX_op_mov_i32: 5079c896fe29Sbellard case INDEX_op_mov_i64: 5080d2fd745fSRichard Henderson case INDEX_op_mov_vec: 5081dd186292SRichard Henderson tcg_reg_alloc_mov(s, op); 5082c896fe29Sbellard break; 5083bab1671fSRichard Henderson case INDEX_op_dup_vec: 5084bab1671fSRichard Henderson tcg_reg_alloc_dup(s, op); 5085bab1671fSRichard Henderson break; 5086765b842aSRichard Henderson case INDEX_op_insn_start: 5087fca8a500SRichard Henderson if (num_insns >= 0) { 50889f754620SRichard Henderson size_t off = tcg_current_code_size(s); 50899f754620SRichard Henderson s->gen_insn_end_off[num_insns] = off; 50909f754620SRichard Henderson /* Assert that we do not overflow our stored offset. */ 50919f754620SRichard Henderson assert(s->gen_insn_end_off[num_insns] == off); 5092fca8a500SRichard Henderson } 5093fca8a500SRichard Henderson num_insns++; 5094bad729e2SRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 5095bad729e2SRichard Henderson target_ulong a; 5096bad729e2SRichard Henderson #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 5097efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 5098bad729e2SRichard Henderson #else 5099efee3746SRichard Henderson a = op->args[i]; 5100bad729e2SRichard Henderson #endif 5101fca8a500SRichard Henderson s->gen_insn_data[num_insns][i] = a; 5102bad729e2SRichard Henderson } 5103c896fe29Sbellard break; 51045ff9d6a4Sbellard case INDEX_op_discard: 510543439139SRichard Henderson temp_dead(s, arg_temp(op->args[0])); 51065ff9d6a4Sbellard break; 5107c896fe29Sbellard case INDEX_op_set_label: 5108e8996ee0Sbellard tcg_reg_alloc_bb_end(s, s->reserved_regs); 510992ab8e7dSRichard Henderson tcg_out_label(s, arg_label(op->args[0])); 5110c896fe29Sbellard break; 5111c896fe29Sbellard case INDEX_op_call: 5112dd186292SRichard Henderson tcg_reg_alloc_call(s, op); 5113c45cb8bbSRichard Henderson break; 5114b55a8d9dSRichard Henderson case INDEX_op_exit_tb: 5115b55a8d9dSRichard Henderson tcg_out_exit_tb(s, op->args[0]); 5116b55a8d9dSRichard Henderson break; 5117cf7d6b8eSRichard Henderson case INDEX_op_goto_tb: 5118cf7d6b8eSRichard Henderson tcg_out_goto_tb(s, op->args[0]); 5119cf7d6b8eSRichard Henderson break; 5120efe86b21SRichard Henderson case INDEX_op_dup2_vec: 5121efe86b21SRichard Henderson if (tcg_reg_alloc_dup2(s, op)) { 5122efe86b21SRichard Henderson break; 5123efe86b21SRichard Henderson } 5124efe86b21SRichard Henderson /* fall through */ 5125c896fe29Sbellard default: 512625c4d9ccSRichard Henderson /* Sanity check that we've not introduced any unhandled opcodes. */ 5127be0f34b5SRichard Henderson tcg_debug_assert(tcg_op_supported(opc)); 5128c896fe29Sbellard /* Note: in order to speed up the code, it would be much 5129c896fe29Sbellard faster to have specialized register allocator functions for 5130c896fe29Sbellard some common argument patterns */ 5131dd186292SRichard Henderson tcg_reg_alloc_op(s, op); 5132c896fe29Sbellard break; 5133c896fe29Sbellard } 5134b125f9dcSRichard Henderson /* Test for (pending) buffer overflow. The assumption is that any 5135b125f9dcSRichard Henderson one operation beginning below the high water mark cannot overrun 5136b125f9dcSRichard Henderson the buffer completely. Thus we can test for overflow after 5137b125f9dcSRichard Henderson generating code without having to check during generation. */ 5138644da9b3SJohn Clarke if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 5139b125f9dcSRichard Henderson return -1; 5140b125f9dcSRichard Henderson } 51416e6c4efeSRichard Henderson /* Test for TB overflow, as seen by gen_insn_end_off. */ 51426e6c4efeSRichard Henderson if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 51436e6c4efeSRichard Henderson return -2; 51446e6c4efeSRichard Henderson } 5145c896fe29Sbellard } 5146fca8a500SRichard Henderson tcg_debug_assert(num_insns >= 0); 5147fca8a500SRichard Henderson s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 5148c45cb8bbSRichard Henderson 5149b76f0d8cSYeongkyoon Lee /* Generate TB finalization at the end of block */ 5150659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 5151aeee05f5SRichard Henderson i = tcg_out_ldst_finalize(s); 5152aeee05f5SRichard Henderson if (i < 0) { 5153aeee05f5SRichard Henderson return i; 515423dceda6SRichard Henderson } 5155659ef5cbSRichard Henderson #endif 515657a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 51571768987bSRichard Henderson i = tcg_out_pool_finalize(s); 51581768987bSRichard Henderson if (i < 0) { 51591768987bSRichard Henderson return i; 516057a26946SRichard Henderson } 516157a26946SRichard Henderson #endif 51627ecd02a0SRichard Henderson if (!tcg_resolve_relocs(s)) { 51637ecd02a0SRichard Henderson return -2; 51647ecd02a0SRichard Henderson } 5165c896fe29Sbellard 5166df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 5167c896fe29Sbellard /* flush instruction cache */ 5168db0c51a3SRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 5169db0c51a3SRichard Henderson (uintptr_t)s->code_buf, 51701da8de39SRichard Henderson tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 5171df5d2b16SRichard Henderson #endif 51722aeabc08SStefan Weil 51731813e175SRichard Henderson return tcg_current_code_size(s); 5174c896fe29Sbellard } 5175c896fe29Sbellard 5176a23a9ec6Sbellard #ifdef CONFIG_PROFILER 51773a841ab5SDaniel P. Berrangé void tcg_dump_info(GString *buf) 5178a23a9ec6Sbellard { 5179c3fac113SEmilio G. Cota TCGProfile prof = {}; 5180c3fac113SEmilio G. Cota const TCGProfile *s; 5181c3fac113SEmilio G. Cota int64_t tb_count; 5182c3fac113SEmilio G. Cota int64_t tb_div_count; 5183c3fac113SEmilio G. Cota int64_t tot; 5184c3fac113SEmilio G. Cota 5185c3fac113SEmilio G. Cota tcg_profile_snapshot_counters(&prof); 5186c3fac113SEmilio G. Cota s = &prof; 5187c3fac113SEmilio G. Cota tb_count = s->tb_count; 5188c3fac113SEmilio G. Cota tb_div_count = tb_count ? tb_count : 1; 5189c3fac113SEmilio G. Cota tot = s->interm_time + s->code_time; 5190a23a9ec6Sbellard 51913a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "JIT cycles %" PRId64 51923a841ab5SDaniel P. Berrangé " (%0.3f s at 2.4 GHz)\n", 5193a23a9ec6Sbellard tot, tot / 2.4e9); 51943a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "translated TBs %" PRId64 51953a841ab5SDaniel P. Berrangé " (aborted=%" PRId64 " %0.1f%%)\n", 5196fca8a500SRichard Henderson tb_count, s->tb_count1 - tb_count, 5197fca8a500SRichard Henderson (double)(s->tb_count1 - s->tb_count) 5198fca8a500SRichard Henderson / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 51993a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n", 5200fca8a500SRichard Henderson (double)s->op_count / tb_div_count, s->op_count_max); 52013a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "deleted ops/TB %0.2f\n", 5202fca8a500SRichard Henderson (double)s->del_op_count / tb_div_count); 52033a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n", 52043a841ab5SDaniel P. Berrangé (double)s->temp_count / tb_div_count, 52053a841ab5SDaniel P. Berrangé s->temp_count_max); 52063a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg host code/TB %0.1f\n", 5207fca8a500SRichard Henderson (double)s->code_out_len / tb_div_count); 52083a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg search data/TB %0.1f\n", 5209fca8a500SRichard Henderson (double)s->search_out_len / tb_div_count); 5210a23a9ec6Sbellard 52113a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/op %0.1f\n", 5212a23a9ec6Sbellard s->op_count ? (double)tot / s->op_count : 0); 52133a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/in byte %0.1f\n", 5214a23a9ec6Sbellard s->code_in_len ? (double)tot / s->code_in_len : 0); 52153a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/out byte %0.1f\n", 5216a23a9ec6Sbellard s->code_out_len ? (double)tot / s->code_out_len : 0); 52173a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/search byte %0.1f\n", 52183a841ab5SDaniel P. Berrangé s->search_out_len ? 52193a841ab5SDaniel P. Berrangé (double)tot / s->search_out_len : 0); 5220fca8a500SRichard Henderson if (tot == 0) { 5221a23a9ec6Sbellard tot = 1; 5222fca8a500SRichard Henderson } 52233a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " gen_interm time %0.1f%%\n", 5224a23a9ec6Sbellard (double)s->interm_time / tot * 100.0); 52253a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " gen_code time %0.1f%%\n", 5226a23a9ec6Sbellard (double)s->code_time / tot * 100.0); 52273a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "optim./code time %0.1f%%\n", 52283a841ab5SDaniel P. Berrangé (double)s->opt_time / (s->code_time ? 52293a841ab5SDaniel P. Berrangé s->code_time : 1) 5230c5cc28ffSAurelien Jarno * 100.0); 52313a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "liveness/code time %0.1f%%\n", 52323a841ab5SDaniel P. Berrangé (double)s->la_time / (s->code_time ? 52333a841ab5SDaniel P. Berrangé s->code_time : 1) * 100.0); 52343a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n", 5235a23a9ec6Sbellard s->restore_count); 52363a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " avg cycles %0.1f\n", 52373a841ab5SDaniel P. Berrangé s->restore_count ? 52383a841ab5SDaniel P. Berrangé (double)s->restore_time / s->restore_count : 0); 5239a23a9ec6Sbellard } 5240a23a9ec6Sbellard #else 52413a841ab5SDaniel P. Berrangé void tcg_dump_info(GString *buf) 5242a23a9ec6Sbellard { 52433a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 5244a23a9ec6Sbellard } 5245a23a9ec6Sbellard #endif 5246813da627SRichard Henderson 5247813da627SRichard Henderson #ifdef ELF_HOST_MACHINE 52485872bbf2SRichard Henderson /* In order to use this feature, the backend needs to do three things: 52495872bbf2SRichard Henderson 52505872bbf2SRichard Henderson (1) Define ELF_HOST_MACHINE to indicate both what value to 52515872bbf2SRichard Henderson put into the ELF image and to indicate support for the feature. 52525872bbf2SRichard Henderson 52535872bbf2SRichard Henderson (2) Define tcg_register_jit. This should create a buffer containing 52545872bbf2SRichard Henderson the contents of a .debug_frame section that describes the post- 52555872bbf2SRichard Henderson prologue unwind info for the tcg machine. 52565872bbf2SRichard Henderson 52575872bbf2SRichard Henderson (3) Call tcg_register_jit_int, with the constructed .debug_frame. 52585872bbf2SRichard Henderson */ 5259813da627SRichard Henderson 5260813da627SRichard Henderson /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 5261813da627SRichard Henderson typedef enum { 5262813da627SRichard Henderson JIT_NOACTION = 0, 5263813da627SRichard Henderson JIT_REGISTER_FN, 5264813da627SRichard Henderson JIT_UNREGISTER_FN 5265813da627SRichard Henderson } jit_actions_t; 5266813da627SRichard Henderson 5267813da627SRichard Henderson struct jit_code_entry { 5268813da627SRichard Henderson struct jit_code_entry *next_entry; 5269813da627SRichard Henderson struct jit_code_entry *prev_entry; 5270813da627SRichard Henderson const void *symfile_addr; 5271813da627SRichard Henderson uint64_t symfile_size; 5272813da627SRichard Henderson }; 5273813da627SRichard Henderson 5274813da627SRichard Henderson struct jit_descriptor { 5275813da627SRichard Henderson uint32_t version; 5276813da627SRichard Henderson uint32_t action_flag; 5277813da627SRichard Henderson struct jit_code_entry *relevant_entry; 5278813da627SRichard Henderson struct jit_code_entry *first_entry; 5279813da627SRichard Henderson }; 5280813da627SRichard Henderson 5281813da627SRichard Henderson void __jit_debug_register_code(void) __attribute__((noinline)); 5282813da627SRichard Henderson void __jit_debug_register_code(void) 5283813da627SRichard Henderson { 5284813da627SRichard Henderson asm(""); 5285813da627SRichard Henderson } 5286813da627SRichard Henderson 5287813da627SRichard Henderson /* Must statically initialize the version, because GDB may check 5288813da627SRichard Henderson the version before we can set it. */ 5289813da627SRichard Henderson struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 5290813da627SRichard Henderson 5291813da627SRichard Henderson /* End GDB interface. */ 5292813da627SRichard Henderson 5293813da627SRichard Henderson static int find_string(const char *strtab, const char *str) 5294813da627SRichard Henderson { 5295813da627SRichard Henderson const char *p = strtab + 1; 5296813da627SRichard Henderson 5297813da627SRichard Henderson while (1) { 5298813da627SRichard Henderson if (strcmp(p, str) == 0) { 5299813da627SRichard Henderson return p - strtab; 5300813da627SRichard Henderson } 5301813da627SRichard Henderson p += strlen(p) + 1; 5302813da627SRichard Henderson } 5303813da627SRichard Henderson } 5304813da627SRichard Henderson 5305755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 53062c90784aSRichard Henderson const void *debug_frame, 53072c90784aSRichard Henderson size_t debug_frame_size) 5308813da627SRichard Henderson { 53095872bbf2SRichard Henderson struct __attribute__((packed)) DebugInfo { 53105872bbf2SRichard Henderson uint32_t len; 53115872bbf2SRichard Henderson uint16_t version; 53125872bbf2SRichard Henderson uint32_t abbrev; 53135872bbf2SRichard Henderson uint8_t ptr_size; 53145872bbf2SRichard Henderson uint8_t cu_die; 53155872bbf2SRichard Henderson uint16_t cu_lang; 53165872bbf2SRichard Henderson uintptr_t cu_low_pc; 53175872bbf2SRichard Henderson uintptr_t cu_high_pc; 53185872bbf2SRichard Henderson uint8_t fn_die; 53195872bbf2SRichard Henderson char fn_name[16]; 53205872bbf2SRichard Henderson uintptr_t fn_low_pc; 53215872bbf2SRichard Henderson uintptr_t fn_high_pc; 53225872bbf2SRichard Henderson uint8_t cu_eoc; 53235872bbf2SRichard Henderson }; 5324813da627SRichard Henderson 5325813da627SRichard Henderson struct ElfImage { 5326813da627SRichard Henderson ElfW(Ehdr) ehdr; 5327813da627SRichard Henderson ElfW(Phdr) phdr; 53285872bbf2SRichard Henderson ElfW(Shdr) shdr[7]; 53295872bbf2SRichard Henderson ElfW(Sym) sym[2]; 53305872bbf2SRichard Henderson struct DebugInfo di; 53315872bbf2SRichard Henderson uint8_t da[24]; 53325872bbf2SRichard Henderson char str[80]; 53335872bbf2SRichard Henderson }; 53345872bbf2SRichard Henderson 53355872bbf2SRichard Henderson struct ElfImage *img; 53365872bbf2SRichard Henderson 53375872bbf2SRichard Henderson static const struct ElfImage img_template = { 53385872bbf2SRichard Henderson .ehdr = { 53395872bbf2SRichard Henderson .e_ident[EI_MAG0] = ELFMAG0, 53405872bbf2SRichard Henderson .e_ident[EI_MAG1] = ELFMAG1, 53415872bbf2SRichard Henderson .e_ident[EI_MAG2] = ELFMAG2, 53425872bbf2SRichard Henderson .e_ident[EI_MAG3] = ELFMAG3, 53435872bbf2SRichard Henderson .e_ident[EI_CLASS] = ELF_CLASS, 53445872bbf2SRichard Henderson .e_ident[EI_DATA] = ELF_DATA, 53455872bbf2SRichard Henderson .e_ident[EI_VERSION] = EV_CURRENT, 53465872bbf2SRichard Henderson .e_type = ET_EXEC, 53475872bbf2SRichard Henderson .e_machine = ELF_HOST_MACHINE, 53485872bbf2SRichard Henderson .e_version = EV_CURRENT, 53495872bbf2SRichard Henderson .e_phoff = offsetof(struct ElfImage, phdr), 53505872bbf2SRichard Henderson .e_shoff = offsetof(struct ElfImage, shdr), 53515872bbf2SRichard Henderson .e_ehsize = sizeof(ElfW(Shdr)), 53525872bbf2SRichard Henderson .e_phentsize = sizeof(ElfW(Phdr)), 53535872bbf2SRichard Henderson .e_phnum = 1, 53545872bbf2SRichard Henderson .e_shentsize = sizeof(ElfW(Shdr)), 53555872bbf2SRichard Henderson .e_shnum = ARRAY_SIZE(img->shdr), 53565872bbf2SRichard Henderson .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 5357abbb3eaeSRichard Henderson #ifdef ELF_HOST_FLAGS 5358abbb3eaeSRichard Henderson .e_flags = ELF_HOST_FLAGS, 5359abbb3eaeSRichard Henderson #endif 5360abbb3eaeSRichard Henderson #ifdef ELF_OSABI 5361abbb3eaeSRichard Henderson .e_ident[EI_OSABI] = ELF_OSABI, 5362abbb3eaeSRichard Henderson #endif 53635872bbf2SRichard Henderson }, 53645872bbf2SRichard Henderson .phdr = { 53655872bbf2SRichard Henderson .p_type = PT_LOAD, 53665872bbf2SRichard Henderson .p_flags = PF_X, 53675872bbf2SRichard Henderson }, 53685872bbf2SRichard Henderson .shdr = { 53695872bbf2SRichard Henderson [0] = { .sh_type = SHT_NULL }, 53705872bbf2SRichard Henderson /* Trick: The contents of code_gen_buffer are not present in 53715872bbf2SRichard Henderson this fake ELF file; that got allocated elsewhere. Therefore 53725872bbf2SRichard Henderson we mark .text as SHT_NOBITS (similar to .bss) so that readers 53735872bbf2SRichard Henderson will not look for contents. We can record any address. */ 53745872bbf2SRichard Henderson [1] = { /* .text */ 53755872bbf2SRichard Henderson .sh_type = SHT_NOBITS, 53765872bbf2SRichard Henderson .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 53775872bbf2SRichard Henderson }, 53785872bbf2SRichard Henderson [2] = { /* .debug_info */ 53795872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 53805872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, di), 53815872bbf2SRichard Henderson .sh_size = sizeof(struct DebugInfo), 53825872bbf2SRichard Henderson }, 53835872bbf2SRichard Henderson [3] = { /* .debug_abbrev */ 53845872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 53855872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, da), 53865872bbf2SRichard Henderson .sh_size = sizeof(img->da), 53875872bbf2SRichard Henderson }, 53885872bbf2SRichard Henderson [4] = { /* .debug_frame */ 53895872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 53905872bbf2SRichard Henderson .sh_offset = sizeof(struct ElfImage), 53915872bbf2SRichard Henderson }, 53925872bbf2SRichard Henderson [5] = { /* .symtab */ 53935872bbf2SRichard Henderson .sh_type = SHT_SYMTAB, 53945872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, sym), 53955872bbf2SRichard Henderson .sh_size = sizeof(img->sym), 53965872bbf2SRichard Henderson .sh_info = 1, 53975872bbf2SRichard Henderson .sh_link = ARRAY_SIZE(img->shdr) - 1, 53985872bbf2SRichard Henderson .sh_entsize = sizeof(ElfW(Sym)), 53995872bbf2SRichard Henderson }, 54005872bbf2SRichard Henderson [6] = { /* .strtab */ 54015872bbf2SRichard Henderson .sh_type = SHT_STRTAB, 54025872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, str), 54035872bbf2SRichard Henderson .sh_size = sizeof(img->str), 54045872bbf2SRichard Henderson } 54055872bbf2SRichard Henderson }, 54065872bbf2SRichard Henderson .sym = { 54075872bbf2SRichard Henderson [1] = { /* code_gen_buffer */ 54085872bbf2SRichard Henderson .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 54095872bbf2SRichard Henderson .st_shndx = 1, 54105872bbf2SRichard Henderson } 54115872bbf2SRichard Henderson }, 54125872bbf2SRichard Henderson .di = { 54135872bbf2SRichard Henderson .len = sizeof(struct DebugInfo) - 4, 54145872bbf2SRichard Henderson .version = 2, 54155872bbf2SRichard Henderson .ptr_size = sizeof(void *), 54165872bbf2SRichard Henderson .cu_die = 1, 54175872bbf2SRichard Henderson .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 54185872bbf2SRichard Henderson .fn_die = 2, 54195872bbf2SRichard Henderson .fn_name = "code_gen_buffer" 54205872bbf2SRichard Henderson }, 54215872bbf2SRichard Henderson .da = { 54225872bbf2SRichard Henderson 1, /* abbrev number (the cu) */ 54235872bbf2SRichard Henderson 0x11, 1, /* DW_TAG_compile_unit, has children */ 54245872bbf2SRichard Henderson 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 54255872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 54265872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 54275872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 54285872bbf2SRichard Henderson 2, /* abbrev number (the fn) */ 54295872bbf2SRichard Henderson 0x2e, 0, /* DW_TAG_subprogram, no children */ 54305872bbf2SRichard Henderson 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 54315872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 54325872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 54335872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 54345872bbf2SRichard Henderson 0 /* no more abbrev */ 54355872bbf2SRichard Henderson }, 54365872bbf2SRichard Henderson .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 54375872bbf2SRichard Henderson ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 5438813da627SRichard Henderson }; 5439813da627SRichard Henderson 5440813da627SRichard Henderson /* We only need a single jit entry; statically allocate it. */ 5441813da627SRichard Henderson static struct jit_code_entry one_entry; 5442813da627SRichard Henderson 54435872bbf2SRichard Henderson uintptr_t buf = (uintptr_t)buf_ptr; 5444813da627SRichard Henderson size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 54452c90784aSRichard Henderson DebugFrameHeader *dfh; 5446813da627SRichard Henderson 54475872bbf2SRichard Henderson img = g_malloc(img_size); 54485872bbf2SRichard Henderson *img = img_template; 5449813da627SRichard Henderson 54505872bbf2SRichard Henderson img->phdr.p_vaddr = buf; 54515872bbf2SRichard Henderson img->phdr.p_paddr = buf; 54525872bbf2SRichard Henderson img->phdr.p_memsz = buf_size; 5453813da627SRichard Henderson 54545872bbf2SRichard Henderson img->shdr[1].sh_name = find_string(img->str, ".text"); 54555872bbf2SRichard Henderson img->shdr[1].sh_addr = buf; 54565872bbf2SRichard Henderson img->shdr[1].sh_size = buf_size; 5457813da627SRichard Henderson 54585872bbf2SRichard Henderson img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 54595872bbf2SRichard Henderson img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 54605872bbf2SRichard Henderson 54615872bbf2SRichard Henderson img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 54625872bbf2SRichard Henderson img->shdr[4].sh_size = debug_frame_size; 54635872bbf2SRichard Henderson 54645872bbf2SRichard Henderson img->shdr[5].sh_name = find_string(img->str, ".symtab"); 54655872bbf2SRichard Henderson img->shdr[6].sh_name = find_string(img->str, ".strtab"); 54665872bbf2SRichard Henderson 54675872bbf2SRichard Henderson img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 54685872bbf2SRichard Henderson img->sym[1].st_value = buf; 54695872bbf2SRichard Henderson img->sym[1].st_size = buf_size; 54705872bbf2SRichard Henderson 54715872bbf2SRichard Henderson img->di.cu_low_pc = buf; 547245aba097SRichard Henderson img->di.cu_high_pc = buf + buf_size; 54735872bbf2SRichard Henderson img->di.fn_low_pc = buf; 547445aba097SRichard Henderson img->di.fn_high_pc = buf + buf_size; 5475813da627SRichard Henderson 54762c90784aSRichard Henderson dfh = (DebugFrameHeader *)(img + 1); 54772c90784aSRichard Henderson memcpy(dfh, debug_frame, debug_frame_size); 54782c90784aSRichard Henderson dfh->fde.func_start = buf; 54792c90784aSRichard Henderson dfh->fde.func_len = buf_size; 54802c90784aSRichard Henderson 5481813da627SRichard Henderson #ifdef DEBUG_JIT 5482813da627SRichard Henderson /* Enable this block to be able to debug the ELF image file creation. 5483813da627SRichard Henderson One can use readelf, objdump, or other inspection utilities. */ 5484813da627SRichard Henderson { 5485eb6b2edfSBin Meng g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 5486eb6b2edfSBin Meng FILE *f = fopen(jit, "w+b"); 5487813da627SRichard Henderson if (f) { 54885872bbf2SRichard Henderson if (fwrite(img, img_size, 1, f) != img_size) { 5489813da627SRichard Henderson /* Avoid stupid unused return value warning for fwrite. */ 5490813da627SRichard Henderson } 5491813da627SRichard Henderson fclose(f); 5492813da627SRichard Henderson } 5493813da627SRichard Henderson } 5494813da627SRichard Henderson #endif 5495813da627SRichard Henderson 5496813da627SRichard Henderson one_entry.symfile_addr = img; 5497813da627SRichard Henderson one_entry.symfile_size = img_size; 5498813da627SRichard Henderson 5499813da627SRichard Henderson __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 5500813da627SRichard Henderson __jit_debug_descriptor.relevant_entry = &one_entry; 5501813da627SRichard Henderson __jit_debug_descriptor.first_entry = &one_entry; 5502813da627SRichard Henderson __jit_debug_register_code(); 5503813da627SRichard Henderson } 5504813da627SRichard Henderson #else 55055872bbf2SRichard Henderson /* No support for the feature. Provide the entry point expected by exec.c, 55065872bbf2SRichard Henderson and implement the internal function we declared earlier. */ 5507813da627SRichard Henderson 5508755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 55092c90784aSRichard Henderson const void *debug_frame, 55102c90784aSRichard Henderson size_t debug_frame_size) 5511813da627SRichard Henderson { 5512813da627SRichard Henderson } 5513813da627SRichard Henderson 5514755bf9e5SRichard Henderson void tcg_register_jit(const void *buf, size_t buf_size) 5515813da627SRichard Henderson { 5516813da627SRichard Henderson } 5517813da627SRichard Henderson #endif /* ELF_HOST_MACHINE */ 5518db432672SRichard Henderson 5519db432672SRichard Henderson #if !TCG_TARGET_MAYBE_vec 5520db432672SRichard Henderson void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 5521db432672SRichard Henderson { 5522db432672SRichard Henderson g_assert_not_reached(); 5523db432672SRichard Henderson } 5524db432672SRichard Henderson #endif 5525