1c896fe29Sbellard /* 2c896fe29Sbellard * Tiny Code Generator for QEMU 3c896fe29Sbellard * 4c896fe29Sbellard * Copyright (c) 2008 Fabrice Bellard 5c896fe29Sbellard * 6c896fe29Sbellard * Permission is hereby granted, free of charge, to any person obtaining a copy 7c896fe29Sbellard * of this software and associated documentation files (the "Software"), to deal 8c896fe29Sbellard * in the Software without restriction, including without limitation the rights 9c896fe29Sbellard * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10c896fe29Sbellard * copies of the Software, and to permit persons to whom the Software is 11c896fe29Sbellard * furnished to do so, subject to the following conditions: 12c896fe29Sbellard * 13c896fe29Sbellard * The above copyright notice and this permission notice shall be included in 14c896fe29Sbellard * all copies or substantial portions of the Software. 15c896fe29Sbellard * 16c896fe29Sbellard * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17c896fe29Sbellard * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18c896fe29Sbellard * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19c896fe29Sbellard * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20c896fe29Sbellard * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21c896fe29Sbellard * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22c896fe29Sbellard * THE SOFTWARE. 23c896fe29Sbellard */ 24c896fe29Sbellard 25c896fe29Sbellard /* define it to use liveness analysis (better code) */ 268f2e8c07SKirill Batuzov #define USE_TCG_OPTIMIZATIONS 27c896fe29Sbellard 28757e725bSPeter Maydell #include "qemu/osdep.h" 29cca82982Saurel32 30813da627SRichard Henderson /* Define to jump the ELF file used to communicate with GDB. */ 31813da627SRichard Henderson #undef DEBUG_JIT 32813da627SRichard Henderson 3372fd2efbSEmilio G. Cota #include "qemu/error-report.h" 34f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 351de7afc9SPaolo Bonzini #include "qemu/host-utils.h" 36d4c51a0aSMarkus Armbruster #include "qemu/qemu-print.h" 371de7afc9SPaolo Bonzini #include "qemu/timer.h" 38084cfca1SRichard Henderson #include "qemu/cacheflush.h" 39c896fe29Sbellard 40c5d3c498SStefan Weil /* Note: the long term plan is to reduce the dependencies on the QEMU 41c896fe29Sbellard CPU definitions. Currently they are used for qemu_ld/st 42c896fe29Sbellard instructions */ 43c896fe29Sbellard #define NO_CPU_IO_DEFS 44c896fe29Sbellard 4563c91552SPaolo Bonzini #include "exec/exec-all.h" 46dcb32f1dSPhilippe Mathieu-Daudé #include "tcg/tcg-op.h" 47813da627SRichard Henderson 48edee2579SRichard Henderson #if UINTPTR_MAX == UINT32_MAX 49813da627SRichard Henderson # define ELF_CLASS ELFCLASS32 50edee2579SRichard Henderson #else 51edee2579SRichard Henderson # define ELF_CLASS ELFCLASS64 52813da627SRichard Henderson #endif 53813da627SRichard Henderson #ifdef HOST_WORDS_BIGENDIAN 54813da627SRichard Henderson # define ELF_DATA ELFDATA2MSB 55813da627SRichard Henderson #else 56813da627SRichard Henderson # define ELF_DATA ELFDATA2LSB 57813da627SRichard Henderson #endif 58813da627SRichard Henderson 59c896fe29Sbellard #include "elf.h" 60508127e2SPaolo Bonzini #include "exec/log.h" 615ff7258cSRichard Henderson #include "tcg-internal.h" 62c896fe29Sbellard 6322f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 6422f15579SRichard Henderson #include <ffi.h> 6522f15579SRichard Henderson #endif 6622f15579SRichard Henderson 67139c1837SPaolo Bonzini /* Forward declarations for functions declared in tcg-target.c.inc and 68ce151109SPeter Maydell used here. */ 69e4d58b41SRichard Henderson static void tcg_target_init(TCGContext *s); 70e4d58b41SRichard Henderson static void tcg_target_qemu_prologue(TCGContext *s); 716ac17786SRichard Henderson static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 722ba7fae2SRichard Henderson intptr_t value, intptr_t addend); 73c896fe29Sbellard 74497a22ebSRichard Henderson /* The CIE and FDE header definitions will be common to all hosts. */ 75497a22ebSRichard Henderson typedef struct { 76497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 77497a22ebSRichard Henderson uint32_t id; 78497a22ebSRichard Henderson uint8_t version; 79497a22ebSRichard Henderson char augmentation[1]; 80497a22ebSRichard Henderson uint8_t code_align; 81497a22ebSRichard Henderson uint8_t data_align; 82497a22ebSRichard Henderson uint8_t return_column; 83497a22ebSRichard Henderson } DebugFrameCIE; 84497a22ebSRichard Henderson 85497a22ebSRichard Henderson typedef struct QEMU_PACKED { 86497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 87497a22ebSRichard Henderson uint32_t cie_offset; 88edee2579SRichard Henderson uintptr_t func_start; 89edee2579SRichard Henderson uintptr_t func_len; 90497a22ebSRichard Henderson } DebugFrameFDEHeader; 91497a22ebSRichard Henderson 922c90784aSRichard Henderson typedef struct QEMU_PACKED { 932c90784aSRichard Henderson DebugFrameCIE cie; 942c90784aSRichard Henderson DebugFrameFDEHeader fde; 952c90784aSRichard Henderson } DebugFrameHeader; 962c90784aSRichard Henderson 97755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 982c90784aSRichard Henderson const void *debug_frame, 992c90784aSRichard Henderson size_t debug_frame_size) 100813da627SRichard Henderson __attribute__((unused)); 101813da627SRichard Henderson 102139c1837SPaolo Bonzini /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 1032a534affSRichard Henderson static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 104a05b5b9bSRichard Henderson intptr_t arg2); 10578113e83SRichard Henderson static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 106c0ad3001SStefan Weil static void tcg_out_movi(TCGContext *s, TCGType type, 1072a534affSRichard Henderson TCGReg ret, tcg_target_long arg); 1085e8892dbSMiroslav Rezanina static void tcg_out_op(TCGContext *s, TCGOpcode opc, 1095e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1105e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 111d2fd745fSRichard Henderson #if TCG_TARGET_MAYBE_vec 112e7632cfaSRichard Henderson static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 113e7632cfaSRichard Henderson TCGReg dst, TCGReg src); 114d6ecb4a9SRichard Henderson static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 115d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset); 1164e186175SRichard Henderson static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1174e186175SRichard Henderson TCGReg dst, int64_t arg); 1185e8892dbSMiroslav Rezanina static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1195e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1205e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1215e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 122d2fd745fSRichard Henderson #else 123e7632cfaSRichard Henderson static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 124e7632cfaSRichard Henderson TCGReg dst, TCGReg src) 125e7632cfaSRichard Henderson { 126e7632cfaSRichard Henderson g_assert_not_reached(); 127e7632cfaSRichard Henderson } 128d6ecb4a9SRichard Henderson static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 129d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset) 130d6ecb4a9SRichard Henderson { 131d6ecb4a9SRichard Henderson g_assert_not_reached(); 132d6ecb4a9SRichard Henderson } 1334e186175SRichard Henderson static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1344e186175SRichard Henderson TCGReg dst, int64_t arg) 135e7632cfaSRichard Henderson { 136e7632cfaSRichard Henderson g_assert_not_reached(); 137e7632cfaSRichard Henderson } 1385e8892dbSMiroslav Rezanina static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1395e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1405e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1415e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]) 142d2fd745fSRichard Henderson { 143d2fd745fSRichard Henderson g_assert_not_reached(); 144d2fd745fSRichard Henderson } 145d2fd745fSRichard Henderson #endif 1462a534affSRichard Henderson static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 147a05b5b9bSRichard Henderson intptr_t arg2); 14859d7c14eSRichard Henderson static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 14959d7c14eSRichard Henderson TCGReg base, intptr_t ofs); 1507b7d8b2dSRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 1517b7d8b2dSRichard Henderson static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 1527b7d8b2dSRichard Henderson ffi_cif *cif); 1537b7d8b2dSRichard Henderson #else 1542be7d76bSRichard Henderson static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target); 1557b7d8b2dSRichard Henderson #endif 156a4fbbd77SRichard Henderson static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 157659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 158aeee05f5SRichard Henderson static int tcg_out_ldst_finalize(TCGContext *s); 159659ef5cbSRichard Henderson #endif 160c896fe29Sbellard 16142eb6dfcSRichard Henderson TCGContext tcg_init_ctx; 16242eb6dfcSRichard Henderson __thread TCGContext *tcg_ctx; 16342eb6dfcSRichard Henderson 1645ff7258cSRichard Henderson TCGContext **tcg_ctxs; 1650e2d61cfSRichard Henderson unsigned int tcg_cur_ctxs; 1660e2d61cfSRichard Henderson unsigned int tcg_max_ctxs; 1671c2adb95SRichard Henderson TCGv_env cpu_env = 0; 168c8bc1168SRichard Henderson const void *tcg_code_gen_epilogue; 169db0c51a3SRichard Henderson uintptr_t tcg_splitwx_diff; 170df2cce29SEmilio G. Cota 171b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 172b91ccb31SRichard Henderson tcg_prologue_fn *tcg_qemu_tb_exec; 173b91ccb31SRichard Henderson #endif 174b91ccb31SRichard Henderson 175d2fd745fSRichard Henderson static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 176b1d8e52eSblueswir1 static TCGRegSet tcg_target_call_clobber_regs; 177c896fe29Sbellard 1781813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE == 1 1794196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 180c896fe29Sbellard { 181c896fe29Sbellard *s->code_ptr++ = v; 182c896fe29Sbellard } 183c896fe29Sbellard 1844196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 1854196dca6SPeter Maydell uint8_t v) 1865c53bb81SPeter Maydell { 1871813e175SRichard Henderson *p = v; 1885c53bb81SPeter Maydell } 1891813e175SRichard Henderson #endif 1905c53bb81SPeter Maydell 1911813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 2 1924196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 193c896fe29Sbellard { 1941813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 1951813e175SRichard Henderson *s->code_ptr++ = v; 1961813e175SRichard Henderson } else { 1971813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 1984387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 1991813e175SRichard Henderson s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 2001813e175SRichard Henderson } 201c896fe29Sbellard } 202c896fe29Sbellard 2034196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 2044196dca6SPeter Maydell uint16_t v) 2055c53bb81SPeter Maydell { 2061813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 2071813e175SRichard Henderson *p = v; 2081813e175SRichard Henderson } else { 2095c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2105c53bb81SPeter Maydell } 2111813e175SRichard Henderson } 2121813e175SRichard Henderson #endif 2135c53bb81SPeter Maydell 2141813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 4 2154196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 216c896fe29Sbellard { 2171813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2181813e175SRichard Henderson *s->code_ptr++ = v; 2191813e175SRichard Henderson } else { 2201813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2214387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2221813e175SRichard Henderson s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 2231813e175SRichard Henderson } 224c896fe29Sbellard } 225c896fe29Sbellard 2264196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 2274196dca6SPeter Maydell uint32_t v) 2285c53bb81SPeter Maydell { 2291813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2301813e175SRichard Henderson *p = v; 2311813e175SRichard Henderson } else { 2325c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2335c53bb81SPeter Maydell } 2341813e175SRichard Henderson } 2351813e175SRichard Henderson #endif 2365c53bb81SPeter Maydell 2371813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 8 2384196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 239ac26eb69SRichard Henderson { 2401813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2411813e175SRichard Henderson *s->code_ptr++ = v; 2421813e175SRichard Henderson } else { 2431813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2444387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2451813e175SRichard Henderson s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 2461813e175SRichard Henderson } 247ac26eb69SRichard Henderson } 248ac26eb69SRichard Henderson 2494196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 2504196dca6SPeter Maydell uint64_t v) 2515c53bb81SPeter Maydell { 2521813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2531813e175SRichard Henderson *p = v; 2541813e175SRichard Henderson } else { 2555c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2565c53bb81SPeter Maydell } 2571813e175SRichard Henderson } 2581813e175SRichard Henderson #endif 2595c53bb81SPeter Maydell 260c896fe29Sbellard /* label relocation processing */ 261c896fe29Sbellard 2621813e175SRichard Henderson static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 263bec16311SRichard Henderson TCGLabel *l, intptr_t addend) 264c896fe29Sbellard { 2657ecd02a0SRichard Henderson TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 266c896fe29Sbellard 267c896fe29Sbellard r->type = type; 268c896fe29Sbellard r->ptr = code_ptr; 269c896fe29Sbellard r->addend = addend; 2707ecd02a0SRichard Henderson QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 271c896fe29Sbellard } 272c896fe29Sbellard 27392ab8e7dSRichard Henderson static void tcg_out_label(TCGContext *s, TCGLabel *l) 274c896fe29Sbellard { 275eabb7b91SAurelien Jarno tcg_debug_assert(!l->has_value); 276c896fe29Sbellard l->has_value = 1; 27792ab8e7dSRichard Henderson l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 278c896fe29Sbellard } 279c896fe29Sbellard 28042a268c2SRichard Henderson TCGLabel *gen_new_label(void) 281c896fe29Sbellard { 282b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 28351e3972cSRichard Henderson TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 284c896fe29Sbellard 2857ecd02a0SRichard Henderson memset(l, 0, sizeof(TCGLabel)); 2867ecd02a0SRichard Henderson l->id = s->nb_labels++; 2877ecd02a0SRichard Henderson QSIMPLEQ_INIT(&l->relocs); 2887ecd02a0SRichard Henderson 289bef16ab4SRichard Henderson QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 29042a268c2SRichard Henderson 29142a268c2SRichard Henderson return l; 292c896fe29Sbellard } 293c896fe29Sbellard 2947ecd02a0SRichard Henderson static bool tcg_resolve_relocs(TCGContext *s) 2957ecd02a0SRichard Henderson { 2967ecd02a0SRichard Henderson TCGLabel *l; 2977ecd02a0SRichard Henderson 2987ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 2997ecd02a0SRichard Henderson TCGRelocation *r; 3007ecd02a0SRichard Henderson uintptr_t value = l->u.value; 3017ecd02a0SRichard Henderson 3027ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(r, &l->relocs, next) { 3037ecd02a0SRichard Henderson if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 3047ecd02a0SRichard Henderson return false; 3057ecd02a0SRichard Henderson } 3067ecd02a0SRichard Henderson } 3077ecd02a0SRichard Henderson } 3087ecd02a0SRichard Henderson return true; 3097ecd02a0SRichard Henderson } 3107ecd02a0SRichard Henderson 3119f754620SRichard Henderson static void set_jmp_reset_offset(TCGContext *s, int which) 3129f754620SRichard Henderson { 313f14bed3fSRichard Henderson /* 314f14bed3fSRichard Henderson * We will check for overflow at the end of the opcode loop in 315f14bed3fSRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 316f14bed3fSRichard Henderson */ 317f14bed3fSRichard Henderson s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); 3189f754620SRichard Henderson } 3199f754620SRichard Henderson 320db6b7d0cSRichard Henderson /* Signal overflow, starting over with fewer guest insns. */ 321db6b7d0cSRichard Henderson static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s) 322db6b7d0cSRichard Henderson { 323db6b7d0cSRichard Henderson siglongjmp(s->jmp_trans, -2); 324db6b7d0cSRichard Henderson } 325db6b7d0cSRichard Henderson 3264c22e840SRichard Henderson #define C_PFX1(P, A) P##A 3274c22e840SRichard Henderson #define C_PFX2(P, A, B) P##A##_##B 3284c22e840SRichard Henderson #define C_PFX3(P, A, B, C) P##A##_##B##_##C 3294c22e840SRichard Henderson #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 3304c22e840SRichard Henderson #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 3314c22e840SRichard Henderson #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 3324c22e840SRichard Henderson 3334c22e840SRichard Henderson /* Define an enumeration for the various combinations. */ 3344c22e840SRichard Henderson 3354c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 3364c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 3374c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 3384c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 3394c22e840SRichard Henderson 3404c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 3414c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 3424c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 3434c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 3444c22e840SRichard Henderson 3454c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 3464c22e840SRichard Henderson 3474c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 3484c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 3494c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 3504c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 3514c22e840SRichard Henderson 3524c22e840SRichard Henderson typedef enum { 3534c22e840SRichard Henderson #include "tcg-target-con-set.h" 3544c22e840SRichard Henderson } TCGConstraintSetIndex; 3554c22e840SRichard Henderson 3564c22e840SRichard Henderson static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 3574c22e840SRichard Henderson 3584c22e840SRichard Henderson #undef C_O0_I1 3594c22e840SRichard Henderson #undef C_O0_I2 3604c22e840SRichard Henderson #undef C_O0_I3 3614c22e840SRichard Henderson #undef C_O0_I4 3624c22e840SRichard Henderson #undef C_O1_I1 3634c22e840SRichard Henderson #undef C_O1_I2 3644c22e840SRichard Henderson #undef C_O1_I3 3654c22e840SRichard Henderson #undef C_O1_I4 3664c22e840SRichard Henderson #undef C_N1_I2 3674c22e840SRichard Henderson #undef C_O2_I1 3684c22e840SRichard Henderson #undef C_O2_I2 3694c22e840SRichard Henderson #undef C_O2_I3 3704c22e840SRichard Henderson #undef C_O2_I4 3714c22e840SRichard Henderson 3724c22e840SRichard Henderson /* Put all of the constraint sets into an array, indexed by the enum. */ 3734c22e840SRichard Henderson 3744c22e840SRichard Henderson #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 3754c22e840SRichard Henderson #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 3764c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 3774c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 3784c22e840SRichard Henderson 3794c22e840SRichard Henderson #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 3804c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 3814c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 3824c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 3834c22e840SRichard Henderson 3844c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 3854c22e840SRichard Henderson 3864c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 3874c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 3884c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 3894c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 3904c22e840SRichard Henderson 3914c22e840SRichard Henderson static const TCGTargetOpDef constraint_sets[] = { 3924c22e840SRichard Henderson #include "tcg-target-con-set.h" 3934c22e840SRichard Henderson }; 3944c22e840SRichard Henderson 3954c22e840SRichard Henderson 3964c22e840SRichard Henderson #undef C_O0_I1 3974c22e840SRichard Henderson #undef C_O0_I2 3984c22e840SRichard Henderson #undef C_O0_I3 3994c22e840SRichard Henderson #undef C_O0_I4 4004c22e840SRichard Henderson #undef C_O1_I1 4014c22e840SRichard Henderson #undef C_O1_I2 4024c22e840SRichard Henderson #undef C_O1_I3 4034c22e840SRichard Henderson #undef C_O1_I4 4044c22e840SRichard Henderson #undef C_N1_I2 4054c22e840SRichard Henderson #undef C_O2_I1 4064c22e840SRichard Henderson #undef C_O2_I2 4074c22e840SRichard Henderson #undef C_O2_I3 4084c22e840SRichard Henderson #undef C_O2_I4 4094c22e840SRichard Henderson 4104c22e840SRichard Henderson /* Expand the enumerator to be returned from tcg_target_op_def(). */ 4114c22e840SRichard Henderson 4124c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 4134c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 4144c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 4154c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 4164c22e840SRichard Henderson 4174c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 4184c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 4194c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 4204c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 4214c22e840SRichard Henderson 4224c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 4234c22e840SRichard Henderson 4244c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 4254c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 4264c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 4274c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 4284c22e840SRichard Henderson 429139c1837SPaolo Bonzini #include "tcg-target.c.inc" 430c896fe29Sbellard 43138b47b19SEmilio G. Cota static void alloc_tcg_plugin_context(TCGContext *s) 43238b47b19SEmilio G. Cota { 43338b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 43438b47b19SEmilio G. Cota s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 43538b47b19SEmilio G. Cota s->plugin_tb->insns = 43638b47b19SEmilio G. Cota g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 43738b47b19SEmilio G. Cota #endif 43838b47b19SEmilio G. Cota } 43938b47b19SEmilio G. Cota 440e8feb96fSEmilio G. Cota /* 4413468b59eSEmilio G. Cota * All TCG threads except the parent (i.e. the one that called tcg_context_init 4423468b59eSEmilio G. Cota * and registered the target's TCG globals) must register with this function 4433468b59eSEmilio G. Cota * before initiating translation. 4443468b59eSEmilio G. Cota * 4453468b59eSEmilio G. Cota * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 4463468b59eSEmilio G. Cota * of tcg_region_init() for the reasoning behind this. 4473468b59eSEmilio G. Cota * 4483468b59eSEmilio G. Cota * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 4493468b59eSEmilio G. Cota * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 4503468b59eSEmilio G. Cota * is not used anymore for translation once this function is called. 4513468b59eSEmilio G. Cota * 4523468b59eSEmilio G. Cota * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 4533468b59eSEmilio G. Cota * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 4543468b59eSEmilio G. Cota */ 4553468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 4563468b59eSEmilio G. Cota void tcg_register_thread(void) 4573468b59eSEmilio G. Cota { 4583468b59eSEmilio G. Cota tcg_ctx = &tcg_init_ctx; 4593468b59eSEmilio G. Cota } 4603468b59eSEmilio G. Cota #else 4613468b59eSEmilio G. Cota void tcg_register_thread(void) 4623468b59eSEmilio G. Cota { 4633468b59eSEmilio G. Cota TCGContext *s = g_malloc(sizeof(*s)); 4643468b59eSEmilio G. Cota unsigned int i, n; 4653468b59eSEmilio G. Cota 4663468b59eSEmilio G. Cota *s = tcg_init_ctx; 4673468b59eSEmilio G. Cota 4683468b59eSEmilio G. Cota /* Relink mem_base. */ 4693468b59eSEmilio G. Cota for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 4703468b59eSEmilio G. Cota if (tcg_init_ctx.temps[i].mem_base) { 4713468b59eSEmilio G. Cota ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 4723468b59eSEmilio G. Cota tcg_debug_assert(b >= 0 && b < n); 4733468b59eSEmilio G. Cota s->temps[i].mem_base = &s->temps[b]; 4743468b59eSEmilio G. Cota } 4753468b59eSEmilio G. Cota } 4763468b59eSEmilio G. Cota 4773468b59eSEmilio G. Cota /* Claim an entry in tcg_ctxs */ 4780e2d61cfSRichard Henderson n = qatomic_fetch_inc(&tcg_cur_ctxs); 4790e2d61cfSRichard Henderson g_assert(n < tcg_max_ctxs); 480d73415a3SStefan Hajnoczi qatomic_set(&tcg_ctxs[n], s); 4813468b59eSEmilio G. Cota 48238b47b19SEmilio G. Cota if (n > 0) { 48338b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 484bf042e8eSRichard Henderson tcg_region_initial_alloc(s); 48538b47b19SEmilio G. Cota } 48638b47b19SEmilio G. Cota 4873468b59eSEmilio G. Cota tcg_ctx = s; 4883468b59eSEmilio G. Cota } 4893468b59eSEmilio G. Cota #endif /* !CONFIG_USER_ONLY */ 4903468b59eSEmilio G. Cota 491c896fe29Sbellard /* pool based memory allocation */ 492c896fe29Sbellard void *tcg_malloc_internal(TCGContext *s, int size) 493c896fe29Sbellard { 494c896fe29Sbellard TCGPool *p; 495c896fe29Sbellard int pool_size; 496c896fe29Sbellard 497c896fe29Sbellard if (size > TCG_POOL_CHUNK_SIZE) { 498c896fe29Sbellard /* big malloc: insert a new pool (XXX: could optimize) */ 4997267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + size); 500c896fe29Sbellard p->size = size; 5014055299eSKirill Batuzov p->next = s->pool_first_large; 5024055299eSKirill Batuzov s->pool_first_large = p; 5034055299eSKirill Batuzov return p->data; 504c896fe29Sbellard } else { 505c896fe29Sbellard p = s->pool_current; 506c896fe29Sbellard if (!p) { 507c896fe29Sbellard p = s->pool_first; 508c896fe29Sbellard if (!p) 509c896fe29Sbellard goto new_pool; 510c896fe29Sbellard } else { 511c896fe29Sbellard if (!p->next) { 512c896fe29Sbellard new_pool: 513c896fe29Sbellard pool_size = TCG_POOL_CHUNK_SIZE; 5147267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + pool_size); 515c896fe29Sbellard p->size = pool_size; 516c896fe29Sbellard p->next = NULL; 517c896fe29Sbellard if (s->pool_current) 518c896fe29Sbellard s->pool_current->next = p; 519c896fe29Sbellard else 520c896fe29Sbellard s->pool_first = p; 521c896fe29Sbellard } else { 522c896fe29Sbellard p = p->next; 523c896fe29Sbellard } 524c896fe29Sbellard } 525c896fe29Sbellard } 526c896fe29Sbellard s->pool_current = p; 527c896fe29Sbellard s->pool_cur = p->data + size; 528c896fe29Sbellard s->pool_end = p->data + p->size; 529c896fe29Sbellard return p->data; 530c896fe29Sbellard } 531c896fe29Sbellard 532c896fe29Sbellard void tcg_pool_reset(TCGContext *s) 533c896fe29Sbellard { 5344055299eSKirill Batuzov TCGPool *p, *t; 5354055299eSKirill Batuzov for (p = s->pool_first_large; p; p = t) { 5364055299eSKirill Batuzov t = p->next; 5374055299eSKirill Batuzov g_free(p); 5384055299eSKirill Batuzov } 5394055299eSKirill Batuzov s->pool_first_large = NULL; 540c896fe29Sbellard s->pool_cur = s->pool_end = NULL; 541c896fe29Sbellard s->pool_current = NULL; 542c896fe29Sbellard } 543c896fe29Sbellard 5442ef6175aSRichard Henderson #include "exec/helper-proto.h" 5452ef6175aSRichard Henderson 546100b5e01SRichard Henderson static const TCGHelperInfo all_helpers[] = { 5472ef6175aSRichard Henderson #include "exec/helper-tcg.h" 548100b5e01SRichard Henderson }; 549619205fdSEmilio G. Cota static GHashTable *helper_table; 550100b5e01SRichard Henderson 55122f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 55222f15579SRichard Henderson static GHashTable *ffi_table; 55322f15579SRichard Henderson 55422f15579SRichard Henderson static ffi_type * const typecode_to_ffi[8] = { 55522f15579SRichard Henderson [dh_typecode_void] = &ffi_type_void, 55622f15579SRichard Henderson [dh_typecode_i32] = &ffi_type_uint32, 55722f15579SRichard Henderson [dh_typecode_s32] = &ffi_type_sint32, 55822f15579SRichard Henderson [dh_typecode_i64] = &ffi_type_uint64, 55922f15579SRichard Henderson [dh_typecode_s64] = &ffi_type_sint64, 56022f15579SRichard Henderson [dh_typecode_ptr] = &ffi_type_pointer, 56122f15579SRichard Henderson }; 56222f15579SRichard Henderson #endif 56322f15579SRichard Henderson 56491478cefSRichard Henderson static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 565f69d277eSRichard Henderson static void process_op_defs(TCGContext *s); 5661c2adb95SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 5671c2adb95SRichard Henderson TCGReg reg, const char *name); 56891478cefSRichard Henderson 56943b972b7SRichard Henderson static void tcg_context_init(unsigned max_cpus) 570c896fe29Sbellard { 571a76aabd3SRichard Henderson TCGContext *s = &tcg_init_ctx; 572100b5e01SRichard Henderson int op, total_args, n, i; 573c896fe29Sbellard TCGOpDef *def; 574c896fe29Sbellard TCGArgConstraint *args_ct; 5751c2adb95SRichard Henderson TCGTemp *ts; 576c896fe29Sbellard 577c896fe29Sbellard memset(s, 0, sizeof(*s)); 578c896fe29Sbellard s->nb_globals = 0; 579c896fe29Sbellard 580c896fe29Sbellard /* Count total number of arguments and allocate the corresponding 581c896fe29Sbellard space */ 582c896fe29Sbellard total_args = 0; 583c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 584c896fe29Sbellard def = &tcg_op_defs[op]; 585c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 586c896fe29Sbellard total_args += n; 587c896fe29Sbellard } 588c896fe29Sbellard 589bc2b17e6SRichard Henderson args_ct = g_new0(TCGArgConstraint, total_args); 590c896fe29Sbellard 591c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 592c896fe29Sbellard def = &tcg_op_defs[op]; 593c896fe29Sbellard def->args_ct = args_ct; 594c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 595c896fe29Sbellard args_ct += n; 596c896fe29Sbellard } 597c896fe29Sbellard 5985cd8f621SRichard Henderson /* Register helpers. */ 59984fd9dd3SRichard Henderson /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 600619205fdSEmilio G. Cota helper_table = g_hash_table_new(NULL, NULL); 60184fd9dd3SRichard Henderson 602100b5e01SRichard Henderson for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 60384fd9dd3SRichard Henderson g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 60472866e82SRichard Henderson (gpointer)&all_helpers[i]); 605100b5e01SRichard Henderson } 6065cd8f621SRichard Henderson 60722f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 60822f15579SRichard Henderson /* g_direct_hash/equal for direct comparisons on uint32_t. */ 60922f15579SRichard Henderson ffi_table = g_hash_table_new(NULL, NULL); 61022f15579SRichard Henderson for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 61122f15579SRichard Henderson struct { 61222f15579SRichard Henderson ffi_cif cif; 61322f15579SRichard Henderson ffi_type *args[]; 61422f15579SRichard Henderson } *ca; 61522f15579SRichard Henderson uint32_t typemask = all_helpers[i].typemask; 61622f15579SRichard Henderson gpointer hash = (gpointer)(uintptr_t)typemask; 61722f15579SRichard Henderson ffi_status status; 61822f15579SRichard Henderson int nargs; 61922f15579SRichard Henderson 62022f15579SRichard Henderson if (g_hash_table_lookup(ffi_table, hash)) { 62122f15579SRichard Henderson continue; 62222f15579SRichard Henderson } 62322f15579SRichard Henderson 62422f15579SRichard Henderson /* Ignoring the return type, find the last non-zero field. */ 62522f15579SRichard Henderson nargs = 32 - clz32(typemask >> 3); 62622f15579SRichard Henderson nargs = DIV_ROUND_UP(nargs, 3); 62722f15579SRichard Henderson 62822f15579SRichard Henderson ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 62922f15579SRichard Henderson ca->cif.rtype = typecode_to_ffi[typemask & 7]; 63022f15579SRichard Henderson ca->cif.nargs = nargs; 63122f15579SRichard Henderson 63222f15579SRichard Henderson if (nargs != 0) { 63322f15579SRichard Henderson ca->cif.arg_types = ca->args; 63422f15579SRichard Henderson for (i = 0; i < nargs; ++i) { 63522f15579SRichard Henderson int typecode = extract32(typemask, (i + 1) * 3, 3); 63622f15579SRichard Henderson ca->args[i] = typecode_to_ffi[typecode]; 63722f15579SRichard Henderson } 63822f15579SRichard Henderson } 63922f15579SRichard Henderson 64022f15579SRichard Henderson status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 64122f15579SRichard Henderson ca->cif.rtype, ca->cif.arg_types); 64222f15579SRichard Henderson assert(status == FFI_OK); 64322f15579SRichard Henderson 64422f15579SRichard Henderson g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); 64522f15579SRichard Henderson } 64622f15579SRichard Henderson #endif 64722f15579SRichard Henderson 648c896fe29Sbellard tcg_target_init(s); 649f69d277eSRichard Henderson process_op_defs(s); 65091478cefSRichard Henderson 65191478cefSRichard Henderson /* Reverse the order of the saved registers, assuming they're all at 65291478cefSRichard Henderson the start of tcg_target_reg_alloc_order. */ 65391478cefSRichard Henderson for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 65491478cefSRichard Henderson int r = tcg_target_reg_alloc_order[n]; 65591478cefSRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 65691478cefSRichard Henderson break; 65791478cefSRichard Henderson } 65891478cefSRichard Henderson } 65991478cefSRichard Henderson for (i = 0; i < n; ++i) { 66091478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 66191478cefSRichard Henderson } 66291478cefSRichard Henderson for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 66391478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 66491478cefSRichard Henderson } 665b1311c4aSEmilio G. Cota 66638b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 66738b47b19SEmilio G. Cota 668b1311c4aSEmilio G. Cota tcg_ctx = s; 6693468b59eSEmilio G. Cota /* 6703468b59eSEmilio G. Cota * In user-mode we simply share the init context among threads, since we 6713468b59eSEmilio G. Cota * use a single region. See the documentation tcg_region_init() for the 6723468b59eSEmilio G. Cota * reasoning behind this. 6733468b59eSEmilio G. Cota * In softmmu we will have at most max_cpus TCG threads. 6743468b59eSEmilio G. Cota */ 6753468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 676df2cce29SEmilio G. Cota tcg_ctxs = &tcg_ctx; 6770e2d61cfSRichard Henderson tcg_cur_ctxs = 1; 6780e2d61cfSRichard Henderson tcg_max_ctxs = 1; 6793468b59eSEmilio G. Cota #else 6800e2d61cfSRichard Henderson tcg_max_ctxs = max_cpus; 6810e2d61cfSRichard Henderson tcg_ctxs = g_new0(TCGContext *, max_cpus); 6823468b59eSEmilio G. Cota #endif 6831c2adb95SRichard Henderson 6841c2adb95SRichard Henderson tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 6851c2adb95SRichard Henderson ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 6861c2adb95SRichard Henderson cpu_env = temp_tcgv_ptr(ts); 6879002ec79SRichard Henderson } 688b03cce8eSbellard 68943b972b7SRichard Henderson void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 690a76aabd3SRichard Henderson { 69143b972b7SRichard Henderson tcg_context_init(max_cpus); 69243b972b7SRichard Henderson tcg_region_init(tb_size, splitwx, max_cpus); 693a76aabd3SRichard Henderson } 694a76aabd3SRichard Henderson 6956e3b2bfdSEmilio G. Cota /* 6966e3b2bfdSEmilio G. Cota * Allocate TBs right before their corresponding translated code, making 6976e3b2bfdSEmilio G. Cota * sure that TBs and code are on different cache lines. 6986e3b2bfdSEmilio G. Cota */ 6996e3b2bfdSEmilio G. Cota TranslationBlock *tcg_tb_alloc(TCGContext *s) 7006e3b2bfdSEmilio G. Cota { 7016e3b2bfdSEmilio G. Cota uintptr_t align = qemu_icache_linesize; 7026e3b2bfdSEmilio G. Cota TranslationBlock *tb; 7036e3b2bfdSEmilio G. Cota void *next; 7046e3b2bfdSEmilio G. Cota 705e8feb96fSEmilio G. Cota retry: 7066e3b2bfdSEmilio G. Cota tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 7076e3b2bfdSEmilio G. Cota next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 7086e3b2bfdSEmilio G. Cota 7096e3b2bfdSEmilio G. Cota if (unlikely(next > s->code_gen_highwater)) { 710e8feb96fSEmilio G. Cota if (tcg_region_alloc(s)) { 7116e3b2bfdSEmilio G. Cota return NULL; 7126e3b2bfdSEmilio G. Cota } 713e8feb96fSEmilio G. Cota goto retry; 714e8feb96fSEmilio G. Cota } 715d73415a3SStefan Hajnoczi qatomic_set(&s->code_gen_ptr, next); 71657a26946SRichard Henderson s->data_gen_ptr = NULL; 7176e3b2bfdSEmilio G. Cota return tb; 7186e3b2bfdSEmilio G. Cota } 7196e3b2bfdSEmilio G. Cota 7209002ec79SRichard Henderson void tcg_prologue_init(TCGContext *s) 7219002ec79SRichard Henderson { 722b0a0794aSRichard Henderson size_t prologue_size; 7238163b749SRichard Henderson 724b0a0794aSRichard Henderson s->code_ptr = s->code_gen_ptr; 725b0a0794aSRichard Henderson s->code_buf = s->code_gen_ptr; 7265b38ee31SRichard Henderson s->data_gen_ptr = NULL; 727b91ccb31SRichard Henderson 728b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 729b0a0794aSRichard Henderson tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 730b91ccb31SRichard Henderson #endif 7318163b749SRichard Henderson 7325b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 7335b38ee31SRichard Henderson s->pool_labels = NULL; 7345b38ee31SRichard Henderson #endif 7355b38ee31SRichard Henderson 736653b87ebSRoman Bolshakov qemu_thread_jit_write(); 7378163b749SRichard Henderson /* Generate the prologue. */ 738b03cce8eSbellard tcg_target_qemu_prologue(s); 7395b38ee31SRichard Henderson 7405b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 7415b38ee31SRichard Henderson /* Allow the prologue to put e.g. guest_base into a pool entry. */ 7425b38ee31SRichard Henderson { 7431768987bSRichard Henderson int result = tcg_out_pool_finalize(s); 7441768987bSRichard Henderson tcg_debug_assert(result == 0); 7455b38ee31SRichard Henderson } 7465b38ee31SRichard Henderson #endif 7475b38ee31SRichard Henderson 748b0a0794aSRichard Henderson prologue_size = tcg_current_code_size(s); 749b0a0794aSRichard Henderson 750df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 751b0a0794aSRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 752b0a0794aSRichard Henderson (uintptr_t)s->code_buf, prologue_size); 753df5d2b16SRichard Henderson #endif 7548163b749SRichard Henderson 755d6b64b2bSRichard Henderson #ifdef DEBUG_DISAS 756d6b64b2bSRichard Henderson if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 757fc59d2d8SRobert Foley FILE *logfile = qemu_log_lock(); 7588163b749SRichard Henderson qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 7595b38ee31SRichard Henderson if (s->data_gen_ptr) { 760b0a0794aSRichard Henderson size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 7615b38ee31SRichard Henderson size_t data_size = prologue_size - code_size; 7625b38ee31SRichard Henderson size_t i; 7635b38ee31SRichard Henderson 764b0a0794aSRichard Henderson log_disas(s->code_gen_ptr, code_size); 7655b38ee31SRichard Henderson 7665b38ee31SRichard Henderson for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 7675b38ee31SRichard Henderson if (sizeof(tcg_target_ulong) == 8) { 7685b38ee31SRichard Henderson qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 7695b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 7705b38ee31SRichard Henderson *(uint64_t *)(s->data_gen_ptr + i)); 7715b38ee31SRichard Henderson } else { 7725b38ee31SRichard Henderson qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 7735b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 7745b38ee31SRichard Henderson *(uint32_t *)(s->data_gen_ptr + i)); 7755b38ee31SRichard Henderson } 7765b38ee31SRichard Henderson } 7775b38ee31SRichard Henderson } else { 778b0a0794aSRichard Henderson log_disas(s->code_gen_ptr, prologue_size); 7795b38ee31SRichard Henderson } 780d6b64b2bSRichard Henderson qemu_log("\n"); 781d6b64b2bSRichard Henderson qemu_log_flush(); 782fc59d2d8SRobert Foley qemu_log_unlock(logfile); 783d6b64b2bSRichard Henderson } 784d6b64b2bSRichard Henderson #endif 785cedbcb01SEmilio G. Cota 7866eea0434SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 7876eea0434SRichard Henderson /* 7886eea0434SRichard Henderson * Assert that goto_ptr is implemented completely, setting an epilogue. 7896eea0434SRichard Henderson * For tci, we use NULL as the signal to return from the interpreter, 7906eea0434SRichard Henderson * so skip this check. 7916eea0434SRichard Henderson */ 7928b5c2b62SRichard Henderson tcg_debug_assert(tcg_code_gen_epilogue != NULL); 7936eea0434SRichard Henderson #endif 794d1c74ab3SRichard Henderson 795d1c74ab3SRichard Henderson tcg_region_prologue_set(s); 796c896fe29Sbellard } 797c896fe29Sbellard 798c896fe29Sbellard void tcg_func_start(TCGContext *s) 799c896fe29Sbellard { 800c896fe29Sbellard tcg_pool_reset(s); 801c896fe29Sbellard s->nb_temps = s->nb_globals; 8020ec9eabcSRichard Henderson 8030ec9eabcSRichard Henderson /* No temps have been previously allocated for size or locality. */ 8040ec9eabcSRichard Henderson memset(s->free_temps, 0, sizeof(s->free_temps)); 8050ec9eabcSRichard Henderson 806c0522136SRichard Henderson /* No constant temps have been previously allocated. */ 807c0522136SRichard Henderson for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 808c0522136SRichard Henderson if (s->const_table[i]) { 809c0522136SRichard Henderson g_hash_table_remove_all(s->const_table[i]); 810c0522136SRichard Henderson } 811c0522136SRichard Henderson } 812c0522136SRichard Henderson 813abebf925SRichard Henderson s->nb_ops = 0; 814c896fe29Sbellard s->nb_labels = 0; 815c896fe29Sbellard s->current_frame_offset = s->frame_start; 816c896fe29Sbellard 8170a209d4bSRichard Henderson #ifdef CONFIG_DEBUG_TCG 8180a209d4bSRichard Henderson s->goto_tb_issue_mask = 0; 8190a209d4bSRichard Henderson #endif 8200a209d4bSRichard Henderson 82115fa08f8SRichard Henderson QTAILQ_INIT(&s->ops); 82215fa08f8SRichard Henderson QTAILQ_INIT(&s->free_ops); 823bef16ab4SRichard Henderson QSIMPLEQ_INIT(&s->labels); 824c896fe29Sbellard } 825c896fe29Sbellard 826ae30e866SRichard Henderson static TCGTemp *tcg_temp_alloc(TCGContext *s) 8277ca4b752SRichard Henderson { 8287ca4b752SRichard Henderson int n = s->nb_temps++; 829ae30e866SRichard Henderson 830ae30e866SRichard Henderson if (n >= TCG_MAX_TEMPS) { 831db6b7d0cSRichard Henderson tcg_raise_tb_overflow(s); 832ae30e866SRichard Henderson } 8337ca4b752SRichard Henderson return memset(&s->temps[n], 0, sizeof(TCGTemp)); 8347ca4b752SRichard Henderson } 8357ca4b752SRichard Henderson 836ae30e866SRichard Henderson static TCGTemp *tcg_global_alloc(TCGContext *s) 8377ca4b752SRichard Henderson { 838fa477d25SRichard Henderson TCGTemp *ts; 839fa477d25SRichard Henderson 8407ca4b752SRichard Henderson tcg_debug_assert(s->nb_globals == s->nb_temps); 841ae30e866SRichard Henderson tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 8427ca4b752SRichard Henderson s->nb_globals++; 843fa477d25SRichard Henderson ts = tcg_temp_alloc(s); 844ee17db83SRichard Henderson ts->kind = TEMP_GLOBAL; 845fa477d25SRichard Henderson 846fa477d25SRichard Henderson return ts; 847c896fe29Sbellard } 848c896fe29Sbellard 849085272b3SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 850b6638662SRichard Henderson TCGReg reg, const char *name) 851c896fe29Sbellard { 852c896fe29Sbellard TCGTemp *ts; 853c896fe29Sbellard 854b3a62939SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 855c896fe29Sbellard tcg_abort(); 856b3a62939SRichard Henderson } 8577ca4b752SRichard Henderson 8587ca4b752SRichard Henderson ts = tcg_global_alloc(s); 859c896fe29Sbellard ts->base_type = type; 860c896fe29Sbellard ts->type = type; 861ee17db83SRichard Henderson ts->kind = TEMP_FIXED; 862c896fe29Sbellard ts->reg = reg; 863c896fe29Sbellard ts->name = name; 864c896fe29Sbellard tcg_regset_set_reg(s->reserved_regs, reg); 8657ca4b752SRichard Henderson 866085272b3SRichard Henderson return ts; 867a7812ae4Spbrook } 868a7812ae4Spbrook 869b6638662SRichard Henderson void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 870a7812ae4Spbrook { 871b3a62939SRichard Henderson s->frame_start = start; 872b3a62939SRichard Henderson s->frame_end = start + size; 873085272b3SRichard Henderson s->frame_temp 874085272b3SRichard Henderson = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 875b3a62939SRichard Henderson } 876a7812ae4Spbrook 877085272b3SRichard Henderson TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 878e1ccc054SRichard Henderson intptr_t offset, const char *name) 879c896fe29Sbellard { 880b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 881dc41aa7dSRichard Henderson TCGTemp *base_ts = tcgv_ptr_temp(base); 8827ca4b752SRichard Henderson TCGTemp *ts = tcg_global_alloc(s); 883b3915dbbSRichard Henderson int indirect_reg = 0, bigendian = 0; 8847ca4b752SRichard Henderson #ifdef HOST_WORDS_BIGENDIAN 8857ca4b752SRichard Henderson bigendian = 1; 8867ca4b752SRichard Henderson #endif 887c896fe29Sbellard 888c0522136SRichard Henderson switch (base_ts->kind) { 889c0522136SRichard Henderson case TEMP_FIXED: 890c0522136SRichard Henderson break; 891c0522136SRichard Henderson case TEMP_GLOBAL: 8925a18407fSRichard Henderson /* We do not support double-indirect registers. */ 8935a18407fSRichard Henderson tcg_debug_assert(!base_ts->indirect_reg); 894b3915dbbSRichard Henderson base_ts->indirect_base = 1; 8955a18407fSRichard Henderson s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 8965a18407fSRichard Henderson ? 2 : 1); 8975a18407fSRichard Henderson indirect_reg = 1; 898c0522136SRichard Henderson break; 899c0522136SRichard Henderson default: 900c0522136SRichard Henderson g_assert_not_reached(); 901b3915dbbSRichard Henderson } 902b3915dbbSRichard Henderson 9037ca4b752SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 9047ca4b752SRichard Henderson TCGTemp *ts2 = tcg_global_alloc(s); 905c896fe29Sbellard char buf[64]; 9067ca4b752SRichard Henderson 9077ca4b752SRichard Henderson ts->base_type = TCG_TYPE_I64; 908c896fe29Sbellard ts->type = TCG_TYPE_I32; 909b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 910c896fe29Sbellard ts->mem_allocated = 1; 911b3a62939SRichard Henderson ts->mem_base = base_ts; 9127ca4b752SRichard Henderson ts->mem_offset = offset + bigendian * 4; 913c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 914c896fe29Sbellard pstrcat(buf, sizeof(buf), "_0"); 915c896fe29Sbellard ts->name = strdup(buf); 916c896fe29Sbellard 9177ca4b752SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 9187ca4b752SRichard Henderson ts2->base_type = TCG_TYPE_I64; 9197ca4b752SRichard Henderson ts2->type = TCG_TYPE_I32; 920b3915dbbSRichard Henderson ts2->indirect_reg = indirect_reg; 9217ca4b752SRichard Henderson ts2->mem_allocated = 1; 9227ca4b752SRichard Henderson ts2->mem_base = base_ts; 9237ca4b752SRichard Henderson ts2->mem_offset = offset + (1 - bigendian) * 4; 924c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 925c896fe29Sbellard pstrcat(buf, sizeof(buf), "_1"); 926120c1084SRichard Henderson ts2->name = strdup(buf); 9277ca4b752SRichard Henderson } else { 928c896fe29Sbellard ts->base_type = type; 929c896fe29Sbellard ts->type = type; 930b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 931c896fe29Sbellard ts->mem_allocated = 1; 932b3a62939SRichard Henderson ts->mem_base = base_ts; 933c896fe29Sbellard ts->mem_offset = offset; 934c896fe29Sbellard ts->name = name; 935c896fe29Sbellard } 936085272b3SRichard Henderson return ts; 937c896fe29Sbellard } 938c896fe29Sbellard 9395bfa8034SRichard Henderson TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) 940c896fe29Sbellard { 941b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 942ee17db83SRichard Henderson TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL; 943c896fe29Sbellard TCGTemp *ts; 944641d5fbeSbellard int idx, k; 945c896fe29Sbellard 9460ec9eabcSRichard Henderson k = type + (temp_local ? TCG_TYPE_COUNT : 0); 9470ec9eabcSRichard Henderson idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 9480ec9eabcSRichard Henderson if (idx < TCG_MAX_TEMPS) { 9490ec9eabcSRichard Henderson /* There is already an available temp with the right type. */ 9500ec9eabcSRichard Henderson clear_bit(idx, s->free_temps[k].l); 9510ec9eabcSRichard Henderson 952e8996ee0Sbellard ts = &s->temps[idx]; 953e8996ee0Sbellard ts->temp_allocated = 1; 9547ca4b752SRichard Henderson tcg_debug_assert(ts->base_type == type); 955ee17db83SRichard Henderson tcg_debug_assert(ts->kind == kind); 956e8996ee0Sbellard } else { 9577ca4b752SRichard Henderson ts = tcg_temp_alloc(s); 9587ca4b752SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 9597ca4b752SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 9607ca4b752SRichard Henderson 961c896fe29Sbellard ts->base_type = type; 962c896fe29Sbellard ts->type = TCG_TYPE_I32; 963e8996ee0Sbellard ts->temp_allocated = 1; 964ee17db83SRichard Henderson ts->kind = kind; 9657ca4b752SRichard Henderson 9667ca4b752SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 9677ca4b752SRichard Henderson ts2->base_type = TCG_TYPE_I64; 9687ca4b752SRichard Henderson ts2->type = TCG_TYPE_I32; 9697ca4b752SRichard Henderson ts2->temp_allocated = 1; 970ee17db83SRichard Henderson ts2->kind = kind; 9717ca4b752SRichard Henderson } else { 972c896fe29Sbellard ts->base_type = type; 973c896fe29Sbellard ts->type = type; 974e8996ee0Sbellard ts->temp_allocated = 1; 975ee17db83SRichard Henderson ts->kind = kind; 976c896fe29Sbellard } 977e8996ee0Sbellard } 97827bfd83cSPeter Maydell 97927bfd83cSPeter Maydell #if defined(CONFIG_DEBUG_TCG) 98027bfd83cSPeter Maydell s->temps_in_use++; 98127bfd83cSPeter Maydell #endif 982085272b3SRichard Henderson return ts; 983c896fe29Sbellard } 984c896fe29Sbellard 985d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec(TCGType type) 986d2fd745fSRichard Henderson { 987d2fd745fSRichard Henderson TCGTemp *t; 988d2fd745fSRichard Henderson 989d2fd745fSRichard Henderson #ifdef CONFIG_DEBUG_TCG 990d2fd745fSRichard Henderson switch (type) { 991d2fd745fSRichard Henderson case TCG_TYPE_V64: 992d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v64); 993d2fd745fSRichard Henderson break; 994d2fd745fSRichard Henderson case TCG_TYPE_V128: 995d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v128); 996d2fd745fSRichard Henderson break; 997d2fd745fSRichard Henderson case TCG_TYPE_V256: 998d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v256); 999d2fd745fSRichard Henderson break; 1000d2fd745fSRichard Henderson default: 1001d2fd745fSRichard Henderson g_assert_not_reached(); 1002d2fd745fSRichard Henderson } 1003d2fd745fSRichard Henderson #endif 1004d2fd745fSRichard Henderson 1005d2fd745fSRichard Henderson t = tcg_temp_new_internal(type, 0); 1006d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1007d2fd745fSRichard Henderson } 1008d2fd745fSRichard Henderson 1009d2fd745fSRichard Henderson /* Create a new temp of the same type as an existing temp. */ 1010d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1011d2fd745fSRichard Henderson { 1012d2fd745fSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 1013d2fd745fSRichard Henderson 1014d2fd745fSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 1015d2fd745fSRichard Henderson 1016d2fd745fSRichard Henderson t = tcg_temp_new_internal(t->base_type, 0); 1017d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1018d2fd745fSRichard Henderson } 1019d2fd745fSRichard Henderson 10205bfa8034SRichard Henderson void tcg_temp_free_internal(TCGTemp *ts) 1021c896fe29Sbellard { 1022b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1023085272b3SRichard Henderson int k, idx; 1024c896fe29Sbellard 1025c0522136SRichard Henderson /* In order to simplify users of tcg_constant_*, silently ignore free. */ 1026c0522136SRichard Henderson if (ts->kind == TEMP_CONST) { 1027c0522136SRichard Henderson return; 1028c0522136SRichard Henderson } 1029c0522136SRichard Henderson 103027bfd83cSPeter Maydell #if defined(CONFIG_DEBUG_TCG) 103127bfd83cSPeter Maydell s->temps_in_use--; 103227bfd83cSPeter Maydell if (s->temps_in_use < 0) { 103327bfd83cSPeter Maydell fprintf(stderr, "More temporaries freed than allocated!\n"); 103427bfd83cSPeter Maydell } 103527bfd83cSPeter Maydell #endif 103627bfd83cSPeter Maydell 1037ee17db83SRichard Henderson tcg_debug_assert(ts->kind < TEMP_GLOBAL); 1038eabb7b91SAurelien Jarno tcg_debug_assert(ts->temp_allocated != 0); 1039e8996ee0Sbellard ts->temp_allocated = 0; 10400ec9eabcSRichard Henderson 1041085272b3SRichard Henderson idx = temp_idx(ts); 1042ee17db83SRichard Henderson k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT); 10430ec9eabcSRichard Henderson set_bit(idx, s->free_temps[k].l); 1044e8996ee0Sbellard } 1045e8996ee0Sbellard 1046c0522136SRichard Henderson TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1047c0522136SRichard Henderson { 1048c0522136SRichard Henderson TCGContext *s = tcg_ctx; 1049c0522136SRichard Henderson GHashTable *h = s->const_table[type]; 1050c0522136SRichard Henderson TCGTemp *ts; 1051c0522136SRichard Henderson 1052c0522136SRichard Henderson if (h == NULL) { 1053c0522136SRichard Henderson h = g_hash_table_new(g_int64_hash, g_int64_equal); 1054c0522136SRichard Henderson s->const_table[type] = h; 1055c0522136SRichard Henderson } 1056c0522136SRichard Henderson 1057c0522136SRichard Henderson ts = g_hash_table_lookup(h, &val); 1058c0522136SRichard Henderson if (ts == NULL) { 1059c0522136SRichard Henderson ts = tcg_temp_alloc(s); 1060c0522136SRichard Henderson 1061c0522136SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1062c0522136SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 1063c0522136SRichard Henderson 1064c0522136SRichard Henderson ts->base_type = TCG_TYPE_I64; 1065c0522136SRichard Henderson ts->type = TCG_TYPE_I32; 1066c0522136SRichard Henderson ts->kind = TEMP_CONST; 1067c0522136SRichard Henderson ts->temp_allocated = 1; 1068c0522136SRichard Henderson /* 1069c0522136SRichard Henderson * Retain the full value of the 64-bit constant in the low 1070c0522136SRichard Henderson * part, so that the hash table works. Actual uses will 1071c0522136SRichard Henderson * truncate the value to the low part. 1072c0522136SRichard Henderson */ 1073c0522136SRichard Henderson ts->val = val; 1074c0522136SRichard Henderson 1075c0522136SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 1076c0522136SRichard Henderson ts2->base_type = TCG_TYPE_I64; 1077c0522136SRichard Henderson ts2->type = TCG_TYPE_I32; 1078c0522136SRichard Henderson ts2->kind = TEMP_CONST; 1079c0522136SRichard Henderson ts2->temp_allocated = 1; 1080c0522136SRichard Henderson ts2->val = val >> 32; 1081c0522136SRichard Henderson } else { 1082c0522136SRichard Henderson ts->base_type = type; 1083c0522136SRichard Henderson ts->type = type; 1084c0522136SRichard Henderson ts->kind = TEMP_CONST; 1085c0522136SRichard Henderson ts->temp_allocated = 1; 1086c0522136SRichard Henderson ts->val = val; 1087c0522136SRichard Henderson } 1088c0522136SRichard Henderson g_hash_table_insert(h, &ts->val, ts); 1089c0522136SRichard Henderson } 1090c0522136SRichard Henderson 1091c0522136SRichard Henderson return ts; 1092c0522136SRichard Henderson } 1093c0522136SRichard Henderson 1094c0522136SRichard Henderson TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1095c0522136SRichard Henderson { 1096c0522136SRichard Henderson val = dup_const(vece, val); 1097c0522136SRichard Henderson return temp_tcgv_vec(tcg_constant_internal(type, val)); 1098c0522136SRichard Henderson } 1099c0522136SRichard Henderson 110088d4005bSRichard Henderson TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 110188d4005bSRichard Henderson { 110288d4005bSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 110388d4005bSRichard Henderson 110488d4005bSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 110588d4005bSRichard Henderson return tcg_constant_vec(t->base_type, vece, val); 110688d4005bSRichard Henderson } 110788d4005bSRichard Henderson 1108a7812ae4Spbrook TCGv_i32 tcg_const_i32(int32_t val) 1109a7812ae4Spbrook { 1110a7812ae4Spbrook TCGv_i32 t0; 1111a7812ae4Spbrook t0 = tcg_temp_new_i32(); 1112e8996ee0Sbellard tcg_gen_movi_i32(t0, val); 1113e8996ee0Sbellard return t0; 1114c896fe29Sbellard } 1115c896fe29Sbellard 1116a7812ae4Spbrook TCGv_i64 tcg_const_i64(int64_t val) 1117c896fe29Sbellard { 1118a7812ae4Spbrook TCGv_i64 t0; 1119a7812ae4Spbrook t0 = tcg_temp_new_i64(); 1120e8996ee0Sbellard tcg_gen_movi_i64(t0, val); 1121e8996ee0Sbellard return t0; 1122c896fe29Sbellard } 1123c896fe29Sbellard 1124a7812ae4Spbrook TCGv_i32 tcg_const_local_i32(int32_t val) 1125bdffd4a9Saurel32 { 1126a7812ae4Spbrook TCGv_i32 t0; 1127a7812ae4Spbrook t0 = tcg_temp_local_new_i32(); 1128bdffd4a9Saurel32 tcg_gen_movi_i32(t0, val); 1129bdffd4a9Saurel32 return t0; 1130bdffd4a9Saurel32 } 1131bdffd4a9Saurel32 1132a7812ae4Spbrook TCGv_i64 tcg_const_local_i64(int64_t val) 1133bdffd4a9Saurel32 { 1134a7812ae4Spbrook TCGv_i64 t0; 1135a7812ae4Spbrook t0 = tcg_temp_local_new_i64(); 1136bdffd4a9Saurel32 tcg_gen_movi_i64(t0, val); 1137bdffd4a9Saurel32 return t0; 1138bdffd4a9Saurel32 } 1139bdffd4a9Saurel32 114027bfd83cSPeter Maydell #if defined(CONFIG_DEBUG_TCG) 114127bfd83cSPeter Maydell void tcg_clear_temp_count(void) 114227bfd83cSPeter Maydell { 1143b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 114427bfd83cSPeter Maydell s->temps_in_use = 0; 114527bfd83cSPeter Maydell } 114627bfd83cSPeter Maydell 114727bfd83cSPeter Maydell int tcg_check_temp_count(void) 114827bfd83cSPeter Maydell { 1149b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 115027bfd83cSPeter Maydell if (s->temps_in_use) { 115127bfd83cSPeter Maydell /* Clear the count so that we don't give another 115227bfd83cSPeter Maydell * warning immediately next time around. 115327bfd83cSPeter Maydell */ 115427bfd83cSPeter Maydell s->temps_in_use = 0; 115527bfd83cSPeter Maydell return 1; 115627bfd83cSPeter Maydell } 115727bfd83cSPeter Maydell return 0; 115827bfd83cSPeter Maydell } 115927bfd83cSPeter Maydell #endif 116027bfd83cSPeter Maydell 1161be0f34b5SRichard Henderson /* Return true if OP may appear in the opcode stream. 1162be0f34b5SRichard Henderson Test the runtime variable that controls each opcode. */ 1163be0f34b5SRichard Henderson bool tcg_op_supported(TCGOpcode op) 1164be0f34b5SRichard Henderson { 1165d2fd745fSRichard Henderson const bool have_vec 1166d2fd745fSRichard Henderson = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1167d2fd745fSRichard Henderson 1168be0f34b5SRichard Henderson switch (op) { 1169be0f34b5SRichard Henderson case INDEX_op_discard: 1170be0f34b5SRichard Henderson case INDEX_op_set_label: 1171be0f34b5SRichard Henderson case INDEX_op_call: 1172be0f34b5SRichard Henderson case INDEX_op_br: 1173be0f34b5SRichard Henderson case INDEX_op_mb: 1174be0f34b5SRichard Henderson case INDEX_op_insn_start: 1175be0f34b5SRichard Henderson case INDEX_op_exit_tb: 1176be0f34b5SRichard Henderson case INDEX_op_goto_tb: 1177f4e01e30SRichard Henderson case INDEX_op_goto_ptr: 1178be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i32: 1179be0f34b5SRichard Henderson case INDEX_op_qemu_st_i32: 1180be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i64: 1181be0f34b5SRichard Henderson case INDEX_op_qemu_st_i64: 1182be0f34b5SRichard Henderson return true; 1183be0f34b5SRichard Henderson 118407ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 118507ce0b05SRichard Henderson return TCG_TARGET_HAS_qemu_st8_i32; 118607ce0b05SRichard Henderson 1187be0f34b5SRichard Henderson case INDEX_op_mov_i32: 1188be0f34b5SRichard Henderson case INDEX_op_setcond_i32: 1189be0f34b5SRichard Henderson case INDEX_op_brcond_i32: 1190be0f34b5SRichard Henderson case INDEX_op_ld8u_i32: 1191be0f34b5SRichard Henderson case INDEX_op_ld8s_i32: 1192be0f34b5SRichard Henderson case INDEX_op_ld16u_i32: 1193be0f34b5SRichard Henderson case INDEX_op_ld16s_i32: 1194be0f34b5SRichard Henderson case INDEX_op_ld_i32: 1195be0f34b5SRichard Henderson case INDEX_op_st8_i32: 1196be0f34b5SRichard Henderson case INDEX_op_st16_i32: 1197be0f34b5SRichard Henderson case INDEX_op_st_i32: 1198be0f34b5SRichard Henderson case INDEX_op_add_i32: 1199be0f34b5SRichard Henderson case INDEX_op_sub_i32: 1200be0f34b5SRichard Henderson case INDEX_op_mul_i32: 1201be0f34b5SRichard Henderson case INDEX_op_and_i32: 1202be0f34b5SRichard Henderson case INDEX_op_or_i32: 1203be0f34b5SRichard Henderson case INDEX_op_xor_i32: 1204be0f34b5SRichard Henderson case INDEX_op_shl_i32: 1205be0f34b5SRichard Henderson case INDEX_op_shr_i32: 1206be0f34b5SRichard Henderson case INDEX_op_sar_i32: 1207be0f34b5SRichard Henderson return true; 1208be0f34b5SRichard Henderson 1209be0f34b5SRichard Henderson case INDEX_op_movcond_i32: 1210be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i32; 1211be0f34b5SRichard Henderson case INDEX_op_div_i32: 1212be0f34b5SRichard Henderson case INDEX_op_divu_i32: 1213be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i32; 1214be0f34b5SRichard Henderson case INDEX_op_rem_i32: 1215be0f34b5SRichard Henderson case INDEX_op_remu_i32: 1216be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i32; 1217be0f34b5SRichard Henderson case INDEX_op_div2_i32: 1218be0f34b5SRichard Henderson case INDEX_op_divu2_i32: 1219be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i32; 1220be0f34b5SRichard Henderson case INDEX_op_rotl_i32: 1221be0f34b5SRichard Henderson case INDEX_op_rotr_i32: 1222be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i32; 1223be0f34b5SRichard Henderson case INDEX_op_deposit_i32: 1224be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i32; 1225be0f34b5SRichard Henderson case INDEX_op_extract_i32: 1226be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i32; 1227be0f34b5SRichard Henderson case INDEX_op_sextract_i32: 1228be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i32; 1229fce1296fSRichard Henderson case INDEX_op_extract2_i32: 1230fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i32; 1231be0f34b5SRichard Henderson case INDEX_op_add2_i32: 1232be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i32; 1233be0f34b5SRichard Henderson case INDEX_op_sub2_i32: 1234be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i32; 1235be0f34b5SRichard Henderson case INDEX_op_mulu2_i32: 1236be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i32; 1237be0f34b5SRichard Henderson case INDEX_op_muls2_i32: 1238be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i32; 1239be0f34b5SRichard Henderson case INDEX_op_muluh_i32: 1240be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i32; 1241be0f34b5SRichard Henderson case INDEX_op_mulsh_i32: 1242be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i32; 1243be0f34b5SRichard Henderson case INDEX_op_ext8s_i32: 1244be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i32; 1245be0f34b5SRichard Henderson case INDEX_op_ext16s_i32: 1246be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i32; 1247be0f34b5SRichard Henderson case INDEX_op_ext8u_i32: 1248be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i32; 1249be0f34b5SRichard Henderson case INDEX_op_ext16u_i32: 1250be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i32; 1251be0f34b5SRichard Henderson case INDEX_op_bswap16_i32: 1252be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i32; 1253be0f34b5SRichard Henderson case INDEX_op_bswap32_i32: 1254be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i32; 1255be0f34b5SRichard Henderson case INDEX_op_not_i32: 1256be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i32; 1257be0f34b5SRichard Henderson case INDEX_op_neg_i32: 1258be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i32; 1259be0f34b5SRichard Henderson case INDEX_op_andc_i32: 1260be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i32; 1261be0f34b5SRichard Henderson case INDEX_op_orc_i32: 1262be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i32; 1263be0f34b5SRichard Henderson case INDEX_op_eqv_i32: 1264be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i32; 1265be0f34b5SRichard Henderson case INDEX_op_nand_i32: 1266be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i32; 1267be0f34b5SRichard Henderson case INDEX_op_nor_i32: 1268be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i32; 1269be0f34b5SRichard Henderson case INDEX_op_clz_i32: 1270be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i32; 1271be0f34b5SRichard Henderson case INDEX_op_ctz_i32: 1272be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i32; 1273be0f34b5SRichard Henderson case INDEX_op_ctpop_i32: 1274be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i32; 1275be0f34b5SRichard Henderson 1276be0f34b5SRichard Henderson case INDEX_op_brcond2_i32: 1277be0f34b5SRichard Henderson case INDEX_op_setcond2_i32: 1278be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 32; 1279be0f34b5SRichard Henderson 1280be0f34b5SRichard Henderson case INDEX_op_mov_i64: 1281be0f34b5SRichard Henderson case INDEX_op_setcond_i64: 1282be0f34b5SRichard Henderson case INDEX_op_brcond_i64: 1283be0f34b5SRichard Henderson case INDEX_op_ld8u_i64: 1284be0f34b5SRichard Henderson case INDEX_op_ld8s_i64: 1285be0f34b5SRichard Henderson case INDEX_op_ld16u_i64: 1286be0f34b5SRichard Henderson case INDEX_op_ld16s_i64: 1287be0f34b5SRichard Henderson case INDEX_op_ld32u_i64: 1288be0f34b5SRichard Henderson case INDEX_op_ld32s_i64: 1289be0f34b5SRichard Henderson case INDEX_op_ld_i64: 1290be0f34b5SRichard Henderson case INDEX_op_st8_i64: 1291be0f34b5SRichard Henderson case INDEX_op_st16_i64: 1292be0f34b5SRichard Henderson case INDEX_op_st32_i64: 1293be0f34b5SRichard Henderson case INDEX_op_st_i64: 1294be0f34b5SRichard Henderson case INDEX_op_add_i64: 1295be0f34b5SRichard Henderson case INDEX_op_sub_i64: 1296be0f34b5SRichard Henderson case INDEX_op_mul_i64: 1297be0f34b5SRichard Henderson case INDEX_op_and_i64: 1298be0f34b5SRichard Henderson case INDEX_op_or_i64: 1299be0f34b5SRichard Henderson case INDEX_op_xor_i64: 1300be0f34b5SRichard Henderson case INDEX_op_shl_i64: 1301be0f34b5SRichard Henderson case INDEX_op_shr_i64: 1302be0f34b5SRichard Henderson case INDEX_op_sar_i64: 1303be0f34b5SRichard Henderson case INDEX_op_ext_i32_i64: 1304be0f34b5SRichard Henderson case INDEX_op_extu_i32_i64: 1305be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 64; 1306be0f34b5SRichard Henderson 1307be0f34b5SRichard Henderson case INDEX_op_movcond_i64: 1308be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i64; 1309be0f34b5SRichard Henderson case INDEX_op_div_i64: 1310be0f34b5SRichard Henderson case INDEX_op_divu_i64: 1311be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i64; 1312be0f34b5SRichard Henderson case INDEX_op_rem_i64: 1313be0f34b5SRichard Henderson case INDEX_op_remu_i64: 1314be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i64; 1315be0f34b5SRichard Henderson case INDEX_op_div2_i64: 1316be0f34b5SRichard Henderson case INDEX_op_divu2_i64: 1317be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i64; 1318be0f34b5SRichard Henderson case INDEX_op_rotl_i64: 1319be0f34b5SRichard Henderson case INDEX_op_rotr_i64: 1320be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i64; 1321be0f34b5SRichard Henderson case INDEX_op_deposit_i64: 1322be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i64; 1323be0f34b5SRichard Henderson case INDEX_op_extract_i64: 1324be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i64; 1325be0f34b5SRichard Henderson case INDEX_op_sextract_i64: 1326be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i64; 1327fce1296fSRichard Henderson case INDEX_op_extract2_i64: 1328fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i64; 1329be0f34b5SRichard Henderson case INDEX_op_extrl_i64_i32: 1330be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrl_i64_i32; 1331be0f34b5SRichard Henderson case INDEX_op_extrh_i64_i32: 1332be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrh_i64_i32; 1333be0f34b5SRichard Henderson case INDEX_op_ext8s_i64: 1334be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i64; 1335be0f34b5SRichard Henderson case INDEX_op_ext16s_i64: 1336be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i64; 1337be0f34b5SRichard Henderson case INDEX_op_ext32s_i64: 1338be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32s_i64; 1339be0f34b5SRichard Henderson case INDEX_op_ext8u_i64: 1340be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i64; 1341be0f34b5SRichard Henderson case INDEX_op_ext16u_i64: 1342be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i64; 1343be0f34b5SRichard Henderson case INDEX_op_ext32u_i64: 1344be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32u_i64; 1345be0f34b5SRichard Henderson case INDEX_op_bswap16_i64: 1346be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i64; 1347be0f34b5SRichard Henderson case INDEX_op_bswap32_i64: 1348be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i64; 1349be0f34b5SRichard Henderson case INDEX_op_bswap64_i64: 1350be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap64_i64; 1351be0f34b5SRichard Henderson case INDEX_op_not_i64: 1352be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i64; 1353be0f34b5SRichard Henderson case INDEX_op_neg_i64: 1354be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i64; 1355be0f34b5SRichard Henderson case INDEX_op_andc_i64: 1356be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i64; 1357be0f34b5SRichard Henderson case INDEX_op_orc_i64: 1358be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i64; 1359be0f34b5SRichard Henderson case INDEX_op_eqv_i64: 1360be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i64; 1361be0f34b5SRichard Henderson case INDEX_op_nand_i64: 1362be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i64; 1363be0f34b5SRichard Henderson case INDEX_op_nor_i64: 1364be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i64; 1365be0f34b5SRichard Henderson case INDEX_op_clz_i64: 1366be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i64; 1367be0f34b5SRichard Henderson case INDEX_op_ctz_i64: 1368be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i64; 1369be0f34b5SRichard Henderson case INDEX_op_ctpop_i64: 1370be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i64; 1371be0f34b5SRichard Henderson case INDEX_op_add2_i64: 1372be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i64; 1373be0f34b5SRichard Henderson case INDEX_op_sub2_i64: 1374be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i64; 1375be0f34b5SRichard Henderson case INDEX_op_mulu2_i64: 1376be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i64; 1377be0f34b5SRichard Henderson case INDEX_op_muls2_i64: 1378be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i64; 1379be0f34b5SRichard Henderson case INDEX_op_muluh_i64: 1380be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i64; 1381be0f34b5SRichard Henderson case INDEX_op_mulsh_i64: 1382be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i64; 1383be0f34b5SRichard Henderson 1384d2fd745fSRichard Henderson case INDEX_op_mov_vec: 1385d2fd745fSRichard Henderson case INDEX_op_dup_vec: 138637ee55a0SRichard Henderson case INDEX_op_dupm_vec: 1387d2fd745fSRichard Henderson case INDEX_op_ld_vec: 1388d2fd745fSRichard Henderson case INDEX_op_st_vec: 1389d2fd745fSRichard Henderson case INDEX_op_add_vec: 1390d2fd745fSRichard Henderson case INDEX_op_sub_vec: 1391d2fd745fSRichard Henderson case INDEX_op_and_vec: 1392d2fd745fSRichard Henderson case INDEX_op_or_vec: 1393d2fd745fSRichard Henderson case INDEX_op_xor_vec: 1394212be173SRichard Henderson case INDEX_op_cmp_vec: 1395d2fd745fSRichard Henderson return have_vec; 1396d2fd745fSRichard Henderson case INDEX_op_dup2_vec: 1397d2fd745fSRichard Henderson return have_vec && TCG_TARGET_REG_BITS == 32; 1398d2fd745fSRichard Henderson case INDEX_op_not_vec: 1399d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_not_vec; 1400d2fd745fSRichard Henderson case INDEX_op_neg_vec: 1401d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_neg_vec; 1402bcefc902SRichard Henderson case INDEX_op_abs_vec: 1403bcefc902SRichard Henderson return have_vec && TCG_TARGET_HAS_abs_vec; 1404d2fd745fSRichard Henderson case INDEX_op_andc_vec: 1405d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_andc_vec; 1406d2fd745fSRichard Henderson case INDEX_op_orc_vec: 1407d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_orc_vec; 14083774030aSRichard Henderson case INDEX_op_mul_vec: 14093774030aSRichard Henderson return have_vec && TCG_TARGET_HAS_mul_vec; 1410d0ec9796SRichard Henderson case INDEX_op_shli_vec: 1411d0ec9796SRichard Henderson case INDEX_op_shri_vec: 1412d0ec9796SRichard Henderson case INDEX_op_sari_vec: 1413d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shi_vec; 1414d0ec9796SRichard Henderson case INDEX_op_shls_vec: 1415d0ec9796SRichard Henderson case INDEX_op_shrs_vec: 1416d0ec9796SRichard Henderson case INDEX_op_sars_vec: 1417d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shs_vec; 1418d0ec9796SRichard Henderson case INDEX_op_shlv_vec: 1419d0ec9796SRichard Henderson case INDEX_op_shrv_vec: 1420d0ec9796SRichard Henderson case INDEX_op_sarv_vec: 1421d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shv_vec; 1422b0f7e744SRichard Henderson case INDEX_op_rotli_vec: 1423b0f7e744SRichard Henderson return have_vec && TCG_TARGET_HAS_roti_vec; 142423850a74SRichard Henderson case INDEX_op_rotls_vec: 142523850a74SRichard Henderson return have_vec && TCG_TARGET_HAS_rots_vec; 14265d0ceda9SRichard Henderson case INDEX_op_rotlv_vec: 14275d0ceda9SRichard Henderson case INDEX_op_rotrv_vec: 14285d0ceda9SRichard Henderson return have_vec && TCG_TARGET_HAS_rotv_vec; 14298afaf050SRichard Henderson case INDEX_op_ssadd_vec: 14308afaf050SRichard Henderson case INDEX_op_usadd_vec: 14318afaf050SRichard Henderson case INDEX_op_sssub_vec: 14328afaf050SRichard Henderson case INDEX_op_ussub_vec: 14338afaf050SRichard Henderson return have_vec && TCG_TARGET_HAS_sat_vec; 1434dd0a0fcdSRichard Henderson case INDEX_op_smin_vec: 1435dd0a0fcdSRichard Henderson case INDEX_op_umin_vec: 1436dd0a0fcdSRichard Henderson case INDEX_op_smax_vec: 1437dd0a0fcdSRichard Henderson case INDEX_op_umax_vec: 1438dd0a0fcdSRichard Henderson return have_vec && TCG_TARGET_HAS_minmax_vec; 143938dc1294SRichard Henderson case INDEX_op_bitsel_vec: 144038dc1294SRichard Henderson return have_vec && TCG_TARGET_HAS_bitsel_vec; 1441f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 1442f75da298SRichard Henderson return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1443d2fd745fSRichard Henderson 1444db432672SRichard Henderson default: 1445db432672SRichard Henderson tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1446db432672SRichard Henderson return true; 1447be0f34b5SRichard Henderson } 1448be0f34b5SRichard Henderson } 1449be0f34b5SRichard Henderson 145039cf05d3Sbellard /* Note: we convert the 64 bit args to 32 bit and do some alignment 145139cf05d3Sbellard and endian swap. Maybe it would be better to do the alignment 145239cf05d3Sbellard and endian swap in tcg_reg_alloc_call(). */ 1453ae8b75dcSRichard Henderson void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1454c896fe29Sbellard { 145575e8b9b7SRichard Henderson int i, real_args, nb_rets, pi; 14563e92aa34SRichard Henderson unsigned typemask; 14573e92aa34SRichard Henderson const TCGHelperInfo *info; 145875e8b9b7SRichard Henderson TCGOp *op; 1459afb49896SRichard Henderson 1460619205fdSEmilio G. Cota info = g_hash_table_lookup(helper_table, (gpointer)func); 14617319d83aSRichard Henderson typemask = info->typemask; 14622bece2c8SRichard Henderson 146338b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 146438b47b19SEmilio G. Cota /* detect non-plugin helpers */ 146538b47b19SEmilio G. Cota if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) { 146638b47b19SEmilio G. Cota tcg_ctx->plugin_insn->calls_helpers = true; 146738b47b19SEmilio G. Cota } 146838b47b19SEmilio G. Cota #endif 146938b47b19SEmilio G. Cota 147034b1a49cSRichard Henderson #if defined(__sparc__) && !defined(__arch64__) \ 147134b1a49cSRichard Henderson && !defined(CONFIG_TCG_INTERPRETER) 147234b1a49cSRichard Henderson /* We have 64-bit values in one register, but need to pass as two 147334b1a49cSRichard Henderson separate parameters. Split them. */ 14747319d83aSRichard Henderson int orig_typemask = typemask; 147534b1a49cSRichard Henderson int orig_nargs = nargs; 147634b1a49cSRichard Henderson TCGv_i64 retl, reth; 1477ae8b75dcSRichard Henderson TCGTemp *split_args[MAX_OPC_PARAM]; 147834b1a49cSRichard Henderson 1479f764718dSRichard Henderson retl = NULL; 1480f764718dSRichard Henderson reth = NULL; 14817319d83aSRichard Henderson typemask = 0; 148234b1a49cSRichard Henderson for (i = real_args = 0; i < nargs; ++i) { 14837319d83aSRichard Henderson int argtype = extract32(orig_typemask, (i + 1) * 3, 3); 14847319d83aSRichard Henderson bool is_64bit = (argtype & ~1) == dh_typecode_i64; 14857319d83aSRichard Henderson 148634b1a49cSRichard Henderson if (is_64bit) { 1487085272b3SRichard Henderson TCGv_i64 orig = temp_tcgv_i64(args[i]); 148834b1a49cSRichard Henderson TCGv_i32 h = tcg_temp_new_i32(); 148934b1a49cSRichard Henderson TCGv_i32 l = tcg_temp_new_i32(); 149034b1a49cSRichard Henderson tcg_gen_extr_i64_i32(l, h, orig); 1491ae8b75dcSRichard Henderson split_args[real_args++] = tcgv_i32_temp(h); 14927319d83aSRichard Henderson typemask |= dh_typecode_i32 << (real_args * 3); 1493ae8b75dcSRichard Henderson split_args[real_args++] = tcgv_i32_temp(l); 14947319d83aSRichard Henderson typemask |= dh_typecode_i32 << (real_args * 3); 149534b1a49cSRichard Henderson } else { 149634b1a49cSRichard Henderson split_args[real_args++] = args[i]; 14977319d83aSRichard Henderson typemask |= argtype << (real_args * 3); 149834b1a49cSRichard Henderson } 149934b1a49cSRichard Henderson } 150034b1a49cSRichard Henderson nargs = real_args; 150134b1a49cSRichard Henderson args = split_args; 150234b1a49cSRichard Henderson #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 15032bece2c8SRichard Henderson for (i = 0; i < nargs; ++i) { 15047319d83aSRichard Henderson int argtype = extract32(typemask, (i + 1) * 3, 3); 15057319d83aSRichard Henderson bool is_32bit = (argtype & ~1) == dh_typecode_i32; 15067319d83aSRichard Henderson bool is_signed = argtype & 1; 15077319d83aSRichard Henderson 15087319d83aSRichard Henderson if (is_32bit) { 15092bece2c8SRichard Henderson TCGv_i64 temp = tcg_temp_new_i64(); 1510085272b3SRichard Henderson TCGv_i64 orig = temp_tcgv_i64(args[i]); 15112bece2c8SRichard Henderson if (is_signed) { 15122bece2c8SRichard Henderson tcg_gen_ext32s_i64(temp, orig); 15132bece2c8SRichard Henderson } else { 15142bece2c8SRichard Henderson tcg_gen_ext32u_i64(temp, orig); 15152bece2c8SRichard Henderson } 1516ae8b75dcSRichard Henderson args[i] = tcgv_i64_temp(temp); 15172bece2c8SRichard Henderson } 15182bece2c8SRichard Henderson } 15192bece2c8SRichard Henderson #endif /* TCG_TARGET_EXTEND_ARGS */ 15202bece2c8SRichard Henderson 152115fa08f8SRichard Henderson op = tcg_emit_op(INDEX_op_call); 152275e8b9b7SRichard Henderson 152375e8b9b7SRichard Henderson pi = 0; 1524ae8b75dcSRichard Henderson if (ret != NULL) { 152534b1a49cSRichard Henderson #if defined(__sparc__) && !defined(__arch64__) \ 152634b1a49cSRichard Henderson && !defined(CONFIG_TCG_INTERPRETER) 15277319d83aSRichard Henderson if ((typemask & 6) == dh_typecode_i64) { 152834b1a49cSRichard Henderson /* The 32-bit ABI is going to return the 64-bit value in 152934b1a49cSRichard Henderson the %o0/%o1 register pair. Prepare for this by using 153034b1a49cSRichard Henderson two return temporaries, and reassemble below. */ 153134b1a49cSRichard Henderson retl = tcg_temp_new_i64(); 153234b1a49cSRichard Henderson reth = tcg_temp_new_i64(); 1533ae8b75dcSRichard Henderson op->args[pi++] = tcgv_i64_arg(reth); 1534ae8b75dcSRichard Henderson op->args[pi++] = tcgv_i64_arg(retl); 153534b1a49cSRichard Henderson nb_rets = 2; 153634b1a49cSRichard Henderson } else { 1537ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(ret); 153834b1a49cSRichard Henderson nb_rets = 1; 153934b1a49cSRichard Henderson } 154034b1a49cSRichard Henderson #else 15417319d83aSRichard Henderson if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { 154202eb19d0SRichard Henderson #ifdef HOST_WORDS_BIGENDIAN 1543ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(ret + 1); 1544ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(ret); 1545a7812ae4Spbrook #else 1546ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(ret); 1547ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(ret + 1); 1548a7812ae4Spbrook #endif 1549a7812ae4Spbrook nb_rets = 2; 155034b1a49cSRichard Henderson } else { 1551ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(ret); 1552a7812ae4Spbrook nb_rets = 1; 1553a7812ae4Spbrook } 155434b1a49cSRichard Henderson #endif 1555a7812ae4Spbrook } else { 1556a7812ae4Spbrook nb_rets = 0; 1557a7812ae4Spbrook } 1558cd9090aaSRichard Henderson TCGOP_CALLO(op) = nb_rets; 155975e8b9b7SRichard Henderson 1560a7812ae4Spbrook real_args = 0; 1561a7812ae4Spbrook for (i = 0; i < nargs; i++) { 15627319d83aSRichard Henderson int argtype = extract32(typemask, (i + 1) * 3, 3); 15637319d83aSRichard Henderson bool is_64bit = (argtype & ~1) == dh_typecode_i64; 15647b7d8b2dSRichard Henderson bool want_align = false; 15657319d83aSRichard Henderson 15667b7d8b2dSRichard Henderson #if defined(CONFIG_TCG_INTERPRETER) 15677b7d8b2dSRichard Henderson /* 15687b7d8b2dSRichard Henderson * Align all arguments, so that they land in predictable places 15697b7d8b2dSRichard Henderson * for passing off to ffi_call. 15707b7d8b2dSRichard Henderson */ 15717b7d8b2dSRichard Henderson want_align = true; 15727b7d8b2dSRichard Henderson #elif defined(TCG_TARGET_CALL_ALIGN_ARGS) 15737b7d8b2dSRichard Henderson /* Some targets want aligned 64 bit args */ 15747b7d8b2dSRichard Henderson want_align = is_64bit; 15757b7d8b2dSRichard Henderson #endif 15767b7d8b2dSRichard Henderson 15777b7d8b2dSRichard Henderson if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) { 157875e8b9b7SRichard Henderson op->args[pi++] = TCG_CALL_DUMMY_ARG; 1579ebd486d5Smalc real_args++; 158039cf05d3Sbellard } 15817b7d8b2dSRichard Henderson 15827b7d8b2dSRichard Henderson if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 15837b7d8b2dSRichard Henderson /* 15847b7d8b2dSRichard Henderson * If stack grows up, then we will be placing successive 15857b7d8b2dSRichard Henderson * arguments at lower addresses, which means we need to 15867b7d8b2dSRichard Henderson * reverse the order compared to how we would normally 15877b7d8b2dSRichard Henderson * treat either big or little-endian. For those arguments 15887b7d8b2dSRichard Henderson * that will wind up in registers, this still works for 15897b7d8b2dSRichard Henderson * HPPA (the only current STACK_GROWSUP target) since the 15907b7d8b2dSRichard Henderson * argument registers are *also* allocated in decreasing 15917b7d8b2dSRichard Henderson * order. If another such target is added, this logic may 15927b7d8b2dSRichard Henderson * have to get more complicated to differentiate between 15937b7d8b2dSRichard Henderson * stack arguments and register arguments. 15947b7d8b2dSRichard Henderson */ 159502eb19d0SRichard Henderson #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 1596ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(args[i] + 1); 1597ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(args[i]); 1598c896fe29Sbellard #else 1599ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(args[i]); 1600ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(args[i] + 1); 1601c896fe29Sbellard #endif 1602a7812ae4Spbrook real_args += 2; 16032bece2c8SRichard Henderson continue; 16042bece2c8SRichard Henderson } 16052bece2c8SRichard Henderson 1606ae8b75dcSRichard Henderson op->args[pi++] = temp_arg(args[i]); 1607a7812ae4Spbrook real_args++; 1608c896fe29Sbellard } 160975e8b9b7SRichard Henderson op->args[pi++] = (uintptr_t)func; 16103e92aa34SRichard Henderson op->args[pi++] = (uintptr_t)info; 1611cd9090aaSRichard Henderson TCGOP_CALLI(op) = real_args; 1612a7812ae4Spbrook 161375e8b9b7SRichard Henderson /* Make sure the fields didn't overflow. */ 1614cd9090aaSRichard Henderson tcg_debug_assert(TCGOP_CALLI(op) == real_args); 161575e8b9b7SRichard Henderson tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); 16162bece2c8SRichard Henderson 161734b1a49cSRichard Henderson #if defined(__sparc__) && !defined(__arch64__) \ 161834b1a49cSRichard Henderson && !defined(CONFIG_TCG_INTERPRETER) 161934b1a49cSRichard Henderson /* Free all of the parts we allocated above. */ 162034b1a49cSRichard Henderson for (i = real_args = 0; i < orig_nargs; ++i) { 16217319d83aSRichard Henderson int argtype = extract32(orig_typemask, (i + 1) * 3, 3); 16227319d83aSRichard Henderson bool is_64bit = (argtype & ~1) == dh_typecode_i64; 16237319d83aSRichard Henderson 162434b1a49cSRichard Henderson if (is_64bit) { 1625085272b3SRichard Henderson tcg_temp_free_internal(args[real_args++]); 1626085272b3SRichard Henderson tcg_temp_free_internal(args[real_args++]); 162734b1a49cSRichard Henderson } else { 162834b1a49cSRichard Henderson real_args++; 162934b1a49cSRichard Henderson } 163034b1a49cSRichard Henderson } 16317319d83aSRichard Henderson if ((orig_typemask & 6) == dh_typecode_i64) { 163234b1a49cSRichard Henderson /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 163334b1a49cSRichard Henderson Note that describing these as TCGv_i64 eliminates an unnecessary 163434b1a49cSRichard Henderson zero-extension that tcg_gen_concat_i32_i64 would create. */ 1635085272b3SRichard Henderson tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth); 163634b1a49cSRichard Henderson tcg_temp_free_i64(retl); 163734b1a49cSRichard Henderson tcg_temp_free_i64(reth); 163834b1a49cSRichard Henderson } 163934b1a49cSRichard Henderson #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 16402bece2c8SRichard Henderson for (i = 0; i < nargs; ++i) { 16417319d83aSRichard Henderson int argtype = extract32(typemask, (i + 1) * 3, 3); 16427319d83aSRichard Henderson bool is_32bit = (argtype & ~1) == dh_typecode_i32; 16437319d83aSRichard Henderson 16447319d83aSRichard Henderson if (is_32bit) { 1645085272b3SRichard Henderson tcg_temp_free_internal(args[i]); 16462bece2c8SRichard Henderson } 16472bece2c8SRichard Henderson } 16482bece2c8SRichard Henderson #endif /* TCG_TARGET_EXTEND_ARGS */ 1649a7812ae4Spbrook } 1650c896fe29Sbellard 16518fcd3692Sblueswir1 static void tcg_reg_alloc_start(TCGContext *s) 1652c896fe29Sbellard { 1653ac3b8891SRichard Henderson int i, n; 1654ac3b8891SRichard Henderson 1655ee17db83SRichard Henderson for (i = 0, n = s->nb_temps; i < n; i++) { 1656ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 1657ee17db83SRichard Henderson TCGTempVal val = TEMP_VAL_MEM; 1658ee17db83SRichard Henderson 1659ee17db83SRichard Henderson switch (ts->kind) { 1660c0522136SRichard Henderson case TEMP_CONST: 1661c0522136SRichard Henderson val = TEMP_VAL_CONST; 1662c0522136SRichard Henderson break; 1663ee17db83SRichard Henderson case TEMP_FIXED: 1664ee17db83SRichard Henderson val = TEMP_VAL_REG; 1665ee17db83SRichard Henderson break; 1666ee17db83SRichard Henderson case TEMP_GLOBAL: 1667ee17db83SRichard Henderson break; 1668ee17db83SRichard Henderson case TEMP_NORMAL: 1669ee17db83SRichard Henderson val = TEMP_VAL_DEAD; 1670ee17db83SRichard Henderson /* fall through */ 1671ee17db83SRichard Henderson case TEMP_LOCAL: 1672e8996ee0Sbellard ts->mem_allocated = 0; 1673ee17db83SRichard Henderson break; 1674ee17db83SRichard Henderson default: 1675ee17db83SRichard Henderson g_assert_not_reached(); 1676ee17db83SRichard Henderson } 1677ee17db83SRichard Henderson ts->val_type = val; 1678e8996ee0Sbellard } 1679f8b2f202SRichard Henderson 1680f8b2f202SRichard Henderson memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1681c896fe29Sbellard } 1682c896fe29Sbellard 1683f8b2f202SRichard Henderson static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1684f8b2f202SRichard Henderson TCGTemp *ts) 1685c896fe29Sbellard { 16861807f4c4SRichard Henderson int idx = temp_idx(ts); 1687ac56dd48Spbrook 1688ee17db83SRichard Henderson switch (ts->kind) { 1689ee17db83SRichard Henderson case TEMP_FIXED: 1690ee17db83SRichard Henderson case TEMP_GLOBAL: 1691ac56dd48Spbrook pstrcpy(buf, buf_size, ts->name); 1692ee17db83SRichard Henderson break; 1693ee17db83SRichard Henderson case TEMP_LOCAL: 1694641d5fbeSbellard snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1695ee17db83SRichard Henderson break; 1696ee17db83SRichard Henderson case TEMP_NORMAL: 1697ac56dd48Spbrook snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1698ee17db83SRichard Henderson break; 1699c0522136SRichard Henderson case TEMP_CONST: 1700c0522136SRichard Henderson switch (ts->type) { 1701c0522136SRichard Henderson case TCG_TYPE_I32: 1702c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1703c0522136SRichard Henderson break; 1704c0522136SRichard Henderson #if TCG_TARGET_REG_BITS > 32 1705c0522136SRichard Henderson case TCG_TYPE_I64: 1706c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1707c0522136SRichard Henderson break; 1708c0522136SRichard Henderson #endif 1709c0522136SRichard Henderson case TCG_TYPE_V64: 1710c0522136SRichard Henderson case TCG_TYPE_V128: 1711c0522136SRichard Henderson case TCG_TYPE_V256: 1712c0522136SRichard Henderson snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1713c0522136SRichard Henderson 64 << (ts->type - TCG_TYPE_V64), ts->val); 1714c0522136SRichard Henderson break; 1715c0522136SRichard Henderson default: 1716c0522136SRichard Henderson g_assert_not_reached(); 1717c0522136SRichard Henderson } 1718c0522136SRichard Henderson break; 1719c896fe29Sbellard } 1720c896fe29Sbellard return buf; 1721c896fe29Sbellard } 1722c896fe29Sbellard 172343439139SRichard Henderson static char *tcg_get_arg_str(TCGContext *s, char *buf, 172443439139SRichard Henderson int buf_size, TCGArg arg) 1725f8b2f202SRichard Henderson { 172643439139SRichard Henderson return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1727f8b2f202SRichard Henderson } 1728f8b2f202SRichard Henderson 1729f48f3edeSblueswir1 static const char * const cond_name[] = 1730f48f3edeSblueswir1 { 17310aed257fSRichard Henderson [TCG_COND_NEVER] = "never", 17320aed257fSRichard Henderson [TCG_COND_ALWAYS] = "always", 1733f48f3edeSblueswir1 [TCG_COND_EQ] = "eq", 1734f48f3edeSblueswir1 [TCG_COND_NE] = "ne", 1735f48f3edeSblueswir1 [TCG_COND_LT] = "lt", 1736f48f3edeSblueswir1 [TCG_COND_GE] = "ge", 1737f48f3edeSblueswir1 [TCG_COND_LE] = "le", 1738f48f3edeSblueswir1 [TCG_COND_GT] = "gt", 1739f48f3edeSblueswir1 [TCG_COND_LTU] = "ltu", 1740f48f3edeSblueswir1 [TCG_COND_GEU] = "geu", 1741f48f3edeSblueswir1 [TCG_COND_LEU] = "leu", 1742f48f3edeSblueswir1 [TCG_COND_GTU] = "gtu" 1743f48f3edeSblueswir1 }; 1744f48f3edeSblueswir1 1745f713d6adSRichard Henderson static const char * const ldst_name[] = 1746f713d6adSRichard Henderson { 1747f713d6adSRichard Henderson [MO_UB] = "ub", 1748f713d6adSRichard Henderson [MO_SB] = "sb", 1749f713d6adSRichard Henderson [MO_LEUW] = "leuw", 1750f713d6adSRichard Henderson [MO_LESW] = "lesw", 1751f713d6adSRichard Henderson [MO_LEUL] = "leul", 1752f713d6adSRichard Henderson [MO_LESL] = "lesl", 1753f713d6adSRichard Henderson [MO_LEQ] = "leq", 1754f713d6adSRichard Henderson [MO_BEUW] = "beuw", 1755f713d6adSRichard Henderson [MO_BESW] = "besw", 1756f713d6adSRichard Henderson [MO_BEUL] = "beul", 1757f713d6adSRichard Henderson [MO_BESL] = "besl", 1758f713d6adSRichard Henderson [MO_BEQ] = "beq", 1759f713d6adSRichard Henderson }; 1760f713d6adSRichard Henderson 17611f00b27fSSergey Sorokin static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 176252bf9771Stony.nguyen@bt.com #ifdef TARGET_ALIGNED_ONLY 17631f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "un+", 17641f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "", 17651f00b27fSSergey Sorokin #else 17661f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "", 17671f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "al+", 17681f00b27fSSergey Sorokin #endif 17691f00b27fSSergey Sorokin [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 17701f00b27fSSergey Sorokin [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 17711f00b27fSSergey Sorokin [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 17721f00b27fSSergey Sorokin [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 17731f00b27fSSergey Sorokin [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 17741f00b27fSSergey Sorokin [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 17751f00b27fSSergey Sorokin }; 17761f00b27fSSergey Sorokin 1777587195bdSRichard Henderson static const char bswap_flag_name[][6] = { 1778587195bdSRichard Henderson [TCG_BSWAP_IZ] = "iz", 1779587195bdSRichard Henderson [TCG_BSWAP_OZ] = "oz", 1780587195bdSRichard Henderson [TCG_BSWAP_OS] = "os", 1781587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1782587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1783587195bdSRichard Henderson }; 1784587195bdSRichard Henderson 1785b016486eSRichard Henderson static inline bool tcg_regset_single(TCGRegSet d) 1786b016486eSRichard Henderson { 1787b016486eSRichard Henderson return (d & (d - 1)) == 0; 1788b016486eSRichard Henderson } 1789b016486eSRichard Henderson 1790b016486eSRichard Henderson static inline TCGReg tcg_regset_first(TCGRegSet d) 1791b016486eSRichard Henderson { 1792b016486eSRichard Henderson if (TCG_TARGET_NB_REGS <= 32) { 1793b016486eSRichard Henderson return ctz32(d); 1794b016486eSRichard Henderson } else { 1795b016486eSRichard Henderson return ctz64(d); 1796b016486eSRichard Henderson } 1797b016486eSRichard Henderson } 1798b016486eSRichard Henderson 17991894f69aSRichard Henderson static void tcg_dump_ops(TCGContext *s, bool have_prefs) 1800c896fe29Sbellard { 1801c896fe29Sbellard char buf[128]; 1802c45cb8bbSRichard Henderson TCGOp *op; 1803c896fe29Sbellard 180415fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 1805c45cb8bbSRichard Henderson int i, k, nb_oargs, nb_iargs, nb_cargs; 1806c45cb8bbSRichard Henderson const TCGOpDef *def; 1807c45cb8bbSRichard Henderson TCGOpcode c; 1808bdfb460eSRichard Henderson int col = 0; 1809c45cb8bbSRichard Henderson 1810c45cb8bbSRichard Henderson c = op->opc; 1811c896fe29Sbellard def = &tcg_op_defs[c]; 1812c45cb8bbSRichard Henderson 1813765b842aSRichard Henderson if (c == INDEX_op_insn_start) { 1814b016486eSRichard Henderson nb_oargs = 0; 181515fa08f8SRichard Henderson col += qemu_log("\n ----"); 18169aef40edSRichard Henderson 18179aef40edSRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 18189aef40edSRichard Henderson target_ulong a; 18197e4597d7Sbellard #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1820efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 18217e4597d7Sbellard #else 1822efee3746SRichard Henderson a = op->args[i]; 18237e4597d7Sbellard #endif 1824bdfb460eSRichard Henderson col += qemu_log(" " TARGET_FMT_lx, a); 1825eeacee4dSBlue Swirl } 18267e4597d7Sbellard } else if (c == INDEX_op_call) { 18273e92aa34SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 1828fa52e660SRichard Henderson void *func = tcg_call_func(op); 18293e92aa34SRichard Henderson 1830c896fe29Sbellard /* variable number of arguments */ 1831cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 1832cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 1833c896fe29Sbellard nb_cargs = def->nb_cargs; 1834b03cce8eSbellard 18353e92aa34SRichard Henderson col += qemu_log(" %s ", def->name); 18363e92aa34SRichard Henderson 18373e92aa34SRichard Henderson /* 18383e92aa34SRichard Henderson * Print the function name from TCGHelperInfo, if available. 18393e92aa34SRichard Henderson * Note that plugins have a template function for the info, 18403e92aa34SRichard Henderson * but the actual function pointer comes from the plugin. 18413e92aa34SRichard Henderson */ 18423e92aa34SRichard Henderson if (func == info->func) { 18433e92aa34SRichard Henderson col += qemu_log("%s", info->name); 18443e92aa34SRichard Henderson } else { 18453e92aa34SRichard Henderson col += qemu_log("plugin(%p)", func); 18463e92aa34SRichard Henderson } 18473e92aa34SRichard Henderson 18488973fe43SRichard Henderson col += qemu_log(",$0x%x,$%d", info->flags, nb_oargs); 1849b03cce8eSbellard for (i = 0; i < nb_oargs; i++) { 185043439139SRichard Henderson col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf), 1851efee3746SRichard Henderson op->args[i])); 1852b03cce8eSbellard } 1853cf066674SRichard Henderson for (i = 0; i < nb_iargs; i++) { 1854efee3746SRichard Henderson TCGArg arg = op->args[nb_oargs + i]; 1855cf066674SRichard Henderson const char *t = "<dummy>"; 1856cf066674SRichard Henderson if (arg != TCG_CALL_DUMMY_ARG) { 185743439139SRichard Henderson t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 1858b03cce8eSbellard } 1859bdfb460eSRichard Henderson col += qemu_log(",%s", t); 1860e8996ee0Sbellard } 1861b03cce8eSbellard } else { 1862bdfb460eSRichard Henderson col += qemu_log(" %s ", def->name); 1863c45cb8bbSRichard Henderson 1864c896fe29Sbellard nb_oargs = def->nb_oargs; 1865c896fe29Sbellard nb_iargs = def->nb_iargs; 1866c896fe29Sbellard nb_cargs = def->nb_cargs; 1867c896fe29Sbellard 1868d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 1869d2fd745fSRichard Henderson col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op), 1870d2fd745fSRichard Henderson 8 << TCGOP_VECE(op)); 1871d2fd745fSRichard Henderson } 1872d2fd745fSRichard Henderson 1873c896fe29Sbellard k = 0; 1874c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 1875eeacee4dSBlue Swirl if (k != 0) { 1876bdfb460eSRichard Henderson col += qemu_log(","); 1877eeacee4dSBlue Swirl } 187843439139SRichard Henderson col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1879efee3746SRichard Henderson op->args[k++])); 1880c896fe29Sbellard } 1881c896fe29Sbellard for (i = 0; i < nb_iargs; i++) { 1882eeacee4dSBlue Swirl if (k != 0) { 1883bdfb460eSRichard Henderson col += qemu_log(","); 1884eeacee4dSBlue Swirl } 188543439139SRichard Henderson col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1886efee3746SRichard Henderson op->args[k++])); 1887c896fe29Sbellard } 1888be210acbSRichard Henderson switch (c) { 1889be210acbSRichard Henderson case INDEX_op_brcond_i32: 1890ffc5ea09SRichard Henderson case INDEX_op_setcond_i32: 1891ffc5ea09SRichard Henderson case INDEX_op_movcond_i32: 1892be210acbSRichard Henderson case INDEX_op_brcond2_i32: 1893be210acbSRichard Henderson case INDEX_op_setcond2_i32: 1894ffc5ea09SRichard Henderson case INDEX_op_brcond_i64: 1895be210acbSRichard Henderson case INDEX_op_setcond_i64: 1896ffc5ea09SRichard Henderson case INDEX_op_movcond_i64: 1897212be173SRichard Henderson case INDEX_op_cmp_vec: 1898f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 1899efee3746SRichard Henderson if (op->args[k] < ARRAY_SIZE(cond_name) 1900efee3746SRichard Henderson && cond_name[op->args[k]]) { 1901efee3746SRichard Henderson col += qemu_log(",%s", cond_name[op->args[k++]]); 1902eeacee4dSBlue Swirl } else { 1903efee3746SRichard Henderson col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]); 1904eeacee4dSBlue Swirl } 1905f48f3edeSblueswir1 i = 1; 1906be210acbSRichard Henderson break; 1907f713d6adSRichard Henderson case INDEX_op_qemu_ld_i32: 1908f713d6adSRichard Henderson case INDEX_op_qemu_st_i32: 190907ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 1910f713d6adSRichard Henderson case INDEX_op_qemu_ld_i64: 1911f713d6adSRichard Henderson case INDEX_op_qemu_st_i64: 191259227d5dSRichard Henderson { 1913*9002ffcbSRichard Henderson MemOpIdx oi = op->args[k++]; 191414776ab5STony Nguyen MemOp op = get_memop(oi); 191559227d5dSRichard Henderson unsigned ix = get_mmuidx(oi); 191659227d5dSRichard Henderson 191759c4b7e8SRichard Henderson if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1918bdfb460eSRichard Henderson col += qemu_log(",$0x%x,%u", op, ix); 191959c4b7e8SRichard Henderson } else { 19201f00b27fSSergey Sorokin const char *s_al, *s_op; 19211f00b27fSSergey Sorokin s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 192259c4b7e8SRichard Henderson s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1923bdfb460eSRichard Henderson col += qemu_log(",%s%s,%u", s_al, s_op, ix); 1924f713d6adSRichard Henderson } 1925f713d6adSRichard Henderson i = 1; 192659227d5dSRichard Henderson } 1927f713d6adSRichard Henderson break; 1928587195bdSRichard Henderson case INDEX_op_bswap16_i32: 1929587195bdSRichard Henderson case INDEX_op_bswap16_i64: 1930587195bdSRichard Henderson case INDEX_op_bswap32_i32: 1931587195bdSRichard Henderson case INDEX_op_bswap32_i64: 1932587195bdSRichard Henderson case INDEX_op_bswap64_i64: 1933587195bdSRichard Henderson { 1934587195bdSRichard Henderson TCGArg flags = op->args[k]; 1935587195bdSRichard Henderson const char *name = NULL; 1936587195bdSRichard Henderson 1937587195bdSRichard Henderson if (flags < ARRAY_SIZE(bswap_flag_name)) { 1938587195bdSRichard Henderson name = bswap_flag_name[flags]; 1939587195bdSRichard Henderson } 1940587195bdSRichard Henderson if (name) { 1941587195bdSRichard Henderson col += qemu_log(",%s", name); 1942587195bdSRichard Henderson } else { 1943587195bdSRichard Henderson col += qemu_log(",$0x%" TCG_PRIlx, flags); 1944587195bdSRichard Henderson } 1945587195bdSRichard Henderson i = k = 1; 1946587195bdSRichard Henderson } 1947587195bdSRichard Henderson break; 1948be210acbSRichard Henderson default: 1949f48f3edeSblueswir1 i = 0; 1950be210acbSRichard Henderson break; 1951be210acbSRichard Henderson } 195251e3972cSRichard Henderson switch (c) { 195351e3972cSRichard Henderson case INDEX_op_set_label: 195451e3972cSRichard Henderson case INDEX_op_br: 195551e3972cSRichard Henderson case INDEX_op_brcond_i32: 195651e3972cSRichard Henderson case INDEX_op_brcond_i64: 195751e3972cSRichard Henderson case INDEX_op_brcond2_i32: 1958efee3746SRichard Henderson col += qemu_log("%s$L%d", k ? "," : "", 1959efee3746SRichard Henderson arg_label(op->args[k])->id); 196051e3972cSRichard Henderson i++, k++; 196151e3972cSRichard Henderson break; 196251e3972cSRichard Henderson default: 196351e3972cSRichard Henderson break; 1964eeacee4dSBlue Swirl } 196551e3972cSRichard Henderson for (; i < nb_cargs; i++, k++) { 1966efee3746SRichard Henderson col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]); 1967bdfb460eSRichard Henderson } 1968bdfb460eSRichard Henderson } 1969bdfb460eSRichard Henderson 19701894f69aSRichard Henderson if (have_prefs || op->life) { 19717606488cSRobert Foley 19727606488cSRobert Foley QemuLogFile *logfile; 19737606488cSRobert Foley 19747606488cSRobert Foley rcu_read_lock(); 1975d73415a3SStefan Hajnoczi logfile = qatomic_rcu_read(&qemu_logfile); 19767606488cSRobert Foley if (logfile) { 19771894f69aSRichard Henderson for (; col < 40; ++col) { 19787606488cSRobert Foley putc(' ', logfile->fd); 1979bdfb460eSRichard Henderson } 19801894f69aSRichard Henderson } 19817606488cSRobert Foley rcu_read_unlock(); 19827606488cSRobert Foley } 19831894f69aSRichard Henderson 19841894f69aSRichard Henderson if (op->life) { 19851894f69aSRichard Henderson unsigned life = op->life; 1986bdfb460eSRichard Henderson 1987bdfb460eSRichard Henderson if (life & (SYNC_ARG * 3)) { 1988bdfb460eSRichard Henderson qemu_log(" sync:"); 1989bdfb460eSRichard Henderson for (i = 0; i < 2; ++i) { 1990bdfb460eSRichard Henderson if (life & (SYNC_ARG << i)) { 1991bdfb460eSRichard Henderson qemu_log(" %d", i); 1992bdfb460eSRichard Henderson } 1993bdfb460eSRichard Henderson } 1994bdfb460eSRichard Henderson } 1995bdfb460eSRichard Henderson life /= DEAD_ARG; 1996bdfb460eSRichard Henderson if (life) { 1997bdfb460eSRichard Henderson qemu_log(" dead:"); 1998bdfb460eSRichard Henderson for (i = 0; life; ++i, life >>= 1) { 1999bdfb460eSRichard Henderson if (life & 1) { 2000bdfb460eSRichard Henderson qemu_log(" %d", i); 2001bdfb460eSRichard Henderson } 2002bdfb460eSRichard Henderson } 2003c896fe29Sbellard } 2004b03cce8eSbellard } 20051894f69aSRichard Henderson 20061894f69aSRichard Henderson if (have_prefs) { 20071894f69aSRichard Henderson for (i = 0; i < nb_oargs; ++i) { 20081894f69aSRichard Henderson TCGRegSet set = op->output_pref[i]; 20091894f69aSRichard Henderson 20101894f69aSRichard Henderson if (i == 0) { 20111894f69aSRichard Henderson qemu_log(" pref="); 20121894f69aSRichard Henderson } else { 20131894f69aSRichard Henderson qemu_log(","); 20141894f69aSRichard Henderson } 20151894f69aSRichard Henderson if (set == 0) { 20161894f69aSRichard Henderson qemu_log("none"); 20171894f69aSRichard Henderson } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 20181894f69aSRichard Henderson qemu_log("all"); 20191894f69aSRichard Henderson #ifdef CONFIG_DEBUG_TCG 20201894f69aSRichard Henderson } else if (tcg_regset_single(set)) { 20211894f69aSRichard Henderson TCGReg reg = tcg_regset_first(set); 20221894f69aSRichard Henderson qemu_log("%s", tcg_target_reg_names[reg]); 20231894f69aSRichard Henderson #endif 20241894f69aSRichard Henderson } else if (TCG_TARGET_NB_REGS <= 32) { 20251894f69aSRichard Henderson qemu_log("%#x", (uint32_t)set); 20261894f69aSRichard Henderson } else { 20271894f69aSRichard Henderson qemu_log("%#" PRIx64, (uint64_t)set); 20281894f69aSRichard Henderson } 20291894f69aSRichard Henderson } 20301894f69aSRichard Henderson } 20311894f69aSRichard Henderson 2032eeacee4dSBlue Swirl qemu_log("\n"); 2033c896fe29Sbellard } 2034c896fe29Sbellard } 2035c896fe29Sbellard 2036c896fe29Sbellard /* we give more priority to constraints with less registers */ 2037c896fe29Sbellard static int get_constraint_priority(const TCGOpDef *def, int k) 2038c896fe29Sbellard { 203974a11790SRichard Henderson const TCGArgConstraint *arg_ct = &def->args_ct[k]; 204074a11790SRichard Henderson int n; 2041c896fe29Sbellard 2042bc2b17e6SRichard Henderson if (arg_ct->oalias) { 2043c896fe29Sbellard /* an alias is equivalent to a single register */ 2044c896fe29Sbellard n = 1; 2045c896fe29Sbellard } else { 204674a11790SRichard Henderson n = ctpop64(arg_ct->regs); 2047c896fe29Sbellard } 2048c896fe29Sbellard return TCG_TARGET_NB_REGS - n + 1; 2049c896fe29Sbellard } 2050c896fe29Sbellard 2051c896fe29Sbellard /* sort from highest priority to lowest */ 2052c896fe29Sbellard static void sort_constraints(TCGOpDef *def, int start, int n) 2053c896fe29Sbellard { 205466792f90SRichard Henderson int i, j; 205566792f90SRichard Henderson TCGArgConstraint *a = def->args_ct; 2056c896fe29Sbellard 205766792f90SRichard Henderson for (i = 0; i < n; i++) { 205866792f90SRichard Henderson a[start + i].sort_index = start + i; 205966792f90SRichard Henderson } 206066792f90SRichard Henderson if (n <= 1) { 2061c896fe29Sbellard return; 206266792f90SRichard Henderson } 2063c896fe29Sbellard for (i = 0; i < n - 1; i++) { 2064c896fe29Sbellard for (j = i + 1; j < n; j++) { 206566792f90SRichard Henderson int p1 = get_constraint_priority(def, a[start + i].sort_index); 206666792f90SRichard Henderson int p2 = get_constraint_priority(def, a[start + j].sort_index); 2067c896fe29Sbellard if (p1 < p2) { 206866792f90SRichard Henderson int tmp = a[start + i].sort_index; 206966792f90SRichard Henderson a[start + i].sort_index = a[start + j].sort_index; 207066792f90SRichard Henderson a[start + j].sort_index = tmp; 2071c896fe29Sbellard } 2072c896fe29Sbellard } 2073c896fe29Sbellard } 2074c896fe29Sbellard } 2075c896fe29Sbellard 2076f69d277eSRichard Henderson static void process_op_defs(TCGContext *s) 2077c896fe29Sbellard { 2078a9751609SRichard Henderson TCGOpcode op; 2079c896fe29Sbellard 2080f69d277eSRichard Henderson for (op = 0; op < NB_OPS; op++) { 2081f69d277eSRichard Henderson TCGOpDef *def = &tcg_op_defs[op]; 2082f69d277eSRichard Henderson const TCGTargetOpDef *tdefs; 2083069ea736SRichard Henderson int i, nb_args; 2084f69d277eSRichard Henderson 2085f69d277eSRichard Henderson if (def->flags & TCG_OPF_NOT_PRESENT) { 2086f69d277eSRichard Henderson continue; 2087f69d277eSRichard Henderson } 2088f69d277eSRichard Henderson 2089c896fe29Sbellard nb_args = def->nb_iargs + def->nb_oargs; 2090f69d277eSRichard Henderson if (nb_args == 0) { 2091f69d277eSRichard Henderson continue; 2092f69d277eSRichard Henderson } 2093f69d277eSRichard Henderson 20944c22e840SRichard Henderson /* 20954c22e840SRichard Henderson * Macro magic should make it impossible, but double-check that 20964c22e840SRichard Henderson * the array index is in range. Since the signness of an enum 20974c22e840SRichard Henderson * is implementation defined, force the result to unsigned. 20984c22e840SRichard Henderson */ 20994c22e840SRichard Henderson unsigned con_set = tcg_target_op_def(op); 21004c22e840SRichard Henderson tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 21014c22e840SRichard Henderson tdefs = &constraint_sets[con_set]; 2102f69d277eSRichard Henderson 2103c896fe29Sbellard for (i = 0; i < nb_args; i++) { 2104f69d277eSRichard Henderson const char *ct_str = tdefs->args_ct_str[i]; 2105f69d277eSRichard Henderson /* Incomplete TCGTargetOpDef entry. */ 2106eabb7b91SAurelien Jarno tcg_debug_assert(ct_str != NULL); 2107f69d277eSRichard Henderson 210817280ff4SRichard Henderson while (*ct_str != '\0') { 210917280ff4SRichard Henderson switch(*ct_str) { 211017280ff4SRichard Henderson case '0' ... '9': 211117280ff4SRichard Henderson { 211217280ff4SRichard Henderson int oarg = *ct_str - '0'; 211317280ff4SRichard Henderson tcg_debug_assert(ct_str == tdefs->args_ct_str[i]); 2114eabb7b91SAurelien Jarno tcg_debug_assert(oarg < def->nb_oargs); 211574a11790SRichard Henderson tcg_debug_assert(def->args_ct[oarg].regs != 0); 2116c896fe29Sbellard def->args_ct[i] = def->args_ct[oarg]; 2117bc2b17e6SRichard Henderson /* The output sets oalias. */ 2118bc2b17e6SRichard Henderson def->args_ct[oarg].oalias = true; 21195ff9d6a4Sbellard def->args_ct[oarg].alias_index = i; 2120bc2b17e6SRichard Henderson /* The input sets ialias. */ 2121bc2b17e6SRichard Henderson def->args_ct[i].ialias = true; 21225ff9d6a4Sbellard def->args_ct[i].alias_index = oarg; 212317280ff4SRichard Henderson } 212417280ff4SRichard Henderson ct_str++; 2125c896fe29Sbellard break; 212682790a87SRichard Henderson case '&': 2127bc2b17e6SRichard Henderson def->args_ct[i].newreg = true; 212882790a87SRichard Henderson ct_str++; 212982790a87SRichard Henderson break; 2130c896fe29Sbellard case 'i': 2131c896fe29Sbellard def->args_ct[i].ct |= TCG_CT_CONST; 2132c896fe29Sbellard ct_str++; 2133c896fe29Sbellard break; 2134358b4923SRichard Henderson 2135358b4923SRichard Henderson /* Include all of the target-specific constraints. */ 2136358b4923SRichard Henderson 2137358b4923SRichard Henderson #undef CONST 2138358b4923SRichard Henderson #define CONST(CASE, MASK) \ 2139358b4923SRichard Henderson case CASE: def->args_ct[i].ct |= MASK; ct_str++; break; 2140358b4923SRichard Henderson #define REGS(CASE, MASK) \ 2141358b4923SRichard Henderson case CASE: def->args_ct[i].regs |= MASK; ct_str++; break; 2142358b4923SRichard Henderson 2143358b4923SRichard Henderson #include "tcg-target-con-str.h" 2144358b4923SRichard Henderson 2145358b4923SRichard Henderson #undef REGS 2146358b4923SRichard Henderson #undef CONST 2147c896fe29Sbellard default: 2148358b4923SRichard Henderson /* Typo in TCGTargetOpDef constraint. */ 2149358b4923SRichard Henderson g_assert_not_reached(); 2150358b4923SRichard Henderson } 2151c896fe29Sbellard } 2152c896fe29Sbellard } 2153c896fe29Sbellard 2154c68aaa18SStefan Weil /* TCGTargetOpDef entry with too much information? */ 2155eabb7b91SAurelien Jarno tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2156c68aaa18SStefan Weil 2157c896fe29Sbellard /* sort the constraints (XXX: this is just an heuristic) */ 2158c896fe29Sbellard sort_constraints(def, 0, def->nb_oargs); 2159c896fe29Sbellard sort_constraints(def, def->nb_oargs, def->nb_iargs); 2160c896fe29Sbellard } 2161c896fe29Sbellard } 2162c896fe29Sbellard 21630c627cdcSRichard Henderson void tcg_op_remove(TCGContext *s, TCGOp *op) 21640c627cdcSRichard Henderson { 2165d88a117eSRichard Henderson TCGLabel *label; 2166d88a117eSRichard Henderson 2167d88a117eSRichard Henderson switch (op->opc) { 2168d88a117eSRichard Henderson case INDEX_op_br: 2169d88a117eSRichard Henderson label = arg_label(op->args[0]); 2170d88a117eSRichard Henderson label->refs--; 2171d88a117eSRichard Henderson break; 2172d88a117eSRichard Henderson case INDEX_op_brcond_i32: 2173d88a117eSRichard Henderson case INDEX_op_brcond_i64: 2174d88a117eSRichard Henderson label = arg_label(op->args[3]); 2175d88a117eSRichard Henderson label->refs--; 2176d88a117eSRichard Henderson break; 2177d88a117eSRichard Henderson case INDEX_op_brcond2_i32: 2178d88a117eSRichard Henderson label = arg_label(op->args[5]); 2179d88a117eSRichard Henderson label->refs--; 2180d88a117eSRichard Henderson break; 2181d88a117eSRichard Henderson default: 2182d88a117eSRichard Henderson break; 2183d88a117eSRichard Henderson } 2184d88a117eSRichard Henderson 218515fa08f8SRichard Henderson QTAILQ_REMOVE(&s->ops, op, link); 218615fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2187abebf925SRichard Henderson s->nb_ops--; 21880c627cdcSRichard Henderson 21890c627cdcSRichard Henderson #ifdef CONFIG_PROFILER 2190d73415a3SStefan Hajnoczi qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 21910c627cdcSRichard Henderson #endif 21920c627cdcSRichard Henderson } 21930c627cdcSRichard Henderson 2194a80cdd31SRichard Henderson void tcg_remove_ops_after(TCGOp *op) 2195a80cdd31SRichard Henderson { 2196a80cdd31SRichard Henderson TCGContext *s = tcg_ctx; 2197a80cdd31SRichard Henderson 2198a80cdd31SRichard Henderson while (true) { 2199a80cdd31SRichard Henderson TCGOp *last = tcg_last_op(); 2200a80cdd31SRichard Henderson if (last == op) { 2201a80cdd31SRichard Henderson return; 2202a80cdd31SRichard Henderson } 2203a80cdd31SRichard Henderson tcg_op_remove(s, last); 2204a80cdd31SRichard Henderson } 2205a80cdd31SRichard Henderson } 2206a80cdd31SRichard Henderson 220715fa08f8SRichard Henderson static TCGOp *tcg_op_alloc(TCGOpcode opc) 220815fa08f8SRichard Henderson { 220915fa08f8SRichard Henderson TCGContext *s = tcg_ctx; 221015fa08f8SRichard Henderson TCGOp *op; 221115fa08f8SRichard Henderson 221215fa08f8SRichard Henderson if (likely(QTAILQ_EMPTY(&s->free_ops))) { 221315fa08f8SRichard Henderson op = tcg_malloc(sizeof(TCGOp)); 221415fa08f8SRichard Henderson } else { 221515fa08f8SRichard Henderson op = QTAILQ_FIRST(&s->free_ops); 221615fa08f8SRichard Henderson QTAILQ_REMOVE(&s->free_ops, op, link); 221715fa08f8SRichard Henderson } 221815fa08f8SRichard Henderson memset(op, 0, offsetof(TCGOp, link)); 221915fa08f8SRichard Henderson op->opc = opc; 2220abebf925SRichard Henderson s->nb_ops++; 222115fa08f8SRichard Henderson 222215fa08f8SRichard Henderson return op; 222315fa08f8SRichard Henderson } 222415fa08f8SRichard Henderson 222515fa08f8SRichard Henderson TCGOp *tcg_emit_op(TCGOpcode opc) 222615fa08f8SRichard Henderson { 222715fa08f8SRichard Henderson TCGOp *op = tcg_op_alloc(opc); 222815fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 222915fa08f8SRichard Henderson return op; 223015fa08f8SRichard Henderson } 223115fa08f8SRichard Henderson 2232ac1043f6SEmilio G. Cota TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 22335a18407fSRichard Henderson { 223415fa08f8SRichard Henderson TCGOp *new_op = tcg_op_alloc(opc); 223515fa08f8SRichard Henderson QTAILQ_INSERT_BEFORE(old_op, new_op, link); 22365a18407fSRichard Henderson return new_op; 22375a18407fSRichard Henderson } 22385a18407fSRichard Henderson 2239ac1043f6SEmilio G. Cota TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 22405a18407fSRichard Henderson { 224115fa08f8SRichard Henderson TCGOp *new_op = tcg_op_alloc(opc); 224215fa08f8SRichard Henderson QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 22435a18407fSRichard Henderson return new_op; 22445a18407fSRichard Henderson } 22455a18407fSRichard Henderson 2246b4fc67c7SRichard Henderson /* Reachable analysis : remove unreachable code. */ 2247b4fc67c7SRichard Henderson static void reachable_code_pass(TCGContext *s) 2248b4fc67c7SRichard Henderson { 2249b4fc67c7SRichard Henderson TCGOp *op, *op_next; 2250b4fc67c7SRichard Henderson bool dead = false; 2251b4fc67c7SRichard Henderson 2252b4fc67c7SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2253b4fc67c7SRichard Henderson bool remove = dead; 2254b4fc67c7SRichard Henderson TCGLabel *label; 2255b4fc67c7SRichard Henderson 2256b4fc67c7SRichard Henderson switch (op->opc) { 2257b4fc67c7SRichard Henderson case INDEX_op_set_label: 2258b4fc67c7SRichard Henderson label = arg_label(op->args[0]); 2259b4fc67c7SRichard Henderson if (label->refs == 0) { 2260b4fc67c7SRichard Henderson /* 2261b4fc67c7SRichard Henderson * While there is an occasional backward branch, virtually 2262b4fc67c7SRichard Henderson * all branches generated by the translators are forward. 2263b4fc67c7SRichard Henderson * Which means that generally we will have already removed 2264b4fc67c7SRichard Henderson * all references to the label that will be, and there is 2265b4fc67c7SRichard Henderson * little to be gained by iterating. 2266b4fc67c7SRichard Henderson */ 2267b4fc67c7SRichard Henderson remove = true; 2268b4fc67c7SRichard Henderson } else { 2269b4fc67c7SRichard Henderson /* Once we see a label, insns become live again. */ 2270b4fc67c7SRichard Henderson dead = false; 2271b4fc67c7SRichard Henderson remove = false; 2272b4fc67c7SRichard Henderson 2273b4fc67c7SRichard Henderson /* 2274b4fc67c7SRichard Henderson * Optimization can fold conditional branches to unconditional. 2275b4fc67c7SRichard Henderson * If we find a label with one reference which is preceded by 2276b4fc67c7SRichard Henderson * an unconditional branch to it, remove both. This needed to 2277b4fc67c7SRichard Henderson * wait until the dead code in between them was removed. 2278b4fc67c7SRichard Henderson */ 2279b4fc67c7SRichard Henderson if (label->refs == 1) { 2280eae3eb3eSPaolo Bonzini TCGOp *op_prev = QTAILQ_PREV(op, link); 2281b4fc67c7SRichard Henderson if (op_prev->opc == INDEX_op_br && 2282b4fc67c7SRichard Henderson label == arg_label(op_prev->args[0])) { 2283b4fc67c7SRichard Henderson tcg_op_remove(s, op_prev); 2284b4fc67c7SRichard Henderson remove = true; 2285b4fc67c7SRichard Henderson } 2286b4fc67c7SRichard Henderson } 2287b4fc67c7SRichard Henderson } 2288b4fc67c7SRichard Henderson break; 2289b4fc67c7SRichard Henderson 2290b4fc67c7SRichard Henderson case INDEX_op_br: 2291b4fc67c7SRichard Henderson case INDEX_op_exit_tb: 2292b4fc67c7SRichard Henderson case INDEX_op_goto_ptr: 2293b4fc67c7SRichard Henderson /* Unconditional branches; everything following is dead. */ 2294b4fc67c7SRichard Henderson dead = true; 2295b4fc67c7SRichard Henderson break; 2296b4fc67c7SRichard Henderson 2297b4fc67c7SRichard Henderson case INDEX_op_call: 2298b4fc67c7SRichard Henderson /* Notice noreturn helper calls, raising exceptions. */ 229990163900SRichard Henderson if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2300b4fc67c7SRichard Henderson dead = true; 2301b4fc67c7SRichard Henderson } 2302b4fc67c7SRichard Henderson break; 2303b4fc67c7SRichard Henderson 2304b4fc67c7SRichard Henderson case INDEX_op_insn_start: 2305b4fc67c7SRichard Henderson /* Never remove -- we need to keep these for unwind. */ 2306b4fc67c7SRichard Henderson remove = false; 2307b4fc67c7SRichard Henderson break; 2308b4fc67c7SRichard Henderson 2309b4fc67c7SRichard Henderson default: 2310b4fc67c7SRichard Henderson break; 2311b4fc67c7SRichard Henderson } 2312b4fc67c7SRichard Henderson 2313b4fc67c7SRichard Henderson if (remove) { 2314b4fc67c7SRichard Henderson tcg_op_remove(s, op); 2315b4fc67c7SRichard Henderson } 2316b4fc67c7SRichard Henderson } 2317b4fc67c7SRichard Henderson } 2318b4fc67c7SRichard Henderson 2319c70fbf0aSRichard Henderson #define TS_DEAD 1 2320c70fbf0aSRichard Henderson #define TS_MEM 2 2321c70fbf0aSRichard Henderson 23225a18407fSRichard Henderson #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 23235a18407fSRichard Henderson #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 23245a18407fSRichard Henderson 232525f49c5fSRichard Henderson /* For liveness_pass_1, the register preferences for a given temp. */ 232625f49c5fSRichard Henderson static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 232725f49c5fSRichard Henderson { 232825f49c5fSRichard Henderson return ts->state_ptr; 232925f49c5fSRichard Henderson } 233025f49c5fSRichard Henderson 233125f49c5fSRichard Henderson /* For liveness_pass_1, reset the preferences for a given temp to the 233225f49c5fSRichard Henderson * maximal regset for its type. 233325f49c5fSRichard Henderson */ 233425f49c5fSRichard Henderson static inline void la_reset_pref(TCGTemp *ts) 233525f49c5fSRichard Henderson { 233625f49c5fSRichard Henderson *la_temp_pref(ts) 233725f49c5fSRichard Henderson = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 233825f49c5fSRichard Henderson } 233925f49c5fSRichard Henderson 23409c43b68dSAurelien Jarno /* liveness analysis: end of function: all temps are dead, and globals 23419c43b68dSAurelien Jarno should be in memory. */ 23422616c808SRichard Henderson static void la_func_end(TCGContext *s, int ng, int nt) 2343c896fe29Sbellard { 2344b83eabeaSRichard Henderson int i; 2345b83eabeaSRichard Henderson 2346b83eabeaSRichard Henderson for (i = 0; i < ng; ++i) { 2347b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 234825f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 2349b83eabeaSRichard Henderson } 2350b83eabeaSRichard Henderson for (i = ng; i < nt; ++i) { 2351b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD; 235225f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 2353b83eabeaSRichard Henderson } 2354c896fe29Sbellard } 2355c896fe29Sbellard 23569c43b68dSAurelien Jarno /* liveness analysis: end of basic block: all temps are dead, globals 23579c43b68dSAurelien Jarno and local temps should be in memory. */ 23582616c808SRichard Henderson static void la_bb_end(TCGContext *s, int ng, int nt) 2359641d5fbeSbellard { 2360b83eabeaSRichard Henderson int i; 2361641d5fbeSbellard 2362ee17db83SRichard Henderson for (i = 0; i < nt; ++i) { 2363ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 2364ee17db83SRichard Henderson int state; 2365ee17db83SRichard Henderson 2366ee17db83SRichard Henderson switch (ts->kind) { 2367ee17db83SRichard Henderson case TEMP_FIXED: 2368ee17db83SRichard Henderson case TEMP_GLOBAL: 2369ee17db83SRichard Henderson case TEMP_LOCAL: 2370ee17db83SRichard Henderson state = TS_DEAD | TS_MEM; 2371ee17db83SRichard Henderson break; 2372ee17db83SRichard Henderson case TEMP_NORMAL: 2373c0522136SRichard Henderson case TEMP_CONST: 2374ee17db83SRichard Henderson state = TS_DEAD; 2375ee17db83SRichard Henderson break; 2376ee17db83SRichard Henderson default: 2377ee17db83SRichard Henderson g_assert_not_reached(); 2378c70fbf0aSRichard Henderson } 2379ee17db83SRichard Henderson ts->state = state; 2380ee17db83SRichard Henderson la_reset_pref(ts); 2381641d5fbeSbellard } 2382641d5fbeSbellard } 2383641d5fbeSbellard 2384f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory. */ 2385f65a061cSRichard Henderson static void la_global_sync(TCGContext *s, int ng) 2386f65a061cSRichard Henderson { 2387f65a061cSRichard Henderson int i; 2388f65a061cSRichard Henderson 2389f65a061cSRichard Henderson for (i = 0; i < ng; ++i) { 239025f49c5fSRichard Henderson int state = s->temps[i].state; 239125f49c5fSRichard Henderson s->temps[i].state = state | TS_MEM; 239225f49c5fSRichard Henderson if (state == TS_DEAD) { 239325f49c5fSRichard Henderson /* If the global was previously dead, reset prefs. */ 239425f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 239525f49c5fSRichard Henderson } 2396f65a061cSRichard Henderson } 2397f65a061cSRichard Henderson } 2398f65a061cSRichard Henderson 2399b4cb76e6SRichard Henderson /* 2400b4cb76e6SRichard Henderson * liveness analysis: conditional branch: all temps are dead, 2401b4cb76e6SRichard Henderson * globals and local temps should be synced. 2402b4cb76e6SRichard Henderson */ 2403b4cb76e6SRichard Henderson static void la_bb_sync(TCGContext *s, int ng, int nt) 2404b4cb76e6SRichard Henderson { 2405b4cb76e6SRichard Henderson la_global_sync(s, ng); 2406b4cb76e6SRichard Henderson 2407b4cb76e6SRichard Henderson for (int i = ng; i < nt; ++i) { 2408c0522136SRichard Henderson TCGTemp *ts = &s->temps[i]; 2409c0522136SRichard Henderson int state; 2410c0522136SRichard Henderson 2411c0522136SRichard Henderson switch (ts->kind) { 2412c0522136SRichard Henderson case TEMP_LOCAL: 2413c0522136SRichard Henderson state = ts->state; 2414c0522136SRichard Henderson ts->state = state | TS_MEM; 2415b4cb76e6SRichard Henderson if (state != TS_DEAD) { 2416b4cb76e6SRichard Henderson continue; 2417b4cb76e6SRichard Henderson } 2418c0522136SRichard Henderson break; 2419c0522136SRichard Henderson case TEMP_NORMAL: 2420b4cb76e6SRichard Henderson s->temps[i].state = TS_DEAD; 2421c0522136SRichard Henderson break; 2422c0522136SRichard Henderson case TEMP_CONST: 2423c0522136SRichard Henderson continue; 2424c0522136SRichard Henderson default: 2425c0522136SRichard Henderson g_assert_not_reached(); 2426b4cb76e6SRichard Henderson } 2427b4cb76e6SRichard Henderson la_reset_pref(&s->temps[i]); 2428b4cb76e6SRichard Henderson } 2429b4cb76e6SRichard Henderson } 2430b4cb76e6SRichard Henderson 2431f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory and kill. */ 2432f65a061cSRichard Henderson static void la_global_kill(TCGContext *s, int ng) 2433f65a061cSRichard Henderson { 2434f65a061cSRichard Henderson int i; 2435f65a061cSRichard Henderson 2436f65a061cSRichard Henderson for (i = 0; i < ng; i++) { 2437f65a061cSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 243825f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 243925f49c5fSRichard Henderson } 244025f49c5fSRichard Henderson } 244125f49c5fSRichard Henderson 244225f49c5fSRichard Henderson /* liveness analysis: note live globals crossing calls. */ 244325f49c5fSRichard Henderson static void la_cross_call(TCGContext *s, int nt) 244425f49c5fSRichard Henderson { 244525f49c5fSRichard Henderson TCGRegSet mask = ~tcg_target_call_clobber_regs; 244625f49c5fSRichard Henderson int i; 244725f49c5fSRichard Henderson 244825f49c5fSRichard Henderson for (i = 0; i < nt; i++) { 244925f49c5fSRichard Henderson TCGTemp *ts = &s->temps[i]; 245025f49c5fSRichard Henderson if (!(ts->state & TS_DEAD)) { 245125f49c5fSRichard Henderson TCGRegSet *pset = la_temp_pref(ts); 245225f49c5fSRichard Henderson TCGRegSet set = *pset; 245325f49c5fSRichard Henderson 245425f49c5fSRichard Henderson set &= mask; 245525f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 245625f49c5fSRichard Henderson if (set == 0) { 245725f49c5fSRichard Henderson set = tcg_target_available_regs[ts->type] & mask; 245825f49c5fSRichard Henderson } 245925f49c5fSRichard Henderson *pset = set; 246025f49c5fSRichard Henderson } 2461f65a061cSRichard Henderson } 2462f65a061cSRichard Henderson } 2463f65a061cSRichard Henderson 2464a1b3c48dSRichard Henderson /* Liveness analysis : update the opc_arg_life array to tell if a 2465c896fe29Sbellard given input arguments is dead. Instructions updating dead 2466c896fe29Sbellard temporaries are removed. */ 2467b83eabeaSRichard Henderson static void liveness_pass_1(TCGContext *s) 2468c896fe29Sbellard { 2469c70fbf0aSRichard Henderson int nb_globals = s->nb_globals; 24702616c808SRichard Henderson int nb_temps = s->nb_temps; 247115fa08f8SRichard Henderson TCGOp *op, *op_prev; 247225f49c5fSRichard Henderson TCGRegSet *prefs; 247325f49c5fSRichard Henderson int i; 247425f49c5fSRichard Henderson 247525f49c5fSRichard Henderson prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 247625f49c5fSRichard Henderson for (i = 0; i < nb_temps; ++i) { 247725f49c5fSRichard Henderson s->temps[i].state_ptr = prefs + i; 247825f49c5fSRichard Henderson } 2479c896fe29Sbellard 2480ae36a246SRichard Henderson /* ??? Should be redundant with the exit_tb that ends the TB. */ 24812616c808SRichard Henderson la_func_end(s, nb_globals, nb_temps); 2482c896fe29Sbellard 2483eae3eb3eSPaolo Bonzini QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 248425f49c5fSRichard Henderson int nb_iargs, nb_oargs; 2485c45cb8bbSRichard Henderson TCGOpcode opc_new, opc_new2; 2486c45cb8bbSRichard Henderson bool have_opc_new2; 2487a1b3c48dSRichard Henderson TCGLifeData arg_life = 0; 248825f49c5fSRichard Henderson TCGTemp *ts; 2489c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 2490c45cb8bbSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 2491c45cb8bbSRichard Henderson 2492c45cb8bbSRichard Henderson switch (opc) { 2493c896fe29Sbellard case INDEX_op_call: 2494c6e113f5Sbellard { 2495c6e113f5Sbellard int call_flags; 249625f49c5fSRichard Henderson int nb_call_regs; 2497c6e113f5Sbellard 2498cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 2499cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 250090163900SRichard Henderson call_flags = tcg_call_flags(op); 2501c6e113f5Sbellard 2502c45cb8bbSRichard Henderson /* pure functions can be removed if their result is unused */ 250378505279SAurelien Jarno if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 2504c6e113f5Sbellard for (i = 0; i < nb_oargs; i++) { 250525f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 250625f49c5fSRichard Henderson if (ts->state != TS_DEAD) { 2507c6e113f5Sbellard goto do_not_remove_call; 2508c6e113f5Sbellard } 25099c43b68dSAurelien Jarno } 2510c45cb8bbSRichard Henderson goto do_remove; 2511152c35aaSRichard Henderson } 2512c6e113f5Sbellard do_not_remove_call: 2513c896fe29Sbellard 251425f49c5fSRichard Henderson /* Output args are dead. */ 2515c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 251625f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 251725f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 2518a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 25196b64b624SAurelien Jarno } 252025f49c5fSRichard Henderson if (ts->state & TS_MEM) { 2521a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 25229c43b68dSAurelien Jarno } 252325f49c5fSRichard Henderson ts->state = TS_DEAD; 252425f49c5fSRichard Henderson la_reset_pref(ts); 252525f49c5fSRichard Henderson 252625f49c5fSRichard Henderson /* Not used -- it will be tcg_target_call_oarg_regs[i]. */ 252725f49c5fSRichard Henderson op->output_pref[i] = 0; 2528c896fe29Sbellard } 2529c896fe29Sbellard 253078505279SAurelien Jarno if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 253178505279SAurelien Jarno TCG_CALL_NO_READ_GLOBALS))) { 2532f65a061cSRichard Henderson la_global_kill(s, nb_globals); 2533c70fbf0aSRichard Henderson } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 2534f65a061cSRichard Henderson la_global_sync(s, nb_globals); 2535b9c18f56Saurel32 } 2536c896fe29Sbellard 253725f49c5fSRichard Henderson /* Record arguments that die in this helper. */ 2538866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 253925f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 254025f49c5fSRichard Henderson if (ts && ts->state & TS_DEAD) { 2541a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 2542c896fe29Sbellard } 2543c896fe29Sbellard } 254425f49c5fSRichard Henderson 254525f49c5fSRichard Henderson /* For all live registers, remove call-clobbered prefs. */ 254625f49c5fSRichard Henderson la_cross_call(s, nb_temps); 254725f49c5fSRichard Henderson 254825f49c5fSRichard Henderson nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 254925f49c5fSRichard Henderson 255025f49c5fSRichard Henderson /* Input arguments are live for preceding opcodes. */ 255125f49c5fSRichard Henderson for (i = 0; i < nb_iargs; i++) { 255225f49c5fSRichard Henderson ts = arg_temp(op->args[i + nb_oargs]); 255325f49c5fSRichard Henderson if (ts && ts->state & TS_DEAD) { 255425f49c5fSRichard Henderson /* For those arguments that die, and will be allocated 255525f49c5fSRichard Henderson * in registers, clear the register set for that arg, 255625f49c5fSRichard Henderson * to be filled in below. For args that will be on 255725f49c5fSRichard Henderson * the stack, reset to any available reg. 255825f49c5fSRichard Henderson */ 255925f49c5fSRichard Henderson *la_temp_pref(ts) 256025f49c5fSRichard Henderson = (i < nb_call_regs ? 0 : 256125f49c5fSRichard Henderson tcg_target_available_regs[ts->type]); 256225f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 256325f49c5fSRichard Henderson } 256425f49c5fSRichard Henderson } 256525f49c5fSRichard Henderson 256625f49c5fSRichard Henderson /* For each input argument, add its input register to prefs. 256725f49c5fSRichard Henderson If a temp is used once, this produces a single set bit. */ 256825f49c5fSRichard Henderson for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) { 256925f49c5fSRichard Henderson ts = arg_temp(op->args[i + nb_oargs]); 257025f49c5fSRichard Henderson if (ts) { 257125f49c5fSRichard Henderson tcg_regset_set_reg(*la_temp_pref(ts), 257225f49c5fSRichard Henderson tcg_target_call_iarg_regs[i]); 2573c70fbf0aSRichard Henderson } 2574c19f47bfSAurelien Jarno } 2575c6e113f5Sbellard } 2576c896fe29Sbellard break; 2577765b842aSRichard Henderson case INDEX_op_insn_start: 2578c896fe29Sbellard break; 25795ff9d6a4Sbellard case INDEX_op_discard: 25805ff9d6a4Sbellard /* mark the temporary as dead */ 258125f49c5fSRichard Henderson ts = arg_temp(op->args[0]); 258225f49c5fSRichard Henderson ts->state = TS_DEAD; 258325f49c5fSRichard Henderson la_reset_pref(ts); 25845ff9d6a4Sbellard break; 25851305c451SRichard Henderson 25861305c451SRichard Henderson case INDEX_op_add2_i32: 2587c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i32; 2588f1fae40cSRichard Henderson goto do_addsub2; 25891305c451SRichard Henderson case INDEX_op_sub2_i32: 2590c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i32; 2591f1fae40cSRichard Henderson goto do_addsub2; 2592f1fae40cSRichard Henderson case INDEX_op_add2_i64: 2593c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i64; 2594f1fae40cSRichard Henderson goto do_addsub2; 2595f1fae40cSRichard Henderson case INDEX_op_sub2_i64: 2596c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i64; 2597f1fae40cSRichard Henderson do_addsub2: 25981305c451SRichard Henderson nb_iargs = 4; 25991305c451SRichard Henderson nb_oargs = 2; 26001305c451SRichard Henderson /* Test if the high part of the operation is dead, but not 26011305c451SRichard Henderson the low part. The result can be optimized to a simple 26021305c451SRichard Henderson add or sub. This happens often for x86_64 guest when the 26031305c451SRichard Henderson cpu mode is set to 32 bit. */ 2604b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 2605b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 26061305c451SRichard Henderson goto do_remove; 26071305c451SRichard Henderson } 2608c45cb8bbSRichard Henderson /* Replace the opcode and adjust the args in place, 2609c45cb8bbSRichard Henderson leaving 3 unused args at the end. */ 2610c45cb8bbSRichard Henderson op->opc = opc = opc_new; 2611efee3746SRichard Henderson op->args[1] = op->args[2]; 2612efee3746SRichard Henderson op->args[2] = op->args[4]; 26131305c451SRichard Henderson /* Fall through and mark the single-word operation live. */ 26141305c451SRichard Henderson nb_iargs = 2; 26151305c451SRichard Henderson nb_oargs = 1; 26161305c451SRichard Henderson } 26171305c451SRichard Henderson goto do_not_remove; 26181305c451SRichard Henderson 26191414968aSRichard Henderson case INDEX_op_mulu2_i32: 2620c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 2621c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i32; 2622c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 262303271524SRichard Henderson goto do_mul2; 2624f1fae40cSRichard Henderson case INDEX_op_muls2_i32: 2625c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 2626c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i32; 2627c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 2628f1fae40cSRichard Henderson goto do_mul2; 2629f1fae40cSRichard Henderson case INDEX_op_mulu2_i64: 2630c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 2631c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i64; 2632c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 263303271524SRichard Henderson goto do_mul2; 2634f1fae40cSRichard Henderson case INDEX_op_muls2_i64: 2635c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 2636c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i64; 2637c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 263803271524SRichard Henderson goto do_mul2; 2639f1fae40cSRichard Henderson do_mul2: 26401414968aSRichard Henderson nb_iargs = 2; 26411414968aSRichard Henderson nb_oargs = 2; 2642b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 2643b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 264403271524SRichard Henderson /* Both parts of the operation are dead. */ 26451414968aSRichard Henderson goto do_remove; 26461414968aSRichard Henderson } 264703271524SRichard Henderson /* The high part of the operation is dead; generate the low. */ 2648c45cb8bbSRichard Henderson op->opc = opc = opc_new; 2649efee3746SRichard Henderson op->args[1] = op->args[2]; 2650efee3746SRichard Henderson op->args[2] = op->args[3]; 2651b83eabeaSRichard Henderson } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 265203271524SRichard Henderson /* The low part of the operation is dead; generate the high. */ 2653c45cb8bbSRichard Henderson op->opc = opc = opc_new2; 2654efee3746SRichard Henderson op->args[0] = op->args[1]; 2655efee3746SRichard Henderson op->args[1] = op->args[2]; 2656efee3746SRichard Henderson op->args[2] = op->args[3]; 265703271524SRichard Henderson } else { 265803271524SRichard Henderson goto do_not_remove; 265903271524SRichard Henderson } 266003271524SRichard Henderson /* Mark the single-word operation live. */ 26611414968aSRichard Henderson nb_oargs = 1; 26621414968aSRichard Henderson goto do_not_remove; 26631414968aSRichard Henderson 2664c896fe29Sbellard default: 26651305c451SRichard Henderson /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 2666c896fe29Sbellard nb_iargs = def->nb_iargs; 2667c896fe29Sbellard nb_oargs = def->nb_oargs; 2668c896fe29Sbellard 2669c896fe29Sbellard /* Test if the operation can be removed because all 26705ff9d6a4Sbellard its outputs are dead. We assume that nb_oargs == 0 26715ff9d6a4Sbellard implies side effects */ 26725ff9d6a4Sbellard if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 2673c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 2674b83eabeaSRichard Henderson if (arg_temp(op->args[i])->state != TS_DEAD) { 2675c896fe29Sbellard goto do_not_remove; 2676c896fe29Sbellard } 26779c43b68dSAurelien Jarno } 2678152c35aaSRichard Henderson goto do_remove; 2679152c35aaSRichard Henderson } 2680152c35aaSRichard Henderson goto do_not_remove; 2681152c35aaSRichard Henderson 26821305c451SRichard Henderson do_remove: 26830c627cdcSRichard Henderson tcg_op_remove(s, op); 2684152c35aaSRichard Henderson break; 2685152c35aaSRichard Henderson 2686c896fe29Sbellard do_not_remove: 2687c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 268825f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 268925f49c5fSRichard Henderson 269025f49c5fSRichard Henderson /* Remember the preference of the uses that followed. */ 269125f49c5fSRichard Henderson op->output_pref[i] = *la_temp_pref(ts); 269225f49c5fSRichard Henderson 269325f49c5fSRichard Henderson /* Output args are dead. */ 269425f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 2695a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 26966b64b624SAurelien Jarno } 269725f49c5fSRichard Henderson if (ts->state & TS_MEM) { 2698a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 26999c43b68dSAurelien Jarno } 270025f49c5fSRichard Henderson ts->state = TS_DEAD; 270125f49c5fSRichard Henderson la_reset_pref(ts); 2702c896fe29Sbellard } 2703c896fe29Sbellard 270425f49c5fSRichard Henderson /* If end of basic block, update. */ 2705ae36a246SRichard Henderson if (def->flags & TCG_OPF_BB_EXIT) { 2706ae36a246SRichard Henderson la_func_end(s, nb_globals, nb_temps); 2707b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_COND_BRANCH) { 2708b4cb76e6SRichard Henderson la_bb_sync(s, nb_globals, nb_temps); 2709ae36a246SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 27102616c808SRichard Henderson la_bb_end(s, nb_globals, nb_temps); 27113d5c5f87SAurelien Jarno } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2712f65a061cSRichard Henderson la_global_sync(s, nb_globals); 271325f49c5fSRichard Henderson if (def->flags & TCG_OPF_CALL_CLOBBER) { 271425f49c5fSRichard Henderson la_cross_call(s, nb_temps); 271525f49c5fSRichard Henderson } 2716c896fe29Sbellard } 2717c896fe29Sbellard 271825f49c5fSRichard Henderson /* Record arguments that die in this opcode. */ 2719866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 272025f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 272125f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 2722a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 2723c896fe29Sbellard } 2724c19f47bfSAurelien Jarno } 272525f49c5fSRichard Henderson 272625f49c5fSRichard Henderson /* Input arguments are live for preceding opcodes. */ 2727c19f47bfSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 272825f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 272925f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 273025f49c5fSRichard Henderson /* For operands that were dead, initially allow 273125f49c5fSRichard Henderson all regs for the type. */ 273225f49c5fSRichard Henderson *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 273325f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 273425f49c5fSRichard Henderson } 273525f49c5fSRichard Henderson } 273625f49c5fSRichard Henderson 273725f49c5fSRichard Henderson /* Incorporate constraints for this operand. */ 273825f49c5fSRichard Henderson switch (opc) { 273925f49c5fSRichard Henderson case INDEX_op_mov_i32: 274025f49c5fSRichard Henderson case INDEX_op_mov_i64: 274125f49c5fSRichard Henderson /* Note that these are TCG_OPF_NOT_PRESENT and do not 274225f49c5fSRichard Henderson have proper constraints. That said, special case 274325f49c5fSRichard Henderson moves to propagate preferences backward. */ 274425f49c5fSRichard Henderson if (IS_DEAD_ARG(1)) { 274525f49c5fSRichard Henderson *la_temp_pref(arg_temp(op->args[0])) 274625f49c5fSRichard Henderson = *la_temp_pref(arg_temp(op->args[1])); 274725f49c5fSRichard Henderson } 274825f49c5fSRichard Henderson break; 274925f49c5fSRichard Henderson 275025f49c5fSRichard Henderson default: 275125f49c5fSRichard Henderson for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 275225f49c5fSRichard Henderson const TCGArgConstraint *ct = &def->args_ct[i]; 275325f49c5fSRichard Henderson TCGRegSet set, *pset; 275425f49c5fSRichard Henderson 275525f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 275625f49c5fSRichard Henderson pset = la_temp_pref(ts); 275725f49c5fSRichard Henderson set = *pset; 275825f49c5fSRichard Henderson 27599be0d080SRichard Henderson set &= ct->regs; 2760bc2b17e6SRichard Henderson if (ct->ialias) { 276125f49c5fSRichard Henderson set &= op->output_pref[ct->alias_index]; 276225f49c5fSRichard Henderson } 276325f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 276425f49c5fSRichard Henderson if (set == 0) { 27659be0d080SRichard Henderson set = ct->regs; 276625f49c5fSRichard Henderson } 276725f49c5fSRichard Henderson *pset = set; 276825f49c5fSRichard Henderson } 276925f49c5fSRichard Henderson break; 2770c896fe29Sbellard } 2771c896fe29Sbellard break; 2772c896fe29Sbellard } 2773bee158cbSRichard Henderson op->life = arg_life; 2774c896fe29Sbellard } 27751ff0a2c5SEvgeny Voevodin } 2776c896fe29Sbellard 27775a18407fSRichard Henderson /* Liveness analysis: Convert indirect regs to direct temporaries. */ 2778b83eabeaSRichard Henderson static bool liveness_pass_2(TCGContext *s) 27795a18407fSRichard Henderson { 27805a18407fSRichard Henderson int nb_globals = s->nb_globals; 278115fa08f8SRichard Henderson int nb_temps, i; 27825a18407fSRichard Henderson bool changes = false; 278315fa08f8SRichard Henderson TCGOp *op, *op_next; 27845a18407fSRichard Henderson 27855a18407fSRichard Henderson /* Create a temporary for each indirect global. */ 27865a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 27875a18407fSRichard Henderson TCGTemp *its = &s->temps[i]; 27885a18407fSRichard Henderson if (its->indirect_reg) { 27895a18407fSRichard Henderson TCGTemp *dts = tcg_temp_alloc(s); 27905a18407fSRichard Henderson dts->type = its->type; 27915a18407fSRichard Henderson dts->base_type = its->base_type; 2792b83eabeaSRichard Henderson its->state_ptr = dts; 2793b83eabeaSRichard Henderson } else { 2794b83eabeaSRichard Henderson its->state_ptr = NULL; 27955a18407fSRichard Henderson } 2796b83eabeaSRichard Henderson /* All globals begin dead. */ 2797b83eabeaSRichard Henderson its->state = TS_DEAD; 27985a18407fSRichard Henderson } 2799b83eabeaSRichard Henderson for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 2800b83eabeaSRichard Henderson TCGTemp *its = &s->temps[i]; 2801b83eabeaSRichard Henderson its->state_ptr = NULL; 2802b83eabeaSRichard Henderson its->state = TS_DEAD; 2803b83eabeaSRichard Henderson } 28045a18407fSRichard Henderson 280515fa08f8SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 28065a18407fSRichard Henderson TCGOpcode opc = op->opc; 28075a18407fSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 28085a18407fSRichard Henderson TCGLifeData arg_life = op->life; 28095a18407fSRichard Henderson int nb_iargs, nb_oargs, call_flags; 2810b83eabeaSRichard Henderson TCGTemp *arg_ts, *dir_ts; 28115a18407fSRichard Henderson 28125a18407fSRichard Henderson if (opc == INDEX_op_call) { 2813cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 2814cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 281590163900SRichard Henderson call_flags = tcg_call_flags(op); 28165a18407fSRichard Henderson } else { 28175a18407fSRichard Henderson nb_iargs = def->nb_iargs; 28185a18407fSRichard Henderson nb_oargs = def->nb_oargs; 28195a18407fSRichard Henderson 28205a18407fSRichard Henderson /* Set flags similar to how calls require. */ 2821b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 2822b4cb76e6SRichard Henderson /* Like reading globals: sync_globals */ 2823b4cb76e6SRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2824b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 28255a18407fSRichard Henderson /* Like writing globals: save_globals */ 28265a18407fSRichard Henderson call_flags = 0; 28275a18407fSRichard Henderson } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 28285a18407fSRichard Henderson /* Like reading globals: sync_globals */ 28295a18407fSRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 28305a18407fSRichard Henderson } else { 28315a18407fSRichard Henderson /* No effect on globals. */ 28325a18407fSRichard Henderson call_flags = (TCG_CALL_NO_READ_GLOBALS | 28335a18407fSRichard Henderson TCG_CALL_NO_WRITE_GLOBALS); 28345a18407fSRichard Henderson } 28355a18407fSRichard Henderson } 28365a18407fSRichard Henderson 28375a18407fSRichard Henderson /* Make sure that input arguments are available. */ 28385a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2839b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 2840b83eabeaSRichard Henderson if (arg_ts) { 2841b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 2842b83eabeaSRichard Henderson if (dir_ts && arg_ts->state == TS_DEAD) { 2843b83eabeaSRichard Henderson TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 28445a18407fSRichard Henderson ? INDEX_op_ld_i32 28455a18407fSRichard Henderson : INDEX_op_ld_i64); 2846ac1043f6SEmilio G. Cota TCGOp *lop = tcg_op_insert_before(s, op, lopc); 28475a18407fSRichard Henderson 2848b83eabeaSRichard Henderson lop->args[0] = temp_arg(dir_ts); 2849b83eabeaSRichard Henderson lop->args[1] = temp_arg(arg_ts->mem_base); 2850b83eabeaSRichard Henderson lop->args[2] = arg_ts->mem_offset; 28515a18407fSRichard Henderson 28525a18407fSRichard Henderson /* Loaded, but synced with memory. */ 2853b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 28545a18407fSRichard Henderson } 28555a18407fSRichard Henderson } 28565a18407fSRichard Henderson } 28575a18407fSRichard Henderson 28585a18407fSRichard Henderson /* Perform input replacement, and mark inputs that became dead. 28595a18407fSRichard Henderson No action is required except keeping temp_state up to date 28605a18407fSRichard Henderson so that we reload when needed. */ 28615a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2862b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 2863b83eabeaSRichard Henderson if (arg_ts) { 2864b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 2865b83eabeaSRichard Henderson if (dir_ts) { 2866b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 28675a18407fSRichard Henderson changes = true; 28685a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 2869b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 28705a18407fSRichard Henderson } 28715a18407fSRichard Henderson } 28725a18407fSRichard Henderson } 28735a18407fSRichard Henderson } 28745a18407fSRichard Henderson 28755a18407fSRichard Henderson /* Liveness analysis should ensure that the following are 28765a18407fSRichard Henderson all correct, for call sites and basic block end points. */ 28775a18407fSRichard Henderson if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 28785a18407fSRichard Henderson /* Nothing to do */ 28795a18407fSRichard Henderson } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 28805a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 28815a18407fSRichard Henderson /* Liveness should see that globals are synced back, 28825a18407fSRichard Henderson that is, either TS_DEAD or TS_MEM. */ 2883b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 2884b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 2885b83eabeaSRichard Henderson || arg_ts->state != 0); 28865a18407fSRichard Henderson } 28875a18407fSRichard Henderson } else { 28885a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 28895a18407fSRichard Henderson /* Liveness should see that globals are saved back, 28905a18407fSRichard Henderson that is, TS_DEAD, waiting to be reloaded. */ 2891b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 2892b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 2893b83eabeaSRichard Henderson || arg_ts->state == TS_DEAD); 28945a18407fSRichard Henderson } 28955a18407fSRichard Henderson } 28965a18407fSRichard Henderson 28975a18407fSRichard Henderson /* Outputs become available. */ 289861f15c48SRichard Henderson if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 289961f15c48SRichard Henderson arg_ts = arg_temp(op->args[0]); 290061f15c48SRichard Henderson dir_ts = arg_ts->state_ptr; 290161f15c48SRichard Henderson if (dir_ts) { 290261f15c48SRichard Henderson op->args[0] = temp_arg(dir_ts); 290361f15c48SRichard Henderson changes = true; 290461f15c48SRichard Henderson 290561f15c48SRichard Henderson /* The output is now live and modified. */ 290661f15c48SRichard Henderson arg_ts->state = 0; 290761f15c48SRichard Henderson 290861f15c48SRichard Henderson if (NEED_SYNC_ARG(0)) { 290961f15c48SRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 291061f15c48SRichard Henderson ? INDEX_op_st_i32 291161f15c48SRichard Henderson : INDEX_op_st_i64); 291261f15c48SRichard Henderson TCGOp *sop = tcg_op_insert_after(s, op, sopc); 291361f15c48SRichard Henderson TCGTemp *out_ts = dir_ts; 291461f15c48SRichard Henderson 291561f15c48SRichard Henderson if (IS_DEAD_ARG(0)) { 291661f15c48SRichard Henderson out_ts = arg_temp(op->args[1]); 291761f15c48SRichard Henderson arg_ts->state = TS_DEAD; 291861f15c48SRichard Henderson tcg_op_remove(s, op); 291961f15c48SRichard Henderson } else { 292061f15c48SRichard Henderson arg_ts->state = TS_MEM; 292161f15c48SRichard Henderson } 292261f15c48SRichard Henderson 292361f15c48SRichard Henderson sop->args[0] = temp_arg(out_ts); 292461f15c48SRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 292561f15c48SRichard Henderson sop->args[2] = arg_ts->mem_offset; 292661f15c48SRichard Henderson } else { 292761f15c48SRichard Henderson tcg_debug_assert(!IS_DEAD_ARG(0)); 292861f15c48SRichard Henderson } 292961f15c48SRichard Henderson } 293061f15c48SRichard Henderson } else { 29315a18407fSRichard Henderson for (i = 0; i < nb_oargs; i++) { 2932b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 2933b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 2934b83eabeaSRichard Henderson if (!dir_ts) { 29355a18407fSRichard Henderson continue; 29365a18407fSRichard Henderson } 2937b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 29385a18407fSRichard Henderson changes = true; 29395a18407fSRichard Henderson 29405a18407fSRichard Henderson /* The output is now live and modified. */ 2941b83eabeaSRichard Henderson arg_ts->state = 0; 29425a18407fSRichard Henderson 29435a18407fSRichard Henderson /* Sync outputs upon their last write. */ 29445a18407fSRichard Henderson if (NEED_SYNC_ARG(i)) { 2945b83eabeaSRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 29465a18407fSRichard Henderson ? INDEX_op_st_i32 29475a18407fSRichard Henderson : INDEX_op_st_i64); 2948ac1043f6SEmilio G. Cota TCGOp *sop = tcg_op_insert_after(s, op, sopc); 29495a18407fSRichard Henderson 2950b83eabeaSRichard Henderson sop->args[0] = temp_arg(dir_ts); 2951b83eabeaSRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 2952b83eabeaSRichard Henderson sop->args[2] = arg_ts->mem_offset; 29535a18407fSRichard Henderson 2954b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 29555a18407fSRichard Henderson } 29565a18407fSRichard Henderson /* Drop outputs that are dead. */ 29575a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 2958b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 29595a18407fSRichard Henderson } 29605a18407fSRichard Henderson } 29615a18407fSRichard Henderson } 296261f15c48SRichard Henderson } 29635a18407fSRichard Henderson 29645a18407fSRichard Henderson return changes; 29655a18407fSRichard Henderson } 29665a18407fSRichard Henderson 29678d8fdbaeSAurelien Jarno #ifdef CONFIG_DEBUG_TCG 2968c896fe29Sbellard static void dump_regs(TCGContext *s) 2969c896fe29Sbellard { 2970c896fe29Sbellard TCGTemp *ts; 2971c896fe29Sbellard int i; 2972c896fe29Sbellard char buf[64]; 2973c896fe29Sbellard 2974c896fe29Sbellard for(i = 0; i < s->nb_temps; i++) { 2975c896fe29Sbellard ts = &s->temps[i]; 297643439139SRichard Henderson printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2977c896fe29Sbellard switch(ts->val_type) { 2978c896fe29Sbellard case TEMP_VAL_REG: 2979c896fe29Sbellard printf("%s", tcg_target_reg_names[ts->reg]); 2980c896fe29Sbellard break; 2981c896fe29Sbellard case TEMP_VAL_MEM: 2982b3a62939SRichard Henderson printf("%d(%s)", (int)ts->mem_offset, 2983b3a62939SRichard Henderson tcg_target_reg_names[ts->mem_base->reg]); 2984c896fe29Sbellard break; 2985c896fe29Sbellard case TEMP_VAL_CONST: 2986bdb38b95SRichard Henderson printf("$0x%" PRIx64, ts->val); 2987c896fe29Sbellard break; 2988c896fe29Sbellard case TEMP_VAL_DEAD: 2989c896fe29Sbellard printf("D"); 2990c896fe29Sbellard break; 2991c896fe29Sbellard default: 2992c896fe29Sbellard printf("???"); 2993c896fe29Sbellard break; 2994c896fe29Sbellard } 2995c896fe29Sbellard printf("\n"); 2996c896fe29Sbellard } 2997c896fe29Sbellard 2998c896fe29Sbellard for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 2999f8b2f202SRichard Henderson if (s->reg_to_temp[i] != NULL) { 3000c896fe29Sbellard printf("%s: %s\n", 3001c896fe29Sbellard tcg_target_reg_names[i], 3002f8b2f202SRichard Henderson tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 3003c896fe29Sbellard } 3004c896fe29Sbellard } 3005c896fe29Sbellard } 3006c896fe29Sbellard 3007c896fe29Sbellard static void check_regs(TCGContext *s) 3008c896fe29Sbellard { 3009869938aeSRichard Henderson int reg; 3010b6638662SRichard Henderson int k; 3011c896fe29Sbellard TCGTemp *ts; 3012c896fe29Sbellard char buf[64]; 3013c896fe29Sbellard 3014c896fe29Sbellard for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 3015f8b2f202SRichard Henderson ts = s->reg_to_temp[reg]; 3016f8b2f202SRichard Henderson if (ts != NULL) { 3017f8b2f202SRichard Henderson if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 3018c896fe29Sbellard printf("Inconsistency for register %s:\n", 3019c896fe29Sbellard tcg_target_reg_names[reg]); 3020b03cce8eSbellard goto fail; 3021c896fe29Sbellard } 3022c896fe29Sbellard } 3023c896fe29Sbellard } 3024c896fe29Sbellard for (k = 0; k < s->nb_temps; k++) { 3025c896fe29Sbellard ts = &s->temps[k]; 3026ee17db83SRichard Henderson if (ts->val_type == TEMP_VAL_REG 3027ee17db83SRichard Henderson && ts->kind != TEMP_FIXED 3028f8b2f202SRichard Henderson && s->reg_to_temp[ts->reg] != ts) { 3029c896fe29Sbellard printf("Inconsistency for temp %s:\n", 3030f8b2f202SRichard Henderson tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 3031b03cce8eSbellard fail: 3032c896fe29Sbellard printf("reg state:\n"); 3033c896fe29Sbellard dump_regs(s); 3034c896fe29Sbellard tcg_abort(); 3035c896fe29Sbellard } 3036c896fe29Sbellard } 3037c896fe29Sbellard } 3038c896fe29Sbellard #endif 3039c896fe29Sbellard 30402272e4a7SRichard Henderson static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3041c896fe29Sbellard { 3042c1c09194SRichard Henderson intptr_t off, size, align; 3043c1c09194SRichard Henderson 3044c1c09194SRichard Henderson switch (ts->type) { 3045c1c09194SRichard Henderson case TCG_TYPE_I32: 3046c1c09194SRichard Henderson size = align = 4; 3047c1c09194SRichard Henderson break; 3048c1c09194SRichard Henderson case TCG_TYPE_I64: 3049c1c09194SRichard Henderson case TCG_TYPE_V64: 3050c1c09194SRichard Henderson size = align = 8; 3051c1c09194SRichard Henderson break; 3052c1c09194SRichard Henderson case TCG_TYPE_V128: 3053c1c09194SRichard Henderson size = align = 16; 3054c1c09194SRichard Henderson break; 3055c1c09194SRichard Henderson case TCG_TYPE_V256: 3056c1c09194SRichard Henderson /* Note that we do not require aligned storage for V256. */ 3057c1c09194SRichard Henderson size = 32, align = 16; 3058c1c09194SRichard Henderson break; 3059c1c09194SRichard Henderson default: 3060c1c09194SRichard Henderson g_assert_not_reached(); 3061b591dc59SBlue Swirl } 3062c1c09194SRichard Henderson 3063c1c09194SRichard Henderson assert(align <= TCG_TARGET_STACK_ALIGN); 3064c1c09194SRichard Henderson off = ROUND_UP(s->current_frame_offset, align); 3065732d5897SRichard Henderson 3066732d5897SRichard Henderson /* If we've exhausted the stack frame, restart with a smaller TB. */ 3067732d5897SRichard Henderson if (off + size > s->frame_end) { 3068732d5897SRichard Henderson tcg_raise_tb_overflow(s); 3069732d5897SRichard Henderson } 3070c1c09194SRichard Henderson s->current_frame_offset = off + size; 3071c1c09194SRichard Henderson 3072c1c09194SRichard Henderson ts->mem_offset = off; 30739defd1bdSRichard Henderson #if defined(__sparc__) 30749defd1bdSRichard Henderson ts->mem_offset += TCG_TARGET_STACK_BIAS; 30759defd1bdSRichard Henderson #endif 3076b3a62939SRichard Henderson ts->mem_base = s->frame_temp; 3077c896fe29Sbellard ts->mem_allocated = 1; 3078c896fe29Sbellard } 3079c896fe29Sbellard 3080b722452aSRichard Henderson static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3081b3915dbbSRichard Henderson 308259d7c14eSRichard Henderson /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 308359d7c14eSRichard Henderson mark it free; otherwise mark it dead. */ 308459d7c14eSRichard Henderson static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3085c896fe29Sbellard { 3086c0522136SRichard Henderson TCGTempVal new_type; 3087c0522136SRichard Henderson 3088c0522136SRichard Henderson switch (ts->kind) { 3089c0522136SRichard Henderson case TEMP_FIXED: 309059d7c14eSRichard Henderson return; 3091c0522136SRichard Henderson case TEMP_GLOBAL: 3092c0522136SRichard Henderson case TEMP_LOCAL: 3093c0522136SRichard Henderson new_type = TEMP_VAL_MEM; 3094c0522136SRichard Henderson break; 3095c0522136SRichard Henderson case TEMP_NORMAL: 3096c0522136SRichard Henderson new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3097c0522136SRichard Henderson break; 3098c0522136SRichard Henderson case TEMP_CONST: 3099c0522136SRichard Henderson new_type = TEMP_VAL_CONST; 3100c0522136SRichard Henderson break; 3101c0522136SRichard Henderson default: 3102c0522136SRichard Henderson g_assert_not_reached(); 310359d7c14eSRichard Henderson } 310459d7c14eSRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 310559d7c14eSRichard Henderson s->reg_to_temp[ts->reg] = NULL; 310659d7c14eSRichard Henderson } 3107c0522136SRichard Henderson ts->val_type = new_type; 310859d7c14eSRichard Henderson } 3109c896fe29Sbellard 311059d7c14eSRichard Henderson /* Mark a temporary as dead. */ 311159d7c14eSRichard Henderson static inline void temp_dead(TCGContext *s, TCGTemp *ts) 311259d7c14eSRichard Henderson { 311359d7c14eSRichard Henderson temp_free_or_dead(s, ts, 1); 311459d7c14eSRichard Henderson } 311559d7c14eSRichard Henderson 311659d7c14eSRichard Henderson /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 311759d7c14eSRichard Henderson registers needs to be allocated to store a constant. If 'free_or_dead' 311859d7c14eSRichard Henderson is non-zero, subsequently release the temporary; if it is positive, the 311959d7c14eSRichard Henderson temp is dead; if it is negative, the temp is free. */ 312098b4e186SRichard Henderson static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 312198b4e186SRichard Henderson TCGRegSet preferred_regs, int free_or_dead) 312259d7c14eSRichard Henderson { 3123c0522136SRichard Henderson if (!temp_readonly(ts) && !ts->mem_coherent) { 31247f6ceedfSAurelien Jarno if (!ts->mem_allocated) { 31252272e4a7SRichard Henderson temp_allocate_frame(s, ts); 312659d7c14eSRichard Henderson } 312759d7c14eSRichard Henderson switch (ts->val_type) { 312859d7c14eSRichard Henderson case TEMP_VAL_CONST: 312959d7c14eSRichard Henderson /* If we're going to free the temp immediately, then we won't 313059d7c14eSRichard Henderson require it later in a register, so attempt to store the 313159d7c14eSRichard Henderson constant to memory directly. */ 313259d7c14eSRichard Henderson if (free_or_dead 313359d7c14eSRichard Henderson && tcg_out_sti(s, ts->type, ts->val, 313459d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset)) { 313559d7c14eSRichard Henderson break; 313659d7c14eSRichard Henderson } 313759d7c14eSRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 313898b4e186SRichard Henderson allocated_regs, preferred_regs); 313959d7c14eSRichard Henderson /* fallthrough */ 314059d7c14eSRichard Henderson 314159d7c14eSRichard Henderson case TEMP_VAL_REG: 314259d7c14eSRichard Henderson tcg_out_st(s, ts->type, ts->reg, 314359d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset); 314459d7c14eSRichard Henderson break; 314559d7c14eSRichard Henderson 314659d7c14eSRichard Henderson case TEMP_VAL_MEM: 314759d7c14eSRichard Henderson break; 314859d7c14eSRichard Henderson 314959d7c14eSRichard Henderson case TEMP_VAL_DEAD: 315059d7c14eSRichard Henderson default: 315159d7c14eSRichard Henderson tcg_abort(); 3152c896fe29Sbellard } 31537f6ceedfSAurelien Jarno ts->mem_coherent = 1; 31547f6ceedfSAurelien Jarno } 315559d7c14eSRichard Henderson if (free_or_dead) { 315659d7c14eSRichard Henderson temp_free_or_dead(s, ts, free_or_dead); 315759d7c14eSRichard Henderson } 315859d7c14eSRichard Henderson } 31597f6ceedfSAurelien Jarno 31607f6ceedfSAurelien Jarno /* free register 'reg' by spilling the corresponding temporary if necessary */ 3161b3915dbbSRichard Henderson static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 31627f6ceedfSAurelien Jarno { 3163f8b2f202SRichard Henderson TCGTemp *ts = s->reg_to_temp[reg]; 3164f8b2f202SRichard Henderson if (ts != NULL) { 316598b4e186SRichard Henderson temp_sync(s, ts, allocated_regs, 0, -1); 3166c896fe29Sbellard } 3167c896fe29Sbellard } 3168c896fe29Sbellard 3169b016486eSRichard Henderson /** 3170b016486eSRichard Henderson * tcg_reg_alloc: 3171b016486eSRichard Henderson * @required_regs: Set of registers in which we must allocate. 3172b016486eSRichard Henderson * @allocated_regs: Set of registers which must be avoided. 3173b016486eSRichard Henderson * @preferred_regs: Set of registers we should prefer. 3174b016486eSRichard Henderson * @rev: True if we search the registers in "indirect" order. 3175b016486eSRichard Henderson * 3176b016486eSRichard Henderson * The allocated register must be in @required_regs & ~@allocated_regs, 3177b016486eSRichard Henderson * but if we can put it in @preferred_regs we may save a move later. 3178b016486eSRichard Henderson */ 3179b016486eSRichard Henderson static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3180b016486eSRichard Henderson TCGRegSet allocated_regs, 3181b016486eSRichard Henderson TCGRegSet preferred_regs, bool rev) 3182c896fe29Sbellard { 3183b016486eSRichard Henderson int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3184b016486eSRichard Henderson TCGRegSet reg_ct[2]; 318591478cefSRichard Henderson const int *order; 3186c896fe29Sbellard 3187b016486eSRichard Henderson reg_ct[1] = required_regs & ~allocated_regs; 3188b016486eSRichard Henderson tcg_debug_assert(reg_ct[1] != 0); 3189b016486eSRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs; 3190b016486eSRichard Henderson 3191b016486eSRichard Henderson /* Skip the preferred_regs option if it cannot be satisfied, 3192b016486eSRichard Henderson or if the preference made no difference. */ 3193b016486eSRichard Henderson f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3194b016486eSRichard Henderson 319591478cefSRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3196c896fe29Sbellard 3197b016486eSRichard Henderson /* Try free registers, preferences first. */ 3198b016486eSRichard Henderson for (j = f; j < 2; j++) { 3199b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 3200b016486eSRichard Henderson 3201b016486eSRichard Henderson if (tcg_regset_single(set)) { 3202b016486eSRichard Henderson /* One register in the set. */ 3203b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 3204b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL) { 3205c896fe29Sbellard return reg; 3206c896fe29Sbellard } 3207b016486eSRichard Henderson } else { 320891478cefSRichard Henderson for (i = 0; i < n; i++) { 3209b016486eSRichard Henderson TCGReg reg = order[i]; 3210b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL && 3211b016486eSRichard Henderson tcg_regset_test_reg(set, reg)) { 3212b016486eSRichard Henderson return reg; 3213b016486eSRichard Henderson } 3214b016486eSRichard Henderson } 3215b016486eSRichard Henderson } 3216b016486eSRichard Henderson } 3217b016486eSRichard Henderson 3218b016486eSRichard Henderson /* We must spill something. */ 3219b016486eSRichard Henderson for (j = f; j < 2; j++) { 3220b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 3221b016486eSRichard Henderson 3222b016486eSRichard Henderson if (tcg_regset_single(set)) { 3223b016486eSRichard Henderson /* One register in the set. */ 3224b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 3225b3915dbbSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 3226c896fe29Sbellard return reg; 3227b016486eSRichard Henderson } else { 3228b016486eSRichard Henderson for (i = 0; i < n; i++) { 3229b016486eSRichard Henderson TCGReg reg = order[i]; 3230b016486eSRichard Henderson if (tcg_regset_test_reg(set, reg)) { 3231b016486eSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 3232b016486eSRichard Henderson return reg; 3233b016486eSRichard Henderson } 3234b016486eSRichard Henderson } 3235c896fe29Sbellard } 3236c896fe29Sbellard } 3237c896fe29Sbellard 3238c896fe29Sbellard tcg_abort(); 3239c896fe29Sbellard } 3240c896fe29Sbellard 324140ae5c62SRichard Henderson /* Make sure the temporary is in a register. If needed, allocate the register 324240ae5c62SRichard Henderson from DESIRED while avoiding ALLOCATED. */ 324340ae5c62SRichard Henderson static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3244b722452aSRichard Henderson TCGRegSet allocated_regs, TCGRegSet preferred_regs) 324540ae5c62SRichard Henderson { 324640ae5c62SRichard Henderson TCGReg reg; 324740ae5c62SRichard Henderson 324840ae5c62SRichard Henderson switch (ts->val_type) { 324940ae5c62SRichard Henderson case TEMP_VAL_REG: 325040ae5c62SRichard Henderson return; 325140ae5c62SRichard Henderson case TEMP_VAL_CONST: 3252b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3253b722452aSRichard Henderson preferred_regs, ts->indirect_base); 32540a6a8bc8SRichard Henderson if (ts->type <= TCG_TYPE_I64) { 325540ae5c62SRichard Henderson tcg_out_movi(s, ts->type, reg, ts->val); 32560a6a8bc8SRichard Henderson } else { 32574e186175SRichard Henderson uint64_t val = ts->val; 32584e186175SRichard Henderson MemOp vece = MO_64; 32594e186175SRichard Henderson 32604e186175SRichard Henderson /* 32614e186175SRichard Henderson * Find the minimal vector element that matches the constant. 32624e186175SRichard Henderson * The targets will, in general, have to do this search anyway, 32634e186175SRichard Henderson * do this generically. 32644e186175SRichard Henderson */ 32654e186175SRichard Henderson if (val == dup_const(MO_8, val)) { 32664e186175SRichard Henderson vece = MO_8; 32674e186175SRichard Henderson } else if (val == dup_const(MO_16, val)) { 32684e186175SRichard Henderson vece = MO_16; 32690b4286ddSRichard Henderson } else if (val == dup_const(MO_32, val)) { 32704e186175SRichard Henderson vece = MO_32; 32714e186175SRichard Henderson } 32724e186175SRichard Henderson 32734e186175SRichard Henderson tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 32740a6a8bc8SRichard Henderson } 327540ae5c62SRichard Henderson ts->mem_coherent = 0; 327640ae5c62SRichard Henderson break; 327740ae5c62SRichard Henderson case TEMP_VAL_MEM: 3278b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3279b722452aSRichard Henderson preferred_regs, ts->indirect_base); 328040ae5c62SRichard Henderson tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 328140ae5c62SRichard Henderson ts->mem_coherent = 1; 328240ae5c62SRichard Henderson break; 328340ae5c62SRichard Henderson case TEMP_VAL_DEAD: 328440ae5c62SRichard Henderson default: 328540ae5c62SRichard Henderson tcg_abort(); 328640ae5c62SRichard Henderson } 328740ae5c62SRichard Henderson ts->reg = reg; 328840ae5c62SRichard Henderson ts->val_type = TEMP_VAL_REG; 328940ae5c62SRichard Henderson s->reg_to_temp[reg] = ts; 329040ae5c62SRichard Henderson } 329140ae5c62SRichard Henderson 329259d7c14eSRichard Henderson /* Save a temporary to memory. 'allocated_regs' is used in case a 3293e8996ee0Sbellard temporary registers needs to be allocated to store a constant. */ 329459d7c14eSRichard Henderson static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 32951ad80729SAurelien Jarno { 32962c0366f0SAurelien Jarno /* The liveness analysis already ensures that globals are back 3297eabb7b91SAurelien Jarno in memory. Keep an tcg_debug_assert for safety. */ 3298e01fa97dSRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 32991ad80729SAurelien Jarno } 33001ad80729SAurelien Jarno 33019814dd27SDong Xu Wang /* save globals to their canonical location and assume they can be 3302641d5fbeSbellard modified be the following code. 'allocated_regs' is used in case a 3303641d5fbeSbellard temporary registers needs to be allocated to store a constant. */ 3304641d5fbeSbellard static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3305641d5fbeSbellard { 3306ac3b8891SRichard Henderson int i, n; 3307641d5fbeSbellard 3308ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 3309b13eb728SRichard Henderson temp_save(s, &s->temps[i], allocated_regs); 3310641d5fbeSbellard } 3311e5097dc8Sbellard } 3312e5097dc8Sbellard 33133d5c5f87SAurelien Jarno /* sync globals to their canonical location and assume they can be 33143d5c5f87SAurelien Jarno read by the following code. 'allocated_regs' is used in case a 33153d5c5f87SAurelien Jarno temporary registers needs to be allocated to store a constant. */ 33163d5c5f87SAurelien Jarno static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 33173d5c5f87SAurelien Jarno { 3318ac3b8891SRichard Henderson int i, n; 33193d5c5f87SAurelien Jarno 3320ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 332112b9b11aSRichard Henderson TCGTemp *ts = &s->temps[i]; 332212b9b11aSRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3323ee17db83SRichard Henderson || ts->kind == TEMP_FIXED 332412b9b11aSRichard Henderson || ts->mem_coherent); 33253d5c5f87SAurelien Jarno } 33263d5c5f87SAurelien Jarno } 33273d5c5f87SAurelien Jarno 3328e5097dc8Sbellard /* at the end of a basic block, we assume all temporaries are dead and 3329e8996ee0Sbellard all globals are stored at their canonical location. */ 3330e8996ee0Sbellard static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3331e5097dc8Sbellard { 3332e5097dc8Sbellard int i; 3333e5097dc8Sbellard 3334c896fe29Sbellard for (i = s->nb_globals; i < s->nb_temps; i++) { 3335b13eb728SRichard Henderson TCGTemp *ts = &s->temps[i]; 3336c0522136SRichard Henderson 3337c0522136SRichard Henderson switch (ts->kind) { 3338c0522136SRichard Henderson case TEMP_LOCAL: 3339b13eb728SRichard Henderson temp_save(s, ts, allocated_regs); 3340c0522136SRichard Henderson break; 3341c0522136SRichard Henderson case TEMP_NORMAL: 33422c0366f0SAurelien Jarno /* The liveness analysis already ensures that temps are dead. 3343eabb7b91SAurelien Jarno Keep an tcg_debug_assert for safety. */ 3344eabb7b91SAurelien Jarno tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3345c0522136SRichard Henderson break; 3346c0522136SRichard Henderson case TEMP_CONST: 3347c0522136SRichard Henderson /* Similarly, we should have freed any allocated register. */ 3348c0522136SRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3349c0522136SRichard Henderson break; 3350c0522136SRichard Henderson default: 3351c0522136SRichard Henderson g_assert_not_reached(); 3352c896fe29Sbellard } 3353641d5fbeSbellard } 3354e8996ee0Sbellard 3355e8996ee0Sbellard save_globals(s, allocated_regs); 3356c896fe29Sbellard } 3357c896fe29Sbellard 3358bab1671fSRichard Henderson /* 3359b4cb76e6SRichard Henderson * At a conditional branch, we assume all temporaries are dead and 3360b4cb76e6SRichard Henderson * all globals and local temps are synced to their location. 3361b4cb76e6SRichard Henderson */ 3362b4cb76e6SRichard Henderson static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3363b4cb76e6SRichard Henderson { 3364b4cb76e6SRichard Henderson sync_globals(s, allocated_regs); 3365b4cb76e6SRichard Henderson 3366b4cb76e6SRichard Henderson for (int i = s->nb_globals; i < s->nb_temps; i++) { 3367b4cb76e6SRichard Henderson TCGTemp *ts = &s->temps[i]; 3368b4cb76e6SRichard Henderson /* 3369b4cb76e6SRichard Henderson * The liveness analysis already ensures that temps are dead. 3370b4cb76e6SRichard Henderson * Keep tcg_debug_asserts for safety. 3371b4cb76e6SRichard Henderson */ 3372c0522136SRichard Henderson switch (ts->kind) { 3373c0522136SRichard Henderson case TEMP_LOCAL: 3374b4cb76e6SRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3375c0522136SRichard Henderson break; 3376c0522136SRichard Henderson case TEMP_NORMAL: 3377b4cb76e6SRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3378c0522136SRichard Henderson break; 3379c0522136SRichard Henderson case TEMP_CONST: 3380c0522136SRichard Henderson break; 3381c0522136SRichard Henderson default: 3382c0522136SRichard Henderson g_assert_not_reached(); 3383b4cb76e6SRichard Henderson } 3384b4cb76e6SRichard Henderson } 3385b4cb76e6SRichard Henderson } 3386b4cb76e6SRichard Henderson 3387b4cb76e6SRichard Henderson /* 3388c58f4c97SRichard Henderson * Specialized code generation for INDEX_op_mov_* with a constant. 3389bab1671fSRichard Henderson */ 33900fe4fca4SPaolo Bonzini static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3391ba87719cSRichard Henderson tcg_target_ulong val, TCGLifeData arg_life, 3392ba87719cSRichard Henderson TCGRegSet preferred_regs) 3393e8996ee0Sbellard { 3394d63e3b6eSRichard Henderson /* ENV should not be modified. */ 3395e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 339659d7c14eSRichard Henderson 339759d7c14eSRichard Henderson /* The movi is not explicitly generated here. */ 3398f8b2f202SRichard Henderson if (ots->val_type == TEMP_VAL_REG) { 3399f8b2f202SRichard Henderson s->reg_to_temp[ots->reg] = NULL; 3400f8b2f202SRichard Henderson } 3401e8996ee0Sbellard ots->val_type = TEMP_VAL_CONST; 3402e8996ee0Sbellard ots->val = val; 340359d7c14eSRichard Henderson ots->mem_coherent = 0; 3404ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 3405ba87719cSRichard Henderson temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 340659d7c14eSRichard Henderson } else if (IS_DEAD_ARG(0)) { 3407f8bf00f1SRichard Henderson temp_dead(s, ots); 34084c4e1ab2SAurelien Jarno } 3409e8996ee0Sbellard } 3410e8996ee0Sbellard 3411bab1671fSRichard Henderson /* 3412bab1671fSRichard Henderson * Specialized code generation for INDEX_op_mov_*. 3413bab1671fSRichard Henderson */ 3414dd186292SRichard Henderson static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 3415c896fe29Sbellard { 3416dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 341769e3706dSRichard Henderson TCGRegSet allocated_regs, preferred_regs; 3418c896fe29Sbellard TCGTemp *ts, *ots; 3419450445d5SRichard Henderson TCGType otype, itype; 3420c896fe29Sbellard 3421d21369f5SRichard Henderson allocated_regs = s->reserved_regs; 342269e3706dSRichard Henderson preferred_regs = op->output_pref[0]; 342343439139SRichard Henderson ots = arg_temp(op->args[0]); 342443439139SRichard Henderson ts = arg_temp(op->args[1]); 3425450445d5SRichard Henderson 3426d63e3b6eSRichard Henderson /* ENV should not be modified. */ 3427e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 3428d63e3b6eSRichard Henderson 3429450445d5SRichard Henderson /* Note that otype != itype for no-op truncation. */ 3430450445d5SRichard Henderson otype = ots->type; 3431450445d5SRichard Henderson itype = ts->type; 3432c896fe29Sbellard 34330fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_CONST) { 34340fe4fca4SPaolo Bonzini /* propagate constant or generate sti */ 34350fe4fca4SPaolo Bonzini tcg_target_ulong val = ts->val; 34360fe4fca4SPaolo Bonzini if (IS_DEAD_ARG(1)) { 34370fe4fca4SPaolo Bonzini temp_dead(s, ts); 34380fe4fca4SPaolo Bonzini } 343969e3706dSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 34400fe4fca4SPaolo Bonzini return; 34410fe4fca4SPaolo Bonzini } 34420fe4fca4SPaolo Bonzini 34430fe4fca4SPaolo Bonzini /* If the source value is in memory we're going to be forced 34440fe4fca4SPaolo Bonzini to have it in a register in order to perform the copy. Copy 34450fe4fca4SPaolo Bonzini the SOURCE value into its own register first, that way we 34460fe4fca4SPaolo Bonzini don't have to reload SOURCE the next time it is used. */ 34470fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_MEM) { 344869e3706dSRichard Henderson temp_load(s, ts, tcg_target_available_regs[itype], 344969e3706dSRichard Henderson allocated_regs, preferred_regs); 3450c29c1d7eSAurelien Jarno } 3451c29c1d7eSAurelien Jarno 34520fe4fca4SPaolo Bonzini tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 3453d63e3b6eSRichard Henderson if (IS_DEAD_ARG(0)) { 3454c29c1d7eSAurelien Jarno /* mov to a non-saved dead register makes no sense (even with 3455c29c1d7eSAurelien Jarno liveness analysis disabled). */ 3456eabb7b91SAurelien Jarno tcg_debug_assert(NEED_SYNC_ARG(0)); 3457c29c1d7eSAurelien Jarno if (!ots->mem_allocated) { 34582272e4a7SRichard Henderson temp_allocate_frame(s, ots); 3459c29c1d7eSAurelien Jarno } 3460b3a62939SRichard Henderson tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 3461c29c1d7eSAurelien Jarno if (IS_DEAD_ARG(1)) { 3462f8bf00f1SRichard Henderson temp_dead(s, ts); 3463c29c1d7eSAurelien Jarno } 3464f8bf00f1SRichard Henderson temp_dead(s, ots); 3465e8996ee0Sbellard } else { 3466ee17db83SRichard Henderson if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 3467c29c1d7eSAurelien Jarno /* the mov can be suppressed */ 3468c29c1d7eSAurelien Jarno if (ots->val_type == TEMP_VAL_REG) { 3469f8b2f202SRichard Henderson s->reg_to_temp[ots->reg] = NULL; 3470c896fe29Sbellard } 3471c29c1d7eSAurelien Jarno ots->reg = ts->reg; 3472f8bf00f1SRichard Henderson temp_dead(s, ts); 3473c29c1d7eSAurelien Jarno } else { 3474c29c1d7eSAurelien Jarno if (ots->val_type != TEMP_VAL_REG) { 3475c29c1d7eSAurelien Jarno /* When allocating a new register, make sure to not spill the 3476c29c1d7eSAurelien Jarno input one. */ 3477c29c1d7eSAurelien Jarno tcg_regset_set_reg(allocated_regs, ts->reg); 3478450445d5SRichard Henderson ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 347969e3706dSRichard Henderson allocated_regs, preferred_regs, 3480b016486eSRichard Henderson ots->indirect_base); 3481c29c1d7eSAurelien Jarno } 348278113e83SRichard Henderson if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) { 3483240c08d0SRichard Henderson /* 3484240c08d0SRichard Henderson * Cross register class move not supported. 3485240c08d0SRichard Henderson * Store the source register into the destination slot 3486240c08d0SRichard Henderson * and leave the destination temp as TEMP_VAL_MEM. 3487240c08d0SRichard Henderson */ 3488e01fa97dSRichard Henderson assert(!temp_readonly(ots)); 3489240c08d0SRichard Henderson if (!ts->mem_allocated) { 3490240c08d0SRichard Henderson temp_allocate_frame(s, ots); 3491240c08d0SRichard Henderson } 3492240c08d0SRichard Henderson tcg_out_st(s, ts->type, ts->reg, 3493240c08d0SRichard Henderson ots->mem_base->reg, ots->mem_offset); 3494240c08d0SRichard Henderson ots->mem_coherent = 1; 3495240c08d0SRichard Henderson temp_free_or_dead(s, ots, -1); 3496240c08d0SRichard Henderson return; 349778113e83SRichard Henderson } 3498c29c1d7eSAurelien Jarno } 3499c896fe29Sbellard ots->val_type = TEMP_VAL_REG; 3500c896fe29Sbellard ots->mem_coherent = 0; 3501f8b2f202SRichard Henderson s->reg_to_temp[ots->reg] = ots; 3502ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 350398b4e186SRichard Henderson temp_sync(s, ots, allocated_regs, 0, 0); 3504c29c1d7eSAurelien Jarno } 3505ec7a869dSAurelien Jarno } 3506c896fe29Sbellard } 3507c896fe29Sbellard 3508bab1671fSRichard Henderson /* 3509bab1671fSRichard Henderson * Specialized code generation for INDEX_op_dup_vec. 3510bab1671fSRichard Henderson */ 3511bab1671fSRichard Henderson static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 3512bab1671fSRichard Henderson { 3513bab1671fSRichard Henderson const TCGLifeData arg_life = op->life; 3514bab1671fSRichard Henderson TCGRegSet dup_out_regs, dup_in_regs; 3515bab1671fSRichard Henderson TCGTemp *its, *ots; 3516bab1671fSRichard Henderson TCGType itype, vtype; 3517d6ecb4a9SRichard Henderson intptr_t endian_fixup; 3518bab1671fSRichard Henderson unsigned vece; 3519bab1671fSRichard Henderson bool ok; 3520bab1671fSRichard Henderson 3521bab1671fSRichard Henderson ots = arg_temp(op->args[0]); 3522bab1671fSRichard Henderson its = arg_temp(op->args[1]); 3523bab1671fSRichard Henderson 3524bab1671fSRichard Henderson /* ENV should not be modified. */ 3525e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 3526bab1671fSRichard Henderson 3527bab1671fSRichard Henderson itype = its->type; 3528bab1671fSRichard Henderson vece = TCGOP_VECE(op); 3529bab1671fSRichard Henderson vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3530bab1671fSRichard Henderson 3531bab1671fSRichard Henderson if (its->val_type == TEMP_VAL_CONST) { 3532bab1671fSRichard Henderson /* Propagate constant via movi -> dupi. */ 3533bab1671fSRichard Henderson tcg_target_ulong val = its->val; 3534bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 3535bab1671fSRichard Henderson temp_dead(s, its); 3536bab1671fSRichard Henderson } 3537bab1671fSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]); 3538bab1671fSRichard Henderson return; 3539bab1671fSRichard Henderson } 3540bab1671fSRichard Henderson 35419be0d080SRichard Henderson dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 35429be0d080SRichard Henderson dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 3543bab1671fSRichard Henderson 3544bab1671fSRichard Henderson /* Allocate the output register now. */ 3545bab1671fSRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 3546bab1671fSRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 3547bab1671fSRichard Henderson 3548bab1671fSRichard Henderson if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 3549bab1671fSRichard Henderson /* Make sure to not spill the input register. */ 3550bab1671fSRichard Henderson tcg_regset_set_reg(allocated_regs, its->reg); 3551bab1671fSRichard Henderson } 3552bab1671fSRichard Henderson ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3553bab1671fSRichard Henderson op->output_pref[0], ots->indirect_base); 3554bab1671fSRichard Henderson ots->val_type = TEMP_VAL_REG; 3555bab1671fSRichard Henderson ots->mem_coherent = 0; 3556bab1671fSRichard Henderson s->reg_to_temp[ots->reg] = ots; 3557bab1671fSRichard Henderson } 3558bab1671fSRichard Henderson 3559bab1671fSRichard Henderson switch (its->val_type) { 3560bab1671fSRichard Henderson case TEMP_VAL_REG: 3561bab1671fSRichard Henderson /* 3562bab1671fSRichard Henderson * The dup constriaints must be broad, covering all possible VECE. 3563bab1671fSRichard Henderson * However, tcg_op_dup_vec() gets to see the VECE and we allow it 3564bab1671fSRichard Henderson * to fail, indicating that extra moves are required for that case. 3565bab1671fSRichard Henderson */ 3566bab1671fSRichard Henderson if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 3567bab1671fSRichard Henderson if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 3568bab1671fSRichard Henderson goto done; 3569bab1671fSRichard Henderson } 3570bab1671fSRichard Henderson /* Try again from memory or a vector input register. */ 3571bab1671fSRichard Henderson } 3572bab1671fSRichard Henderson if (!its->mem_coherent) { 3573bab1671fSRichard Henderson /* 3574bab1671fSRichard Henderson * The input register is not synced, and so an extra store 3575bab1671fSRichard Henderson * would be required to use memory. Attempt an integer-vector 3576bab1671fSRichard Henderson * register move first. We do not have a TCGRegSet for this. 3577bab1671fSRichard Henderson */ 3578bab1671fSRichard Henderson if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 3579bab1671fSRichard Henderson break; 3580bab1671fSRichard Henderson } 3581bab1671fSRichard Henderson /* Sync the temp back to its slot and load from there. */ 3582bab1671fSRichard Henderson temp_sync(s, its, s->reserved_regs, 0, 0); 3583bab1671fSRichard Henderson } 3584bab1671fSRichard Henderson /* fall through */ 3585bab1671fSRichard Henderson 3586bab1671fSRichard Henderson case TEMP_VAL_MEM: 3587d6ecb4a9SRichard Henderson #ifdef HOST_WORDS_BIGENDIAN 3588d6ecb4a9SRichard Henderson endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8; 3589d6ecb4a9SRichard Henderson endian_fixup -= 1 << vece; 3590d6ecb4a9SRichard Henderson #else 3591d6ecb4a9SRichard Henderson endian_fixup = 0; 3592d6ecb4a9SRichard Henderson #endif 3593d6ecb4a9SRichard Henderson if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 3594d6ecb4a9SRichard Henderson its->mem_offset + endian_fixup)) { 3595d6ecb4a9SRichard Henderson goto done; 3596d6ecb4a9SRichard Henderson } 3597bab1671fSRichard Henderson tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 3598bab1671fSRichard Henderson break; 3599bab1671fSRichard Henderson 3600bab1671fSRichard Henderson default: 3601bab1671fSRichard Henderson g_assert_not_reached(); 3602bab1671fSRichard Henderson } 3603bab1671fSRichard Henderson 3604bab1671fSRichard Henderson /* We now have a vector input register, so dup must succeed. */ 3605bab1671fSRichard Henderson ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 3606bab1671fSRichard Henderson tcg_debug_assert(ok); 3607bab1671fSRichard Henderson 3608bab1671fSRichard Henderson done: 3609bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 3610bab1671fSRichard Henderson temp_dead(s, its); 3611bab1671fSRichard Henderson } 3612bab1671fSRichard Henderson if (NEED_SYNC_ARG(0)) { 3613bab1671fSRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, 0); 3614bab1671fSRichard Henderson } 3615bab1671fSRichard Henderson if (IS_DEAD_ARG(0)) { 3616bab1671fSRichard Henderson temp_dead(s, ots); 3617bab1671fSRichard Henderson } 3618bab1671fSRichard Henderson } 3619bab1671fSRichard Henderson 3620dd186292SRichard Henderson static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 3621c896fe29Sbellard { 3622dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 3623dd186292SRichard Henderson const TCGOpDef * const def = &tcg_op_defs[op->opc]; 362482790a87SRichard Henderson TCGRegSet i_allocated_regs; 362582790a87SRichard Henderson TCGRegSet o_allocated_regs; 3626b6638662SRichard Henderson int i, k, nb_iargs, nb_oargs; 3627b6638662SRichard Henderson TCGReg reg; 3628c896fe29Sbellard TCGArg arg; 3629c896fe29Sbellard const TCGArgConstraint *arg_ct; 3630c896fe29Sbellard TCGTemp *ts; 3631c896fe29Sbellard TCGArg new_args[TCG_MAX_OP_ARGS]; 3632c896fe29Sbellard int const_args[TCG_MAX_OP_ARGS]; 3633c896fe29Sbellard 3634c896fe29Sbellard nb_oargs = def->nb_oargs; 3635c896fe29Sbellard nb_iargs = def->nb_iargs; 3636c896fe29Sbellard 3637c896fe29Sbellard /* copy constants */ 3638c896fe29Sbellard memcpy(new_args + nb_oargs + nb_iargs, 3639dd186292SRichard Henderson op->args + nb_oargs + nb_iargs, 3640c896fe29Sbellard sizeof(TCGArg) * def->nb_cargs); 3641c896fe29Sbellard 3642d21369f5SRichard Henderson i_allocated_regs = s->reserved_regs; 3643d21369f5SRichard Henderson o_allocated_regs = s->reserved_regs; 364482790a87SRichard Henderson 3645c896fe29Sbellard /* satisfy input constraints */ 3646c896fe29Sbellard for (k = 0; k < nb_iargs; k++) { 3647d62816f2SRichard Henderson TCGRegSet i_preferred_regs, o_preferred_regs; 3648d62816f2SRichard Henderson 364966792f90SRichard Henderson i = def->args_ct[nb_oargs + k].sort_index; 3650dd186292SRichard Henderson arg = op->args[i]; 3651c896fe29Sbellard arg_ct = &def->args_ct[i]; 365243439139SRichard Henderson ts = arg_temp(arg); 365340ae5c62SRichard Henderson 365440ae5c62SRichard Henderson if (ts->val_type == TEMP_VAL_CONST 3655a4fbbd77SRichard Henderson && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 3656c896fe29Sbellard /* constant is OK for instruction */ 3657c896fe29Sbellard const_args[i] = 1; 3658c896fe29Sbellard new_args[i] = ts->val; 3659d62816f2SRichard Henderson continue; 3660c896fe29Sbellard } 366140ae5c62SRichard Henderson 3662d62816f2SRichard Henderson i_preferred_regs = o_preferred_regs = 0; 3663bc2b17e6SRichard Henderson if (arg_ct->ialias) { 3664d62816f2SRichard Henderson o_preferred_regs = op->output_pref[arg_ct->alias_index]; 3665c0522136SRichard Henderson 3666c0522136SRichard Henderson /* 3667c0522136SRichard Henderson * If the input is readonly, then it cannot also be an 3668c0522136SRichard Henderson * output and aliased to itself. If the input is not 3669c0522136SRichard Henderson * dead after the instruction, we must allocate a new 3670c0522136SRichard Henderson * register and move it. 3671c0522136SRichard Henderson */ 3672c0522136SRichard Henderson if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 3673c896fe29Sbellard goto allocate_in_reg; 3674c896fe29Sbellard } 3675d62816f2SRichard Henderson 3676c0522136SRichard Henderson /* 3677c0522136SRichard Henderson * Check if the current register has already been allocated 3678c0522136SRichard Henderson * for another input aliased to an output. 3679c0522136SRichard Henderson */ 3680d62816f2SRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 3681d62816f2SRichard Henderson reg = ts->reg; 3682c0522136SRichard Henderson for (int k2 = 0; k2 < k; k2++) { 3683c0522136SRichard Henderson int i2 = def->args_ct[nb_oargs + k2].sort_index; 3684bc2b17e6SRichard Henderson if (def->args_ct[i2].ialias && reg == new_args[i2]) { 36857e1df267SAurelien Jarno goto allocate_in_reg; 36867e1df267SAurelien Jarno } 36877e1df267SAurelien Jarno } 36885ff9d6a4Sbellard } 3689d62816f2SRichard Henderson i_preferred_regs = o_preferred_regs; 3690866cb6cbSAurelien Jarno } 3691d62816f2SRichard Henderson 36929be0d080SRichard Henderson temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs); 3693c896fe29Sbellard reg = ts->reg; 3694d62816f2SRichard Henderson 3695c0522136SRichard Henderson if (!tcg_regset_test_reg(arg_ct->regs, reg)) { 3696c896fe29Sbellard allocate_in_reg: 3697c0522136SRichard Henderson /* 3698c0522136SRichard Henderson * Allocate a new register matching the constraint 3699c0522136SRichard Henderson * and move the temporary register into it. 3700c0522136SRichard Henderson */ 3701d62816f2SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 3702d62816f2SRichard Henderson i_allocated_regs, 0); 37039be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs, 3704d62816f2SRichard Henderson o_preferred_regs, ts->indirect_base); 370578113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 3706240c08d0SRichard Henderson /* 3707240c08d0SRichard Henderson * Cross register class move not supported. Sync the 3708240c08d0SRichard Henderson * temp back to its slot and load from there. 3709240c08d0SRichard Henderson */ 3710240c08d0SRichard Henderson temp_sync(s, ts, i_allocated_regs, 0, 0); 3711240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 3712240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 371378113e83SRichard Henderson } 3714c896fe29Sbellard } 3715c896fe29Sbellard new_args[i] = reg; 3716c896fe29Sbellard const_args[i] = 0; 371782790a87SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg); 3718c896fe29Sbellard } 3719c896fe29Sbellard 3720c896fe29Sbellard /* mark dead temporaries and free the associated registers */ 3721866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3722866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 372343439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 3724c896fe29Sbellard } 3725c896fe29Sbellard } 3726c896fe29Sbellard 3727b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 3728b4cb76e6SRichard Henderson tcg_reg_alloc_cbranch(s, i_allocated_regs); 3729b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 373082790a87SRichard Henderson tcg_reg_alloc_bb_end(s, i_allocated_regs); 3731a52ad07eSAurelien Jarno } else { 3732c896fe29Sbellard if (def->flags & TCG_OPF_CALL_CLOBBER) { 3733b03cce8eSbellard /* XXX: permit generic clobber register list ? */ 3734c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3735c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 373682790a87SRichard Henderson tcg_reg_free(s, i, i_allocated_regs); 3737c896fe29Sbellard } 3738c896fe29Sbellard } 37393d5c5f87SAurelien Jarno } 37403d5c5f87SAurelien Jarno if (def->flags & TCG_OPF_SIDE_EFFECTS) { 37413d5c5f87SAurelien Jarno /* sync globals if the op has side effects and might trigger 37423d5c5f87SAurelien Jarno an exception. */ 374382790a87SRichard Henderson sync_globals(s, i_allocated_regs); 3744c896fe29Sbellard } 3745c896fe29Sbellard 3746c896fe29Sbellard /* satisfy the output constraints */ 3747c896fe29Sbellard for(k = 0; k < nb_oargs; k++) { 374866792f90SRichard Henderson i = def->args_ct[k].sort_index; 3749dd186292SRichard Henderson arg = op->args[i]; 3750c896fe29Sbellard arg_ct = &def->args_ct[i]; 375143439139SRichard Henderson ts = arg_temp(arg); 3752d63e3b6eSRichard Henderson 3753d63e3b6eSRichard Henderson /* ENV should not be modified. */ 3754e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 3755d63e3b6eSRichard Henderson 3756bc2b17e6SRichard Henderson if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 37575ff9d6a4Sbellard reg = new_args[arg_ct->alias_index]; 3758bc2b17e6SRichard Henderson } else if (arg_ct->newreg) { 37599be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, 376082790a87SRichard Henderson i_allocated_regs | o_allocated_regs, 376169e3706dSRichard Henderson op->output_pref[k], ts->indirect_base); 3762c896fe29Sbellard } else { 37639be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 376469e3706dSRichard Henderson op->output_pref[k], ts->indirect_base); 3765c896fe29Sbellard } 376682790a87SRichard Henderson tcg_regset_set_reg(o_allocated_regs, reg); 3767639368ddSAurelien Jarno if (ts->val_type == TEMP_VAL_REG) { 3768f8b2f202SRichard Henderson s->reg_to_temp[ts->reg] = NULL; 3769639368ddSAurelien Jarno } 3770c896fe29Sbellard ts->val_type = TEMP_VAL_REG; 3771c896fe29Sbellard ts->reg = reg; 3772d63e3b6eSRichard Henderson /* 3773d63e3b6eSRichard Henderson * Temp value is modified, so the value kept in memory is 3774d63e3b6eSRichard Henderson * potentially not the same. 3775d63e3b6eSRichard Henderson */ 3776c896fe29Sbellard ts->mem_coherent = 0; 3777f8b2f202SRichard Henderson s->reg_to_temp[reg] = ts; 3778c896fe29Sbellard new_args[i] = reg; 3779c896fe29Sbellard } 3780e8996ee0Sbellard } 3781c896fe29Sbellard 3782c896fe29Sbellard /* emit instruction */ 3783d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 3784d2fd745fSRichard Henderson tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 3785d2fd745fSRichard Henderson new_args, const_args); 3786d2fd745fSRichard Henderson } else { 3787dd186292SRichard Henderson tcg_out_op(s, op->opc, new_args, const_args); 3788d2fd745fSRichard Henderson } 3789c896fe29Sbellard 3790c896fe29Sbellard /* move the outputs in the correct register if needed */ 3791c896fe29Sbellard for(i = 0; i < nb_oargs; i++) { 379243439139SRichard Henderson ts = arg_temp(op->args[i]); 3793d63e3b6eSRichard Henderson 3794d63e3b6eSRichard Henderson /* ENV should not be modified. */ 3795e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 3796d63e3b6eSRichard Henderson 3797ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 379898b4e186SRichard Henderson temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 379959d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 3800f8bf00f1SRichard Henderson temp_dead(s, ts); 3801ec7a869dSAurelien Jarno } 3802c896fe29Sbellard } 3803c896fe29Sbellard } 3804c896fe29Sbellard 3805efe86b21SRichard Henderson static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 3806efe86b21SRichard Henderson { 3807efe86b21SRichard Henderson const TCGLifeData arg_life = op->life; 3808efe86b21SRichard Henderson TCGTemp *ots, *itsl, *itsh; 3809efe86b21SRichard Henderson TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3810efe86b21SRichard Henderson 3811efe86b21SRichard Henderson /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 3812efe86b21SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 3813efe86b21SRichard Henderson tcg_debug_assert(TCGOP_VECE(op) == MO_64); 3814efe86b21SRichard Henderson 3815efe86b21SRichard Henderson ots = arg_temp(op->args[0]); 3816efe86b21SRichard Henderson itsl = arg_temp(op->args[1]); 3817efe86b21SRichard Henderson itsh = arg_temp(op->args[2]); 3818efe86b21SRichard Henderson 3819efe86b21SRichard Henderson /* ENV should not be modified. */ 3820efe86b21SRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 3821efe86b21SRichard Henderson 3822efe86b21SRichard Henderson /* Allocate the output register now. */ 3823efe86b21SRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 3824efe86b21SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 3825efe86b21SRichard Henderson TCGRegSet dup_out_regs = 3826efe86b21SRichard Henderson tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 3827efe86b21SRichard Henderson 3828efe86b21SRichard Henderson /* Make sure to not spill the input registers. */ 3829efe86b21SRichard Henderson if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 3830efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsl->reg); 3831efe86b21SRichard Henderson } 3832efe86b21SRichard Henderson if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 3833efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsh->reg); 3834efe86b21SRichard Henderson } 3835efe86b21SRichard Henderson 3836efe86b21SRichard Henderson ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3837efe86b21SRichard Henderson op->output_pref[0], ots->indirect_base); 3838efe86b21SRichard Henderson ots->val_type = TEMP_VAL_REG; 3839efe86b21SRichard Henderson ots->mem_coherent = 0; 3840efe86b21SRichard Henderson s->reg_to_temp[ots->reg] = ots; 3841efe86b21SRichard Henderson } 3842efe86b21SRichard Henderson 3843efe86b21SRichard Henderson /* Promote dup2 of immediates to dupi_vec. */ 3844efe86b21SRichard Henderson if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 3845efe86b21SRichard Henderson uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 3846efe86b21SRichard Henderson MemOp vece = MO_64; 3847efe86b21SRichard Henderson 3848efe86b21SRichard Henderson if (val == dup_const(MO_8, val)) { 3849efe86b21SRichard Henderson vece = MO_8; 3850efe86b21SRichard Henderson } else if (val == dup_const(MO_16, val)) { 3851efe86b21SRichard Henderson vece = MO_16; 3852efe86b21SRichard Henderson } else if (val == dup_const(MO_32, val)) { 3853efe86b21SRichard Henderson vece = MO_32; 3854efe86b21SRichard Henderson } 3855efe86b21SRichard Henderson 3856efe86b21SRichard Henderson tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 3857efe86b21SRichard Henderson goto done; 3858efe86b21SRichard Henderson } 3859efe86b21SRichard Henderson 3860efe86b21SRichard Henderson /* If the two inputs form one 64-bit value, try dupm_vec. */ 3861efe86b21SRichard Henderson if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) { 3862efe86b21SRichard Henderson if (!itsl->mem_coherent) { 3863efe86b21SRichard Henderson temp_sync(s, itsl, s->reserved_regs, 0, 0); 3864efe86b21SRichard Henderson } 3865efe86b21SRichard Henderson if (!itsh->mem_coherent) { 3866efe86b21SRichard Henderson temp_sync(s, itsh, s->reserved_regs, 0, 0); 3867efe86b21SRichard Henderson } 3868efe86b21SRichard Henderson #ifdef HOST_WORDS_BIGENDIAN 3869efe86b21SRichard Henderson TCGTemp *its = itsh; 3870efe86b21SRichard Henderson #else 3871efe86b21SRichard Henderson TCGTemp *its = itsl; 3872efe86b21SRichard Henderson #endif 3873efe86b21SRichard Henderson if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 3874efe86b21SRichard Henderson its->mem_base->reg, its->mem_offset)) { 3875efe86b21SRichard Henderson goto done; 3876efe86b21SRichard Henderson } 3877efe86b21SRichard Henderson } 3878efe86b21SRichard Henderson 3879efe86b21SRichard Henderson /* Fall back to generic expansion. */ 3880efe86b21SRichard Henderson return false; 3881efe86b21SRichard Henderson 3882efe86b21SRichard Henderson done: 3883efe86b21SRichard Henderson if (IS_DEAD_ARG(1)) { 3884efe86b21SRichard Henderson temp_dead(s, itsl); 3885efe86b21SRichard Henderson } 3886efe86b21SRichard Henderson if (IS_DEAD_ARG(2)) { 3887efe86b21SRichard Henderson temp_dead(s, itsh); 3888efe86b21SRichard Henderson } 3889efe86b21SRichard Henderson if (NEED_SYNC_ARG(0)) { 3890efe86b21SRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 3891efe86b21SRichard Henderson } else if (IS_DEAD_ARG(0)) { 3892efe86b21SRichard Henderson temp_dead(s, ots); 3893efe86b21SRichard Henderson } 3894efe86b21SRichard Henderson return true; 3895efe86b21SRichard Henderson } 3896efe86b21SRichard Henderson 3897b03cce8eSbellard #ifdef TCG_TARGET_STACK_GROWSUP 3898b03cce8eSbellard #define STACK_DIR(x) (-(x)) 3899b03cce8eSbellard #else 3900b03cce8eSbellard #define STACK_DIR(x) (x) 3901b03cce8eSbellard #endif 3902b03cce8eSbellard 3903dd186292SRichard Henderson static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 3904c896fe29Sbellard { 3905cd9090aaSRichard Henderson const int nb_oargs = TCGOP_CALLO(op); 3906cd9090aaSRichard Henderson const int nb_iargs = TCGOP_CALLI(op); 3907dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 39087b7d8b2dSRichard Henderson const TCGHelperInfo *info; 3909b6638662SRichard Henderson int flags, nb_regs, i; 3910b6638662SRichard Henderson TCGReg reg; 3911cf066674SRichard Henderson TCGArg arg; 3912c896fe29Sbellard TCGTemp *ts; 3913d3452f1fSRichard Henderson intptr_t stack_offset; 3914d3452f1fSRichard Henderson size_t call_stack_size; 3915cf066674SRichard Henderson tcg_insn_unit *func_addr; 3916cf066674SRichard Henderson int allocate_args; 3917c896fe29Sbellard TCGRegSet allocated_regs; 3918c896fe29Sbellard 3919fa52e660SRichard Henderson func_addr = tcg_call_func(op); 39207b7d8b2dSRichard Henderson info = tcg_call_info(op); 39217b7d8b2dSRichard Henderson flags = info->flags; 3922c896fe29Sbellard 39236e17d0c5SStefan Weil nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 3924c45cb8bbSRichard Henderson if (nb_regs > nb_iargs) { 3925c45cb8bbSRichard Henderson nb_regs = nb_iargs; 3926cf066674SRichard Henderson } 3927c896fe29Sbellard 3928c896fe29Sbellard /* assign stack slots first */ 3929c45cb8bbSRichard Henderson call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 3930c896fe29Sbellard call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 3931c896fe29Sbellard ~(TCG_TARGET_STACK_ALIGN - 1); 3932b03cce8eSbellard allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 3933b03cce8eSbellard if (allocate_args) { 3934345649c0SBlue Swirl /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 3935345649c0SBlue Swirl preallocate call stack */ 3936345649c0SBlue Swirl tcg_abort(); 3937b03cce8eSbellard } 393839cf05d3Sbellard 393939cf05d3Sbellard stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 3940c45cb8bbSRichard Henderson for (i = nb_regs; i < nb_iargs; i++) { 3941dd186292SRichard Henderson arg = op->args[nb_oargs + i]; 394239cf05d3Sbellard #ifdef TCG_TARGET_STACK_GROWSUP 394339cf05d3Sbellard stack_offset -= sizeof(tcg_target_long); 394439cf05d3Sbellard #endif 394539cf05d3Sbellard if (arg != TCG_CALL_DUMMY_ARG) { 394643439139SRichard Henderson ts = arg_temp(arg); 394740ae5c62SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 3948b722452aSRichard Henderson s->reserved_regs, 0); 3949e4d5434cSblueswir1 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 395039cf05d3Sbellard } 395139cf05d3Sbellard #ifndef TCG_TARGET_STACK_GROWSUP 395239cf05d3Sbellard stack_offset += sizeof(tcg_target_long); 395339cf05d3Sbellard #endif 3954c896fe29Sbellard } 3955c896fe29Sbellard 3956c896fe29Sbellard /* assign input registers */ 3957d21369f5SRichard Henderson allocated_regs = s->reserved_regs; 3958c896fe29Sbellard for (i = 0; i < nb_regs; i++) { 3959dd186292SRichard Henderson arg = op->args[nb_oargs + i]; 396039cf05d3Sbellard if (arg != TCG_CALL_DUMMY_ARG) { 396143439139SRichard Henderson ts = arg_temp(arg); 3962c896fe29Sbellard reg = tcg_target_call_iarg_regs[i]; 396340ae5c62SRichard Henderson 3964c896fe29Sbellard if (ts->val_type == TEMP_VAL_REG) { 3965c896fe29Sbellard if (ts->reg != reg) { 39664250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 396778113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 3968240c08d0SRichard Henderson /* 3969240c08d0SRichard Henderson * Cross register class move not supported. Sync the 3970240c08d0SRichard Henderson * temp back to its slot and load from there. 3971240c08d0SRichard Henderson */ 3972240c08d0SRichard Henderson temp_sync(s, ts, allocated_regs, 0, 0); 3973240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 3974240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 397578113e83SRichard Henderson } 3976c896fe29Sbellard } 3977c896fe29Sbellard } else { 3978ccb1bb66SRichard Henderson TCGRegSet arg_set = 0; 397940ae5c62SRichard Henderson 39804250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 398140ae5c62SRichard Henderson tcg_regset_set_reg(arg_set, reg); 3982b722452aSRichard Henderson temp_load(s, ts, arg_set, allocated_regs, 0); 3983c896fe29Sbellard } 398440ae5c62SRichard Henderson 3985c896fe29Sbellard tcg_regset_set_reg(allocated_regs, reg); 3986c896fe29Sbellard } 398739cf05d3Sbellard } 3988c896fe29Sbellard 3989c896fe29Sbellard /* mark dead temporaries and free the associated registers */ 3990866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3991866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 399243439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 3993c896fe29Sbellard } 3994c896fe29Sbellard } 3995c896fe29Sbellard 3996c896fe29Sbellard /* clobber call registers */ 3997c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3998c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3999b3915dbbSRichard Henderson tcg_reg_free(s, i, allocated_regs); 4000c896fe29Sbellard } 4001c896fe29Sbellard } 4002c896fe29Sbellard 400378505279SAurelien Jarno /* Save globals if they might be written by the helper, sync them if 400478505279SAurelien Jarno they might be read. */ 400578505279SAurelien Jarno if (flags & TCG_CALL_NO_READ_GLOBALS) { 400678505279SAurelien Jarno /* Nothing to do */ 400778505279SAurelien Jarno } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 400878505279SAurelien Jarno sync_globals(s, allocated_regs); 400978505279SAurelien Jarno } else { 4010e8996ee0Sbellard save_globals(s, allocated_regs); 4011b9c18f56Saurel32 } 4012c896fe29Sbellard 40137b7d8b2dSRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 40147b7d8b2dSRichard Henderson { 40157b7d8b2dSRichard Henderson gpointer hash = (gpointer)(uintptr_t)info->typemask; 40167b7d8b2dSRichard Henderson ffi_cif *cif = g_hash_table_lookup(ffi_table, hash); 40177b7d8b2dSRichard Henderson assert(cif != NULL); 40187b7d8b2dSRichard Henderson tcg_out_call(s, func_addr, cif); 40197b7d8b2dSRichard Henderson } 40207b7d8b2dSRichard Henderson #else 4021cf066674SRichard Henderson tcg_out_call(s, func_addr); 40227b7d8b2dSRichard Henderson #endif 4023c896fe29Sbellard 4024c896fe29Sbellard /* assign output registers and emit moves if needed */ 4025c896fe29Sbellard for(i = 0; i < nb_oargs; i++) { 4026dd186292SRichard Henderson arg = op->args[i]; 402743439139SRichard Henderson ts = arg_temp(arg); 4028d63e3b6eSRichard Henderson 4029d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4030e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4031d63e3b6eSRichard Henderson 4032c896fe29Sbellard reg = tcg_target_call_oarg_regs[i]; 4033eabb7b91SAurelien Jarno tcg_debug_assert(s->reg_to_temp[reg] == NULL); 4034639368ddSAurelien Jarno if (ts->val_type == TEMP_VAL_REG) { 4035f8b2f202SRichard Henderson s->reg_to_temp[ts->reg] = NULL; 4036639368ddSAurelien Jarno } 4037c896fe29Sbellard ts->val_type = TEMP_VAL_REG; 4038c896fe29Sbellard ts->reg = reg; 4039c896fe29Sbellard ts->mem_coherent = 0; 4040f8b2f202SRichard Henderson s->reg_to_temp[reg] = ts; 4041ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 404298b4e186SRichard Henderson temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i)); 404359d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 4044f8bf00f1SRichard Henderson temp_dead(s, ts); 4045c896fe29Sbellard } 4046c896fe29Sbellard } 40478c11ad25SAurelien Jarno } 4048c896fe29Sbellard 4049c896fe29Sbellard #ifdef CONFIG_PROFILER 4050c896fe29Sbellard 4051c3fac113SEmilio G. Cota /* avoid copy/paste errors */ 4052c3fac113SEmilio G. Cota #define PROF_ADD(to, from, field) \ 4053c3fac113SEmilio G. Cota do { \ 4054d73415a3SStefan Hajnoczi (to)->field += qatomic_read(&((from)->field)); \ 4055c3fac113SEmilio G. Cota } while (0) 4056c896fe29Sbellard 4057c3fac113SEmilio G. Cota #define PROF_MAX(to, from, field) \ 4058c3fac113SEmilio G. Cota do { \ 4059d73415a3SStefan Hajnoczi typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4060c3fac113SEmilio G. Cota if (val__ > (to)->field) { \ 4061c3fac113SEmilio G. Cota (to)->field = val__; \ 4062c3fac113SEmilio G. Cota } \ 4063c3fac113SEmilio G. Cota } while (0) 4064c3fac113SEmilio G. Cota 4065c3fac113SEmilio G. Cota /* Pass in a zero'ed @prof */ 4066c3fac113SEmilio G. Cota static inline 4067c3fac113SEmilio G. Cota void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4068c896fe29Sbellard { 40690e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4070c3fac113SEmilio G. Cota unsigned int i; 4071c3fac113SEmilio G. Cota 40723468b59eSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 4073d73415a3SStefan Hajnoczi TCGContext *s = qatomic_read(&tcg_ctxs[i]); 40743468b59eSEmilio G. Cota const TCGProfile *orig = &s->prof; 4075c3fac113SEmilio G. Cota 4076c3fac113SEmilio G. Cota if (counters) { 407772fd2efbSEmilio G. Cota PROF_ADD(prof, orig, cpu_exec_time); 4078c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count1); 4079c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count); 4080c3fac113SEmilio G. Cota PROF_ADD(prof, orig, op_count); 4081c3fac113SEmilio G. Cota PROF_MAX(prof, orig, op_count_max); 4082c3fac113SEmilio G. Cota PROF_ADD(prof, orig, temp_count); 4083c3fac113SEmilio G. Cota PROF_MAX(prof, orig, temp_count_max); 4084c3fac113SEmilio G. Cota PROF_ADD(prof, orig, del_op_count); 4085c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_in_len); 4086c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_out_len); 4087c3fac113SEmilio G. Cota PROF_ADD(prof, orig, search_out_len); 4088c3fac113SEmilio G. Cota PROF_ADD(prof, orig, interm_time); 4089c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_time); 4090c3fac113SEmilio G. Cota PROF_ADD(prof, orig, la_time); 4091c3fac113SEmilio G. Cota PROF_ADD(prof, orig, opt_time); 4092c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_count); 4093c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_time); 4094c3fac113SEmilio G. Cota } 4095c3fac113SEmilio G. Cota if (table) { 4096c896fe29Sbellard int i; 4097d70724ceSzhanghailiang 409815fc7daaSRichard Henderson for (i = 0; i < NB_OPS; i++) { 4099c3fac113SEmilio G. Cota PROF_ADD(prof, orig, table_op_count[i]); 4100c3fac113SEmilio G. Cota } 4101c3fac113SEmilio G. Cota } 4102c3fac113SEmilio G. Cota } 4103c3fac113SEmilio G. Cota } 4104c3fac113SEmilio G. Cota 4105c3fac113SEmilio G. Cota #undef PROF_ADD 4106c3fac113SEmilio G. Cota #undef PROF_MAX 4107c3fac113SEmilio G. Cota 4108c3fac113SEmilio G. Cota static void tcg_profile_snapshot_counters(TCGProfile *prof) 4109c3fac113SEmilio G. Cota { 4110c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, true, false); 4111c3fac113SEmilio G. Cota } 4112c3fac113SEmilio G. Cota 4113c3fac113SEmilio G. Cota static void tcg_profile_snapshot_table(TCGProfile *prof) 4114c3fac113SEmilio G. Cota { 4115c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, false, true); 4116c3fac113SEmilio G. Cota } 4117c3fac113SEmilio G. Cota 4118d4c51a0aSMarkus Armbruster void tcg_dump_op_count(void) 4119c3fac113SEmilio G. Cota { 4120c3fac113SEmilio G. Cota TCGProfile prof = {}; 4121c3fac113SEmilio G. Cota int i; 4122c3fac113SEmilio G. Cota 4123c3fac113SEmilio G. Cota tcg_profile_snapshot_table(&prof); 4124c3fac113SEmilio G. Cota for (i = 0; i < NB_OPS; i++) { 4125d4c51a0aSMarkus Armbruster qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name, 4126c3fac113SEmilio G. Cota prof.table_op_count[i]); 4127c896fe29Sbellard } 4128c896fe29Sbellard } 412972fd2efbSEmilio G. Cota 413072fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 413172fd2efbSEmilio G. Cota { 41320e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 413372fd2efbSEmilio G. Cota unsigned int i; 413472fd2efbSEmilio G. Cota int64_t ret = 0; 413572fd2efbSEmilio G. Cota 413672fd2efbSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 4137d73415a3SStefan Hajnoczi const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 413872fd2efbSEmilio G. Cota const TCGProfile *prof = &s->prof; 413972fd2efbSEmilio G. Cota 4140d73415a3SStefan Hajnoczi ret += qatomic_read(&prof->cpu_exec_time); 414172fd2efbSEmilio G. Cota } 414272fd2efbSEmilio G. Cota return ret; 414372fd2efbSEmilio G. Cota } 4144246ae24dSMax Filippov #else 4145d4c51a0aSMarkus Armbruster void tcg_dump_op_count(void) 4146246ae24dSMax Filippov { 4147d4c51a0aSMarkus Armbruster qemu_printf("[TCG profiler not compiled]\n"); 4148246ae24dSMax Filippov } 414972fd2efbSEmilio G. Cota 415072fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 415172fd2efbSEmilio G. Cota { 415272fd2efbSEmilio G. Cota error_report("%s: TCG profiler not compiled", __func__); 415372fd2efbSEmilio G. Cota exit(EXIT_FAILURE); 415472fd2efbSEmilio G. Cota } 4155c896fe29Sbellard #endif 4156c896fe29Sbellard 4157c896fe29Sbellard 41585bd2ec3dSAlex Bennée int tcg_gen_code(TCGContext *s, TranslationBlock *tb) 4159c896fe29Sbellard { 4160c3fac113SEmilio G. Cota #ifdef CONFIG_PROFILER 4161c3fac113SEmilio G. Cota TCGProfile *prof = &s->prof; 4162c3fac113SEmilio G. Cota #endif 416315fa08f8SRichard Henderson int i, num_insns; 416415fa08f8SRichard Henderson TCGOp *op; 4165c896fe29Sbellard 416604fe6400SRichard Henderson #ifdef CONFIG_PROFILER 416704fe6400SRichard Henderson { 4168c1f543b7SEmilio G. Cota int n = 0; 416904fe6400SRichard Henderson 417015fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 417115fa08f8SRichard Henderson n++; 417215fa08f8SRichard Henderson } 4173d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count, prof->op_count + n); 4174c3fac113SEmilio G. Cota if (n > prof->op_count_max) { 4175d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count_max, n); 417604fe6400SRichard Henderson } 417704fe6400SRichard Henderson 417804fe6400SRichard Henderson n = s->nb_temps; 4179d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count, prof->temp_count + n); 4180c3fac113SEmilio G. Cota if (n > prof->temp_count_max) { 4181d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count_max, n); 418204fe6400SRichard Henderson } 418304fe6400SRichard Henderson } 418404fe6400SRichard Henderson #endif 418504fe6400SRichard Henderson 4186c896fe29Sbellard #ifdef DEBUG_DISAS 4187d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4188d977e1c2SAlex Bennée && qemu_log_in_addr_range(tb->pc))) { 4189fc59d2d8SRobert Foley FILE *logfile = qemu_log_lock(); 419093fcfe39Saliguori qemu_log("OP:\n"); 41911894f69aSRichard Henderson tcg_dump_ops(s, false); 419293fcfe39Saliguori qemu_log("\n"); 4193fc59d2d8SRobert Foley qemu_log_unlock(logfile); 4194c896fe29Sbellard } 4195c896fe29Sbellard #endif 4196c896fe29Sbellard 4197bef16ab4SRichard Henderson #ifdef CONFIG_DEBUG_TCG 4198bef16ab4SRichard Henderson /* Ensure all labels referenced have been emitted. */ 4199bef16ab4SRichard Henderson { 4200bef16ab4SRichard Henderson TCGLabel *l; 4201bef16ab4SRichard Henderson bool error = false; 4202bef16ab4SRichard Henderson 4203bef16ab4SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 4204bef16ab4SRichard Henderson if (unlikely(!l->present) && l->refs) { 4205bef16ab4SRichard Henderson qemu_log_mask(CPU_LOG_TB_OP, 4206bef16ab4SRichard Henderson "$L%d referenced but not present.\n", l->id); 4207bef16ab4SRichard Henderson error = true; 4208bef16ab4SRichard Henderson } 4209bef16ab4SRichard Henderson } 4210bef16ab4SRichard Henderson assert(!error); 4211bef16ab4SRichard Henderson } 4212bef16ab4SRichard Henderson #endif 4213bef16ab4SRichard Henderson 4214c5cc28ffSAurelien Jarno #ifdef CONFIG_PROFILER 4215d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 4216c5cc28ffSAurelien Jarno #endif 4217c5cc28ffSAurelien Jarno 42188f2e8c07SKirill Batuzov #ifdef USE_TCG_OPTIMIZATIONS 4219c45cb8bbSRichard Henderson tcg_optimize(s); 42208f2e8c07SKirill Batuzov #endif 42218f2e8c07SKirill Batuzov 4222a23a9ec6Sbellard #ifdef CONFIG_PROFILER 4223d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 4224d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 4225a23a9ec6Sbellard #endif 4226c5cc28ffSAurelien Jarno 4227b4fc67c7SRichard Henderson reachable_code_pass(s); 4228b83eabeaSRichard Henderson liveness_pass_1(s); 42295a18407fSRichard Henderson 42305a18407fSRichard Henderson if (s->nb_indirects > 0) { 42315a18407fSRichard Henderson #ifdef DEBUG_DISAS 42325a18407fSRichard Henderson if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 42335a18407fSRichard Henderson && qemu_log_in_addr_range(tb->pc))) { 4234fc59d2d8SRobert Foley FILE *logfile = qemu_log_lock(); 42355a18407fSRichard Henderson qemu_log("OP before indirect lowering:\n"); 42361894f69aSRichard Henderson tcg_dump_ops(s, false); 42375a18407fSRichard Henderson qemu_log("\n"); 4238fc59d2d8SRobert Foley qemu_log_unlock(logfile); 42395a18407fSRichard Henderson } 42405a18407fSRichard Henderson #endif 42415a18407fSRichard Henderson /* Replace indirect temps with direct temps. */ 4242b83eabeaSRichard Henderson if (liveness_pass_2(s)) { 42435a18407fSRichard Henderson /* If changes were made, re-run liveness. */ 4244b83eabeaSRichard Henderson liveness_pass_1(s); 42455a18407fSRichard Henderson } 42465a18407fSRichard Henderson } 4247c5cc28ffSAurelien Jarno 4248a23a9ec6Sbellard #ifdef CONFIG_PROFILER 4249d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 4250a23a9ec6Sbellard #endif 4251c896fe29Sbellard 4252c896fe29Sbellard #ifdef DEBUG_DISAS 4253d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 4254d977e1c2SAlex Bennée && qemu_log_in_addr_range(tb->pc))) { 4255fc59d2d8SRobert Foley FILE *logfile = qemu_log_lock(); 4256c5cc28ffSAurelien Jarno qemu_log("OP after optimization and liveness analysis:\n"); 42571894f69aSRichard Henderson tcg_dump_ops(s, true); 425893fcfe39Saliguori qemu_log("\n"); 4259fc59d2d8SRobert Foley qemu_log_unlock(logfile); 4260c896fe29Sbellard } 4261c896fe29Sbellard #endif 4262c896fe29Sbellard 4263c896fe29Sbellard tcg_reg_alloc_start(s); 4264c896fe29Sbellard 4265db0c51a3SRichard Henderson /* 4266db0c51a3SRichard Henderson * Reset the buffer pointers when restarting after overflow. 4267db0c51a3SRichard Henderson * TODO: Move this into translate-all.c with the rest of the 4268db0c51a3SRichard Henderson * buffer management. Having only this done here is confusing. 4269db0c51a3SRichard Henderson */ 4270db0c51a3SRichard Henderson s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 4271db0c51a3SRichard Henderson s->code_ptr = s->code_buf; 4272c896fe29Sbellard 4273659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 42746001f772SLaurent Vivier QSIMPLEQ_INIT(&s->ldst_labels); 4275659ef5cbSRichard Henderson #endif 427657a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 427757a26946SRichard Henderson s->pool_labels = NULL; 427857a26946SRichard Henderson #endif 42799ecefc84SRichard Henderson 4280fca8a500SRichard Henderson num_insns = -1; 428115fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 4282c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 4283b3db8758Sblueswir1 4284c896fe29Sbellard #ifdef CONFIG_PROFILER 4285d73415a3SStefan Hajnoczi qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 4286c896fe29Sbellard #endif 4287c45cb8bbSRichard Henderson 4288c896fe29Sbellard switch (opc) { 4289c896fe29Sbellard case INDEX_op_mov_i32: 4290c896fe29Sbellard case INDEX_op_mov_i64: 4291d2fd745fSRichard Henderson case INDEX_op_mov_vec: 4292dd186292SRichard Henderson tcg_reg_alloc_mov(s, op); 4293c896fe29Sbellard break; 4294bab1671fSRichard Henderson case INDEX_op_dup_vec: 4295bab1671fSRichard Henderson tcg_reg_alloc_dup(s, op); 4296bab1671fSRichard Henderson break; 4297765b842aSRichard Henderson case INDEX_op_insn_start: 4298fca8a500SRichard Henderson if (num_insns >= 0) { 42999f754620SRichard Henderson size_t off = tcg_current_code_size(s); 43009f754620SRichard Henderson s->gen_insn_end_off[num_insns] = off; 43019f754620SRichard Henderson /* Assert that we do not overflow our stored offset. */ 43029f754620SRichard Henderson assert(s->gen_insn_end_off[num_insns] == off); 4303fca8a500SRichard Henderson } 4304fca8a500SRichard Henderson num_insns++; 4305bad729e2SRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 4306bad729e2SRichard Henderson target_ulong a; 4307bad729e2SRichard Henderson #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 4308efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 4309bad729e2SRichard Henderson #else 4310efee3746SRichard Henderson a = op->args[i]; 4311bad729e2SRichard Henderson #endif 4312fca8a500SRichard Henderson s->gen_insn_data[num_insns][i] = a; 4313bad729e2SRichard Henderson } 4314c896fe29Sbellard break; 43155ff9d6a4Sbellard case INDEX_op_discard: 431643439139SRichard Henderson temp_dead(s, arg_temp(op->args[0])); 43175ff9d6a4Sbellard break; 4318c896fe29Sbellard case INDEX_op_set_label: 4319e8996ee0Sbellard tcg_reg_alloc_bb_end(s, s->reserved_regs); 432092ab8e7dSRichard Henderson tcg_out_label(s, arg_label(op->args[0])); 4321c896fe29Sbellard break; 4322c896fe29Sbellard case INDEX_op_call: 4323dd186292SRichard Henderson tcg_reg_alloc_call(s, op); 4324c45cb8bbSRichard Henderson break; 4325efe86b21SRichard Henderson case INDEX_op_dup2_vec: 4326efe86b21SRichard Henderson if (tcg_reg_alloc_dup2(s, op)) { 4327efe86b21SRichard Henderson break; 4328efe86b21SRichard Henderson } 4329efe86b21SRichard Henderson /* fall through */ 4330c896fe29Sbellard default: 433125c4d9ccSRichard Henderson /* Sanity check that we've not introduced any unhandled opcodes. */ 4332be0f34b5SRichard Henderson tcg_debug_assert(tcg_op_supported(opc)); 4333c896fe29Sbellard /* Note: in order to speed up the code, it would be much 4334c896fe29Sbellard faster to have specialized register allocator functions for 4335c896fe29Sbellard some common argument patterns */ 4336dd186292SRichard Henderson tcg_reg_alloc_op(s, op); 4337c896fe29Sbellard break; 4338c896fe29Sbellard } 43398d8fdbaeSAurelien Jarno #ifdef CONFIG_DEBUG_TCG 4340c896fe29Sbellard check_regs(s); 4341c896fe29Sbellard #endif 4342b125f9dcSRichard Henderson /* Test for (pending) buffer overflow. The assumption is that any 4343b125f9dcSRichard Henderson one operation beginning below the high water mark cannot overrun 4344b125f9dcSRichard Henderson the buffer completely. Thus we can test for overflow after 4345b125f9dcSRichard Henderson generating code without having to check during generation. */ 4346644da9b3SJohn Clarke if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 4347b125f9dcSRichard Henderson return -1; 4348b125f9dcSRichard Henderson } 43496e6c4efeSRichard Henderson /* Test for TB overflow, as seen by gen_insn_end_off. */ 43506e6c4efeSRichard Henderson if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 43516e6c4efeSRichard Henderson return -2; 43526e6c4efeSRichard Henderson } 4353c896fe29Sbellard } 4354fca8a500SRichard Henderson tcg_debug_assert(num_insns >= 0); 4355fca8a500SRichard Henderson s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 4356c45cb8bbSRichard Henderson 4357b76f0d8cSYeongkyoon Lee /* Generate TB finalization at the end of block */ 4358659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 4359aeee05f5SRichard Henderson i = tcg_out_ldst_finalize(s); 4360aeee05f5SRichard Henderson if (i < 0) { 4361aeee05f5SRichard Henderson return i; 436223dceda6SRichard Henderson } 4363659ef5cbSRichard Henderson #endif 436457a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 43651768987bSRichard Henderson i = tcg_out_pool_finalize(s); 43661768987bSRichard Henderson if (i < 0) { 43671768987bSRichard Henderson return i; 436857a26946SRichard Henderson } 436957a26946SRichard Henderson #endif 43707ecd02a0SRichard Henderson if (!tcg_resolve_relocs(s)) { 43717ecd02a0SRichard Henderson return -2; 43727ecd02a0SRichard Henderson } 4373c896fe29Sbellard 4374df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 4375c896fe29Sbellard /* flush instruction cache */ 4376db0c51a3SRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 4377db0c51a3SRichard Henderson (uintptr_t)s->code_buf, 43781da8de39SRichard Henderson tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 4379df5d2b16SRichard Henderson #endif 43802aeabc08SStefan Weil 43811813e175SRichard Henderson return tcg_current_code_size(s); 4382c896fe29Sbellard } 4383c896fe29Sbellard 4384a23a9ec6Sbellard #ifdef CONFIG_PROFILER 43853de2faa9SMarkus Armbruster void tcg_dump_info(void) 4386a23a9ec6Sbellard { 4387c3fac113SEmilio G. Cota TCGProfile prof = {}; 4388c3fac113SEmilio G. Cota const TCGProfile *s; 4389c3fac113SEmilio G. Cota int64_t tb_count; 4390c3fac113SEmilio G. Cota int64_t tb_div_count; 4391c3fac113SEmilio G. Cota int64_t tot; 4392c3fac113SEmilio G. Cota 4393c3fac113SEmilio G. Cota tcg_profile_snapshot_counters(&prof); 4394c3fac113SEmilio G. Cota s = &prof; 4395c3fac113SEmilio G. Cota tb_count = s->tb_count; 4396c3fac113SEmilio G. Cota tb_div_count = tb_count ? tb_count : 1; 4397c3fac113SEmilio G. Cota tot = s->interm_time + s->code_time; 4398a23a9ec6Sbellard 43993de2faa9SMarkus Armbruster qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 4400a23a9ec6Sbellard tot, tot / 2.4e9); 44013de2faa9SMarkus Armbruster qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64 44023de2faa9SMarkus Armbruster " %0.1f%%)\n", 4403fca8a500SRichard Henderson tb_count, s->tb_count1 - tb_count, 4404fca8a500SRichard Henderson (double)(s->tb_count1 - s->tb_count) 4405fca8a500SRichard Henderson / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 44063de2faa9SMarkus Armbruster qemu_printf("avg ops/TB %0.1f max=%d\n", 4407fca8a500SRichard Henderson (double)s->op_count / tb_div_count, s->op_count_max); 44083de2faa9SMarkus Armbruster qemu_printf("deleted ops/TB %0.2f\n", 4409fca8a500SRichard Henderson (double)s->del_op_count / tb_div_count); 44103de2faa9SMarkus Armbruster qemu_printf("avg temps/TB %0.2f max=%d\n", 4411fca8a500SRichard Henderson (double)s->temp_count / tb_div_count, s->temp_count_max); 44123de2faa9SMarkus Armbruster qemu_printf("avg host code/TB %0.1f\n", 4413fca8a500SRichard Henderson (double)s->code_out_len / tb_div_count); 44143de2faa9SMarkus Armbruster qemu_printf("avg search data/TB %0.1f\n", 4415fca8a500SRichard Henderson (double)s->search_out_len / tb_div_count); 4416a23a9ec6Sbellard 44173de2faa9SMarkus Armbruster qemu_printf("cycles/op %0.1f\n", 4418a23a9ec6Sbellard s->op_count ? (double)tot / s->op_count : 0); 44193de2faa9SMarkus Armbruster qemu_printf("cycles/in byte %0.1f\n", 4420a23a9ec6Sbellard s->code_in_len ? (double)tot / s->code_in_len : 0); 44213de2faa9SMarkus Armbruster qemu_printf("cycles/out byte %0.1f\n", 4422a23a9ec6Sbellard s->code_out_len ? (double)tot / s->code_out_len : 0); 44233de2faa9SMarkus Armbruster qemu_printf("cycles/search byte %0.1f\n", 4424fca8a500SRichard Henderson s->search_out_len ? (double)tot / s->search_out_len : 0); 4425fca8a500SRichard Henderson if (tot == 0) { 4426a23a9ec6Sbellard tot = 1; 4427fca8a500SRichard Henderson } 44283de2faa9SMarkus Armbruster qemu_printf(" gen_interm time %0.1f%%\n", 4429a23a9ec6Sbellard (double)s->interm_time / tot * 100.0); 44303de2faa9SMarkus Armbruster qemu_printf(" gen_code time %0.1f%%\n", 4431a23a9ec6Sbellard (double)s->code_time / tot * 100.0); 44323de2faa9SMarkus Armbruster qemu_printf("optim./code time %0.1f%%\n", 4433c5cc28ffSAurelien Jarno (double)s->opt_time / (s->code_time ? s->code_time : 1) 4434c5cc28ffSAurelien Jarno * 100.0); 44353de2faa9SMarkus Armbruster qemu_printf("liveness/code time %0.1f%%\n", 4436a23a9ec6Sbellard (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 44373de2faa9SMarkus Armbruster qemu_printf("cpu_restore count %" PRId64 "\n", 4438a23a9ec6Sbellard s->restore_count); 44393de2faa9SMarkus Armbruster qemu_printf(" avg cycles %0.1f\n", 4440a23a9ec6Sbellard s->restore_count ? (double)s->restore_time / s->restore_count : 0); 4441a23a9ec6Sbellard } 4442a23a9ec6Sbellard #else 44433de2faa9SMarkus Armbruster void tcg_dump_info(void) 4444a23a9ec6Sbellard { 44453de2faa9SMarkus Armbruster qemu_printf("[TCG profiler not compiled]\n"); 4446a23a9ec6Sbellard } 4447a23a9ec6Sbellard #endif 4448813da627SRichard Henderson 4449813da627SRichard Henderson #ifdef ELF_HOST_MACHINE 44505872bbf2SRichard Henderson /* In order to use this feature, the backend needs to do three things: 44515872bbf2SRichard Henderson 44525872bbf2SRichard Henderson (1) Define ELF_HOST_MACHINE to indicate both what value to 44535872bbf2SRichard Henderson put into the ELF image and to indicate support for the feature. 44545872bbf2SRichard Henderson 44555872bbf2SRichard Henderson (2) Define tcg_register_jit. This should create a buffer containing 44565872bbf2SRichard Henderson the contents of a .debug_frame section that describes the post- 44575872bbf2SRichard Henderson prologue unwind info for the tcg machine. 44585872bbf2SRichard Henderson 44595872bbf2SRichard Henderson (3) Call tcg_register_jit_int, with the constructed .debug_frame. 44605872bbf2SRichard Henderson */ 4461813da627SRichard Henderson 4462813da627SRichard Henderson /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 4463813da627SRichard Henderson typedef enum { 4464813da627SRichard Henderson JIT_NOACTION = 0, 4465813da627SRichard Henderson JIT_REGISTER_FN, 4466813da627SRichard Henderson JIT_UNREGISTER_FN 4467813da627SRichard Henderson } jit_actions_t; 4468813da627SRichard Henderson 4469813da627SRichard Henderson struct jit_code_entry { 4470813da627SRichard Henderson struct jit_code_entry *next_entry; 4471813da627SRichard Henderson struct jit_code_entry *prev_entry; 4472813da627SRichard Henderson const void *symfile_addr; 4473813da627SRichard Henderson uint64_t symfile_size; 4474813da627SRichard Henderson }; 4475813da627SRichard Henderson 4476813da627SRichard Henderson struct jit_descriptor { 4477813da627SRichard Henderson uint32_t version; 4478813da627SRichard Henderson uint32_t action_flag; 4479813da627SRichard Henderson struct jit_code_entry *relevant_entry; 4480813da627SRichard Henderson struct jit_code_entry *first_entry; 4481813da627SRichard Henderson }; 4482813da627SRichard Henderson 4483813da627SRichard Henderson void __jit_debug_register_code(void) __attribute__((noinline)); 4484813da627SRichard Henderson void __jit_debug_register_code(void) 4485813da627SRichard Henderson { 4486813da627SRichard Henderson asm(""); 4487813da627SRichard Henderson } 4488813da627SRichard Henderson 4489813da627SRichard Henderson /* Must statically initialize the version, because GDB may check 4490813da627SRichard Henderson the version before we can set it. */ 4491813da627SRichard Henderson struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 4492813da627SRichard Henderson 4493813da627SRichard Henderson /* End GDB interface. */ 4494813da627SRichard Henderson 4495813da627SRichard Henderson static int find_string(const char *strtab, const char *str) 4496813da627SRichard Henderson { 4497813da627SRichard Henderson const char *p = strtab + 1; 4498813da627SRichard Henderson 4499813da627SRichard Henderson while (1) { 4500813da627SRichard Henderson if (strcmp(p, str) == 0) { 4501813da627SRichard Henderson return p - strtab; 4502813da627SRichard Henderson } 4503813da627SRichard Henderson p += strlen(p) + 1; 4504813da627SRichard Henderson } 4505813da627SRichard Henderson } 4506813da627SRichard Henderson 4507755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 45082c90784aSRichard Henderson const void *debug_frame, 45092c90784aSRichard Henderson size_t debug_frame_size) 4510813da627SRichard Henderson { 45115872bbf2SRichard Henderson struct __attribute__((packed)) DebugInfo { 45125872bbf2SRichard Henderson uint32_t len; 45135872bbf2SRichard Henderson uint16_t version; 45145872bbf2SRichard Henderson uint32_t abbrev; 45155872bbf2SRichard Henderson uint8_t ptr_size; 45165872bbf2SRichard Henderson uint8_t cu_die; 45175872bbf2SRichard Henderson uint16_t cu_lang; 45185872bbf2SRichard Henderson uintptr_t cu_low_pc; 45195872bbf2SRichard Henderson uintptr_t cu_high_pc; 45205872bbf2SRichard Henderson uint8_t fn_die; 45215872bbf2SRichard Henderson char fn_name[16]; 45225872bbf2SRichard Henderson uintptr_t fn_low_pc; 45235872bbf2SRichard Henderson uintptr_t fn_high_pc; 45245872bbf2SRichard Henderson uint8_t cu_eoc; 45255872bbf2SRichard Henderson }; 4526813da627SRichard Henderson 4527813da627SRichard Henderson struct ElfImage { 4528813da627SRichard Henderson ElfW(Ehdr) ehdr; 4529813da627SRichard Henderson ElfW(Phdr) phdr; 45305872bbf2SRichard Henderson ElfW(Shdr) shdr[7]; 45315872bbf2SRichard Henderson ElfW(Sym) sym[2]; 45325872bbf2SRichard Henderson struct DebugInfo di; 45335872bbf2SRichard Henderson uint8_t da[24]; 45345872bbf2SRichard Henderson char str[80]; 45355872bbf2SRichard Henderson }; 45365872bbf2SRichard Henderson 45375872bbf2SRichard Henderson struct ElfImage *img; 45385872bbf2SRichard Henderson 45395872bbf2SRichard Henderson static const struct ElfImage img_template = { 45405872bbf2SRichard Henderson .ehdr = { 45415872bbf2SRichard Henderson .e_ident[EI_MAG0] = ELFMAG0, 45425872bbf2SRichard Henderson .e_ident[EI_MAG1] = ELFMAG1, 45435872bbf2SRichard Henderson .e_ident[EI_MAG2] = ELFMAG2, 45445872bbf2SRichard Henderson .e_ident[EI_MAG3] = ELFMAG3, 45455872bbf2SRichard Henderson .e_ident[EI_CLASS] = ELF_CLASS, 45465872bbf2SRichard Henderson .e_ident[EI_DATA] = ELF_DATA, 45475872bbf2SRichard Henderson .e_ident[EI_VERSION] = EV_CURRENT, 45485872bbf2SRichard Henderson .e_type = ET_EXEC, 45495872bbf2SRichard Henderson .e_machine = ELF_HOST_MACHINE, 45505872bbf2SRichard Henderson .e_version = EV_CURRENT, 45515872bbf2SRichard Henderson .e_phoff = offsetof(struct ElfImage, phdr), 45525872bbf2SRichard Henderson .e_shoff = offsetof(struct ElfImage, shdr), 45535872bbf2SRichard Henderson .e_ehsize = sizeof(ElfW(Shdr)), 45545872bbf2SRichard Henderson .e_phentsize = sizeof(ElfW(Phdr)), 45555872bbf2SRichard Henderson .e_phnum = 1, 45565872bbf2SRichard Henderson .e_shentsize = sizeof(ElfW(Shdr)), 45575872bbf2SRichard Henderson .e_shnum = ARRAY_SIZE(img->shdr), 45585872bbf2SRichard Henderson .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 4559abbb3eaeSRichard Henderson #ifdef ELF_HOST_FLAGS 4560abbb3eaeSRichard Henderson .e_flags = ELF_HOST_FLAGS, 4561abbb3eaeSRichard Henderson #endif 4562abbb3eaeSRichard Henderson #ifdef ELF_OSABI 4563abbb3eaeSRichard Henderson .e_ident[EI_OSABI] = ELF_OSABI, 4564abbb3eaeSRichard Henderson #endif 45655872bbf2SRichard Henderson }, 45665872bbf2SRichard Henderson .phdr = { 45675872bbf2SRichard Henderson .p_type = PT_LOAD, 45685872bbf2SRichard Henderson .p_flags = PF_X, 45695872bbf2SRichard Henderson }, 45705872bbf2SRichard Henderson .shdr = { 45715872bbf2SRichard Henderson [0] = { .sh_type = SHT_NULL }, 45725872bbf2SRichard Henderson /* Trick: The contents of code_gen_buffer are not present in 45735872bbf2SRichard Henderson this fake ELF file; that got allocated elsewhere. Therefore 45745872bbf2SRichard Henderson we mark .text as SHT_NOBITS (similar to .bss) so that readers 45755872bbf2SRichard Henderson will not look for contents. We can record any address. */ 45765872bbf2SRichard Henderson [1] = { /* .text */ 45775872bbf2SRichard Henderson .sh_type = SHT_NOBITS, 45785872bbf2SRichard Henderson .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 45795872bbf2SRichard Henderson }, 45805872bbf2SRichard Henderson [2] = { /* .debug_info */ 45815872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 45825872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, di), 45835872bbf2SRichard Henderson .sh_size = sizeof(struct DebugInfo), 45845872bbf2SRichard Henderson }, 45855872bbf2SRichard Henderson [3] = { /* .debug_abbrev */ 45865872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 45875872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, da), 45885872bbf2SRichard Henderson .sh_size = sizeof(img->da), 45895872bbf2SRichard Henderson }, 45905872bbf2SRichard Henderson [4] = { /* .debug_frame */ 45915872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 45925872bbf2SRichard Henderson .sh_offset = sizeof(struct ElfImage), 45935872bbf2SRichard Henderson }, 45945872bbf2SRichard Henderson [5] = { /* .symtab */ 45955872bbf2SRichard Henderson .sh_type = SHT_SYMTAB, 45965872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, sym), 45975872bbf2SRichard Henderson .sh_size = sizeof(img->sym), 45985872bbf2SRichard Henderson .sh_info = 1, 45995872bbf2SRichard Henderson .sh_link = ARRAY_SIZE(img->shdr) - 1, 46005872bbf2SRichard Henderson .sh_entsize = sizeof(ElfW(Sym)), 46015872bbf2SRichard Henderson }, 46025872bbf2SRichard Henderson [6] = { /* .strtab */ 46035872bbf2SRichard Henderson .sh_type = SHT_STRTAB, 46045872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, str), 46055872bbf2SRichard Henderson .sh_size = sizeof(img->str), 46065872bbf2SRichard Henderson } 46075872bbf2SRichard Henderson }, 46085872bbf2SRichard Henderson .sym = { 46095872bbf2SRichard Henderson [1] = { /* code_gen_buffer */ 46105872bbf2SRichard Henderson .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 46115872bbf2SRichard Henderson .st_shndx = 1, 46125872bbf2SRichard Henderson } 46135872bbf2SRichard Henderson }, 46145872bbf2SRichard Henderson .di = { 46155872bbf2SRichard Henderson .len = sizeof(struct DebugInfo) - 4, 46165872bbf2SRichard Henderson .version = 2, 46175872bbf2SRichard Henderson .ptr_size = sizeof(void *), 46185872bbf2SRichard Henderson .cu_die = 1, 46195872bbf2SRichard Henderson .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 46205872bbf2SRichard Henderson .fn_die = 2, 46215872bbf2SRichard Henderson .fn_name = "code_gen_buffer" 46225872bbf2SRichard Henderson }, 46235872bbf2SRichard Henderson .da = { 46245872bbf2SRichard Henderson 1, /* abbrev number (the cu) */ 46255872bbf2SRichard Henderson 0x11, 1, /* DW_TAG_compile_unit, has children */ 46265872bbf2SRichard Henderson 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 46275872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 46285872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 46295872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 46305872bbf2SRichard Henderson 2, /* abbrev number (the fn) */ 46315872bbf2SRichard Henderson 0x2e, 0, /* DW_TAG_subprogram, no children */ 46325872bbf2SRichard Henderson 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 46335872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 46345872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 46355872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 46365872bbf2SRichard Henderson 0 /* no more abbrev */ 46375872bbf2SRichard Henderson }, 46385872bbf2SRichard Henderson .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 46395872bbf2SRichard Henderson ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 4640813da627SRichard Henderson }; 4641813da627SRichard Henderson 4642813da627SRichard Henderson /* We only need a single jit entry; statically allocate it. */ 4643813da627SRichard Henderson static struct jit_code_entry one_entry; 4644813da627SRichard Henderson 46455872bbf2SRichard Henderson uintptr_t buf = (uintptr_t)buf_ptr; 4646813da627SRichard Henderson size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 46472c90784aSRichard Henderson DebugFrameHeader *dfh; 4648813da627SRichard Henderson 46495872bbf2SRichard Henderson img = g_malloc(img_size); 46505872bbf2SRichard Henderson *img = img_template; 4651813da627SRichard Henderson 46525872bbf2SRichard Henderson img->phdr.p_vaddr = buf; 46535872bbf2SRichard Henderson img->phdr.p_paddr = buf; 46545872bbf2SRichard Henderson img->phdr.p_memsz = buf_size; 4655813da627SRichard Henderson 46565872bbf2SRichard Henderson img->shdr[1].sh_name = find_string(img->str, ".text"); 46575872bbf2SRichard Henderson img->shdr[1].sh_addr = buf; 46585872bbf2SRichard Henderson img->shdr[1].sh_size = buf_size; 4659813da627SRichard Henderson 46605872bbf2SRichard Henderson img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 46615872bbf2SRichard Henderson img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 46625872bbf2SRichard Henderson 46635872bbf2SRichard Henderson img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 46645872bbf2SRichard Henderson img->shdr[4].sh_size = debug_frame_size; 46655872bbf2SRichard Henderson 46665872bbf2SRichard Henderson img->shdr[5].sh_name = find_string(img->str, ".symtab"); 46675872bbf2SRichard Henderson img->shdr[6].sh_name = find_string(img->str, ".strtab"); 46685872bbf2SRichard Henderson 46695872bbf2SRichard Henderson img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 46705872bbf2SRichard Henderson img->sym[1].st_value = buf; 46715872bbf2SRichard Henderson img->sym[1].st_size = buf_size; 46725872bbf2SRichard Henderson 46735872bbf2SRichard Henderson img->di.cu_low_pc = buf; 467445aba097SRichard Henderson img->di.cu_high_pc = buf + buf_size; 46755872bbf2SRichard Henderson img->di.fn_low_pc = buf; 467645aba097SRichard Henderson img->di.fn_high_pc = buf + buf_size; 4677813da627SRichard Henderson 46782c90784aSRichard Henderson dfh = (DebugFrameHeader *)(img + 1); 46792c90784aSRichard Henderson memcpy(dfh, debug_frame, debug_frame_size); 46802c90784aSRichard Henderson dfh->fde.func_start = buf; 46812c90784aSRichard Henderson dfh->fde.func_len = buf_size; 46822c90784aSRichard Henderson 4683813da627SRichard Henderson #ifdef DEBUG_JIT 4684813da627SRichard Henderson /* Enable this block to be able to debug the ELF image file creation. 4685813da627SRichard Henderson One can use readelf, objdump, or other inspection utilities. */ 4686813da627SRichard Henderson { 4687813da627SRichard Henderson FILE *f = fopen("/tmp/qemu.jit", "w+b"); 4688813da627SRichard Henderson if (f) { 46895872bbf2SRichard Henderson if (fwrite(img, img_size, 1, f) != img_size) { 4690813da627SRichard Henderson /* Avoid stupid unused return value warning for fwrite. */ 4691813da627SRichard Henderson } 4692813da627SRichard Henderson fclose(f); 4693813da627SRichard Henderson } 4694813da627SRichard Henderson } 4695813da627SRichard Henderson #endif 4696813da627SRichard Henderson 4697813da627SRichard Henderson one_entry.symfile_addr = img; 4698813da627SRichard Henderson one_entry.symfile_size = img_size; 4699813da627SRichard Henderson 4700813da627SRichard Henderson __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 4701813da627SRichard Henderson __jit_debug_descriptor.relevant_entry = &one_entry; 4702813da627SRichard Henderson __jit_debug_descriptor.first_entry = &one_entry; 4703813da627SRichard Henderson __jit_debug_register_code(); 4704813da627SRichard Henderson } 4705813da627SRichard Henderson #else 47065872bbf2SRichard Henderson /* No support for the feature. Provide the entry point expected by exec.c, 47075872bbf2SRichard Henderson and implement the internal function we declared earlier. */ 4708813da627SRichard Henderson 4709755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 47102c90784aSRichard Henderson const void *debug_frame, 47112c90784aSRichard Henderson size_t debug_frame_size) 4712813da627SRichard Henderson { 4713813da627SRichard Henderson } 4714813da627SRichard Henderson 4715755bf9e5SRichard Henderson void tcg_register_jit(const void *buf, size_t buf_size) 4716813da627SRichard Henderson { 4717813da627SRichard Henderson } 4718813da627SRichard Henderson #endif /* ELF_HOST_MACHINE */ 4719db432672SRichard Henderson 4720db432672SRichard Henderson #if !TCG_TARGET_MAYBE_vec 4721db432672SRichard Henderson void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 4722db432672SRichard Henderson { 4723db432672SRichard Henderson g_assert_not_reached(); 4724db432672SRichard Henderson } 4725db432672SRichard Henderson #endif 4726