1c896fe29Sbellard /* 2c896fe29Sbellard * Tiny Code Generator for QEMU 3c896fe29Sbellard * 4c896fe29Sbellard * Copyright (c) 2008 Fabrice Bellard 5c896fe29Sbellard * 6c896fe29Sbellard * Permission is hereby granted, free of charge, to any person obtaining a copy 7c896fe29Sbellard * of this software and associated documentation files (the "Software"), to deal 8c896fe29Sbellard * in the Software without restriction, including without limitation the rights 9c896fe29Sbellard * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10c896fe29Sbellard * copies of the Software, and to permit persons to whom the Software is 11c896fe29Sbellard * furnished to do so, subject to the following conditions: 12c896fe29Sbellard * 13c896fe29Sbellard * The above copyright notice and this permission notice shall be included in 14c896fe29Sbellard * all copies or substantial portions of the Software. 15c896fe29Sbellard * 16c896fe29Sbellard * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17c896fe29Sbellard * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18c896fe29Sbellard * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19c896fe29Sbellard * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20c896fe29Sbellard * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21c896fe29Sbellard * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22c896fe29Sbellard * THE SOFTWARE. 23c896fe29Sbellard */ 24c896fe29Sbellard 25c896fe29Sbellard /* define it to use liveness analysis (better code) */ 268f2e8c07SKirill Batuzov #define USE_TCG_OPTIMIZATIONS 27c896fe29Sbellard 28757e725bSPeter Maydell #include "qemu/osdep.h" 29cca82982Saurel32 30813da627SRichard Henderson /* Define to jump the ELF file used to communicate with GDB. */ 31813da627SRichard Henderson #undef DEBUG_JIT 32813da627SRichard Henderson 3372fd2efbSEmilio G. Cota #include "qemu/error-report.h" 34f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 351de7afc9SPaolo Bonzini #include "qemu/host-utils.h" 36d4c51a0aSMarkus Armbruster #include "qemu/qemu-print.h" 37084cfca1SRichard Henderson #include "qemu/cacheflush.h" 38ad768e6fSPeter Maydell #include "qemu/cacheinfo.h" 39533206f0SRichard W.M. Jones #include "qemu/timer.h" 40c896fe29Sbellard 41c5d3c498SStefan Weil /* Note: the long term plan is to reduce the dependencies on the QEMU 42c896fe29Sbellard CPU definitions. Currently they are used for qemu_ld/st 43c896fe29Sbellard instructions */ 44c896fe29Sbellard #define NO_CPU_IO_DEFS 45c896fe29Sbellard 4663c91552SPaolo Bonzini #include "exec/exec-all.h" 47dcb32f1dSPhilippe Mathieu-Daudé #include "tcg/tcg-op.h" 48813da627SRichard Henderson 49edee2579SRichard Henderson #if UINTPTR_MAX == UINT32_MAX 50813da627SRichard Henderson # define ELF_CLASS ELFCLASS32 51edee2579SRichard Henderson #else 52edee2579SRichard Henderson # define ELF_CLASS ELFCLASS64 53813da627SRichard Henderson #endif 54e03b5686SMarc-André Lureau #if HOST_BIG_ENDIAN 55813da627SRichard Henderson # define ELF_DATA ELFDATA2MSB 56813da627SRichard Henderson #else 57813da627SRichard Henderson # define ELF_DATA ELFDATA2LSB 58813da627SRichard Henderson #endif 59813da627SRichard Henderson 60c896fe29Sbellard #include "elf.h" 61508127e2SPaolo Bonzini #include "exec/log.h" 62d2ba8026SRichard Henderson #include "tcg/tcg-ldst.h" 6347f7313dSRichard Henderson #include "tcg/tcg-temp-internal.h" 645ff7258cSRichard Henderson #include "tcg-internal.h" 655584e2dbSIlya Leoshkevich #include "accel/tcg/perf.h" 66c896fe29Sbellard 67139c1837SPaolo Bonzini /* Forward declarations for functions declared in tcg-target.c.inc and 68ce151109SPeter Maydell used here. */ 69e4d58b41SRichard Henderson static void tcg_target_init(TCGContext *s); 70e4d58b41SRichard Henderson static void tcg_target_qemu_prologue(TCGContext *s); 716ac17786SRichard Henderson static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 722ba7fae2SRichard Henderson intptr_t value, intptr_t addend); 73c896fe29Sbellard 74497a22ebSRichard Henderson /* The CIE and FDE header definitions will be common to all hosts. */ 75497a22ebSRichard Henderson typedef struct { 76497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 77497a22ebSRichard Henderson uint32_t id; 78497a22ebSRichard Henderson uint8_t version; 79497a22ebSRichard Henderson char augmentation[1]; 80497a22ebSRichard Henderson uint8_t code_align; 81497a22ebSRichard Henderson uint8_t data_align; 82497a22ebSRichard Henderson uint8_t return_column; 83497a22ebSRichard Henderson } DebugFrameCIE; 84497a22ebSRichard Henderson 85497a22ebSRichard Henderson typedef struct QEMU_PACKED { 86497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 87497a22ebSRichard Henderson uint32_t cie_offset; 88edee2579SRichard Henderson uintptr_t func_start; 89edee2579SRichard Henderson uintptr_t func_len; 90497a22ebSRichard Henderson } DebugFrameFDEHeader; 91497a22ebSRichard Henderson 922c90784aSRichard Henderson typedef struct QEMU_PACKED { 932c90784aSRichard Henderson DebugFrameCIE cie; 942c90784aSRichard Henderson DebugFrameFDEHeader fde; 952c90784aSRichard Henderson } DebugFrameHeader; 962c90784aSRichard Henderson 972528f771SRichard Henderson typedef struct TCGLabelQemuLdst { 982528f771SRichard Henderson bool is_ld; /* qemu_ld: true, qemu_st: false */ 992528f771SRichard Henderson MemOpIdx oi; 1002528f771SRichard Henderson TCGType type; /* result type of a load */ 1012528f771SRichard Henderson TCGReg addrlo_reg; /* reg index for low word of guest virtual addr */ 1022528f771SRichard Henderson TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */ 1032528f771SRichard Henderson TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ 1042528f771SRichard Henderson TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ 1052528f771SRichard Henderson const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ 1062528f771SRichard Henderson tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ 1072528f771SRichard Henderson QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; 1082528f771SRichard Henderson } TCGLabelQemuLdst; 1092528f771SRichard Henderson 110755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 1112c90784aSRichard Henderson const void *debug_frame, 1122c90784aSRichard Henderson size_t debug_frame_size) 113813da627SRichard Henderson __attribute__((unused)); 114813da627SRichard Henderson 115139c1837SPaolo Bonzini /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 1162a534affSRichard Henderson static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 117a05b5b9bSRichard Henderson intptr_t arg2); 11878113e83SRichard Henderson static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 119c0ad3001SStefan Weil static void tcg_out_movi(TCGContext *s, TCGType type, 1202a534affSRichard Henderson TCGReg ret, tcg_target_long arg); 121678155b2SRichard Henderson static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 122753e42eaSRichard Henderson static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 123d0e66c89SRichard Henderson static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg); 124379afdffSRichard Henderson static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg); 12552bf3398SRichard Henderson static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg); 1269ecf5f61SRichard Henderson static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg); 1279c6aa274SRichard Henderson static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); 128b9bfe000SRichard Henderson static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); 129b8b94ac6SRichard Henderson static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg); 130313bdea8SRichard Henderson static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); 131129f1f9eSRichard Henderson static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2); 132b55a8d9dSRichard Henderson static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); 133cf7d6b8eSRichard Henderson static void tcg_out_goto_tb(TCGContext *s, int which); 1345e8892dbSMiroslav Rezanina static void tcg_out_op(TCGContext *s, TCGOpcode opc, 1355e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1365e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 137d2fd745fSRichard Henderson #if TCG_TARGET_MAYBE_vec 138e7632cfaSRichard Henderson static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 139e7632cfaSRichard Henderson TCGReg dst, TCGReg src); 140d6ecb4a9SRichard Henderson static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 141d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset); 1424e186175SRichard Henderson static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1434e186175SRichard Henderson TCGReg dst, int64_t arg); 1445e8892dbSMiroslav Rezanina static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1455e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1465e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1475e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 148d2fd745fSRichard Henderson #else 149e7632cfaSRichard Henderson static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 150e7632cfaSRichard Henderson TCGReg dst, TCGReg src) 151e7632cfaSRichard Henderson { 152e7632cfaSRichard Henderson g_assert_not_reached(); 153e7632cfaSRichard Henderson } 154d6ecb4a9SRichard Henderson static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 155d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset) 156d6ecb4a9SRichard Henderson { 157d6ecb4a9SRichard Henderson g_assert_not_reached(); 158d6ecb4a9SRichard Henderson } 1594e186175SRichard Henderson static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1604e186175SRichard Henderson TCGReg dst, int64_t arg) 161e7632cfaSRichard Henderson { 162e7632cfaSRichard Henderson g_assert_not_reached(); 163e7632cfaSRichard Henderson } 1645e8892dbSMiroslav Rezanina static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1655e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1665e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1675e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]) 168d2fd745fSRichard Henderson { 169d2fd745fSRichard Henderson g_assert_not_reached(); 170d2fd745fSRichard Henderson } 171d2fd745fSRichard Henderson #endif 1722a534affSRichard Henderson static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 173a05b5b9bSRichard Henderson intptr_t arg2); 17459d7c14eSRichard Henderson static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 17559d7c14eSRichard Henderson TCGReg base, intptr_t ofs); 1767b7d8b2dSRichard Henderson static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 177cee44b03SRichard Henderson const TCGHelperInfo *info); 1785e3d0c19SRichard Henderson static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); 179a4fbbd77SRichard Henderson static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 180659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 181aeee05f5SRichard Henderson static int tcg_out_ldst_finalize(TCGContext *s); 182659ef5cbSRichard Henderson #endif 183c896fe29Sbellard 184*8429a1caSRichard Henderson typedef struct TCGLdstHelperParam { 185*8429a1caSRichard Henderson TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg); 186*8429a1caSRichard Henderson unsigned ntmp; 187*8429a1caSRichard Henderson int tmp[3]; 188*8429a1caSRichard Henderson } TCGLdstHelperParam; 189*8429a1caSRichard Henderson 190*8429a1caSRichard Henderson static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, 191*8429a1caSRichard Henderson const TCGLdstHelperParam *p) 192*8429a1caSRichard Henderson __attribute__((unused)); 193*8429a1caSRichard Henderson static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l, 194*8429a1caSRichard Henderson bool load_sign, const TCGLdstHelperParam *p) 195*8429a1caSRichard Henderson __attribute__((unused)); 196*8429a1caSRichard Henderson static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, 197*8429a1caSRichard Henderson const TCGLdstHelperParam *p) 198*8429a1caSRichard Henderson __attribute__((unused)); 199*8429a1caSRichard Henderson 20042eb6dfcSRichard Henderson TCGContext tcg_init_ctx; 20142eb6dfcSRichard Henderson __thread TCGContext *tcg_ctx; 20242eb6dfcSRichard Henderson 2035ff7258cSRichard Henderson TCGContext **tcg_ctxs; 2040e2d61cfSRichard Henderson unsigned int tcg_cur_ctxs; 2050e2d61cfSRichard Henderson unsigned int tcg_max_ctxs; 2061c2adb95SRichard Henderson TCGv_env cpu_env = 0; 207c8bc1168SRichard Henderson const void *tcg_code_gen_epilogue; 208db0c51a3SRichard Henderson uintptr_t tcg_splitwx_diff; 209df2cce29SEmilio G. Cota 210b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 211b91ccb31SRichard Henderson tcg_prologue_fn *tcg_qemu_tb_exec; 212b91ccb31SRichard Henderson #endif 213b91ccb31SRichard Henderson 214d2fd745fSRichard Henderson static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 215b1d8e52eSblueswir1 static TCGRegSet tcg_target_call_clobber_regs; 216c896fe29Sbellard 2171813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE == 1 2184196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 219c896fe29Sbellard { 220c896fe29Sbellard *s->code_ptr++ = v; 221c896fe29Sbellard } 222c896fe29Sbellard 2234196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 2244196dca6SPeter Maydell uint8_t v) 2255c53bb81SPeter Maydell { 2261813e175SRichard Henderson *p = v; 2275c53bb81SPeter Maydell } 2281813e175SRichard Henderson #endif 2295c53bb81SPeter Maydell 2301813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 2 2314196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 232c896fe29Sbellard { 2331813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 2341813e175SRichard Henderson *s->code_ptr++ = v; 2351813e175SRichard Henderson } else { 2361813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2374387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2381813e175SRichard Henderson s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 2391813e175SRichard Henderson } 240c896fe29Sbellard } 241c896fe29Sbellard 2424196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 2434196dca6SPeter Maydell uint16_t v) 2445c53bb81SPeter Maydell { 2451813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 2461813e175SRichard Henderson *p = v; 2471813e175SRichard Henderson } else { 2485c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2495c53bb81SPeter Maydell } 2501813e175SRichard Henderson } 2511813e175SRichard Henderson #endif 2525c53bb81SPeter Maydell 2531813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 4 2544196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 255c896fe29Sbellard { 2561813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2571813e175SRichard Henderson *s->code_ptr++ = v; 2581813e175SRichard Henderson } else { 2591813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2604387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2611813e175SRichard Henderson s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 2621813e175SRichard Henderson } 263c896fe29Sbellard } 264c896fe29Sbellard 2654196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 2664196dca6SPeter Maydell uint32_t v) 2675c53bb81SPeter Maydell { 2681813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2691813e175SRichard Henderson *p = v; 2701813e175SRichard Henderson } else { 2715c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2725c53bb81SPeter Maydell } 2731813e175SRichard Henderson } 2741813e175SRichard Henderson #endif 2755c53bb81SPeter Maydell 2761813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 8 2774196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 278ac26eb69SRichard Henderson { 2791813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2801813e175SRichard Henderson *s->code_ptr++ = v; 2811813e175SRichard Henderson } else { 2821813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2834387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2841813e175SRichard Henderson s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 2851813e175SRichard Henderson } 286ac26eb69SRichard Henderson } 287ac26eb69SRichard Henderson 2884196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 2894196dca6SPeter Maydell uint64_t v) 2905c53bb81SPeter Maydell { 2911813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2921813e175SRichard Henderson *p = v; 2931813e175SRichard Henderson } else { 2945c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2955c53bb81SPeter Maydell } 2961813e175SRichard Henderson } 2971813e175SRichard Henderson #endif 2985c53bb81SPeter Maydell 299c896fe29Sbellard /* label relocation processing */ 300c896fe29Sbellard 3011813e175SRichard Henderson static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 302bec16311SRichard Henderson TCGLabel *l, intptr_t addend) 303c896fe29Sbellard { 3047ecd02a0SRichard Henderson TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 305c896fe29Sbellard 306c896fe29Sbellard r->type = type; 307c896fe29Sbellard r->ptr = code_ptr; 308c896fe29Sbellard r->addend = addend; 3097ecd02a0SRichard Henderson QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 310c896fe29Sbellard } 311c896fe29Sbellard 31292ab8e7dSRichard Henderson static void tcg_out_label(TCGContext *s, TCGLabel *l) 313c896fe29Sbellard { 314eabb7b91SAurelien Jarno tcg_debug_assert(!l->has_value); 315c896fe29Sbellard l->has_value = 1; 31692ab8e7dSRichard Henderson l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 317c896fe29Sbellard } 318c896fe29Sbellard 31942a268c2SRichard Henderson TCGLabel *gen_new_label(void) 320c896fe29Sbellard { 321b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 32251e3972cSRichard Henderson TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 323c896fe29Sbellard 3247ecd02a0SRichard Henderson memset(l, 0, sizeof(TCGLabel)); 3257ecd02a0SRichard Henderson l->id = s->nb_labels++; 326f85b1fc4SRichard Henderson QSIMPLEQ_INIT(&l->branches); 3277ecd02a0SRichard Henderson QSIMPLEQ_INIT(&l->relocs); 3287ecd02a0SRichard Henderson 329bef16ab4SRichard Henderson QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 33042a268c2SRichard Henderson 33142a268c2SRichard Henderson return l; 332c896fe29Sbellard } 333c896fe29Sbellard 3347ecd02a0SRichard Henderson static bool tcg_resolve_relocs(TCGContext *s) 3357ecd02a0SRichard Henderson { 3367ecd02a0SRichard Henderson TCGLabel *l; 3377ecd02a0SRichard Henderson 3387ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 3397ecd02a0SRichard Henderson TCGRelocation *r; 3407ecd02a0SRichard Henderson uintptr_t value = l->u.value; 3417ecd02a0SRichard Henderson 3427ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(r, &l->relocs, next) { 3437ecd02a0SRichard Henderson if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 3447ecd02a0SRichard Henderson return false; 3457ecd02a0SRichard Henderson } 3467ecd02a0SRichard Henderson } 3477ecd02a0SRichard Henderson } 3487ecd02a0SRichard Henderson return true; 3497ecd02a0SRichard Henderson } 3507ecd02a0SRichard Henderson 3519f754620SRichard Henderson static void set_jmp_reset_offset(TCGContext *s, int which) 3529f754620SRichard Henderson { 353f14bed3fSRichard Henderson /* 354f14bed3fSRichard Henderson * We will check for overflow at the end of the opcode loop in 355f14bed3fSRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 356f14bed3fSRichard Henderson */ 357b7e4afbdSRichard Henderson s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); 3589f754620SRichard Henderson } 3599f754620SRichard Henderson 360b52a2c03SRichard Henderson static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) 361b52a2c03SRichard Henderson { 362b52a2c03SRichard Henderson /* 363b52a2c03SRichard Henderson * We will check for overflow at the end of the opcode loop in 364b52a2c03SRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 365b52a2c03SRichard Henderson */ 3669da6079bSRichard Henderson s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); 367b52a2c03SRichard Henderson } 368b52a2c03SRichard Henderson 369becc452aSRichard Henderson static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) 370becc452aSRichard Henderson { 371becc452aSRichard Henderson /* 372becc452aSRichard Henderson * Return the read-execute version of the pointer, for the benefit 373becc452aSRichard Henderson * of any pc-relative addressing mode. 374becc452aSRichard Henderson */ 3759da6079bSRichard Henderson return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); 376becc452aSRichard Henderson } 377becc452aSRichard Henderson 378db6b7d0cSRichard Henderson /* Signal overflow, starting over with fewer guest insns. */ 3798905770bSMarc-André Lureau static G_NORETURN 3808905770bSMarc-André Lureau void tcg_raise_tb_overflow(TCGContext *s) 381db6b7d0cSRichard Henderson { 382db6b7d0cSRichard Henderson siglongjmp(s->jmp_trans, -2); 383db6b7d0cSRichard Henderson } 384db6b7d0cSRichard Henderson 385*8429a1caSRichard Henderson /* 386*8429a1caSRichard Henderson * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext. 387*8429a1caSRichard Henderson * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg. 388*8429a1caSRichard Henderson * 389*8429a1caSRichard Henderson * However, tcg_out_helper_load_slots reuses this field to hold an 390*8429a1caSRichard Henderson * argument slot number (which may designate a argument register or an 391*8429a1caSRichard Henderson * argument stack slot), converting to TCGReg once all arguments that 392*8429a1caSRichard Henderson * are destined for the stack are processed. 393*8429a1caSRichard Henderson */ 394129f1f9eSRichard Henderson typedef struct TCGMovExtend { 395*8429a1caSRichard Henderson unsigned dst; 396129f1f9eSRichard Henderson TCGReg src; 397129f1f9eSRichard Henderson TCGType dst_type; 398129f1f9eSRichard Henderson TCGType src_type; 399129f1f9eSRichard Henderson MemOp src_ext; 400129f1f9eSRichard Henderson } TCGMovExtend; 401129f1f9eSRichard Henderson 402b3dfd5fcSRichard Henderson /** 403b3dfd5fcSRichard Henderson * tcg_out_movext -- move and extend 404b3dfd5fcSRichard Henderson * @s: tcg context 405b3dfd5fcSRichard Henderson * @dst_type: integral type for destination 406b3dfd5fcSRichard Henderson * @dst: destination register 407b3dfd5fcSRichard Henderson * @src_type: integral type for source 408b3dfd5fcSRichard Henderson * @src_ext: extension to apply to source 409b3dfd5fcSRichard Henderson * @src: source register 410b3dfd5fcSRichard Henderson * 411b3dfd5fcSRichard Henderson * Move or extend @src into @dst, depending on @src_ext and the types. 412b3dfd5fcSRichard Henderson */ 413129f1f9eSRichard Henderson static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst, 414b3dfd5fcSRichard Henderson TCGType src_type, MemOp src_ext, TCGReg src) 415b3dfd5fcSRichard Henderson { 416b3dfd5fcSRichard Henderson switch (src_ext) { 417b3dfd5fcSRichard Henderson case MO_UB: 418b3dfd5fcSRichard Henderson tcg_out_ext8u(s, dst, src); 419b3dfd5fcSRichard Henderson break; 420b3dfd5fcSRichard Henderson case MO_SB: 421b3dfd5fcSRichard Henderson tcg_out_ext8s(s, dst_type, dst, src); 422b3dfd5fcSRichard Henderson break; 423b3dfd5fcSRichard Henderson case MO_UW: 424b3dfd5fcSRichard Henderson tcg_out_ext16u(s, dst, src); 425b3dfd5fcSRichard Henderson break; 426b3dfd5fcSRichard Henderson case MO_SW: 427b3dfd5fcSRichard Henderson tcg_out_ext16s(s, dst_type, dst, src); 428b3dfd5fcSRichard Henderson break; 429b3dfd5fcSRichard Henderson case MO_UL: 430b3dfd5fcSRichard Henderson case MO_SL: 431b3dfd5fcSRichard Henderson if (dst_type == TCG_TYPE_I32) { 432b3dfd5fcSRichard Henderson if (src_type == TCG_TYPE_I32) { 433b3dfd5fcSRichard Henderson tcg_out_mov(s, TCG_TYPE_I32, dst, src); 434b3dfd5fcSRichard Henderson } else { 435b3dfd5fcSRichard Henderson tcg_out_extrl_i64_i32(s, dst, src); 436b3dfd5fcSRichard Henderson } 437b3dfd5fcSRichard Henderson } else if (src_type == TCG_TYPE_I32) { 438b3dfd5fcSRichard Henderson if (src_ext & MO_SIGN) { 439b3dfd5fcSRichard Henderson tcg_out_exts_i32_i64(s, dst, src); 440b3dfd5fcSRichard Henderson } else { 441b3dfd5fcSRichard Henderson tcg_out_extu_i32_i64(s, dst, src); 442b3dfd5fcSRichard Henderson } 443b3dfd5fcSRichard Henderson } else { 444b3dfd5fcSRichard Henderson if (src_ext & MO_SIGN) { 445b3dfd5fcSRichard Henderson tcg_out_ext32s(s, dst, src); 446b3dfd5fcSRichard Henderson } else { 447b3dfd5fcSRichard Henderson tcg_out_ext32u(s, dst, src); 448b3dfd5fcSRichard Henderson } 449b3dfd5fcSRichard Henderson } 450b3dfd5fcSRichard Henderson break; 451b3dfd5fcSRichard Henderson case MO_UQ: 452b3dfd5fcSRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 453b3dfd5fcSRichard Henderson if (dst_type == TCG_TYPE_I32) { 454b3dfd5fcSRichard Henderson tcg_out_extrl_i64_i32(s, dst, src); 455b3dfd5fcSRichard Henderson } else { 456b3dfd5fcSRichard Henderson tcg_out_mov(s, TCG_TYPE_I64, dst, src); 457b3dfd5fcSRichard Henderson } 458b3dfd5fcSRichard Henderson break; 459b3dfd5fcSRichard Henderson default: 460b3dfd5fcSRichard Henderson g_assert_not_reached(); 461b3dfd5fcSRichard Henderson } 462b3dfd5fcSRichard Henderson } 463b3dfd5fcSRichard Henderson 464129f1f9eSRichard Henderson /* Minor variations on a theme, using a structure. */ 465129f1f9eSRichard Henderson static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i, 466129f1f9eSRichard Henderson TCGReg src) 467129f1f9eSRichard Henderson { 468129f1f9eSRichard Henderson tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src); 469129f1f9eSRichard Henderson } 470129f1f9eSRichard Henderson 471129f1f9eSRichard Henderson static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i) 472129f1f9eSRichard Henderson { 473129f1f9eSRichard Henderson tcg_out_movext1_new_src(s, i, i->src); 474129f1f9eSRichard Henderson } 475129f1f9eSRichard Henderson 476129f1f9eSRichard Henderson /** 477129f1f9eSRichard Henderson * tcg_out_movext2 -- move and extend two pair 478129f1f9eSRichard Henderson * @s: tcg context 479129f1f9eSRichard Henderson * @i1: first move description 480129f1f9eSRichard Henderson * @i2: second move description 481129f1f9eSRichard Henderson * @scratch: temporary register, or -1 for none 482129f1f9eSRichard Henderson * 483129f1f9eSRichard Henderson * As tcg_out_movext, for both @i1 and @i2, caring for overlap 484129f1f9eSRichard Henderson * between the sources and destinations. 485129f1f9eSRichard Henderson */ 486129f1f9eSRichard Henderson 487*8429a1caSRichard Henderson static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1, 488129f1f9eSRichard Henderson const TCGMovExtend *i2, int scratch) 489129f1f9eSRichard Henderson { 490129f1f9eSRichard Henderson TCGReg src1 = i1->src; 491129f1f9eSRichard Henderson TCGReg src2 = i2->src; 492129f1f9eSRichard Henderson 493129f1f9eSRichard Henderson if (i1->dst != src2) { 494129f1f9eSRichard Henderson tcg_out_movext1(s, i1); 495129f1f9eSRichard Henderson tcg_out_movext1(s, i2); 496129f1f9eSRichard Henderson return; 497129f1f9eSRichard Henderson } 498129f1f9eSRichard Henderson if (i2->dst == src1) { 499129f1f9eSRichard Henderson TCGType src1_type = i1->src_type; 500129f1f9eSRichard Henderson TCGType src2_type = i2->src_type; 501129f1f9eSRichard Henderson 502129f1f9eSRichard Henderson if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) { 503129f1f9eSRichard Henderson /* The data is now in the correct registers, now extend. */ 504129f1f9eSRichard Henderson src1 = i2->src; 505129f1f9eSRichard Henderson src2 = i1->src; 506129f1f9eSRichard Henderson } else { 507129f1f9eSRichard Henderson tcg_debug_assert(scratch >= 0); 508129f1f9eSRichard Henderson tcg_out_mov(s, src1_type, scratch, src1); 509129f1f9eSRichard Henderson src1 = scratch; 510129f1f9eSRichard Henderson } 511129f1f9eSRichard Henderson } 512129f1f9eSRichard Henderson tcg_out_movext1_new_src(s, i2, src2); 513129f1f9eSRichard Henderson tcg_out_movext1_new_src(s, i1, src1); 514129f1f9eSRichard Henderson } 515129f1f9eSRichard Henderson 5164c22e840SRichard Henderson #define C_PFX1(P, A) P##A 5174c22e840SRichard Henderson #define C_PFX2(P, A, B) P##A##_##B 5184c22e840SRichard Henderson #define C_PFX3(P, A, B, C) P##A##_##B##_##C 5194c22e840SRichard Henderson #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 5204c22e840SRichard Henderson #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 5214c22e840SRichard Henderson #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 5224c22e840SRichard Henderson 5234c22e840SRichard Henderson /* Define an enumeration for the various combinations. */ 5244c22e840SRichard Henderson 5254c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 5264c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 5274c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 5284c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 5294c22e840SRichard Henderson 5304c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 5314c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 5324c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 5334c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 5344c22e840SRichard Henderson 5354c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 5364c22e840SRichard Henderson 5374c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 5384c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 5394c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 5404c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 5414c22e840SRichard Henderson 5424c22e840SRichard Henderson typedef enum { 5434c22e840SRichard Henderson #include "tcg-target-con-set.h" 5444c22e840SRichard Henderson } TCGConstraintSetIndex; 5454c22e840SRichard Henderson 5464c22e840SRichard Henderson static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 5474c22e840SRichard Henderson 5484c22e840SRichard Henderson #undef C_O0_I1 5494c22e840SRichard Henderson #undef C_O0_I2 5504c22e840SRichard Henderson #undef C_O0_I3 5514c22e840SRichard Henderson #undef C_O0_I4 5524c22e840SRichard Henderson #undef C_O1_I1 5534c22e840SRichard Henderson #undef C_O1_I2 5544c22e840SRichard Henderson #undef C_O1_I3 5554c22e840SRichard Henderson #undef C_O1_I4 5564c22e840SRichard Henderson #undef C_N1_I2 5574c22e840SRichard Henderson #undef C_O2_I1 5584c22e840SRichard Henderson #undef C_O2_I2 5594c22e840SRichard Henderson #undef C_O2_I3 5604c22e840SRichard Henderson #undef C_O2_I4 5614c22e840SRichard Henderson 5624c22e840SRichard Henderson /* Put all of the constraint sets into an array, indexed by the enum. */ 5634c22e840SRichard Henderson 5644c22e840SRichard Henderson #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 5654c22e840SRichard Henderson #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 5664c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 5674c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 5684c22e840SRichard Henderson 5694c22e840SRichard Henderson #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 5704c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 5714c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 5724c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 5734c22e840SRichard Henderson 5744c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 5754c22e840SRichard Henderson 5764c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 5774c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 5784c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 5794c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 5804c22e840SRichard Henderson 5814c22e840SRichard Henderson static const TCGTargetOpDef constraint_sets[] = { 5824c22e840SRichard Henderson #include "tcg-target-con-set.h" 5834c22e840SRichard Henderson }; 5844c22e840SRichard Henderson 5854c22e840SRichard Henderson 5864c22e840SRichard Henderson #undef C_O0_I1 5874c22e840SRichard Henderson #undef C_O0_I2 5884c22e840SRichard Henderson #undef C_O0_I3 5894c22e840SRichard Henderson #undef C_O0_I4 5904c22e840SRichard Henderson #undef C_O1_I1 5914c22e840SRichard Henderson #undef C_O1_I2 5924c22e840SRichard Henderson #undef C_O1_I3 5934c22e840SRichard Henderson #undef C_O1_I4 5944c22e840SRichard Henderson #undef C_N1_I2 5954c22e840SRichard Henderson #undef C_O2_I1 5964c22e840SRichard Henderson #undef C_O2_I2 5974c22e840SRichard Henderson #undef C_O2_I3 5984c22e840SRichard Henderson #undef C_O2_I4 5994c22e840SRichard Henderson 6004c22e840SRichard Henderson /* Expand the enumerator to be returned from tcg_target_op_def(). */ 6014c22e840SRichard Henderson 6024c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 6034c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 6044c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 6054c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 6064c22e840SRichard Henderson 6074c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 6084c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 6094c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 6104c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 6114c22e840SRichard Henderson 6124c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 6134c22e840SRichard Henderson 6144c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 6154c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 6164c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 6174c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 6184c22e840SRichard Henderson 619139c1837SPaolo Bonzini #include "tcg-target.c.inc" 620c896fe29Sbellard 62138b47b19SEmilio G. Cota static void alloc_tcg_plugin_context(TCGContext *s) 62238b47b19SEmilio G. Cota { 62338b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 62438b47b19SEmilio G. Cota s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 62538b47b19SEmilio G. Cota s->plugin_tb->insns = 62638b47b19SEmilio G. Cota g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 62738b47b19SEmilio G. Cota #endif 62838b47b19SEmilio G. Cota } 62938b47b19SEmilio G. Cota 630e8feb96fSEmilio G. Cota /* 6313468b59eSEmilio G. Cota * All TCG threads except the parent (i.e. the one that called tcg_context_init 6323468b59eSEmilio G. Cota * and registered the target's TCG globals) must register with this function 6333468b59eSEmilio G. Cota * before initiating translation. 6343468b59eSEmilio G. Cota * 6353468b59eSEmilio G. Cota * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 6363468b59eSEmilio G. Cota * of tcg_region_init() for the reasoning behind this. 6373468b59eSEmilio G. Cota * 6383468b59eSEmilio G. Cota * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 6393468b59eSEmilio G. Cota * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 6403468b59eSEmilio G. Cota * is not used anymore for translation once this function is called. 6413468b59eSEmilio G. Cota * 6423468b59eSEmilio G. Cota * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 6433468b59eSEmilio G. Cota * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 6443468b59eSEmilio G. Cota */ 6453468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 6463468b59eSEmilio G. Cota void tcg_register_thread(void) 6473468b59eSEmilio G. Cota { 6483468b59eSEmilio G. Cota tcg_ctx = &tcg_init_ctx; 6493468b59eSEmilio G. Cota } 6503468b59eSEmilio G. Cota #else 6513468b59eSEmilio G. Cota void tcg_register_thread(void) 6523468b59eSEmilio G. Cota { 6533468b59eSEmilio G. Cota TCGContext *s = g_malloc(sizeof(*s)); 6543468b59eSEmilio G. Cota unsigned int i, n; 6553468b59eSEmilio G. Cota 6563468b59eSEmilio G. Cota *s = tcg_init_ctx; 6573468b59eSEmilio G. Cota 6583468b59eSEmilio G. Cota /* Relink mem_base. */ 6593468b59eSEmilio G. Cota for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 6603468b59eSEmilio G. Cota if (tcg_init_ctx.temps[i].mem_base) { 6613468b59eSEmilio G. Cota ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 6623468b59eSEmilio G. Cota tcg_debug_assert(b >= 0 && b < n); 6633468b59eSEmilio G. Cota s->temps[i].mem_base = &s->temps[b]; 6643468b59eSEmilio G. Cota } 6653468b59eSEmilio G. Cota } 6663468b59eSEmilio G. Cota 6673468b59eSEmilio G. Cota /* Claim an entry in tcg_ctxs */ 6680e2d61cfSRichard Henderson n = qatomic_fetch_inc(&tcg_cur_ctxs); 6690e2d61cfSRichard Henderson g_assert(n < tcg_max_ctxs); 670d73415a3SStefan Hajnoczi qatomic_set(&tcg_ctxs[n], s); 6713468b59eSEmilio G. Cota 67238b47b19SEmilio G. Cota if (n > 0) { 67338b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 674bf042e8eSRichard Henderson tcg_region_initial_alloc(s); 67538b47b19SEmilio G. Cota } 67638b47b19SEmilio G. Cota 6773468b59eSEmilio G. Cota tcg_ctx = s; 6783468b59eSEmilio G. Cota } 6793468b59eSEmilio G. Cota #endif /* !CONFIG_USER_ONLY */ 6803468b59eSEmilio G. Cota 681c896fe29Sbellard /* pool based memory allocation */ 682c896fe29Sbellard void *tcg_malloc_internal(TCGContext *s, int size) 683c896fe29Sbellard { 684c896fe29Sbellard TCGPool *p; 685c896fe29Sbellard int pool_size; 686c896fe29Sbellard 687c896fe29Sbellard if (size > TCG_POOL_CHUNK_SIZE) { 688c896fe29Sbellard /* big malloc: insert a new pool (XXX: could optimize) */ 6897267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + size); 690c896fe29Sbellard p->size = size; 6914055299eSKirill Batuzov p->next = s->pool_first_large; 6924055299eSKirill Batuzov s->pool_first_large = p; 6934055299eSKirill Batuzov return p->data; 694c896fe29Sbellard } else { 695c896fe29Sbellard p = s->pool_current; 696c896fe29Sbellard if (!p) { 697c896fe29Sbellard p = s->pool_first; 698c896fe29Sbellard if (!p) 699c896fe29Sbellard goto new_pool; 700c896fe29Sbellard } else { 701c896fe29Sbellard if (!p->next) { 702c896fe29Sbellard new_pool: 703c896fe29Sbellard pool_size = TCG_POOL_CHUNK_SIZE; 7047267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + pool_size); 705c896fe29Sbellard p->size = pool_size; 706c896fe29Sbellard p->next = NULL; 707a813e36fSRichard Henderson if (s->pool_current) { 708c896fe29Sbellard s->pool_current->next = p; 709a813e36fSRichard Henderson } else { 710c896fe29Sbellard s->pool_first = p; 711a813e36fSRichard Henderson } 712c896fe29Sbellard } else { 713c896fe29Sbellard p = p->next; 714c896fe29Sbellard } 715c896fe29Sbellard } 716c896fe29Sbellard } 717c896fe29Sbellard s->pool_current = p; 718c896fe29Sbellard s->pool_cur = p->data + size; 719c896fe29Sbellard s->pool_end = p->data + p->size; 720c896fe29Sbellard return p->data; 721c896fe29Sbellard } 722c896fe29Sbellard 723c896fe29Sbellard void tcg_pool_reset(TCGContext *s) 724c896fe29Sbellard { 7254055299eSKirill Batuzov TCGPool *p, *t; 7264055299eSKirill Batuzov for (p = s->pool_first_large; p; p = t) { 7274055299eSKirill Batuzov t = p->next; 7284055299eSKirill Batuzov g_free(p); 7294055299eSKirill Batuzov } 7304055299eSKirill Batuzov s->pool_first_large = NULL; 731c896fe29Sbellard s->pool_cur = s->pool_end = NULL; 732c896fe29Sbellard s->pool_current = NULL; 733c896fe29Sbellard } 734c896fe29Sbellard 7352ef6175aSRichard Henderson #include "exec/helper-proto.h" 7362ef6175aSRichard Henderson 73739004a71SRichard Henderson static TCGHelperInfo all_helpers[] = { 7382ef6175aSRichard Henderson #include "exec/helper-tcg.h" 739100b5e01SRichard Henderson }; 740619205fdSEmilio G. Cota static GHashTable *helper_table; 741100b5e01SRichard Henderson 742*8429a1caSRichard Henderson /* 743*8429a1caSRichard Henderson * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions, 744*8429a1caSRichard Henderson * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N. 745*8429a1caSRichard Henderson * We only use these for layout in tcg_out_ld_helper_ret and 746*8429a1caSRichard Henderson * tcg_out_st_helper_args, and share them between several of 747*8429a1caSRichard Henderson * the helpers, with the end result that it's easier to build manually. 748*8429a1caSRichard Henderson */ 749*8429a1caSRichard Henderson 750*8429a1caSRichard Henderson #if TCG_TARGET_REG_BITS == 32 751*8429a1caSRichard Henderson # define dh_typecode_ttl dh_typecode_i32 752*8429a1caSRichard Henderson #else 753*8429a1caSRichard Henderson # define dh_typecode_ttl dh_typecode_i64 754*8429a1caSRichard Henderson #endif 755*8429a1caSRichard Henderson 756*8429a1caSRichard Henderson static TCGHelperInfo info_helper_ld32_mmu = { 757*8429a1caSRichard Henderson .flags = TCG_CALL_NO_WG, 758*8429a1caSRichard Henderson .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */ 759*8429a1caSRichard Henderson | dh_typemask(env, 1) 760*8429a1caSRichard Henderson | dh_typemask(tl, 2) /* target_ulong addr */ 761*8429a1caSRichard Henderson | dh_typemask(i32, 3) /* unsigned oi */ 762*8429a1caSRichard Henderson | dh_typemask(ptr, 4) /* uintptr_t ra */ 763*8429a1caSRichard Henderson }; 764*8429a1caSRichard Henderson 765*8429a1caSRichard Henderson static TCGHelperInfo info_helper_ld64_mmu = { 766*8429a1caSRichard Henderson .flags = TCG_CALL_NO_WG, 767*8429a1caSRichard Henderson .typemask = dh_typemask(i64, 0) /* return uint64_t */ 768*8429a1caSRichard Henderson | dh_typemask(env, 1) 769*8429a1caSRichard Henderson | dh_typemask(tl, 2) /* target_ulong addr */ 770*8429a1caSRichard Henderson | dh_typemask(i32, 3) /* unsigned oi */ 771*8429a1caSRichard Henderson | dh_typemask(ptr, 4) /* uintptr_t ra */ 772*8429a1caSRichard Henderson }; 773*8429a1caSRichard Henderson 774*8429a1caSRichard Henderson static TCGHelperInfo info_helper_st32_mmu = { 775*8429a1caSRichard Henderson .flags = TCG_CALL_NO_WG, 776*8429a1caSRichard Henderson .typemask = dh_typemask(void, 0) 777*8429a1caSRichard Henderson | dh_typemask(env, 1) 778*8429a1caSRichard Henderson | dh_typemask(tl, 2) /* target_ulong addr */ 779*8429a1caSRichard Henderson | dh_typemask(i32, 3) /* uint32_t data */ 780*8429a1caSRichard Henderson | dh_typemask(i32, 4) /* unsigned oi */ 781*8429a1caSRichard Henderson | dh_typemask(ptr, 5) /* uintptr_t ra */ 782*8429a1caSRichard Henderson }; 783*8429a1caSRichard Henderson 784*8429a1caSRichard Henderson static TCGHelperInfo info_helper_st64_mmu = { 785*8429a1caSRichard Henderson .flags = TCG_CALL_NO_WG, 786*8429a1caSRichard Henderson .typemask = dh_typemask(void, 0) 787*8429a1caSRichard Henderson | dh_typemask(env, 1) 788*8429a1caSRichard Henderson | dh_typemask(tl, 2) /* target_ulong addr */ 789*8429a1caSRichard Henderson | dh_typemask(i64, 3) /* uint64_t data */ 790*8429a1caSRichard Henderson | dh_typemask(i32, 4) /* unsigned oi */ 791*8429a1caSRichard Henderson | dh_typemask(ptr, 5) /* uintptr_t ra */ 792*8429a1caSRichard Henderson }; 793*8429a1caSRichard Henderson 79422f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 795c6ef8c7bSPhilippe Mathieu-Daudé static ffi_type *typecode_to_ffi(int argmask) 796c6ef8c7bSPhilippe Mathieu-Daudé { 797e9709e17SRichard Henderson /* 798e9709e17SRichard Henderson * libffi does not support __int128_t, so we have forced Int128 799e9709e17SRichard Henderson * to use the structure definition instead of the builtin type. 800e9709e17SRichard Henderson */ 801e9709e17SRichard Henderson static ffi_type *ffi_type_i128_elements[3] = { 802e9709e17SRichard Henderson &ffi_type_uint64, 803e9709e17SRichard Henderson &ffi_type_uint64, 804e9709e17SRichard Henderson NULL 805e9709e17SRichard Henderson }; 806e9709e17SRichard Henderson static ffi_type ffi_type_i128 = { 807e9709e17SRichard Henderson .size = 16, 808e9709e17SRichard Henderson .alignment = __alignof__(Int128), 809e9709e17SRichard Henderson .type = FFI_TYPE_STRUCT, 810e9709e17SRichard Henderson .elements = ffi_type_i128_elements, 811e9709e17SRichard Henderson }; 812e9709e17SRichard Henderson 813c6ef8c7bSPhilippe Mathieu-Daudé switch (argmask) { 814c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_void: 815c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_void; 816c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i32: 817c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint32; 818c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s32: 819c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint32; 820c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i64: 821c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint64; 822c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s64: 823c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint64; 824c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_ptr: 825c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_pointer; 826e9709e17SRichard Henderson case dh_typecode_i128: 827e9709e17SRichard Henderson return &ffi_type_i128; 828c6ef8c7bSPhilippe Mathieu-Daudé } 829c6ef8c7bSPhilippe Mathieu-Daudé g_assert_not_reached(); 830c6ef8c7bSPhilippe Mathieu-Daudé } 8310c22e176SPhilippe Mathieu-Daudé 8320c22e176SPhilippe Mathieu-Daudé static void init_ffi_layouts(void) 8330c22e176SPhilippe Mathieu-Daudé { 8340c22e176SPhilippe Mathieu-Daudé /* g_direct_hash/equal for direct comparisons on uint32_t. */ 835f9c4bb80SRichard Henderson GHashTable *ffi_table = g_hash_table_new(NULL, NULL); 836f9c4bb80SRichard Henderson 8370c22e176SPhilippe Mathieu-Daudé for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 838f9c4bb80SRichard Henderson TCGHelperInfo *info = &all_helpers[i]; 839f9c4bb80SRichard Henderson unsigned typemask = info->typemask; 8400c22e176SPhilippe Mathieu-Daudé gpointer hash = (gpointer)(uintptr_t)typemask; 8410c22e176SPhilippe Mathieu-Daudé struct { 8420c22e176SPhilippe Mathieu-Daudé ffi_cif cif; 8430c22e176SPhilippe Mathieu-Daudé ffi_type *args[]; 8440c22e176SPhilippe Mathieu-Daudé } *ca; 8450c22e176SPhilippe Mathieu-Daudé ffi_status status; 8460c22e176SPhilippe Mathieu-Daudé int nargs; 847f9c4bb80SRichard Henderson ffi_cif *cif; 8480c22e176SPhilippe Mathieu-Daudé 849f9c4bb80SRichard Henderson cif = g_hash_table_lookup(ffi_table, hash); 850f9c4bb80SRichard Henderson if (cif) { 851f9c4bb80SRichard Henderson info->cif = cif; 8520c22e176SPhilippe Mathieu-Daudé continue; 8530c22e176SPhilippe Mathieu-Daudé } 8540c22e176SPhilippe Mathieu-Daudé 8550c22e176SPhilippe Mathieu-Daudé /* Ignoring the return type, find the last non-zero field. */ 8560c22e176SPhilippe Mathieu-Daudé nargs = 32 - clz32(typemask >> 3); 8570c22e176SPhilippe Mathieu-Daudé nargs = DIV_ROUND_UP(nargs, 3); 858e9709e17SRichard Henderson assert(nargs <= MAX_CALL_IARGS); 8590c22e176SPhilippe Mathieu-Daudé 8600c22e176SPhilippe Mathieu-Daudé ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 8610c22e176SPhilippe Mathieu-Daudé ca->cif.rtype = typecode_to_ffi(typemask & 7); 8620c22e176SPhilippe Mathieu-Daudé ca->cif.nargs = nargs; 8630c22e176SPhilippe Mathieu-Daudé 8640c22e176SPhilippe Mathieu-Daudé if (nargs != 0) { 8650c22e176SPhilippe Mathieu-Daudé ca->cif.arg_types = ca->args; 8660c22e176SPhilippe Mathieu-Daudé for (int j = 0; j < nargs; ++j) { 8670c22e176SPhilippe Mathieu-Daudé int typecode = extract32(typemask, (j + 1) * 3, 3); 8680c22e176SPhilippe Mathieu-Daudé ca->args[j] = typecode_to_ffi(typecode); 8690c22e176SPhilippe Mathieu-Daudé } 8700c22e176SPhilippe Mathieu-Daudé } 8710c22e176SPhilippe Mathieu-Daudé 8720c22e176SPhilippe Mathieu-Daudé status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 8730c22e176SPhilippe Mathieu-Daudé ca->cif.rtype, ca->cif.arg_types); 8740c22e176SPhilippe Mathieu-Daudé assert(status == FFI_OK); 8750c22e176SPhilippe Mathieu-Daudé 876f9c4bb80SRichard Henderson cif = &ca->cif; 877f9c4bb80SRichard Henderson info->cif = cif; 878f9c4bb80SRichard Henderson g_hash_table_insert(ffi_table, hash, (gpointer)cif); 8790c22e176SPhilippe Mathieu-Daudé } 880f9c4bb80SRichard Henderson 881f9c4bb80SRichard Henderson g_hash_table_destroy(ffi_table); 8820c22e176SPhilippe Mathieu-Daudé } 8830c22e176SPhilippe Mathieu-Daudé #endif /* CONFIG_TCG_INTERPRETER */ 88422f15579SRichard Henderson 885338b61e9SRichard Henderson static inline bool arg_slot_reg_p(unsigned arg_slot) 886338b61e9SRichard Henderson { 887338b61e9SRichard Henderson /* 888338b61e9SRichard Henderson * Split the sizeof away from the comparison to avoid Werror from 889338b61e9SRichard Henderson * "unsigned < 0 is always false", when iarg_regs is empty. 890338b61e9SRichard Henderson */ 891338b61e9SRichard Henderson unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs); 892338b61e9SRichard Henderson return arg_slot < nreg; 893338b61e9SRichard Henderson } 894338b61e9SRichard Henderson 895d78e4a4fSRichard Henderson static inline int arg_slot_stk_ofs(unsigned arg_slot) 896d78e4a4fSRichard Henderson { 897d78e4a4fSRichard Henderson unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 898d78e4a4fSRichard Henderson unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); 899d78e4a4fSRichard Henderson 900d78e4a4fSRichard Henderson tcg_debug_assert(stk_slot < max); 901d78e4a4fSRichard Henderson return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long); 902d78e4a4fSRichard Henderson } 903d78e4a4fSRichard Henderson 90439004a71SRichard Henderson typedef struct TCGCumulativeArgs { 90539004a71SRichard Henderson int arg_idx; /* tcg_gen_callN args[] */ 90639004a71SRichard Henderson int info_in_idx; /* TCGHelperInfo in[] */ 90739004a71SRichard Henderson int arg_slot; /* regs+stack slot */ 90839004a71SRichard Henderson int ref_slot; /* stack slots for references */ 90939004a71SRichard Henderson } TCGCumulativeArgs; 91039004a71SRichard Henderson 91139004a71SRichard Henderson static void layout_arg_even(TCGCumulativeArgs *cum) 91239004a71SRichard Henderson { 91339004a71SRichard Henderson cum->arg_slot += cum->arg_slot & 1; 91439004a71SRichard Henderson } 91539004a71SRichard Henderson 91639004a71SRichard Henderson static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, 91739004a71SRichard Henderson TCGCallArgumentKind kind) 91839004a71SRichard Henderson { 91939004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 92039004a71SRichard Henderson 92139004a71SRichard Henderson *loc = (TCGCallArgumentLoc){ 92239004a71SRichard Henderson .kind = kind, 92339004a71SRichard Henderson .arg_idx = cum->arg_idx, 92439004a71SRichard Henderson .arg_slot = cum->arg_slot, 92539004a71SRichard Henderson }; 92639004a71SRichard Henderson cum->info_in_idx++; 92739004a71SRichard Henderson cum->arg_slot++; 92839004a71SRichard Henderson } 92939004a71SRichard Henderson 93039004a71SRichard Henderson static void layout_arg_normal_n(TCGCumulativeArgs *cum, 93139004a71SRichard Henderson TCGHelperInfo *info, int n) 93239004a71SRichard Henderson { 93339004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 93439004a71SRichard Henderson 93539004a71SRichard Henderson for (int i = 0; i < n; ++i) { 93639004a71SRichard Henderson /* Layout all using the same arg_idx, adjusting the subindex. */ 93739004a71SRichard Henderson loc[i] = (TCGCallArgumentLoc){ 93839004a71SRichard Henderson .kind = TCG_CALL_ARG_NORMAL, 93939004a71SRichard Henderson .arg_idx = cum->arg_idx, 94039004a71SRichard Henderson .tmp_subindex = i, 94139004a71SRichard Henderson .arg_slot = cum->arg_slot + i, 94239004a71SRichard Henderson }; 94339004a71SRichard Henderson } 94439004a71SRichard Henderson cum->info_in_idx += n; 94539004a71SRichard Henderson cum->arg_slot += n; 94639004a71SRichard Henderson } 94739004a71SRichard Henderson 948313bdea8SRichard Henderson static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info) 949313bdea8SRichard Henderson { 950313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 951313bdea8SRichard Henderson int n = 128 / TCG_TARGET_REG_BITS; 952313bdea8SRichard Henderson 953313bdea8SRichard Henderson /* The first subindex carries the pointer. */ 954313bdea8SRichard Henderson layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF); 955313bdea8SRichard Henderson 956313bdea8SRichard Henderson /* 957313bdea8SRichard Henderson * The callee is allowed to clobber memory associated with 958313bdea8SRichard Henderson * structure pass by-reference. Therefore we must make copies. 959313bdea8SRichard Henderson * Allocate space from "ref_slot", which will be adjusted to 960313bdea8SRichard Henderson * follow the parameters on the stack. 961313bdea8SRichard Henderson */ 962313bdea8SRichard Henderson loc[0].ref_slot = cum->ref_slot; 963313bdea8SRichard Henderson 964313bdea8SRichard Henderson /* 965313bdea8SRichard Henderson * Subsequent words also go into the reference slot, but 966313bdea8SRichard Henderson * do not accumulate into the regular arguments. 967313bdea8SRichard Henderson */ 968313bdea8SRichard Henderson for (int i = 1; i < n; ++i) { 969313bdea8SRichard Henderson loc[i] = (TCGCallArgumentLoc){ 970313bdea8SRichard Henderson .kind = TCG_CALL_ARG_BY_REF_N, 971313bdea8SRichard Henderson .arg_idx = cum->arg_idx, 972313bdea8SRichard Henderson .tmp_subindex = i, 973313bdea8SRichard Henderson .ref_slot = cum->ref_slot + i, 974313bdea8SRichard Henderson }; 975313bdea8SRichard Henderson } 976313bdea8SRichard Henderson cum->info_in_idx += n; 977313bdea8SRichard Henderson cum->ref_slot += n; 978313bdea8SRichard Henderson } 979313bdea8SRichard Henderson 98039004a71SRichard Henderson static void init_call_layout(TCGHelperInfo *info) 98139004a71SRichard Henderson { 98239004a71SRichard Henderson int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); 98339004a71SRichard Henderson int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 98439004a71SRichard Henderson unsigned typemask = info->typemask; 98539004a71SRichard Henderson unsigned typecode; 98639004a71SRichard Henderson TCGCumulativeArgs cum = { }; 98739004a71SRichard Henderson 98839004a71SRichard Henderson /* 98939004a71SRichard Henderson * Parse and place any function return value. 99039004a71SRichard Henderson */ 99139004a71SRichard Henderson typecode = typemask & 7; 99239004a71SRichard Henderson switch (typecode) { 99339004a71SRichard Henderson case dh_typecode_void: 99439004a71SRichard Henderson info->nr_out = 0; 99539004a71SRichard Henderson break; 99639004a71SRichard Henderson case dh_typecode_i32: 99739004a71SRichard Henderson case dh_typecode_s32: 99839004a71SRichard Henderson case dh_typecode_ptr: 99939004a71SRichard Henderson info->nr_out = 1; 100039004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL; 100139004a71SRichard Henderson break; 100239004a71SRichard Henderson case dh_typecode_i64: 100339004a71SRichard Henderson case dh_typecode_s64: 100439004a71SRichard Henderson info->nr_out = 64 / TCG_TARGET_REG_BITS; 100539004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL; 10065e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */ 10075e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 1008466d3759SRichard Henderson break; 1009466d3759SRichard Henderson case dh_typecode_i128: 1010466d3759SRichard Henderson info->nr_out = 128 / TCG_TARGET_REG_BITS; 10115427a9a7SRichard Henderson info->out_kind = TCG_TARGET_CALL_RET_I128; 10125427a9a7SRichard Henderson switch (TCG_TARGET_CALL_RET_I128) { 1013466d3759SRichard Henderson case TCG_CALL_RET_NORMAL: 10145e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */ 10155e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 1016466d3759SRichard Henderson break; 1017c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC: 1018c6556aa0SRichard Henderson /* Query the single register now to trigger any assert early. */ 1019c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0); 1020c6556aa0SRichard Henderson break; 1021313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF: 1022313bdea8SRichard Henderson /* 1023313bdea8SRichard Henderson * Allocate the first argument to the output. 1024313bdea8SRichard Henderson * We don't need to store this anywhere, just make it 1025313bdea8SRichard Henderson * unavailable for use in the input loop below. 1026313bdea8SRichard Henderson */ 1027313bdea8SRichard Henderson cum.arg_slot = 1; 1028313bdea8SRichard Henderson break; 1029466d3759SRichard Henderson default: 1030466d3759SRichard Henderson qemu_build_not_reached(); 1031466d3759SRichard Henderson } 103239004a71SRichard Henderson break; 103339004a71SRichard Henderson default: 103439004a71SRichard Henderson g_assert_not_reached(); 103539004a71SRichard Henderson } 103639004a71SRichard Henderson 103739004a71SRichard Henderson /* 103839004a71SRichard Henderson * Parse and place function arguments. 103939004a71SRichard Henderson */ 104039004a71SRichard Henderson for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { 104139004a71SRichard Henderson TCGCallArgumentKind kind; 104239004a71SRichard Henderson TCGType type; 104339004a71SRichard Henderson 104439004a71SRichard Henderson typecode = typemask & 7; 104539004a71SRichard Henderson switch (typecode) { 104639004a71SRichard Henderson case dh_typecode_i32: 104739004a71SRichard Henderson case dh_typecode_s32: 104839004a71SRichard Henderson type = TCG_TYPE_I32; 104939004a71SRichard Henderson break; 105039004a71SRichard Henderson case dh_typecode_i64: 105139004a71SRichard Henderson case dh_typecode_s64: 105239004a71SRichard Henderson type = TCG_TYPE_I64; 105339004a71SRichard Henderson break; 105439004a71SRichard Henderson case dh_typecode_ptr: 105539004a71SRichard Henderson type = TCG_TYPE_PTR; 105639004a71SRichard Henderson break; 1057466d3759SRichard Henderson case dh_typecode_i128: 1058466d3759SRichard Henderson type = TCG_TYPE_I128; 1059466d3759SRichard Henderson break; 106039004a71SRichard Henderson default: 106139004a71SRichard Henderson g_assert_not_reached(); 106239004a71SRichard Henderson } 106339004a71SRichard Henderson 106439004a71SRichard Henderson switch (type) { 106539004a71SRichard Henderson case TCG_TYPE_I32: 106639004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I32) { 106739004a71SRichard Henderson case TCG_CALL_ARG_EVEN: 106839004a71SRichard Henderson layout_arg_even(&cum); 106939004a71SRichard Henderson /* fall through */ 107039004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 107139004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 107239004a71SRichard Henderson break; 107339004a71SRichard Henderson case TCG_CALL_ARG_EXTEND: 107439004a71SRichard Henderson kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); 107539004a71SRichard Henderson layout_arg_1(&cum, info, kind); 107639004a71SRichard Henderson break; 107739004a71SRichard Henderson default: 107839004a71SRichard Henderson qemu_build_not_reached(); 107939004a71SRichard Henderson } 108039004a71SRichard Henderson break; 108139004a71SRichard Henderson 108239004a71SRichard Henderson case TCG_TYPE_I64: 108339004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I64) { 108439004a71SRichard Henderson case TCG_CALL_ARG_EVEN: 108539004a71SRichard Henderson layout_arg_even(&cum); 108639004a71SRichard Henderson /* fall through */ 108739004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 108839004a71SRichard Henderson if (TCG_TARGET_REG_BITS == 32) { 108939004a71SRichard Henderson layout_arg_normal_n(&cum, info, 2); 109039004a71SRichard Henderson } else { 109139004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 109239004a71SRichard Henderson } 109339004a71SRichard Henderson break; 109439004a71SRichard Henderson default: 109539004a71SRichard Henderson qemu_build_not_reached(); 109639004a71SRichard Henderson } 109739004a71SRichard Henderson break; 109839004a71SRichard Henderson 1099466d3759SRichard Henderson case TCG_TYPE_I128: 11005427a9a7SRichard Henderson switch (TCG_TARGET_CALL_ARG_I128) { 1101466d3759SRichard Henderson case TCG_CALL_ARG_EVEN: 1102466d3759SRichard Henderson layout_arg_even(&cum); 1103466d3759SRichard Henderson /* fall through */ 1104466d3759SRichard Henderson case TCG_CALL_ARG_NORMAL: 1105466d3759SRichard Henderson layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS); 1106466d3759SRichard Henderson break; 1107313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 1108313bdea8SRichard Henderson layout_arg_by_ref(&cum, info); 1109313bdea8SRichard Henderson break; 1110466d3759SRichard Henderson default: 1111466d3759SRichard Henderson qemu_build_not_reached(); 1112466d3759SRichard Henderson } 1113466d3759SRichard Henderson break; 1114466d3759SRichard Henderson 111539004a71SRichard Henderson default: 111639004a71SRichard Henderson g_assert_not_reached(); 111739004a71SRichard Henderson } 111839004a71SRichard Henderson } 111939004a71SRichard Henderson info->nr_in = cum.info_in_idx; 112039004a71SRichard Henderson 112139004a71SRichard Henderson /* Validate that we didn't overrun the input array. */ 112239004a71SRichard Henderson assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); 112339004a71SRichard Henderson /* Validate the backend has enough argument space. */ 112439004a71SRichard Henderson assert(cum.arg_slot <= max_reg_slots + max_stk_slots); 1125313bdea8SRichard Henderson 1126313bdea8SRichard Henderson /* 1127313bdea8SRichard Henderson * Relocate the "ref_slot" area to the end of the parameters. 1128313bdea8SRichard Henderson * Minimizing this stack offset helps code size for x86, 1129313bdea8SRichard Henderson * which has a signed 8-bit offset encoding. 1130313bdea8SRichard Henderson */ 1131313bdea8SRichard Henderson if (cum.ref_slot != 0) { 1132313bdea8SRichard Henderson int ref_base = 0; 1133313bdea8SRichard Henderson 1134313bdea8SRichard Henderson if (cum.arg_slot > max_reg_slots) { 1135313bdea8SRichard Henderson int align = __alignof(Int128) / sizeof(tcg_target_long); 1136313bdea8SRichard Henderson 1137313bdea8SRichard Henderson ref_base = cum.arg_slot - max_reg_slots; 1138313bdea8SRichard Henderson if (align > 1) { 1139313bdea8SRichard Henderson ref_base = ROUND_UP(ref_base, align); 1140313bdea8SRichard Henderson } 1141313bdea8SRichard Henderson } 1142313bdea8SRichard Henderson assert(ref_base + cum.ref_slot <= max_stk_slots); 1143d78e4a4fSRichard Henderson ref_base += max_reg_slots; 1144313bdea8SRichard Henderson 1145313bdea8SRichard Henderson if (ref_base != 0) { 1146313bdea8SRichard Henderson for (int i = cum.info_in_idx - 1; i >= 0; --i) { 1147313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[i]; 1148313bdea8SRichard Henderson switch (loc->kind) { 1149313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 1150313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 1151313bdea8SRichard Henderson loc->ref_slot += ref_base; 1152313bdea8SRichard Henderson break; 1153313bdea8SRichard Henderson default: 1154313bdea8SRichard Henderson break; 1155313bdea8SRichard Henderson } 1156313bdea8SRichard Henderson } 1157313bdea8SRichard Henderson } 1158313bdea8SRichard Henderson } 115939004a71SRichard Henderson } 116039004a71SRichard Henderson 116191478cefSRichard Henderson static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 1162f69d277eSRichard Henderson static void process_op_defs(TCGContext *s); 11631c2adb95SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 11641c2adb95SRichard Henderson TCGReg reg, const char *name); 116591478cefSRichard Henderson 116643b972b7SRichard Henderson static void tcg_context_init(unsigned max_cpus) 1167c896fe29Sbellard { 1168a76aabd3SRichard Henderson TCGContext *s = &tcg_init_ctx; 1169100b5e01SRichard Henderson int op, total_args, n, i; 1170c896fe29Sbellard TCGOpDef *def; 1171c896fe29Sbellard TCGArgConstraint *args_ct; 11721c2adb95SRichard Henderson TCGTemp *ts; 1173c896fe29Sbellard 1174c896fe29Sbellard memset(s, 0, sizeof(*s)); 1175c896fe29Sbellard s->nb_globals = 0; 1176c896fe29Sbellard 1177c896fe29Sbellard /* Count total number of arguments and allocate the corresponding 1178c896fe29Sbellard space */ 1179c896fe29Sbellard total_args = 0; 1180c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 1181c896fe29Sbellard def = &tcg_op_defs[op]; 1182c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 1183c896fe29Sbellard total_args += n; 1184c896fe29Sbellard } 1185c896fe29Sbellard 1186bc2b17e6SRichard Henderson args_ct = g_new0(TCGArgConstraint, total_args); 1187c896fe29Sbellard 1188c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 1189c896fe29Sbellard def = &tcg_op_defs[op]; 1190c896fe29Sbellard def->args_ct = args_ct; 1191c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 1192c896fe29Sbellard args_ct += n; 1193c896fe29Sbellard } 1194c896fe29Sbellard 11955cd8f621SRichard Henderson /* Register helpers. */ 119684fd9dd3SRichard Henderson /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 1197619205fdSEmilio G. Cota helper_table = g_hash_table_new(NULL, NULL); 119884fd9dd3SRichard Henderson 1199100b5e01SRichard Henderson for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 120039004a71SRichard Henderson init_call_layout(&all_helpers[i]); 120184fd9dd3SRichard Henderson g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 120272866e82SRichard Henderson (gpointer)&all_helpers[i]); 1203100b5e01SRichard Henderson } 12045cd8f621SRichard Henderson 1205*8429a1caSRichard Henderson init_call_layout(&info_helper_ld32_mmu); 1206*8429a1caSRichard Henderson init_call_layout(&info_helper_ld64_mmu); 1207*8429a1caSRichard Henderson init_call_layout(&info_helper_st32_mmu); 1208*8429a1caSRichard Henderson init_call_layout(&info_helper_st64_mmu); 1209*8429a1caSRichard Henderson 121022f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 12110c22e176SPhilippe Mathieu-Daudé init_ffi_layouts(); 121222f15579SRichard Henderson #endif 121322f15579SRichard Henderson 1214c896fe29Sbellard tcg_target_init(s); 1215f69d277eSRichard Henderson process_op_defs(s); 121691478cefSRichard Henderson 121791478cefSRichard Henderson /* Reverse the order of the saved registers, assuming they're all at 121891478cefSRichard Henderson the start of tcg_target_reg_alloc_order. */ 121991478cefSRichard Henderson for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 122091478cefSRichard Henderson int r = tcg_target_reg_alloc_order[n]; 122191478cefSRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 122291478cefSRichard Henderson break; 122391478cefSRichard Henderson } 122491478cefSRichard Henderson } 122591478cefSRichard Henderson for (i = 0; i < n; ++i) { 122691478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 122791478cefSRichard Henderson } 122891478cefSRichard Henderson for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 122991478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 123091478cefSRichard Henderson } 1231b1311c4aSEmilio G. Cota 123238b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 123338b47b19SEmilio G. Cota 1234b1311c4aSEmilio G. Cota tcg_ctx = s; 12353468b59eSEmilio G. Cota /* 12363468b59eSEmilio G. Cota * In user-mode we simply share the init context among threads, since we 12373468b59eSEmilio G. Cota * use a single region. See the documentation tcg_region_init() for the 12383468b59eSEmilio G. Cota * reasoning behind this. 12393468b59eSEmilio G. Cota * In softmmu we will have at most max_cpus TCG threads. 12403468b59eSEmilio G. Cota */ 12413468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 1242df2cce29SEmilio G. Cota tcg_ctxs = &tcg_ctx; 12430e2d61cfSRichard Henderson tcg_cur_ctxs = 1; 12440e2d61cfSRichard Henderson tcg_max_ctxs = 1; 12453468b59eSEmilio G. Cota #else 12460e2d61cfSRichard Henderson tcg_max_ctxs = max_cpus; 12470e2d61cfSRichard Henderson tcg_ctxs = g_new0(TCGContext *, max_cpus); 12483468b59eSEmilio G. Cota #endif 12491c2adb95SRichard Henderson 12501c2adb95SRichard Henderson tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 12511c2adb95SRichard Henderson ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 12521c2adb95SRichard Henderson cpu_env = temp_tcgv_ptr(ts); 12539002ec79SRichard Henderson } 1254b03cce8eSbellard 125543b972b7SRichard Henderson void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 1256a76aabd3SRichard Henderson { 125743b972b7SRichard Henderson tcg_context_init(max_cpus); 125843b972b7SRichard Henderson tcg_region_init(tb_size, splitwx, max_cpus); 1259a76aabd3SRichard Henderson } 1260a76aabd3SRichard Henderson 12616e3b2bfdSEmilio G. Cota /* 12626e3b2bfdSEmilio G. Cota * Allocate TBs right before their corresponding translated code, making 12636e3b2bfdSEmilio G. Cota * sure that TBs and code are on different cache lines. 12646e3b2bfdSEmilio G. Cota */ 12656e3b2bfdSEmilio G. Cota TranslationBlock *tcg_tb_alloc(TCGContext *s) 12666e3b2bfdSEmilio G. Cota { 12676e3b2bfdSEmilio G. Cota uintptr_t align = qemu_icache_linesize; 12686e3b2bfdSEmilio G. Cota TranslationBlock *tb; 12696e3b2bfdSEmilio G. Cota void *next; 12706e3b2bfdSEmilio G. Cota 1271e8feb96fSEmilio G. Cota retry: 12726e3b2bfdSEmilio G. Cota tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 12736e3b2bfdSEmilio G. Cota next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 12746e3b2bfdSEmilio G. Cota 12756e3b2bfdSEmilio G. Cota if (unlikely(next > s->code_gen_highwater)) { 1276e8feb96fSEmilio G. Cota if (tcg_region_alloc(s)) { 12776e3b2bfdSEmilio G. Cota return NULL; 12786e3b2bfdSEmilio G. Cota } 1279e8feb96fSEmilio G. Cota goto retry; 1280e8feb96fSEmilio G. Cota } 1281d73415a3SStefan Hajnoczi qatomic_set(&s->code_gen_ptr, next); 128257a26946SRichard Henderson s->data_gen_ptr = NULL; 12836e3b2bfdSEmilio G. Cota return tb; 12846e3b2bfdSEmilio G. Cota } 12856e3b2bfdSEmilio G. Cota 12869002ec79SRichard Henderson void tcg_prologue_init(TCGContext *s) 12879002ec79SRichard Henderson { 1288b0a0794aSRichard Henderson size_t prologue_size; 12898163b749SRichard Henderson 1290b0a0794aSRichard Henderson s->code_ptr = s->code_gen_ptr; 1291b0a0794aSRichard Henderson s->code_buf = s->code_gen_ptr; 12925b38ee31SRichard Henderson s->data_gen_ptr = NULL; 1293b91ccb31SRichard Henderson 1294b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 1295b0a0794aSRichard Henderson tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 1296b91ccb31SRichard Henderson #endif 12978163b749SRichard Henderson 12985b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 12995b38ee31SRichard Henderson s->pool_labels = NULL; 13005b38ee31SRichard Henderson #endif 13015b38ee31SRichard Henderson 1302653b87ebSRoman Bolshakov qemu_thread_jit_write(); 13038163b749SRichard Henderson /* Generate the prologue. */ 1304b03cce8eSbellard tcg_target_qemu_prologue(s); 13055b38ee31SRichard Henderson 13065b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 13075b38ee31SRichard Henderson /* Allow the prologue to put e.g. guest_base into a pool entry. */ 13085b38ee31SRichard Henderson { 13091768987bSRichard Henderson int result = tcg_out_pool_finalize(s); 13101768987bSRichard Henderson tcg_debug_assert(result == 0); 13115b38ee31SRichard Henderson } 13125b38ee31SRichard Henderson #endif 13135b38ee31SRichard Henderson 1314b0a0794aSRichard Henderson prologue_size = tcg_current_code_size(s); 13155584e2dbSIlya Leoshkevich perf_report_prologue(s->code_gen_ptr, prologue_size); 1316b0a0794aSRichard Henderson 1317df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 1318b0a0794aSRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 1319b0a0794aSRichard Henderson (uintptr_t)s->code_buf, prologue_size); 1320df5d2b16SRichard Henderson #endif 13218163b749SRichard Henderson 1322d6b64b2bSRichard Henderson #ifdef DEBUG_DISAS 1323d6b64b2bSRichard Henderson if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 1324c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 132578b54858SRichard Henderson if (logfile) { 132678b54858SRichard Henderson fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 13275b38ee31SRichard Henderson if (s->data_gen_ptr) { 1328b0a0794aSRichard Henderson size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 13295b38ee31SRichard Henderson size_t data_size = prologue_size - code_size; 13305b38ee31SRichard Henderson size_t i; 13315b38ee31SRichard Henderson 133278b54858SRichard Henderson disas(logfile, s->code_gen_ptr, code_size); 13335b38ee31SRichard Henderson 13345b38ee31SRichard Henderson for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 13355b38ee31SRichard Henderson if (sizeof(tcg_target_ulong) == 8) { 133678b54858SRichard Henderson fprintf(logfile, 133778b54858SRichard Henderson "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 13385b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 13395b38ee31SRichard Henderson *(uint64_t *)(s->data_gen_ptr + i)); 13405b38ee31SRichard Henderson } else { 134178b54858SRichard Henderson fprintf(logfile, 134278b54858SRichard Henderson "0x%08" PRIxPTR ": .long 0x%08x\n", 13435b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 13445b38ee31SRichard Henderson *(uint32_t *)(s->data_gen_ptr + i)); 13455b38ee31SRichard Henderson } 13465b38ee31SRichard Henderson } 13475b38ee31SRichard Henderson } else { 134878b54858SRichard Henderson disas(logfile, s->code_gen_ptr, prologue_size); 13495b38ee31SRichard Henderson } 135078b54858SRichard Henderson fprintf(logfile, "\n"); 1351fc59d2d8SRobert Foley qemu_log_unlock(logfile); 1352d6b64b2bSRichard Henderson } 135378b54858SRichard Henderson } 1354d6b64b2bSRichard Henderson #endif 1355cedbcb01SEmilio G. Cota 13566eea0434SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 13576eea0434SRichard Henderson /* 13586eea0434SRichard Henderson * Assert that goto_ptr is implemented completely, setting an epilogue. 13596eea0434SRichard Henderson * For tci, we use NULL as the signal to return from the interpreter, 13606eea0434SRichard Henderson * so skip this check. 13616eea0434SRichard Henderson */ 13628b5c2b62SRichard Henderson tcg_debug_assert(tcg_code_gen_epilogue != NULL); 13636eea0434SRichard Henderson #endif 1364d1c74ab3SRichard Henderson 1365d1c74ab3SRichard Henderson tcg_region_prologue_set(s); 1366c896fe29Sbellard } 1367c896fe29Sbellard 1368c896fe29Sbellard void tcg_func_start(TCGContext *s) 1369c896fe29Sbellard { 1370c896fe29Sbellard tcg_pool_reset(s); 1371c896fe29Sbellard s->nb_temps = s->nb_globals; 13720ec9eabcSRichard Henderson 13730ec9eabcSRichard Henderson /* No temps have been previously allocated for size or locality. */ 13740ec9eabcSRichard Henderson memset(s->free_temps, 0, sizeof(s->free_temps)); 13750ec9eabcSRichard Henderson 1376c0522136SRichard Henderson /* No constant temps have been previously allocated. */ 1377c0522136SRichard Henderson for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 1378c0522136SRichard Henderson if (s->const_table[i]) { 1379c0522136SRichard Henderson g_hash_table_remove_all(s->const_table[i]); 1380c0522136SRichard Henderson } 1381c0522136SRichard Henderson } 1382c0522136SRichard Henderson 1383abebf925SRichard Henderson s->nb_ops = 0; 1384c896fe29Sbellard s->nb_labels = 0; 1385c896fe29Sbellard s->current_frame_offset = s->frame_start; 1386c896fe29Sbellard 13870a209d4bSRichard Henderson #ifdef CONFIG_DEBUG_TCG 13880a209d4bSRichard Henderson s->goto_tb_issue_mask = 0; 13890a209d4bSRichard Henderson #endif 13900a209d4bSRichard Henderson 139115fa08f8SRichard Henderson QTAILQ_INIT(&s->ops); 139215fa08f8SRichard Henderson QTAILQ_INIT(&s->free_ops); 1393bef16ab4SRichard Henderson QSIMPLEQ_INIT(&s->labels); 1394c896fe29Sbellard } 1395c896fe29Sbellard 1396ae30e866SRichard Henderson static TCGTemp *tcg_temp_alloc(TCGContext *s) 13977ca4b752SRichard Henderson { 13987ca4b752SRichard Henderson int n = s->nb_temps++; 1399ae30e866SRichard Henderson 1400ae30e866SRichard Henderson if (n >= TCG_MAX_TEMPS) { 1401db6b7d0cSRichard Henderson tcg_raise_tb_overflow(s); 1402ae30e866SRichard Henderson } 14037ca4b752SRichard Henderson return memset(&s->temps[n], 0, sizeof(TCGTemp)); 14047ca4b752SRichard Henderson } 14057ca4b752SRichard Henderson 1406ae30e866SRichard Henderson static TCGTemp *tcg_global_alloc(TCGContext *s) 14077ca4b752SRichard Henderson { 1408fa477d25SRichard Henderson TCGTemp *ts; 1409fa477d25SRichard Henderson 14107ca4b752SRichard Henderson tcg_debug_assert(s->nb_globals == s->nb_temps); 1411ae30e866SRichard Henderson tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 14127ca4b752SRichard Henderson s->nb_globals++; 1413fa477d25SRichard Henderson ts = tcg_temp_alloc(s); 1414ee17db83SRichard Henderson ts->kind = TEMP_GLOBAL; 1415fa477d25SRichard Henderson 1416fa477d25SRichard Henderson return ts; 1417c896fe29Sbellard } 1418c896fe29Sbellard 1419085272b3SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1420b6638662SRichard Henderson TCGReg reg, const char *name) 1421c896fe29Sbellard { 1422c896fe29Sbellard TCGTemp *ts; 1423c896fe29Sbellard 14241a057554SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); 14257ca4b752SRichard Henderson 14267ca4b752SRichard Henderson ts = tcg_global_alloc(s); 1427c896fe29Sbellard ts->base_type = type; 1428c896fe29Sbellard ts->type = type; 1429ee17db83SRichard Henderson ts->kind = TEMP_FIXED; 1430c896fe29Sbellard ts->reg = reg; 1431c896fe29Sbellard ts->name = name; 1432c896fe29Sbellard tcg_regset_set_reg(s->reserved_regs, reg); 14337ca4b752SRichard Henderson 1434085272b3SRichard Henderson return ts; 1435a7812ae4Spbrook } 1436a7812ae4Spbrook 1437b6638662SRichard Henderson void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 1438a7812ae4Spbrook { 1439b3a62939SRichard Henderson s->frame_start = start; 1440b3a62939SRichard Henderson s->frame_end = start + size; 1441085272b3SRichard Henderson s->frame_temp 1442085272b3SRichard Henderson = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 1443b3a62939SRichard Henderson } 1444a7812ae4Spbrook 1445085272b3SRichard Henderson TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 1446e1ccc054SRichard Henderson intptr_t offset, const char *name) 1447c896fe29Sbellard { 1448b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1449dc41aa7dSRichard Henderson TCGTemp *base_ts = tcgv_ptr_temp(base); 14507ca4b752SRichard Henderson TCGTemp *ts = tcg_global_alloc(s); 1451aef85402SRichard Henderson int indirect_reg = 0; 1452c896fe29Sbellard 1453c0522136SRichard Henderson switch (base_ts->kind) { 1454c0522136SRichard Henderson case TEMP_FIXED: 1455c0522136SRichard Henderson break; 1456c0522136SRichard Henderson case TEMP_GLOBAL: 14575a18407fSRichard Henderson /* We do not support double-indirect registers. */ 14585a18407fSRichard Henderson tcg_debug_assert(!base_ts->indirect_reg); 1459b3915dbbSRichard Henderson base_ts->indirect_base = 1; 14605a18407fSRichard Henderson s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 14615a18407fSRichard Henderson ? 2 : 1); 14625a18407fSRichard Henderson indirect_reg = 1; 1463c0522136SRichard Henderson break; 1464c0522136SRichard Henderson default: 1465c0522136SRichard Henderson g_assert_not_reached(); 1466b3915dbbSRichard Henderson } 1467b3915dbbSRichard Henderson 14687ca4b752SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 14697ca4b752SRichard Henderson TCGTemp *ts2 = tcg_global_alloc(s); 1470c896fe29Sbellard char buf[64]; 14717ca4b752SRichard Henderson 14727ca4b752SRichard Henderson ts->base_type = TCG_TYPE_I64; 1473c896fe29Sbellard ts->type = TCG_TYPE_I32; 1474b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 1475c896fe29Sbellard ts->mem_allocated = 1; 1476b3a62939SRichard Henderson ts->mem_base = base_ts; 1477aef85402SRichard Henderson ts->mem_offset = offset; 1478c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 1479c896fe29Sbellard pstrcat(buf, sizeof(buf), "_0"); 1480c896fe29Sbellard ts->name = strdup(buf); 1481c896fe29Sbellard 14827ca4b752SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 14837ca4b752SRichard Henderson ts2->base_type = TCG_TYPE_I64; 14847ca4b752SRichard Henderson ts2->type = TCG_TYPE_I32; 1485b3915dbbSRichard Henderson ts2->indirect_reg = indirect_reg; 14867ca4b752SRichard Henderson ts2->mem_allocated = 1; 14877ca4b752SRichard Henderson ts2->mem_base = base_ts; 1488aef85402SRichard Henderson ts2->mem_offset = offset + 4; 1489fac87bd2SRichard Henderson ts2->temp_subindex = 1; 1490c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 1491c896fe29Sbellard pstrcat(buf, sizeof(buf), "_1"); 1492120c1084SRichard Henderson ts2->name = strdup(buf); 14937ca4b752SRichard Henderson } else { 1494c896fe29Sbellard ts->base_type = type; 1495c896fe29Sbellard ts->type = type; 1496b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 1497c896fe29Sbellard ts->mem_allocated = 1; 1498b3a62939SRichard Henderson ts->mem_base = base_ts; 1499c896fe29Sbellard ts->mem_offset = offset; 1500c896fe29Sbellard ts->name = name; 1501c896fe29Sbellard } 1502085272b3SRichard Henderson return ts; 1503c896fe29Sbellard } 1504c896fe29Sbellard 1505bbf989bfSRichard Henderson TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) 1506c896fe29Sbellard { 1507b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1508c896fe29Sbellard TCGTemp *ts; 1509e1c08b00SRichard Henderson int n; 1510c896fe29Sbellard 1511e1c08b00SRichard Henderson if (kind == TEMP_EBB) { 1512e1c08b00SRichard Henderson int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); 1513e1c08b00SRichard Henderson 15140ec9eabcSRichard Henderson if (idx < TCG_MAX_TEMPS) { 15150ec9eabcSRichard Henderson /* There is already an available temp with the right type. */ 1516e1c08b00SRichard Henderson clear_bit(idx, s->free_temps[type].l); 15170ec9eabcSRichard Henderson 1518e8996ee0Sbellard ts = &s->temps[idx]; 1519e8996ee0Sbellard ts->temp_allocated = 1; 15207ca4b752SRichard Henderson tcg_debug_assert(ts->base_type == type); 1521ee17db83SRichard Henderson tcg_debug_assert(ts->kind == kind); 15222f2e911dSRichard Henderson return ts; 1523e1c08b00SRichard Henderson } 1524e8996ee0Sbellard } else { 1525e1c08b00SRichard Henderson tcg_debug_assert(kind == TEMP_TB); 1526e1c08b00SRichard Henderson } 152743eef72fSRichard Henderson 152843eef72fSRichard Henderson switch (type) { 152943eef72fSRichard Henderson case TCG_TYPE_I32: 153043eef72fSRichard Henderson case TCG_TYPE_V64: 153143eef72fSRichard Henderson case TCG_TYPE_V128: 153243eef72fSRichard Henderson case TCG_TYPE_V256: 153343eef72fSRichard Henderson n = 1; 153443eef72fSRichard Henderson break; 153543eef72fSRichard Henderson case TCG_TYPE_I64: 153643eef72fSRichard Henderson n = 64 / TCG_TARGET_REG_BITS; 153743eef72fSRichard Henderson break; 153843eef72fSRichard Henderson case TCG_TYPE_I128: 153943eef72fSRichard Henderson n = 128 / TCG_TARGET_REG_BITS; 154043eef72fSRichard Henderson break; 154143eef72fSRichard Henderson default: 154243eef72fSRichard Henderson g_assert_not_reached(); 154343eef72fSRichard Henderson } 154443eef72fSRichard Henderson 15457ca4b752SRichard Henderson ts = tcg_temp_alloc(s); 154643eef72fSRichard Henderson ts->base_type = type; 154743eef72fSRichard Henderson ts->temp_allocated = 1; 154843eef72fSRichard Henderson ts->kind = kind; 154943eef72fSRichard Henderson 155043eef72fSRichard Henderson if (n == 1) { 155143eef72fSRichard Henderson ts->type = type; 155243eef72fSRichard Henderson } else { 155343eef72fSRichard Henderson ts->type = TCG_TYPE_REG; 155443eef72fSRichard Henderson 1555e1c08b00SRichard Henderson for (int i = 1; i < n; ++i) { 15567ca4b752SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 15577ca4b752SRichard Henderson 155843eef72fSRichard Henderson tcg_debug_assert(ts2 == ts + i); 155943eef72fSRichard Henderson ts2->base_type = type; 156043eef72fSRichard Henderson ts2->type = TCG_TYPE_REG; 15617ca4b752SRichard Henderson ts2->temp_allocated = 1; 156243eef72fSRichard Henderson ts2->temp_subindex = i; 1563ee17db83SRichard Henderson ts2->kind = kind; 156443eef72fSRichard Henderson } 1565c896fe29Sbellard } 1566085272b3SRichard Henderson return ts; 1567c896fe29Sbellard } 1568c896fe29Sbellard 1569d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec(TCGType type) 1570d2fd745fSRichard Henderson { 1571d2fd745fSRichard Henderson TCGTemp *t; 1572d2fd745fSRichard Henderson 1573d2fd745fSRichard Henderson #ifdef CONFIG_DEBUG_TCG 1574d2fd745fSRichard Henderson switch (type) { 1575d2fd745fSRichard Henderson case TCG_TYPE_V64: 1576d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v64); 1577d2fd745fSRichard Henderson break; 1578d2fd745fSRichard Henderson case TCG_TYPE_V128: 1579d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v128); 1580d2fd745fSRichard Henderson break; 1581d2fd745fSRichard Henderson case TCG_TYPE_V256: 1582d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v256); 1583d2fd745fSRichard Henderson break; 1584d2fd745fSRichard Henderson default: 1585d2fd745fSRichard Henderson g_assert_not_reached(); 1586d2fd745fSRichard Henderson } 1587d2fd745fSRichard Henderson #endif 1588d2fd745fSRichard Henderson 1589bbf989bfSRichard Henderson t = tcg_temp_new_internal(type, TEMP_EBB); 1590d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1591d2fd745fSRichard Henderson } 1592d2fd745fSRichard Henderson 1593d2fd745fSRichard Henderson /* Create a new temp of the same type as an existing temp. */ 1594d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1595d2fd745fSRichard Henderson { 1596d2fd745fSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 1597d2fd745fSRichard Henderson 1598d2fd745fSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 1599d2fd745fSRichard Henderson 1600bbf989bfSRichard Henderson t = tcg_temp_new_internal(t->base_type, TEMP_EBB); 1601d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1602d2fd745fSRichard Henderson } 1603d2fd745fSRichard Henderson 16045bfa8034SRichard Henderson void tcg_temp_free_internal(TCGTemp *ts) 1605c896fe29Sbellard { 1606b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1607c896fe29Sbellard 1608c7482438SRichard Henderson switch (ts->kind) { 1609c7482438SRichard Henderson case TEMP_CONST: 1610f57c6915SRichard Henderson case TEMP_TB: 16112f2e911dSRichard Henderson /* Silently ignore free. */ 1612c7482438SRichard Henderson break; 16132f2e911dSRichard Henderson case TEMP_EBB: 1614eabb7b91SAurelien Jarno tcg_debug_assert(ts->temp_allocated != 0); 1615e8996ee0Sbellard ts->temp_allocated = 0; 16162f2e911dSRichard Henderson set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); 16172f2e911dSRichard Henderson break; 16182f2e911dSRichard Henderson default: 16192f2e911dSRichard Henderson /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */ 16202f2e911dSRichard Henderson g_assert_not_reached(); 1621e1c08b00SRichard Henderson } 1622e8996ee0Sbellard } 1623e8996ee0Sbellard 1624c0522136SRichard Henderson TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1625c0522136SRichard Henderson { 1626c0522136SRichard Henderson TCGContext *s = tcg_ctx; 1627c0522136SRichard Henderson GHashTable *h = s->const_table[type]; 1628c0522136SRichard Henderson TCGTemp *ts; 1629c0522136SRichard Henderson 1630c0522136SRichard Henderson if (h == NULL) { 1631c0522136SRichard Henderson h = g_hash_table_new(g_int64_hash, g_int64_equal); 1632c0522136SRichard Henderson s->const_table[type] = h; 1633c0522136SRichard Henderson } 1634c0522136SRichard Henderson 1635c0522136SRichard Henderson ts = g_hash_table_lookup(h, &val); 1636c0522136SRichard Henderson if (ts == NULL) { 1637aef85402SRichard Henderson int64_t *val_ptr; 1638aef85402SRichard Henderson 1639c0522136SRichard Henderson ts = tcg_temp_alloc(s); 1640c0522136SRichard Henderson 1641c0522136SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1642c0522136SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 1643c0522136SRichard Henderson 1644aef85402SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 1645aef85402SRichard Henderson 1646c0522136SRichard Henderson ts->base_type = TCG_TYPE_I64; 1647c0522136SRichard Henderson ts->type = TCG_TYPE_I32; 1648c0522136SRichard Henderson ts->kind = TEMP_CONST; 1649c0522136SRichard Henderson ts->temp_allocated = 1; 1650c0522136SRichard Henderson 1651c0522136SRichard Henderson ts2->base_type = TCG_TYPE_I64; 1652c0522136SRichard Henderson ts2->type = TCG_TYPE_I32; 1653c0522136SRichard Henderson ts2->kind = TEMP_CONST; 1654c0522136SRichard Henderson ts2->temp_allocated = 1; 1655fac87bd2SRichard Henderson ts2->temp_subindex = 1; 1656aef85402SRichard Henderson 1657aef85402SRichard Henderson /* 1658aef85402SRichard Henderson * Retain the full value of the 64-bit constant in the low 1659aef85402SRichard Henderson * part, so that the hash table works. Actual uses will 1660aef85402SRichard Henderson * truncate the value to the low part. 1661aef85402SRichard Henderson */ 1662aef85402SRichard Henderson ts[HOST_BIG_ENDIAN].val = val; 1663aef85402SRichard Henderson ts[!HOST_BIG_ENDIAN].val = val >> 32; 1664aef85402SRichard Henderson val_ptr = &ts[HOST_BIG_ENDIAN].val; 1665c0522136SRichard Henderson } else { 1666c0522136SRichard Henderson ts->base_type = type; 1667c0522136SRichard Henderson ts->type = type; 1668c0522136SRichard Henderson ts->kind = TEMP_CONST; 1669c0522136SRichard Henderson ts->temp_allocated = 1; 1670c0522136SRichard Henderson ts->val = val; 1671aef85402SRichard Henderson val_ptr = &ts->val; 1672c0522136SRichard Henderson } 1673aef85402SRichard Henderson g_hash_table_insert(h, val_ptr, ts); 1674c0522136SRichard Henderson } 1675c0522136SRichard Henderson 1676c0522136SRichard Henderson return ts; 1677c0522136SRichard Henderson } 1678c0522136SRichard Henderson 1679c0522136SRichard Henderson TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1680c0522136SRichard Henderson { 1681c0522136SRichard Henderson val = dup_const(vece, val); 1682c0522136SRichard Henderson return temp_tcgv_vec(tcg_constant_internal(type, val)); 1683c0522136SRichard Henderson } 1684c0522136SRichard Henderson 168588d4005bSRichard Henderson TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 168688d4005bSRichard Henderson { 168788d4005bSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 168888d4005bSRichard Henderson 168988d4005bSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 169088d4005bSRichard Henderson return tcg_constant_vec(t->base_type, vece, val); 169188d4005bSRichard Henderson } 169288d4005bSRichard Henderson 1693be0f34b5SRichard Henderson /* Return true if OP may appear in the opcode stream. 1694be0f34b5SRichard Henderson Test the runtime variable that controls each opcode. */ 1695be0f34b5SRichard Henderson bool tcg_op_supported(TCGOpcode op) 1696be0f34b5SRichard Henderson { 1697d2fd745fSRichard Henderson const bool have_vec 1698d2fd745fSRichard Henderson = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1699d2fd745fSRichard Henderson 1700be0f34b5SRichard Henderson switch (op) { 1701be0f34b5SRichard Henderson case INDEX_op_discard: 1702be0f34b5SRichard Henderson case INDEX_op_set_label: 1703be0f34b5SRichard Henderson case INDEX_op_call: 1704be0f34b5SRichard Henderson case INDEX_op_br: 1705be0f34b5SRichard Henderson case INDEX_op_mb: 1706be0f34b5SRichard Henderson case INDEX_op_insn_start: 1707be0f34b5SRichard Henderson case INDEX_op_exit_tb: 1708be0f34b5SRichard Henderson case INDEX_op_goto_tb: 1709f4e01e30SRichard Henderson case INDEX_op_goto_ptr: 1710be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i32: 1711be0f34b5SRichard Henderson case INDEX_op_qemu_st_i32: 1712be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i64: 1713be0f34b5SRichard Henderson case INDEX_op_qemu_st_i64: 1714be0f34b5SRichard Henderson return true; 1715be0f34b5SRichard Henderson 171607ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 171707ce0b05SRichard Henderson return TCG_TARGET_HAS_qemu_st8_i32; 171807ce0b05SRichard Henderson 1719be0f34b5SRichard Henderson case INDEX_op_mov_i32: 1720be0f34b5SRichard Henderson case INDEX_op_setcond_i32: 1721be0f34b5SRichard Henderson case INDEX_op_brcond_i32: 1722be0f34b5SRichard Henderson case INDEX_op_ld8u_i32: 1723be0f34b5SRichard Henderson case INDEX_op_ld8s_i32: 1724be0f34b5SRichard Henderson case INDEX_op_ld16u_i32: 1725be0f34b5SRichard Henderson case INDEX_op_ld16s_i32: 1726be0f34b5SRichard Henderson case INDEX_op_ld_i32: 1727be0f34b5SRichard Henderson case INDEX_op_st8_i32: 1728be0f34b5SRichard Henderson case INDEX_op_st16_i32: 1729be0f34b5SRichard Henderson case INDEX_op_st_i32: 1730be0f34b5SRichard Henderson case INDEX_op_add_i32: 1731be0f34b5SRichard Henderson case INDEX_op_sub_i32: 1732be0f34b5SRichard Henderson case INDEX_op_mul_i32: 1733be0f34b5SRichard Henderson case INDEX_op_and_i32: 1734be0f34b5SRichard Henderson case INDEX_op_or_i32: 1735be0f34b5SRichard Henderson case INDEX_op_xor_i32: 1736be0f34b5SRichard Henderson case INDEX_op_shl_i32: 1737be0f34b5SRichard Henderson case INDEX_op_shr_i32: 1738be0f34b5SRichard Henderson case INDEX_op_sar_i32: 1739be0f34b5SRichard Henderson return true; 1740be0f34b5SRichard Henderson 1741be0f34b5SRichard Henderson case INDEX_op_movcond_i32: 1742be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i32; 1743be0f34b5SRichard Henderson case INDEX_op_div_i32: 1744be0f34b5SRichard Henderson case INDEX_op_divu_i32: 1745be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i32; 1746be0f34b5SRichard Henderson case INDEX_op_rem_i32: 1747be0f34b5SRichard Henderson case INDEX_op_remu_i32: 1748be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i32; 1749be0f34b5SRichard Henderson case INDEX_op_div2_i32: 1750be0f34b5SRichard Henderson case INDEX_op_divu2_i32: 1751be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i32; 1752be0f34b5SRichard Henderson case INDEX_op_rotl_i32: 1753be0f34b5SRichard Henderson case INDEX_op_rotr_i32: 1754be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i32; 1755be0f34b5SRichard Henderson case INDEX_op_deposit_i32: 1756be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i32; 1757be0f34b5SRichard Henderson case INDEX_op_extract_i32: 1758be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i32; 1759be0f34b5SRichard Henderson case INDEX_op_sextract_i32: 1760be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i32; 1761fce1296fSRichard Henderson case INDEX_op_extract2_i32: 1762fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i32; 1763be0f34b5SRichard Henderson case INDEX_op_add2_i32: 1764be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i32; 1765be0f34b5SRichard Henderson case INDEX_op_sub2_i32: 1766be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i32; 1767be0f34b5SRichard Henderson case INDEX_op_mulu2_i32: 1768be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i32; 1769be0f34b5SRichard Henderson case INDEX_op_muls2_i32: 1770be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i32; 1771be0f34b5SRichard Henderson case INDEX_op_muluh_i32: 1772be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i32; 1773be0f34b5SRichard Henderson case INDEX_op_mulsh_i32: 1774be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i32; 1775be0f34b5SRichard Henderson case INDEX_op_ext8s_i32: 1776be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i32; 1777be0f34b5SRichard Henderson case INDEX_op_ext16s_i32: 1778be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i32; 1779be0f34b5SRichard Henderson case INDEX_op_ext8u_i32: 1780be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i32; 1781be0f34b5SRichard Henderson case INDEX_op_ext16u_i32: 1782be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i32; 1783be0f34b5SRichard Henderson case INDEX_op_bswap16_i32: 1784be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i32; 1785be0f34b5SRichard Henderson case INDEX_op_bswap32_i32: 1786be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i32; 1787be0f34b5SRichard Henderson case INDEX_op_not_i32: 1788be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i32; 1789be0f34b5SRichard Henderson case INDEX_op_neg_i32: 1790be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i32; 1791be0f34b5SRichard Henderson case INDEX_op_andc_i32: 1792be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i32; 1793be0f34b5SRichard Henderson case INDEX_op_orc_i32: 1794be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i32; 1795be0f34b5SRichard Henderson case INDEX_op_eqv_i32: 1796be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i32; 1797be0f34b5SRichard Henderson case INDEX_op_nand_i32: 1798be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i32; 1799be0f34b5SRichard Henderson case INDEX_op_nor_i32: 1800be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i32; 1801be0f34b5SRichard Henderson case INDEX_op_clz_i32: 1802be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i32; 1803be0f34b5SRichard Henderson case INDEX_op_ctz_i32: 1804be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i32; 1805be0f34b5SRichard Henderson case INDEX_op_ctpop_i32: 1806be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i32; 1807be0f34b5SRichard Henderson 1808be0f34b5SRichard Henderson case INDEX_op_brcond2_i32: 1809be0f34b5SRichard Henderson case INDEX_op_setcond2_i32: 1810be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 32; 1811be0f34b5SRichard Henderson 1812be0f34b5SRichard Henderson case INDEX_op_mov_i64: 1813be0f34b5SRichard Henderson case INDEX_op_setcond_i64: 1814be0f34b5SRichard Henderson case INDEX_op_brcond_i64: 1815be0f34b5SRichard Henderson case INDEX_op_ld8u_i64: 1816be0f34b5SRichard Henderson case INDEX_op_ld8s_i64: 1817be0f34b5SRichard Henderson case INDEX_op_ld16u_i64: 1818be0f34b5SRichard Henderson case INDEX_op_ld16s_i64: 1819be0f34b5SRichard Henderson case INDEX_op_ld32u_i64: 1820be0f34b5SRichard Henderson case INDEX_op_ld32s_i64: 1821be0f34b5SRichard Henderson case INDEX_op_ld_i64: 1822be0f34b5SRichard Henderson case INDEX_op_st8_i64: 1823be0f34b5SRichard Henderson case INDEX_op_st16_i64: 1824be0f34b5SRichard Henderson case INDEX_op_st32_i64: 1825be0f34b5SRichard Henderson case INDEX_op_st_i64: 1826be0f34b5SRichard Henderson case INDEX_op_add_i64: 1827be0f34b5SRichard Henderson case INDEX_op_sub_i64: 1828be0f34b5SRichard Henderson case INDEX_op_mul_i64: 1829be0f34b5SRichard Henderson case INDEX_op_and_i64: 1830be0f34b5SRichard Henderson case INDEX_op_or_i64: 1831be0f34b5SRichard Henderson case INDEX_op_xor_i64: 1832be0f34b5SRichard Henderson case INDEX_op_shl_i64: 1833be0f34b5SRichard Henderson case INDEX_op_shr_i64: 1834be0f34b5SRichard Henderson case INDEX_op_sar_i64: 1835be0f34b5SRichard Henderson case INDEX_op_ext_i32_i64: 1836be0f34b5SRichard Henderson case INDEX_op_extu_i32_i64: 1837be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 64; 1838be0f34b5SRichard Henderson 1839be0f34b5SRichard Henderson case INDEX_op_movcond_i64: 1840be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i64; 1841be0f34b5SRichard Henderson case INDEX_op_div_i64: 1842be0f34b5SRichard Henderson case INDEX_op_divu_i64: 1843be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i64; 1844be0f34b5SRichard Henderson case INDEX_op_rem_i64: 1845be0f34b5SRichard Henderson case INDEX_op_remu_i64: 1846be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i64; 1847be0f34b5SRichard Henderson case INDEX_op_div2_i64: 1848be0f34b5SRichard Henderson case INDEX_op_divu2_i64: 1849be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i64; 1850be0f34b5SRichard Henderson case INDEX_op_rotl_i64: 1851be0f34b5SRichard Henderson case INDEX_op_rotr_i64: 1852be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i64; 1853be0f34b5SRichard Henderson case INDEX_op_deposit_i64: 1854be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i64; 1855be0f34b5SRichard Henderson case INDEX_op_extract_i64: 1856be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i64; 1857be0f34b5SRichard Henderson case INDEX_op_sextract_i64: 1858be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i64; 1859fce1296fSRichard Henderson case INDEX_op_extract2_i64: 1860fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i64; 1861be0f34b5SRichard Henderson case INDEX_op_extrl_i64_i32: 1862be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrl_i64_i32; 1863be0f34b5SRichard Henderson case INDEX_op_extrh_i64_i32: 1864be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrh_i64_i32; 1865be0f34b5SRichard Henderson case INDEX_op_ext8s_i64: 1866be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i64; 1867be0f34b5SRichard Henderson case INDEX_op_ext16s_i64: 1868be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i64; 1869be0f34b5SRichard Henderson case INDEX_op_ext32s_i64: 1870be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32s_i64; 1871be0f34b5SRichard Henderson case INDEX_op_ext8u_i64: 1872be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i64; 1873be0f34b5SRichard Henderson case INDEX_op_ext16u_i64: 1874be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i64; 1875be0f34b5SRichard Henderson case INDEX_op_ext32u_i64: 1876be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32u_i64; 1877be0f34b5SRichard Henderson case INDEX_op_bswap16_i64: 1878be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i64; 1879be0f34b5SRichard Henderson case INDEX_op_bswap32_i64: 1880be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i64; 1881be0f34b5SRichard Henderson case INDEX_op_bswap64_i64: 1882be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap64_i64; 1883be0f34b5SRichard Henderson case INDEX_op_not_i64: 1884be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i64; 1885be0f34b5SRichard Henderson case INDEX_op_neg_i64: 1886be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i64; 1887be0f34b5SRichard Henderson case INDEX_op_andc_i64: 1888be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i64; 1889be0f34b5SRichard Henderson case INDEX_op_orc_i64: 1890be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i64; 1891be0f34b5SRichard Henderson case INDEX_op_eqv_i64: 1892be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i64; 1893be0f34b5SRichard Henderson case INDEX_op_nand_i64: 1894be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i64; 1895be0f34b5SRichard Henderson case INDEX_op_nor_i64: 1896be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i64; 1897be0f34b5SRichard Henderson case INDEX_op_clz_i64: 1898be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i64; 1899be0f34b5SRichard Henderson case INDEX_op_ctz_i64: 1900be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i64; 1901be0f34b5SRichard Henderson case INDEX_op_ctpop_i64: 1902be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i64; 1903be0f34b5SRichard Henderson case INDEX_op_add2_i64: 1904be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i64; 1905be0f34b5SRichard Henderson case INDEX_op_sub2_i64: 1906be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i64; 1907be0f34b5SRichard Henderson case INDEX_op_mulu2_i64: 1908be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i64; 1909be0f34b5SRichard Henderson case INDEX_op_muls2_i64: 1910be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i64; 1911be0f34b5SRichard Henderson case INDEX_op_muluh_i64: 1912be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i64; 1913be0f34b5SRichard Henderson case INDEX_op_mulsh_i64: 1914be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i64; 1915be0f34b5SRichard Henderson 1916d2fd745fSRichard Henderson case INDEX_op_mov_vec: 1917d2fd745fSRichard Henderson case INDEX_op_dup_vec: 191837ee55a0SRichard Henderson case INDEX_op_dupm_vec: 1919d2fd745fSRichard Henderson case INDEX_op_ld_vec: 1920d2fd745fSRichard Henderson case INDEX_op_st_vec: 1921d2fd745fSRichard Henderson case INDEX_op_add_vec: 1922d2fd745fSRichard Henderson case INDEX_op_sub_vec: 1923d2fd745fSRichard Henderson case INDEX_op_and_vec: 1924d2fd745fSRichard Henderson case INDEX_op_or_vec: 1925d2fd745fSRichard Henderson case INDEX_op_xor_vec: 1926212be173SRichard Henderson case INDEX_op_cmp_vec: 1927d2fd745fSRichard Henderson return have_vec; 1928d2fd745fSRichard Henderson case INDEX_op_dup2_vec: 1929d2fd745fSRichard Henderson return have_vec && TCG_TARGET_REG_BITS == 32; 1930d2fd745fSRichard Henderson case INDEX_op_not_vec: 1931d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_not_vec; 1932d2fd745fSRichard Henderson case INDEX_op_neg_vec: 1933d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_neg_vec; 1934bcefc902SRichard Henderson case INDEX_op_abs_vec: 1935bcefc902SRichard Henderson return have_vec && TCG_TARGET_HAS_abs_vec; 1936d2fd745fSRichard Henderson case INDEX_op_andc_vec: 1937d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_andc_vec; 1938d2fd745fSRichard Henderson case INDEX_op_orc_vec: 1939d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_orc_vec; 1940ed523473SRichard Henderson case INDEX_op_nand_vec: 1941ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nand_vec; 1942ed523473SRichard Henderson case INDEX_op_nor_vec: 1943ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nor_vec; 1944ed523473SRichard Henderson case INDEX_op_eqv_vec: 1945ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_eqv_vec; 19463774030aSRichard Henderson case INDEX_op_mul_vec: 19473774030aSRichard Henderson return have_vec && TCG_TARGET_HAS_mul_vec; 1948d0ec9796SRichard Henderson case INDEX_op_shli_vec: 1949d0ec9796SRichard Henderson case INDEX_op_shri_vec: 1950d0ec9796SRichard Henderson case INDEX_op_sari_vec: 1951d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shi_vec; 1952d0ec9796SRichard Henderson case INDEX_op_shls_vec: 1953d0ec9796SRichard Henderson case INDEX_op_shrs_vec: 1954d0ec9796SRichard Henderson case INDEX_op_sars_vec: 1955d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shs_vec; 1956d0ec9796SRichard Henderson case INDEX_op_shlv_vec: 1957d0ec9796SRichard Henderson case INDEX_op_shrv_vec: 1958d0ec9796SRichard Henderson case INDEX_op_sarv_vec: 1959d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shv_vec; 1960b0f7e744SRichard Henderson case INDEX_op_rotli_vec: 1961b0f7e744SRichard Henderson return have_vec && TCG_TARGET_HAS_roti_vec; 196223850a74SRichard Henderson case INDEX_op_rotls_vec: 196323850a74SRichard Henderson return have_vec && TCG_TARGET_HAS_rots_vec; 19645d0ceda9SRichard Henderson case INDEX_op_rotlv_vec: 19655d0ceda9SRichard Henderson case INDEX_op_rotrv_vec: 19665d0ceda9SRichard Henderson return have_vec && TCG_TARGET_HAS_rotv_vec; 19678afaf050SRichard Henderson case INDEX_op_ssadd_vec: 19688afaf050SRichard Henderson case INDEX_op_usadd_vec: 19698afaf050SRichard Henderson case INDEX_op_sssub_vec: 19708afaf050SRichard Henderson case INDEX_op_ussub_vec: 19718afaf050SRichard Henderson return have_vec && TCG_TARGET_HAS_sat_vec; 1972dd0a0fcdSRichard Henderson case INDEX_op_smin_vec: 1973dd0a0fcdSRichard Henderson case INDEX_op_umin_vec: 1974dd0a0fcdSRichard Henderson case INDEX_op_smax_vec: 1975dd0a0fcdSRichard Henderson case INDEX_op_umax_vec: 1976dd0a0fcdSRichard Henderson return have_vec && TCG_TARGET_HAS_minmax_vec; 197738dc1294SRichard Henderson case INDEX_op_bitsel_vec: 197838dc1294SRichard Henderson return have_vec && TCG_TARGET_HAS_bitsel_vec; 1979f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 1980f75da298SRichard Henderson return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1981d2fd745fSRichard Henderson 1982db432672SRichard Henderson default: 1983db432672SRichard Henderson tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1984db432672SRichard Henderson return true; 1985be0f34b5SRichard Henderson } 1986be0f34b5SRichard Henderson } 1987be0f34b5SRichard Henderson 198839004a71SRichard Henderson static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); 198939004a71SRichard Henderson 1990ae8b75dcSRichard Henderson void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1991c896fe29Sbellard { 19923e92aa34SRichard Henderson const TCGHelperInfo *info; 199339004a71SRichard Henderson TCGv_i64 extend_free[MAX_CALL_IARGS]; 199439004a71SRichard Henderson int n_extend = 0; 199575e8b9b7SRichard Henderson TCGOp *op; 199639004a71SRichard Henderson int i, n, pi = 0, total_args; 1997afb49896SRichard Henderson 1998619205fdSEmilio G. Cota info = g_hash_table_lookup(helper_table, (gpointer)func); 199939004a71SRichard Henderson total_args = info->nr_out + info->nr_in + 2; 200039004a71SRichard Henderson op = tcg_op_alloc(INDEX_op_call, total_args); 20012bece2c8SRichard Henderson 200238b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 200317083f6fSEmilio Cota /* Flag helpers that may affect guest state */ 200417083f6fSEmilio Cota if (tcg_ctx->plugin_insn && 200517083f6fSEmilio Cota !(info->flags & TCG_CALL_PLUGIN) && 200617083f6fSEmilio Cota !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { 200738b47b19SEmilio G. Cota tcg_ctx->plugin_insn->calls_helpers = true; 200838b47b19SEmilio G. Cota } 200938b47b19SEmilio G. Cota #endif 201038b47b19SEmilio G. Cota 201139004a71SRichard Henderson TCGOP_CALLO(op) = n = info->nr_out; 201239004a71SRichard Henderson switch (n) { 201339004a71SRichard Henderson case 0: 201439004a71SRichard Henderson tcg_debug_assert(ret == NULL); 201539004a71SRichard Henderson break; 201639004a71SRichard Henderson case 1: 201739004a71SRichard Henderson tcg_debug_assert(ret != NULL); 201839004a71SRichard Henderson op->args[pi++] = temp_arg(ret); 201939004a71SRichard Henderson break; 202039004a71SRichard Henderson case 2: 2021466d3759SRichard Henderson case 4: 202239004a71SRichard Henderson tcg_debug_assert(ret != NULL); 2023466d3759SRichard Henderson tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); 202439004a71SRichard Henderson tcg_debug_assert(ret->temp_subindex == 0); 2025466d3759SRichard Henderson for (i = 0; i < n; ++i) { 2026466d3759SRichard Henderson op->args[pi++] = temp_arg(ret + i); 2027466d3759SRichard Henderson } 202839004a71SRichard Henderson break; 202939004a71SRichard Henderson default: 203039004a71SRichard Henderson g_assert_not_reached(); 203139004a71SRichard Henderson } 20327319d83aSRichard Henderson 203339004a71SRichard Henderson TCGOP_CALLI(op) = n = info->nr_in; 203439004a71SRichard Henderson for (i = 0; i < n; i++) { 203539004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 203639004a71SRichard Henderson TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; 203739004a71SRichard Henderson 203839004a71SRichard Henderson switch (loc->kind) { 203939004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 2040313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 2041313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 204239004a71SRichard Henderson op->args[pi++] = temp_arg(ts); 204339004a71SRichard Henderson break; 204439004a71SRichard Henderson 204539004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 204639004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 204739004a71SRichard Henderson { 20485dd48602SRichard Henderson TCGv_i64 temp = tcg_temp_ebb_new_i64(); 204939004a71SRichard Henderson TCGv_i32 orig = temp_tcgv_i32(ts); 205039004a71SRichard Henderson 205139004a71SRichard Henderson if (loc->kind == TCG_CALL_ARG_EXTEND_S) { 205218cf3d07SRichard Henderson tcg_gen_ext_i32_i64(temp, orig); 20532bece2c8SRichard Henderson } else { 205418cf3d07SRichard Henderson tcg_gen_extu_i32_i64(temp, orig); 20552bece2c8SRichard Henderson } 205639004a71SRichard Henderson op->args[pi++] = tcgv_i64_arg(temp); 205739004a71SRichard Henderson extend_free[n_extend++] = temp; 20582bece2c8SRichard Henderson } 205939004a71SRichard Henderson break; 20602bece2c8SRichard Henderson 2061e2a9dd6bSRichard Henderson default: 2062e2a9dd6bSRichard Henderson g_assert_not_reached(); 2063e2a9dd6bSRichard Henderson } 2064c896fe29Sbellard } 206575e8b9b7SRichard Henderson op->args[pi++] = (uintptr_t)func; 20663e92aa34SRichard Henderson op->args[pi++] = (uintptr_t)info; 206739004a71SRichard Henderson tcg_debug_assert(pi == total_args); 2068a7812ae4Spbrook 206939004a71SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 20702bece2c8SRichard Henderson 207139004a71SRichard Henderson tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); 207239004a71SRichard Henderson for (i = 0; i < n_extend; ++i) { 207339004a71SRichard Henderson tcg_temp_free_i64(extend_free[i]); 2074eb8b0224SRichard Henderson } 2075a7812ae4Spbrook } 2076c896fe29Sbellard 20778fcd3692Sblueswir1 static void tcg_reg_alloc_start(TCGContext *s) 2078c896fe29Sbellard { 2079ac3b8891SRichard Henderson int i, n; 2080ac3b8891SRichard Henderson 2081ee17db83SRichard Henderson for (i = 0, n = s->nb_temps; i < n; i++) { 2082ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 2083ee17db83SRichard Henderson TCGTempVal val = TEMP_VAL_MEM; 2084ee17db83SRichard Henderson 2085ee17db83SRichard Henderson switch (ts->kind) { 2086c0522136SRichard Henderson case TEMP_CONST: 2087c0522136SRichard Henderson val = TEMP_VAL_CONST; 2088c0522136SRichard Henderson break; 2089ee17db83SRichard Henderson case TEMP_FIXED: 2090ee17db83SRichard Henderson val = TEMP_VAL_REG; 2091ee17db83SRichard Henderson break; 2092ee17db83SRichard Henderson case TEMP_GLOBAL: 2093ee17db83SRichard Henderson break; 2094c7482438SRichard Henderson case TEMP_EBB: 2095ee17db83SRichard Henderson val = TEMP_VAL_DEAD; 2096ee17db83SRichard Henderson /* fall through */ 2097f57c6915SRichard Henderson case TEMP_TB: 2098e8996ee0Sbellard ts->mem_allocated = 0; 2099ee17db83SRichard Henderson break; 2100ee17db83SRichard Henderson default: 2101ee17db83SRichard Henderson g_assert_not_reached(); 2102ee17db83SRichard Henderson } 2103ee17db83SRichard Henderson ts->val_type = val; 2104e8996ee0Sbellard } 2105f8b2f202SRichard Henderson 2106f8b2f202SRichard Henderson memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 2107c896fe29Sbellard } 2108c896fe29Sbellard 2109f8b2f202SRichard Henderson static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 2110f8b2f202SRichard Henderson TCGTemp *ts) 2111c896fe29Sbellard { 21121807f4c4SRichard Henderson int idx = temp_idx(ts); 2113ac56dd48Spbrook 2114ee17db83SRichard Henderson switch (ts->kind) { 2115ee17db83SRichard Henderson case TEMP_FIXED: 2116ee17db83SRichard Henderson case TEMP_GLOBAL: 2117ac56dd48Spbrook pstrcpy(buf, buf_size, ts->name); 2118ee17db83SRichard Henderson break; 2119f57c6915SRichard Henderson case TEMP_TB: 2120641d5fbeSbellard snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 2121ee17db83SRichard Henderson break; 2122c7482438SRichard Henderson case TEMP_EBB: 2123ac56dd48Spbrook snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 2124ee17db83SRichard Henderson break; 2125c0522136SRichard Henderson case TEMP_CONST: 2126c0522136SRichard Henderson switch (ts->type) { 2127c0522136SRichard Henderson case TCG_TYPE_I32: 2128c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 2129c0522136SRichard Henderson break; 2130c0522136SRichard Henderson #if TCG_TARGET_REG_BITS > 32 2131c0522136SRichard Henderson case TCG_TYPE_I64: 2132c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 2133c0522136SRichard Henderson break; 2134c0522136SRichard Henderson #endif 2135c0522136SRichard Henderson case TCG_TYPE_V64: 2136c0522136SRichard Henderson case TCG_TYPE_V128: 2137c0522136SRichard Henderson case TCG_TYPE_V256: 2138c0522136SRichard Henderson snprintf(buf, buf_size, "v%d$0x%" PRIx64, 2139c0522136SRichard Henderson 64 << (ts->type - TCG_TYPE_V64), ts->val); 2140c0522136SRichard Henderson break; 2141c0522136SRichard Henderson default: 2142c0522136SRichard Henderson g_assert_not_reached(); 2143c0522136SRichard Henderson } 2144c0522136SRichard Henderson break; 2145c896fe29Sbellard } 2146c896fe29Sbellard return buf; 2147c896fe29Sbellard } 2148c896fe29Sbellard 214943439139SRichard Henderson static char *tcg_get_arg_str(TCGContext *s, char *buf, 215043439139SRichard Henderson int buf_size, TCGArg arg) 2151f8b2f202SRichard Henderson { 215243439139SRichard Henderson return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 2153f8b2f202SRichard Henderson } 2154f8b2f202SRichard Henderson 2155f48f3edeSblueswir1 static const char * const cond_name[] = 2156f48f3edeSblueswir1 { 21570aed257fSRichard Henderson [TCG_COND_NEVER] = "never", 21580aed257fSRichard Henderson [TCG_COND_ALWAYS] = "always", 2159f48f3edeSblueswir1 [TCG_COND_EQ] = "eq", 2160f48f3edeSblueswir1 [TCG_COND_NE] = "ne", 2161f48f3edeSblueswir1 [TCG_COND_LT] = "lt", 2162f48f3edeSblueswir1 [TCG_COND_GE] = "ge", 2163f48f3edeSblueswir1 [TCG_COND_LE] = "le", 2164f48f3edeSblueswir1 [TCG_COND_GT] = "gt", 2165f48f3edeSblueswir1 [TCG_COND_LTU] = "ltu", 2166f48f3edeSblueswir1 [TCG_COND_GEU] = "geu", 2167f48f3edeSblueswir1 [TCG_COND_LEU] = "leu", 2168f48f3edeSblueswir1 [TCG_COND_GTU] = "gtu" 2169f48f3edeSblueswir1 }; 2170f48f3edeSblueswir1 2171f713d6adSRichard Henderson static const char * const ldst_name[] = 2172f713d6adSRichard Henderson { 2173f713d6adSRichard Henderson [MO_UB] = "ub", 2174f713d6adSRichard Henderson [MO_SB] = "sb", 2175f713d6adSRichard Henderson [MO_LEUW] = "leuw", 2176f713d6adSRichard Henderson [MO_LESW] = "lesw", 2177f713d6adSRichard Henderson [MO_LEUL] = "leul", 2178f713d6adSRichard Henderson [MO_LESL] = "lesl", 2179fc313c64SFrédéric Pétrot [MO_LEUQ] = "leq", 2180f713d6adSRichard Henderson [MO_BEUW] = "beuw", 2181f713d6adSRichard Henderson [MO_BESW] = "besw", 2182f713d6adSRichard Henderson [MO_BEUL] = "beul", 2183f713d6adSRichard Henderson [MO_BESL] = "besl", 2184fc313c64SFrédéric Pétrot [MO_BEUQ] = "beq", 2185f713d6adSRichard Henderson }; 2186f713d6adSRichard Henderson 21871f00b27fSSergey Sorokin static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 218852bf9771Stony.nguyen@bt.com #ifdef TARGET_ALIGNED_ONLY 21891f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "un+", 21901f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "", 21911f00b27fSSergey Sorokin #else 21921f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "", 21931f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "al+", 21941f00b27fSSergey Sorokin #endif 21951f00b27fSSergey Sorokin [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 21961f00b27fSSergey Sorokin [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 21971f00b27fSSergey Sorokin [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 21981f00b27fSSergey Sorokin [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 21991f00b27fSSergey Sorokin [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 22001f00b27fSSergey Sorokin [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 22011f00b27fSSergey Sorokin }; 22021f00b27fSSergey Sorokin 2203587195bdSRichard Henderson static const char bswap_flag_name[][6] = { 2204587195bdSRichard Henderson [TCG_BSWAP_IZ] = "iz", 2205587195bdSRichard Henderson [TCG_BSWAP_OZ] = "oz", 2206587195bdSRichard Henderson [TCG_BSWAP_OS] = "os", 2207587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 2208587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 2209587195bdSRichard Henderson }; 2210587195bdSRichard Henderson 2211b016486eSRichard Henderson static inline bool tcg_regset_single(TCGRegSet d) 2212b016486eSRichard Henderson { 2213b016486eSRichard Henderson return (d & (d - 1)) == 0; 2214b016486eSRichard Henderson } 2215b016486eSRichard Henderson 2216b016486eSRichard Henderson static inline TCGReg tcg_regset_first(TCGRegSet d) 2217b016486eSRichard Henderson { 2218b016486eSRichard Henderson if (TCG_TARGET_NB_REGS <= 32) { 2219b016486eSRichard Henderson return ctz32(d); 2220b016486eSRichard Henderson } else { 2221b016486eSRichard Henderson return ctz64(d); 2222b016486eSRichard Henderson } 2223b016486eSRichard Henderson } 2224b016486eSRichard Henderson 2225b7a83ff8SRichard Henderson /* Return only the number of characters output -- no error return. */ 2226b7a83ff8SRichard Henderson #define ne_fprintf(...) \ 2227b7a83ff8SRichard Henderson ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 2228b7a83ff8SRichard Henderson 2229b7a83ff8SRichard Henderson static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 2230c896fe29Sbellard { 2231c896fe29Sbellard char buf[128]; 2232c45cb8bbSRichard Henderson TCGOp *op; 2233c896fe29Sbellard 223415fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 2235c45cb8bbSRichard Henderson int i, k, nb_oargs, nb_iargs, nb_cargs; 2236c45cb8bbSRichard Henderson const TCGOpDef *def; 2237c45cb8bbSRichard Henderson TCGOpcode c; 2238bdfb460eSRichard Henderson int col = 0; 2239c45cb8bbSRichard Henderson 2240c45cb8bbSRichard Henderson c = op->opc; 2241c896fe29Sbellard def = &tcg_op_defs[c]; 2242c45cb8bbSRichard Henderson 2243765b842aSRichard Henderson if (c == INDEX_op_insn_start) { 2244b016486eSRichard Henderson nb_oargs = 0; 2245b7a83ff8SRichard Henderson col += ne_fprintf(f, "\n ----"); 22469aef40edSRichard Henderson 22479aef40edSRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 22489aef40edSRichard Henderson target_ulong a; 22497e4597d7Sbellard #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 2250efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 22517e4597d7Sbellard #else 2252efee3746SRichard Henderson a = op->args[i]; 22537e4597d7Sbellard #endif 2254b7a83ff8SRichard Henderson col += ne_fprintf(f, " " TARGET_FMT_lx, a); 2255eeacee4dSBlue Swirl } 22567e4597d7Sbellard } else if (c == INDEX_op_call) { 22573e92aa34SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 2258fa52e660SRichard Henderson void *func = tcg_call_func(op); 22593e92aa34SRichard Henderson 2260c896fe29Sbellard /* variable number of arguments */ 2261cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 2262cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 2263c896fe29Sbellard nb_cargs = def->nb_cargs; 2264b03cce8eSbellard 2265b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name); 22663e92aa34SRichard Henderson 22673e92aa34SRichard Henderson /* 22683e92aa34SRichard Henderson * Print the function name from TCGHelperInfo, if available. 22693e92aa34SRichard Henderson * Note that plugins have a template function for the info, 22703e92aa34SRichard Henderson * but the actual function pointer comes from the plugin. 22713e92aa34SRichard Henderson */ 22723e92aa34SRichard Henderson if (func == info->func) { 2273b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s", info->name); 22743e92aa34SRichard Henderson } else { 2275b7a83ff8SRichard Henderson col += ne_fprintf(f, "plugin(%p)", func); 22763e92aa34SRichard Henderson } 22773e92aa34SRichard Henderson 2278b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 2279b03cce8eSbellard for (i = 0; i < nb_oargs; i++) { 2280b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 2281efee3746SRichard Henderson op->args[i])); 2282b03cce8eSbellard } 2283cf066674SRichard Henderson for (i = 0; i < nb_iargs; i++) { 2284efee3746SRichard Henderson TCGArg arg = op->args[nb_oargs + i]; 228539004a71SRichard Henderson const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 2286b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", t); 2287e8996ee0Sbellard } 2288b03cce8eSbellard } else { 2289b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name); 2290c45cb8bbSRichard Henderson 2291c896fe29Sbellard nb_oargs = def->nb_oargs; 2292c896fe29Sbellard nb_iargs = def->nb_iargs; 2293c896fe29Sbellard nb_cargs = def->nb_cargs; 2294c896fe29Sbellard 2295d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 2296b7a83ff8SRichard Henderson col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op), 2297d2fd745fSRichard Henderson 8 << TCGOP_VECE(op)); 2298d2fd745fSRichard Henderson } 2299d2fd745fSRichard Henderson 2300c896fe29Sbellard k = 0; 2301c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 2302b7a83ff8SRichard Henderson const char *sep = k ? "," : ""; 2303b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep, 2304b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf), 2305efee3746SRichard Henderson op->args[k++])); 2306c896fe29Sbellard } 2307c896fe29Sbellard for (i = 0; i < nb_iargs; i++) { 2308b7a83ff8SRichard Henderson const char *sep = k ? "," : ""; 2309b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep, 2310b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf), 2311efee3746SRichard Henderson op->args[k++])); 2312c896fe29Sbellard } 2313be210acbSRichard Henderson switch (c) { 2314be210acbSRichard Henderson case INDEX_op_brcond_i32: 2315ffc5ea09SRichard Henderson case INDEX_op_setcond_i32: 2316ffc5ea09SRichard Henderson case INDEX_op_movcond_i32: 2317be210acbSRichard Henderson case INDEX_op_brcond2_i32: 2318be210acbSRichard Henderson case INDEX_op_setcond2_i32: 2319ffc5ea09SRichard Henderson case INDEX_op_brcond_i64: 2320be210acbSRichard Henderson case INDEX_op_setcond_i64: 2321ffc5ea09SRichard Henderson case INDEX_op_movcond_i64: 2322212be173SRichard Henderson case INDEX_op_cmp_vec: 2323f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 2324efee3746SRichard Henderson if (op->args[k] < ARRAY_SIZE(cond_name) 2325efee3746SRichard Henderson && cond_name[op->args[k]]) { 2326b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 2327eeacee4dSBlue Swirl } else { 2328b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 2329eeacee4dSBlue Swirl } 2330f48f3edeSblueswir1 i = 1; 2331be210acbSRichard Henderson break; 2332f713d6adSRichard Henderson case INDEX_op_qemu_ld_i32: 2333f713d6adSRichard Henderson case INDEX_op_qemu_st_i32: 233407ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 2335f713d6adSRichard Henderson case INDEX_op_qemu_ld_i64: 2336f713d6adSRichard Henderson case INDEX_op_qemu_st_i64: 233759227d5dSRichard Henderson { 23389002ffcbSRichard Henderson MemOpIdx oi = op->args[k++]; 233914776ab5STony Nguyen MemOp op = get_memop(oi); 234059227d5dSRichard Henderson unsigned ix = get_mmuidx(oi); 234159227d5dSRichard Henderson 234259c4b7e8SRichard Henderson if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 2343b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%x,%u", op, ix); 234459c4b7e8SRichard Henderson } else { 23451f00b27fSSergey Sorokin const char *s_al, *s_op; 23461f00b27fSSergey Sorokin s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 234759c4b7e8SRichard Henderson s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 2348b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix); 2349f713d6adSRichard Henderson } 2350f713d6adSRichard Henderson i = 1; 235159227d5dSRichard Henderson } 2352f713d6adSRichard Henderson break; 2353587195bdSRichard Henderson case INDEX_op_bswap16_i32: 2354587195bdSRichard Henderson case INDEX_op_bswap16_i64: 2355587195bdSRichard Henderson case INDEX_op_bswap32_i32: 2356587195bdSRichard Henderson case INDEX_op_bswap32_i64: 2357587195bdSRichard Henderson case INDEX_op_bswap64_i64: 2358587195bdSRichard Henderson { 2359587195bdSRichard Henderson TCGArg flags = op->args[k]; 2360587195bdSRichard Henderson const char *name = NULL; 2361587195bdSRichard Henderson 2362587195bdSRichard Henderson if (flags < ARRAY_SIZE(bswap_flag_name)) { 2363587195bdSRichard Henderson name = bswap_flag_name[flags]; 2364587195bdSRichard Henderson } 2365587195bdSRichard Henderson if (name) { 2366b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", name); 2367587195bdSRichard Henderson } else { 2368b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 2369587195bdSRichard Henderson } 2370587195bdSRichard Henderson i = k = 1; 2371587195bdSRichard Henderson } 2372587195bdSRichard Henderson break; 2373be210acbSRichard Henderson default: 2374f48f3edeSblueswir1 i = 0; 2375be210acbSRichard Henderson break; 2376be210acbSRichard Henderson } 237751e3972cSRichard Henderson switch (c) { 237851e3972cSRichard Henderson case INDEX_op_set_label: 237951e3972cSRichard Henderson case INDEX_op_br: 238051e3972cSRichard Henderson case INDEX_op_brcond_i32: 238151e3972cSRichard Henderson case INDEX_op_brcond_i64: 238251e3972cSRichard Henderson case INDEX_op_brcond2_i32: 2383b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$L%d", k ? "," : "", 2384efee3746SRichard Henderson arg_label(op->args[k])->id); 238551e3972cSRichard Henderson i++, k++; 238651e3972cSRichard Henderson break; 23873470867bSRichard Henderson case INDEX_op_mb: 23883470867bSRichard Henderson { 23893470867bSRichard Henderson TCGBar membar = op->args[k]; 23903470867bSRichard Henderson const char *b_op, *m_op; 23913470867bSRichard Henderson 23923470867bSRichard Henderson switch (membar & TCG_BAR_SC) { 23933470867bSRichard Henderson case 0: 23943470867bSRichard Henderson b_op = "none"; 23953470867bSRichard Henderson break; 23963470867bSRichard Henderson case TCG_BAR_LDAQ: 23973470867bSRichard Henderson b_op = "acq"; 23983470867bSRichard Henderson break; 23993470867bSRichard Henderson case TCG_BAR_STRL: 24003470867bSRichard Henderson b_op = "rel"; 24013470867bSRichard Henderson break; 24023470867bSRichard Henderson case TCG_BAR_SC: 24033470867bSRichard Henderson b_op = "seq"; 24043470867bSRichard Henderson break; 24053470867bSRichard Henderson default: 24063470867bSRichard Henderson g_assert_not_reached(); 24073470867bSRichard Henderson } 24083470867bSRichard Henderson 24093470867bSRichard Henderson switch (membar & TCG_MO_ALL) { 24103470867bSRichard Henderson case 0: 24113470867bSRichard Henderson m_op = "none"; 24123470867bSRichard Henderson break; 24133470867bSRichard Henderson case TCG_MO_LD_LD: 24143470867bSRichard Henderson m_op = "rr"; 24153470867bSRichard Henderson break; 24163470867bSRichard Henderson case TCG_MO_LD_ST: 24173470867bSRichard Henderson m_op = "rw"; 24183470867bSRichard Henderson break; 24193470867bSRichard Henderson case TCG_MO_ST_LD: 24203470867bSRichard Henderson m_op = "wr"; 24213470867bSRichard Henderson break; 24223470867bSRichard Henderson case TCG_MO_ST_ST: 24233470867bSRichard Henderson m_op = "ww"; 24243470867bSRichard Henderson break; 24253470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST: 24263470867bSRichard Henderson m_op = "rr+rw"; 24273470867bSRichard Henderson break; 24283470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD: 24293470867bSRichard Henderson m_op = "rr+wr"; 24303470867bSRichard Henderson break; 24313470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_ST: 24323470867bSRichard Henderson m_op = "rr+ww"; 24333470867bSRichard Henderson break; 24343470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD: 24353470867bSRichard Henderson m_op = "rw+wr"; 24363470867bSRichard Henderson break; 24373470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_ST: 24383470867bSRichard Henderson m_op = "rw+ww"; 24393470867bSRichard Henderson break; 24403470867bSRichard Henderson case TCG_MO_ST_LD | TCG_MO_ST_ST: 24413470867bSRichard Henderson m_op = "wr+ww"; 24423470867bSRichard Henderson break; 24433470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD: 24443470867bSRichard Henderson m_op = "rr+rw+wr"; 24453470867bSRichard Henderson break; 24463470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST: 24473470867bSRichard Henderson m_op = "rr+rw+ww"; 24483470867bSRichard Henderson break; 24493470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST: 24503470867bSRichard Henderson m_op = "rr+wr+ww"; 24513470867bSRichard Henderson break; 24523470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST: 24533470867bSRichard Henderson m_op = "rw+wr+ww"; 24543470867bSRichard Henderson break; 24553470867bSRichard Henderson case TCG_MO_ALL: 24563470867bSRichard Henderson m_op = "all"; 24573470867bSRichard Henderson break; 24583470867bSRichard Henderson default: 24593470867bSRichard Henderson g_assert_not_reached(); 24603470867bSRichard Henderson } 24613470867bSRichard Henderson 24623470867bSRichard Henderson col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op); 24633470867bSRichard Henderson i++, k++; 24643470867bSRichard Henderson } 24653470867bSRichard Henderson break; 246651e3972cSRichard Henderson default: 246751e3972cSRichard Henderson break; 2468eeacee4dSBlue Swirl } 246951e3972cSRichard Henderson for (; i < nb_cargs; i++, k++) { 2470b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 2471b7a83ff8SRichard Henderson op->args[k]); 2472bdfb460eSRichard Henderson } 2473bdfb460eSRichard Henderson } 2474bdfb460eSRichard Henderson 24751894f69aSRichard Henderson if (have_prefs || op->life) { 24761894f69aSRichard Henderson for (; col < 40; ++col) { 2477b7a83ff8SRichard Henderson putc(' ', f); 2478bdfb460eSRichard Henderson } 24791894f69aSRichard Henderson } 24801894f69aSRichard Henderson 24811894f69aSRichard Henderson if (op->life) { 24821894f69aSRichard Henderson unsigned life = op->life; 2483bdfb460eSRichard Henderson 2484bdfb460eSRichard Henderson if (life & (SYNC_ARG * 3)) { 2485b7a83ff8SRichard Henderson ne_fprintf(f, " sync:"); 2486bdfb460eSRichard Henderson for (i = 0; i < 2; ++i) { 2487bdfb460eSRichard Henderson if (life & (SYNC_ARG << i)) { 2488b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i); 2489bdfb460eSRichard Henderson } 2490bdfb460eSRichard Henderson } 2491bdfb460eSRichard Henderson } 2492bdfb460eSRichard Henderson life /= DEAD_ARG; 2493bdfb460eSRichard Henderson if (life) { 2494b7a83ff8SRichard Henderson ne_fprintf(f, " dead:"); 2495bdfb460eSRichard Henderson for (i = 0; life; ++i, life >>= 1) { 2496bdfb460eSRichard Henderson if (life & 1) { 2497b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i); 2498bdfb460eSRichard Henderson } 2499bdfb460eSRichard Henderson } 2500c896fe29Sbellard } 2501b03cce8eSbellard } 25021894f69aSRichard Henderson 25031894f69aSRichard Henderson if (have_prefs) { 25041894f69aSRichard Henderson for (i = 0; i < nb_oargs; ++i) { 250531fd884bSRichard Henderson TCGRegSet set = output_pref(op, i); 25061894f69aSRichard Henderson 25071894f69aSRichard Henderson if (i == 0) { 2508b7a83ff8SRichard Henderson ne_fprintf(f, " pref="); 25091894f69aSRichard Henderson } else { 2510b7a83ff8SRichard Henderson ne_fprintf(f, ","); 25111894f69aSRichard Henderson } 25121894f69aSRichard Henderson if (set == 0) { 2513b7a83ff8SRichard Henderson ne_fprintf(f, "none"); 25141894f69aSRichard Henderson } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 2515b7a83ff8SRichard Henderson ne_fprintf(f, "all"); 25161894f69aSRichard Henderson #ifdef CONFIG_DEBUG_TCG 25171894f69aSRichard Henderson } else if (tcg_regset_single(set)) { 25181894f69aSRichard Henderson TCGReg reg = tcg_regset_first(set); 2519b7a83ff8SRichard Henderson ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 25201894f69aSRichard Henderson #endif 25211894f69aSRichard Henderson } else if (TCG_TARGET_NB_REGS <= 32) { 2522b7a83ff8SRichard Henderson ne_fprintf(f, "0x%x", (uint32_t)set); 25231894f69aSRichard Henderson } else { 2524b7a83ff8SRichard Henderson ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 25251894f69aSRichard Henderson } 25261894f69aSRichard Henderson } 25271894f69aSRichard Henderson } 25281894f69aSRichard Henderson 2529b7a83ff8SRichard Henderson putc('\n', f); 2530c896fe29Sbellard } 2531c896fe29Sbellard } 2532c896fe29Sbellard 2533c896fe29Sbellard /* we give more priority to constraints with less registers */ 2534c896fe29Sbellard static int get_constraint_priority(const TCGOpDef *def, int k) 2535c896fe29Sbellard { 253674a11790SRichard Henderson const TCGArgConstraint *arg_ct = &def->args_ct[k]; 253729f5e925SRichard Henderson int n = ctpop64(arg_ct->regs); 2538c896fe29Sbellard 253929f5e925SRichard Henderson /* 254029f5e925SRichard Henderson * Sort constraints of a single register first, which includes output 254129f5e925SRichard Henderson * aliases (which must exactly match the input already allocated). 254229f5e925SRichard Henderson */ 254329f5e925SRichard Henderson if (n == 1 || arg_ct->oalias) { 254429f5e925SRichard Henderson return INT_MAX; 2545c896fe29Sbellard } 254629f5e925SRichard Henderson 254729f5e925SRichard Henderson /* 254829f5e925SRichard Henderson * Sort register pairs next, first then second immediately after. 254929f5e925SRichard Henderson * Arbitrarily sort multiple pairs by the index of the first reg; 255029f5e925SRichard Henderson * there shouldn't be many pairs. 255129f5e925SRichard Henderson */ 255229f5e925SRichard Henderson switch (arg_ct->pair) { 255329f5e925SRichard Henderson case 1: 255429f5e925SRichard Henderson case 3: 255529f5e925SRichard Henderson return (k + 1) * 2; 255629f5e925SRichard Henderson case 2: 255729f5e925SRichard Henderson return (arg_ct->pair_index + 1) * 2 - 1; 255829f5e925SRichard Henderson } 255929f5e925SRichard Henderson 256029f5e925SRichard Henderson /* Finally, sort by decreasing register count. */ 256129f5e925SRichard Henderson assert(n > 1); 256229f5e925SRichard Henderson return -n; 2563c896fe29Sbellard } 2564c896fe29Sbellard 2565c896fe29Sbellard /* sort from highest priority to lowest */ 2566c896fe29Sbellard static void sort_constraints(TCGOpDef *def, int start, int n) 2567c896fe29Sbellard { 256866792f90SRichard Henderson int i, j; 256966792f90SRichard Henderson TCGArgConstraint *a = def->args_ct; 2570c896fe29Sbellard 257166792f90SRichard Henderson for (i = 0; i < n; i++) { 257266792f90SRichard Henderson a[start + i].sort_index = start + i; 257366792f90SRichard Henderson } 257466792f90SRichard Henderson if (n <= 1) { 2575c896fe29Sbellard return; 257666792f90SRichard Henderson } 2577c896fe29Sbellard for (i = 0; i < n - 1; i++) { 2578c896fe29Sbellard for (j = i + 1; j < n; j++) { 257966792f90SRichard Henderson int p1 = get_constraint_priority(def, a[start + i].sort_index); 258066792f90SRichard Henderson int p2 = get_constraint_priority(def, a[start + j].sort_index); 2581c896fe29Sbellard if (p1 < p2) { 258266792f90SRichard Henderson int tmp = a[start + i].sort_index; 258366792f90SRichard Henderson a[start + i].sort_index = a[start + j].sort_index; 258466792f90SRichard Henderson a[start + j].sort_index = tmp; 2585c896fe29Sbellard } 2586c896fe29Sbellard } 2587c896fe29Sbellard } 2588c896fe29Sbellard } 2589c896fe29Sbellard 2590f69d277eSRichard Henderson static void process_op_defs(TCGContext *s) 2591c896fe29Sbellard { 2592a9751609SRichard Henderson TCGOpcode op; 2593c896fe29Sbellard 2594f69d277eSRichard Henderson for (op = 0; op < NB_OPS; op++) { 2595f69d277eSRichard Henderson TCGOpDef *def = &tcg_op_defs[op]; 2596f69d277eSRichard Henderson const TCGTargetOpDef *tdefs; 259729f5e925SRichard Henderson bool saw_alias_pair = false; 259829f5e925SRichard Henderson int i, o, i2, o2, nb_args; 2599f69d277eSRichard Henderson 2600f69d277eSRichard Henderson if (def->flags & TCG_OPF_NOT_PRESENT) { 2601f69d277eSRichard Henderson continue; 2602f69d277eSRichard Henderson } 2603f69d277eSRichard Henderson 2604c896fe29Sbellard nb_args = def->nb_iargs + def->nb_oargs; 2605f69d277eSRichard Henderson if (nb_args == 0) { 2606f69d277eSRichard Henderson continue; 2607f69d277eSRichard Henderson } 2608f69d277eSRichard Henderson 26094c22e840SRichard Henderson /* 26104c22e840SRichard Henderson * Macro magic should make it impossible, but double-check that 26114c22e840SRichard Henderson * the array index is in range. Since the signness of an enum 26124c22e840SRichard Henderson * is implementation defined, force the result to unsigned. 26134c22e840SRichard Henderson */ 26144c22e840SRichard Henderson unsigned con_set = tcg_target_op_def(op); 26154c22e840SRichard Henderson tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 26164c22e840SRichard Henderson tdefs = &constraint_sets[con_set]; 2617f69d277eSRichard Henderson 2618c896fe29Sbellard for (i = 0; i < nb_args; i++) { 2619f69d277eSRichard Henderson const char *ct_str = tdefs->args_ct_str[i]; 26208940ea0dSPhilippe Mathieu-Daudé bool input_p = i >= def->nb_oargs; 26218940ea0dSPhilippe Mathieu-Daudé 2622f69d277eSRichard Henderson /* Incomplete TCGTargetOpDef entry. */ 2623eabb7b91SAurelien Jarno tcg_debug_assert(ct_str != NULL); 2624f69d277eSRichard Henderson 262517280ff4SRichard Henderson switch (*ct_str) { 262617280ff4SRichard Henderson case '0' ... '9': 26278940ea0dSPhilippe Mathieu-Daudé o = *ct_str - '0'; 26288940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(input_p); 26298940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(o < def->nb_oargs); 26308940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(def->args_ct[o].regs != 0); 26318940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!def->args_ct[o].oalias); 26328940ea0dSPhilippe Mathieu-Daudé def->args_ct[i] = def->args_ct[o]; 2633bc2b17e6SRichard Henderson /* The output sets oalias. */ 26348940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].oalias = 1; 26358940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].alias_index = i; 2636bc2b17e6SRichard Henderson /* The input sets ialias. */ 26378940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].ialias = 1; 26388940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].alias_index = o; 263929f5e925SRichard Henderson if (def->args_ct[i].pair) { 264029f5e925SRichard Henderson saw_alias_pair = true; 264129f5e925SRichard Henderson } 26428940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(ct_str[1] == '\0'); 26438940ea0dSPhilippe Mathieu-Daudé continue; 26448940ea0dSPhilippe Mathieu-Daudé 264582790a87SRichard Henderson case '&': 26468940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!input_p); 2647bc2b17e6SRichard Henderson def->args_ct[i].newreg = true; 264882790a87SRichard Henderson ct_str++; 264982790a87SRichard Henderson break; 265029f5e925SRichard Henderson 265129f5e925SRichard Henderson case 'p': /* plus */ 265229f5e925SRichard Henderson /* Allocate to the register after the previous. */ 265329f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 265429f5e925SRichard Henderson o = i - 1; 265529f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair); 265629f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct); 265729f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){ 265829f5e925SRichard Henderson .pair = 2, 265929f5e925SRichard Henderson .pair_index = o, 266029f5e925SRichard Henderson .regs = def->args_ct[o].regs << 1, 266129f5e925SRichard Henderson }; 266229f5e925SRichard Henderson def->args_ct[o].pair = 1; 266329f5e925SRichard Henderson def->args_ct[o].pair_index = i; 266429f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0'); 266529f5e925SRichard Henderson continue; 266629f5e925SRichard Henderson 266729f5e925SRichard Henderson case 'm': /* minus */ 266829f5e925SRichard Henderson /* Allocate to the register before the previous. */ 266929f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 267029f5e925SRichard Henderson o = i - 1; 267129f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair); 267229f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct); 267329f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){ 267429f5e925SRichard Henderson .pair = 1, 267529f5e925SRichard Henderson .pair_index = o, 267629f5e925SRichard Henderson .regs = def->args_ct[o].regs >> 1, 267729f5e925SRichard Henderson }; 267829f5e925SRichard Henderson def->args_ct[o].pair = 2; 267929f5e925SRichard Henderson def->args_ct[o].pair_index = i; 268029f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0'); 268129f5e925SRichard Henderson continue; 26828940ea0dSPhilippe Mathieu-Daudé } 26838940ea0dSPhilippe Mathieu-Daudé 26848940ea0dSPhilippe Mathieu-Daudé do { 26858940ea0dSPhilippe Mathieu-Daudé switch (*ct_str) { 2686c896fe29Sbellard case 'i': 2687c896fe29Sbellard def->args_ct[i].ct |= TCG_CT_CONST; 2688c896fe29Sbellard break; 2689358b4923SRichard Henderson 2690358b4923SRichard Henderson /* Include all of the target-specific constraints. */ 2691358b4923SRichard Henderson 2692358b4923SRichard Henderson #undef CONST 2693358b4923SRichard Henderson #define CONST(CASE, MASK) \ 26948940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].ct |= MASK; break; 2695358b4923SRichard Henderson #define REGS(CASE, MASK) \ 26968940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].regs |= MASK; break; 2697358b4923SRichard Henderson 2698358b4923SRichard Henderson #include "tcg-target-con-str.h" 2699358b4923SRichard Henderson 2700358b4923SRichard Henderson #undef REGS 2701358b4923SRichard Henderson #undef CONST 2702c896fe29Sbellard default: 27038940ea0dSPhilippe Mathieu-Daudé case '0' ... '9': 27048940ea0dSPhilippe Mathieu-Daudé case '&': 270529f5e925SRichard Henderson case 'p': 270629f5e925SRichard Henderson case 'm': 2707358b4923SRichard Henderson /* Typo in TCGTargetOpDef constraint. */ 2708358b4923SRichard Henderson g_assert_not_reached(); 2709358b4923SRichard Henderson } 27108940ea0dSPhilippe Mathieu-Daudé } while (*++ct_str != '\0'); 2711c896fe29Sbellard } 2712c896fe29Sbellard 2713c68aaa18SStefan Weil /* TCGTargetOpDef entry with too much information? */ 2714eabb7b91SAurelien Jarno tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2715c68aaa18SStefan Weil 271629f5e925SRichard Henderson /* 271729f5e925SRichard Henderson * Fix up output pairs that are aliased with inputs. 271829f5e925SRichard Henderson * When we created the alias, we copied pair from the output. 271929f5e925SRichard Henderson * There are three cases: 272029f5e925SRichard Henderson * (1a) Pairs of inputs alias pairs of outputs. 272129f5e925SRichard Henderson * (1b) One input aliases the first of a pair of outputs. 272229f5e925SRichard Henderson * (2) One input aliases the second of a pair of outputs. 272329f5e925SRichard Henderson * 272429f5e925SRichard Henderson * Case 1a is handled by making sure that the pair_index'es are 272529f5e925SRichard Henderson * properly updated so that they appear the same as a pair of inputs. 272629f5e925SRichard Henderson * 272729f5e925SRichard Henderson * Case 1b is handled by setting the pair_index of the input to 272829f5e925SRichard Henderson * itself, simply so it doesn't point to an unrelated argument. 272929f5e925SRichard Henderson * Since we don't encounter the "second" during the input allocation 273029f5e925SRichard Henderson * phase, nothing happens with the second half of the input pair. 273129f5e925SRichard Henderson * 273229f5e925SRichard Henderson * Case 2 is handled by setting the second input to pair=3, the 273329f5e925SRichard Henderson * first output to pair=3, and the pair_index'es to match. 273429f5e925SRichard Henderson */ 273529f5e925SRichard Henderson if (saw_alias_pair) { 273629f5e925SRichard Henderson for (i = def->nb_oargs; i < nb_args; i++) { 273729f5e925SRichard Henderson /* 273829f5e925SRichard Henderson * Since [0-9pm] must be alone in the constraint string, 273929f5e925SRichard Henderson * the only way they can both be set is if the pair comes 274029f5e925SRichard Henderson * from the output alias. 274129f5e925SRichard Henderson */ 274229f5e925SRichard Henderson if (!def->args_ct[i].ialias) { 274329f5e925SRichard Henderson continue; 274429f5e925SRichard Henderson } 274529f5e925SRichard Henderson switch (def->args_ct[i].pair) { 274629f5e925SRichard Henderson case 0: 274729f5e925SRichard Henderson break; 274829f5e925SRichard Henderson case 1: 274929f5e925SRichard Henderson o = def->args_ct[i].alias_index; 275029f5e925SRichard Henderson o2 = def->args_ct[o].pair_index; 275129f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 1); 275229f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 2); 275329f5e925SRichard Henderson if (def->args_ct[o2].oalias) { 275429f5e925SRichard Henderson /* Case 1a */ 275529f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index; 275629f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 2); 275729f5e925SRichard Henderson def->args_ct[i2].pair_index = i; 275829f5e925SRichard Henderson def->args_ct[i].pair_index = i2; 275929f5e925SRichard Henderson } else { 276029f5e925SRichard Henderson /* Case 1b */ 276129f5e925SRichard Henderson def->args_ct[i].pair_index = i; 276229f5e925SRichard Henderson } 276329f5e925SRichard Henderson break; 276429f5e925SRichard Henderson case 2: 276529f5e925SRichard Henderson o = def->args_ct[i].alias_index; 276629f5e925SRichard Henderson o2 = def->args_ct[o].pair_index; 276729f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 2); 276829f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 1); 276929f5e925SRichard Henderson if (def->args_ct[o2].oalias) { 277029f5e925SRichard Henderson /* Case 1a */ 277129f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index; 277229f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 1); 277329f5e925SRichard Henderson def->args_ct[i2].pair_index = i; 277429f5e925SRichard Henderson def->args_ct[i].pair_index = i2; 277529f5e925SRichard Henderson } else { 277629f5e925SRichard Henderson /* Case 2 */ 277729f5e925SRichard Henderson def->args_ct[i].pair = 3; 277829f5e925SRichard Henderson def->args_ct[o2].pair = 3; 277929f5e925SRichard Henderson def->args_ct[i].pair_index = o2; 278029f5e925SRichard Henderson def->args_ct[o2].pair_index = i; 278129f5e925SRichard Henderson } 278229f5e925SRichard Henderson break; 278329f5e925SRichard Henderson default: 278429f5e925SRichard Henderson g_assert_not_reached(); 278529f5e925SRichard Henderson } 278629f5e925SRichard Henderson } 278729f5e925SRichard Henderson } 278829f5e925SRichard Henderson 2789c896fe29Sbellard /* sort the constraints (XXX: this is just an heuristic) */ 2790c896fe29Sbellard sort_constraints(def, 0, def->nb_oargs); 2791c896fe29Sbellard sort_constraints(def, def->nb_oargs, def->nb_iargs); 2792c896fe29Sbellard } 2793c896fe29Sbellard } 2794c896fe29Sbellard 2795f85b1fc4SRichard Henderson static void remove_label_use(TCGOp *op, int idx) 2796f85b1fc4SRichard Henderson { 2797f85b1fc4SRichard Henderson TCGLabel *label = arg_label(op->args[idx]); 2798f85b1fc4SRichard Henderson TCGLabelUse *use; 2799f85b1fc4SRichard Henderson 2800f85b1fc4SRichard Henderson QSIMPLEQ_FOREACH(use, &label->branches, next) { 2801f85b1fc4SRichard Henderson if (use->op == op) { 2802f85b1fc4SRichard Henderson QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); 2803f85b1fc4SRichard Henderson return; 2804f85b1fc4SRichard Henderson } 2805f85b1fc4SRichard Henderson } 2806f85b1fc4SRichard Henderson g_assert_not_reached(); 2807f85b1fc4SRichard Henderson } 2808f85b1fc4SRichard Henderson 28090c627cdcSRichard Henderson void tcg_op_remove(TCGContext *s, TCGOp *op) 28100c627cdcSRichard Henderson { 2811d88a117eSRichard Henderson switch (op->opc) { 2812d88a117eSRichard Henderson case INDEX_op_br: 2813f85b1fc4SRichard Henderson remove_label_use(op, 0); 2814d88a117eSRichard Henderson break; 2815d88a117eSRichard Henderson case INDEX_op_brcond_i32: 2816d88a117eSRichard Henderson case INDEX_op_brcond_i64: 2817f85b1fc4SRichard Henderson remove_label_use(op, 3); 2818d88a117eSRichard Henderson break; 2819d88a117eSRichard Henderson case INDEX_op_brcond2_i32: 2820f85b1fc4SRichard Henderson remove_label_use(op, 5); 2821d88a117eSRichard Henderson break; 2822d88a117eSRichard Henderson default: 2823d88a117eSRichard Henderson break; 2824d88a117eSRichard Henderson } 2825d88a117eSRichard Henderson 282615fa08f8SRichard Henderson QTAILQ_REMOVE(&s->ops, op, link); 282715fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2828abebf925SRichard Henderson s->nb_ops--; 28290c627cdcSRichard Henderson 28300c627cdcSRichard Henderson #ifdef CONFIG_PROFILER 2831d73415a3SStefan Hajnoczi qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 28320c627cdcSRichard Henderson #endif 28330c627cdcSRichard Henderson } 28340c627cdcSRichard Henderson 2835a80cdd31SRichard Henderson void tcg_remove_ops_after(TCGOp *op) 2836a80cdd31SRichard Henderson { 2837a80cdd31SRichard Henderson TCGContext *s = tcg_ctx; 2838a80cdd31SRichard Henderson 2839a80cdd31SRichard Henderson while (true) { 2840a80cdd31SRichard Henderson TCGOp *last = tcg_last_op(); 2841a80cdd31SRichard Henderson if (last == op) { 2842a80cdd31SRichard Henderson return; 2843a80cdd31SRichard Henderson } 2844a80cdd31SRichard Henderson tcg_op_remove(s, last); 2845a80cdd31SRichard Henderson } 2846a80cdd31SRichard Henderson } 2847a80cdd31SRichard Henderson 2848d4478943SPhilippe Mathieu-Daudé static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) 284915fa08f8SRichard Henderson { 285015fa08f8SRichard Henderson TCGContext *s = tcg_ctx; 2851cb10bc63SRichard Henderson TCGOp *op = NULL; 285215fa08f8SRichard Henderson 2853cb10bc63SRichard Henderson if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { 2854cb10bc63SRichard Henderson QTAILQ_FOREACH(op, &s->free_ops, link) { 2855cb10bc63SRichard Henderson if (nargs <= op->nargs) { 285615fa08f8SRichard Henderson QTAILQ_REMOVE(&s->free_ops, op, link); 2857cb10bc63SRichard Henderson nargs = op->nargs; 2858cb10bc63SRichard Henderson goto found; 285915fa08f8SRichard Henderson } 2860cb10bc63SRichard Henderson } 2861cb10bc63SRichard Henderson } 2862cb10bc63SRichard Henderson 2863cb10bc63SRichard Henderson /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ 2864cb10bc63SRichard Henderson nargs = MAX(4, nargs); 2865cb10bc63SRichard Henderson op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); 2866cb10bc63SRichard Henderson 2867cb10bc63SRichard Henderson found: 286815fa08f8SRichard Henderson memset(op, 0, offsetof(TCGOp, link)); 286915fa08f8SRichard Henderson op->opc = opc; 2870cb10bc63SRichard Henderson op->nargs = nargs; 287115fa08f8SRichard Henderson 2872cb10bc63SRichard Henderson /* Check for bitfield overflow. */ 2873cb10bc63SRichard Henderson tcg_debug_assert(op->nargs == nargs); 2874cb10bc63SRichard Henderson 2875cb10bc63SRichard Henderson s->nb_ops++; 287615fa08f8SRichard Henderson return op; 287715fa08f8SRichard Henderson } 287815fa08f8SRichard Henderson 2879d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) 288015fa08f8SRichard Henderson { 2881d4478943SPhilippe Mathieu-Daudé TCGOp *op = tcg_op_alloc(opc, nargs); 288215fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 288315fa08f8SRichard Henderson return op; 288415fa08f8SRichard Henderson } 288515fa08f8SRichard Henderson 2886d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 2887d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs) 28885a18407fSRichard Henderson { 2889d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs); 289015fa08f8SRichard Henderson QTAILQ_INSERT_BEFORE(old_op, new_op, link); 28915a18407fSRichard Henderson return new_op; 28925a18407fSRichard Henderson } 28935a18407fSRichard Henderson 2894d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 2895d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs) 28965a18407fSRichard Henderson { 2897d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs); 289815fa08f8SRichard Henderson QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 28995a18407fSRichard Henderson return new_op; 29005a18407fSRichard Henderson } 29015a18407fSRichard Henderson 2902968f305eSRichard Henderson static void move_label_uses(TCGLabel *to, TCGLabel *from) 2903968f305eSRichard Henderson { 2904968f305eSRichard Henderson TCGLabelUse *u; 2905968f305eSRichard Henderson 2906968f305eSRichard Henderson QSIMPLEQ_FOREACH(u, &from->branches, next) { 2907968f305eSRichard Henderson TCGOp *op = u->op; 2908968f305eSRichard Henderson switch (op->opc) { 2909968f305eSRichard Henderson case INDEX_op_br: 2910968f305eSRichard Henderson op->args[0] = label_arg(to); 2911968f305eSRichard Henderson break; 2912968f305eSRichard Henderson case INDEX_op_brcond_i32: 2913968f305eSRichard Henderson case INDEX_op_brcond_i64: 2914968f305eSRichard Henderson op->args[3] = label_arg(to); 2915968f305eSRichard Henderson break; 2916968f305eSRichard Henderson case INDEX_op_brcond2_i32: 2917968f305eSRichard Henderson op->args[5] = label_arg(to); 2918968f305eSRichard Henderson break; 2919968f305eSRichard Henderson default: 2920968f305eSRichard Henderson g_assert_not_reached(); 2921968f305eSRichard Henderson } 2922968f305eSRichard Henderson } 2923968f305eSRichard Henderson 2924968f305eSRichard Henderson QSIMPLEQ_CONCAT(&to->branches, &from->branches); 2925968f305eSRichard Henderson } 2926968f305eSRichard Henderson 2927b4fc67c7SRichard Henderson /* Reachable analysis : remove unreachable code. */ 29289bbee4c0SRichard Henderson static void __attribute__((noinline)) 29299bbee4c0SRichard Henderson reachable_code_pass(TCGContext *s) 2930b4fc67c7SRichard Henderson { 29314d89d0bbSRichard Henderson TCGOp *op, *op_next, *op_prev; 2932b4fc67c7SRichard Henderson bool dead = false; 2933b4fc67c7SRichard Henderson 2934b4fc67c7SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2935b4fc67c7SRichard Henderson bool remove = dead; 2936b4fc67c7SRichard Henderson TCGLabel *label; 2937b4fc67c7SRichard Henderson 2938b4fc67c7SRichard Henderson switch (op->opc) { 2939b4fc67c7SRichard Henderson case INDEX_op_set_label: 2940b4fc67c7SRichard Henderson label = arg_label(op->args[0]); 29414d89d0bbSRichard Henderson 29424d89d0bbSRichard Henderson /* 2943968f305eSRichard Henderson * Note that the first op in the TB is always a load, 2944968f305eSRichard Henderson * so there is always something before a label. 2945968f305eSRichard Henderson */ 2946968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link); 2947968f305eSRichard Henderson 2948968f305eSRichard Henderson /* 2949968f305eSRichard Henderson * If we find two sequential labels, move all branches to 2950968f305eSRichard Henderson * reference the second label and remove the first label. 2951968f305eSRichard Henderson * Do this before branch to next optimization, so that the 2952968f305eSRichard Henderson * middle label is out of the way. 2953968f305eSRichard Henderson */ 2954968f305eSRichard Henderson if (op_prev->opc == INDEX_op_set_label) { 2955968f305eSRichard Henderson move_label_uses(label, arg_label(op_prev->args[0])); 2956968f305eSRichard Henderson tcg_op_remove(s, op_prev); 2957968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link); 2958968f305eSRichard Henderson } 2959968f305eSRichard Henderson 2960968f305eSRichard Henderson /* 29614d89d0bbSRichard Henderson * Optimization can fold conditional branches to unconditional. 29624d89d0bbSRichard Henderson * If we find a label which is preceded by an unconditional 29634d89d0bbSRichard Henderson * branch to next, remove the branch. We couldn't do this when 29644d89d0bbSRichard Henderson * processing the branch because any dead code between the branch 29654d89d0bbSRichard Henderson * and label had not yet been removed. 29664d89d0bbSRichard Henderson */ 29674d89d0bbSRichard Henderson if (op_prev->opc == INDEX_op_br && 29684d89d0bbSRichard Henderson label == arg_label(op_prev->args[0])) { 29694d89d0bbSRichard Henderson tcg_op_remove(s, op_prev); 29704d89d0bbSRichard Henderson /* Fall through means insns become live again. */ 29714d89d0bbSRichard Henderson dead = false; 29724d89d0bbSRichard Henderson } 29734d89d0bbSRichard Henderson 2974f85b1fc4SRichard Henderson if (QSIMPLEQ_EMPTY(&label->branches)) { 2975b4fc67c7SRichard Henderson /* 2976b4fc67c7SRichard Henderson * While there is an occasional backward branch, virtually 2977b4fc67c7SRichard Henderson * all branches generated by the translators are forward. 2978b4fc67c7SRichard Henderson * Which means that generally we will have already removed 2979b4fc67c7SRichard Henderson * all references to the label that will be, and there is 2980b4fc67c7SRichard Henderson * little to be gained by iterating. 2981b4fc67c7SRichard Henderson */ 2982b4fc67c7SRichard Henderson remove = true; 2983b4fc67c7SRichard Henderson } else { 2984b4fc67c7SRichard Henderson /* Once we see a label, insns become live again. */ 2985b4fc67c7SRichard Henderson dead = false; 2986b4fc67c7SRichard Henderson remove = false; 2987b4fc67c7SRichard Henderson } 2988b4fc67c7SRichard Henderson break; 2989b4fc67c7SRichard Henderson 2990b4fc67c7SRichard Henderson case INDEX_op_br: 2991b4fc67c7SRichard Henderson case INDEX_op_exit_tb: 2992b4fc67c7SRichard Henderson case INDEX_op_goto_ptr: 2993b4fc67c7SRichard Henderson /* Unconditional branches; everything following is dead. */ 2994b4fc67c7SRichard Henderson dead = true; 2995b4fc67c7SRichard Henderson break; 2996b4fc67c7SRichard Henderson 2997b4fc67c7SRichard Henderson case INDEX_op_call: 2998b4fc67c7SRichard Henderson /* Notice noreturn helper calls, raising exceptions. */ 299990163900SRichard Henderson if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 3000b4fc67c7SRichard Henderson dead = true; 3001b4fc67c7SRichard Henderson } 3002b4fc67c7SRichard Henderson break; 3003b4fc67c7SRichard Henderson 3004b4fc67c7SRichard Henderson case INDEX_op_insn_start: 3005b4fc67c7SRichard Henderson /* Never remove -- we need to keep these for unwind. */ 3006b4fc67c7SRichard Henderson remove = false; 3007b4fc67c7SRichard Henderson break; 3008b4fc67c7SRichard Henderson 3009b4fc67c7SRichard Henderson default: 3010b4fc67c7SRichard Henderson break; 3011b4fc67c7SRichard Henderson } 3012b4fc67c7SRichard Henderson 3013b4fc67c7SRichard Henderson if (remove) { 3014b4fc67c7SRichard Henderson tcg_op_remove(s, op); 3015b4fc67c7SRichard Henderson } 3016b4fc67c7SRichard Henderson } 3017b4fc67c7SRichard Henderson } 3018b4fc67c7SRichard Henderson 3019c70fbf0aSRichard Henderson #define TS_DEAD 1 3020c70fbf0aSRichard Henderson #define TS_MEM 2 3021c70fbf0aSRichard Henderson 30225a18407fSRichard Henderson #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 30235a18407fSRichard Henderson #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 30245a18407fSRichard Henderson 302525f49c5fSRichard Henderson /* For liveness_pass_1, the register preferences for a given temp. */ 302625f49c5fSRichard Henderson static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 302725f49c5fSRichard Henderson { 302825f49c5fSRichard Henderson return ts->state_ptr; 302925f49c5fSRichard Henderson } 303025f49c5fSRichard Henderson 303125f49c5fSRichard Henderson /* For liveness_pass_1, reset the preferences for a given temp to the 303225f49c5fSRichard Henderson * maximal regset for its type. 303325f49c5fSRichard Henderson */ 303425f49c5fSRichard Henderson static inline void la_reset_pref(TCGTemp *ts) 303525f49c5fSRichard Henderson { 303625f49c5fSRichard Henderson *la_temp_pref(ts) 303725f49c5fSRichard Henderson = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 303825f49c5fSRichard Henderson } 303925f49c5fSRichard Henderson 30409c43b68dSAurelien Jarno /* liveness analysis: end of function: all temps are dead, and globals 30419c43b68dSAurelien Jarno should be in memory. */ 30422616c808SRichard Henderson static void la_func_end(TCGContext *s, int ng, int nt) 3043c896fe29Sbellard { 3044b83eabeaSRichard Henderson int i; 3045b83eabeaSRichard Henderson 3046b83eabeaSRichard Henderson for (i = 0; i < ng; ++i) { 3047b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 304825f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 3049b83eabeaSRichard Henderson } 3050b83eabeaSRichard Henderson for (i = ng; i < nt; ++i) { 3051b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD; 305225f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 3053b83eabeaSRichard Henderson } 3054c896fe29Sbellard } 3055c896fe29Sbellard 30569c43b68dSAurelien Jarno /* liveness analysis: end of basic block: all temps are dead, globals 30579c43b68dSAurelien Jarno and local temps should be in memory. */ 30582616c808SRichard Henderson static void la_bb_end(TCGContext *s, int ng, int nt) 3059641d5fbeSbellard { 3060b83eabeaSRichard Henderson int i; 3061641d5fbeSbellard 3062ee17db83SRichard Henderson for (i = 0; i < nt; ++i) { 3063ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 3064ee17db83SRichard Henderson int state; 3065ee17db83SRichard Henderson 3066ee17db83SRichard Henderson switch (ts->kind) { 3067ee17db83SRichard Henderson case TEMP_FIXED: 3068ee17db83SRichard Henderson case TEMP_GLOBAL: 3069f57c6915SRichard Henderson case TEMP_TB: 3070ee17db83SRichard Henderson state = TS_DEAD | TS_MEM; 3071ee17db83SRichard Henderson break; 3072c7482438SRichard Henderson case TEMP_EBB: 3073c0522136SRichard Henderson case TEMP_CONST: 3074ee17db83SRichard Henderson state = TS_DEAD; 3075ee17db83SRichard Henderson break; 3076ee17db83SRichard Henderson default: 3077ee17db83SRichard Henderson g_assert_not_reached(); 3078c70fbf0aSRichard Henderson } 3079ee17db83SRichard Henderson ts->state = state; 3080ee17db83SRichard Henderson la_reset_pref(ts); 3081641d5fbeSbellard } 3082641d5fbeSbellard } 3083641d5fbeSbellard 3084f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory. */ 3085f65a061cSRichard Henderson static void la_global_sync(TCGContext *s, int ng) 3086f65a061cSRichard Henderson { 3087f65a061cSRichard Henderson int i; 3088f65a061cSRichard Henderson 3089f65a061cSRichard Henderson for (i = 0; i < ng; ++i) { 309025f49c5fSRichard Henderson int state = s->temps[i].state; 309125f49c5fSRichard Henderson s->temps[i].state = state | TS_MEM; 309225f49c5fSRichard Henderson if (state == TS_DEAD) { 309325f49c5fSRichard Henderson /* If the global was previously dead, reset prefs. */ 309425f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 309525f49c5fSRichard Henderson } 3096f65a061cSRichard Henderson } 3097f65a061cSRichard Henderson } 3098f65a061cSRichard Henderson 3099b4cb76e6SRichard Henderson /* 3100c7482438SRichard Henderson * liveness analysis: conditional branch: all temps are dead unless 3101c7482438SRichard Henderson * explicitly live-across-conditional-branch, globals and local temps 3102c7482438SRichard Henderson * should be synced. 3103b4cb76e6SRichard Henderson */ 3104b4cb76e6SRichard Henderson static void la_bb_sync(TCGContext *s, int ng, int nt) 3105b4cb76e6SRichard Henderson { 3106b4cb76e6SRichard Henderson la_global_sync(s, ng); 3107b4cb76e6SRichard Henderson 3108b4cb76e6SRichard Henderson for (int i = ng; i < nt; ++i) { 3109c0522136SRichard Henderson TCGTemp *ts = &s->temps[i]; 3110c0522136SRichard Henderson int state; 3111c0522136SRichard Henderson 3112c0522136SRichard Henderson switch (ts->kind) { 3113f57c6915SRichard Henderson case TEMP_TB: 3114c0522136SRichard Henderson state = ts->state; 3115c0522136SRichard Henderson ts->state = state | TS_MEM; 3116b4cb76e6SRichard Henderson if (state != TS_DEAD) { 3117b4cb76e6SRichard Henderson continue; 3118b4cb76e6SRichard Henderson } 3119c0522136SRichard Henderson break; 3120c7482438SRichard Henderson case TEMP_EBB: 3121c0522136SRichard Henderson case TEMP_CONST: 3122c0522136SRichard Henderson continue; 3123c0522136SRichard Henderson default: 3124c0522136SRichard Henderson g_assert_not_reached(); 3125b4cb76e6SRichard Henderson } 3126b4cb76e6SRichard Henderson la_reset_pref(&s->temps[i]); 3127b4cb76e6SRichard Henderson } 3128b4cb76e6SRichard Henderson } 3129b4cb76e6SRichard Henderson 3130f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory and kill. */ 3131f65a061cSRichard Henderson static void la_global_kill(TCGContext *s, int ng) 3132f65a061cSRichard Henderson { 3133f65a061cSRichard Henderson int i; 3134f65a061cSRichard Henderson 3135f65a061cSRichard Henderson for (i = 0; i < ng; i++) { 3136f65a061cSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 313725f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 313825f49c5fSRichard Henderson } 313925f49c5fSRichard Henderson } 314025f49c5fSRichard Henderson 314125f49c5fSRichard Henderson /* liveness analysis: note live globals crossing calls. */ 314225f49c5fSRichard Henderson static void la_cross_call(TCGContext *s, int nt) 314325f49c5fSRichard Henderson { 314425f49c5fSRichard Henderson TCGRegSet mask = ~tcg_target_call_clobber_regs; 314525f49c5fSRichard Henderson int i; 314625f49c5fSRichard Henderson 314725f49c5fSRichard Henderson for (i = 0; i < nt; i++) { 314825f49c5fSRichard Henderson TCGTemp *ts = &s->temps[i]; 314925f49c5fSRichard Henderson if (!(ts->state & TS_DEAD)) { 315025f49c5fSRichard Henderson TCGRegSet *pset = la_temp_pref(ts); 315125f49c5fSRichard Henderson TCGRegSet set = *pset; 315225f49c5fSRichard Henderson 315325f49c5fSRichard Henderson set &= mask; 315425f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 315525f49c5fSRichard Henderson if (set == 0) { 315625f49c5fSRichard Henderson set = tcg_target_available_regs[ts->type] & mask; 315725f49c5fSRichard Henderson } 315825f49c5fSRichard Henderson *pset = set; 315925f49c5fSRichard Henderson } 3160f65a061cSRichard Henderson } 3161f65a061cSRichard Henderson } 3162f65a061cSRichard Henderson 3163874b8574SRichard Henderson /* 3164874b8574SRichard Henderson * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce 3165874b8574SRichard Henderson * to TEMP_EBB, if possible. 3166874b8574SRichard Henderson */ 3167874b8574SRichard Henderson static void __attribute__((noinline)) 3168874b8574SRichard Henderson liveness_pass_0(TCGContext *s) 3169874b8574SRichard Henderson { 3170874b8574SRichard Henderson void * const multiple_ebb = (void *)(uintptr_t)-1; 3171874b8574SRichard Henderson int nb_temps = s->nb_temps; 3172874b8574SRichard Henderson TCGOp *op, *ebb; 3173874b8574SRichard Henderson 3174874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) { 3175874b8574SRichard Henderson s->temps[i].state_ptr = NULL; 3176874b8574SRichard Henderson } 3177874b8574SRichard Henderson 3178874b8574SRichard Henderson /* 3179874b8574SRichard Henderson * Represent each EBB by the op at which it begins. In the case of 3180874b8574SRichard Henderson * the first EBB, this is the first op, otherwise it is a label. 3181874b8574SRichard Henderson * Collect the uses of each TEMP_TB: NULL for unused, EBB for use 3182874b8574SRichard Henderson * within a single EBB, else MULTIPLE_EBB. 3183874b8574SRichard Henderson */ 3184874b8574SRichard Henderson ebb = QTAILQ_FIRST(&s->ops); 3185874b8574SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 3186874b8574SRichard Henderson const TCGOpDef *def; 3187874b8574SRichard Henderson int nb_oargs, nb_iargs; 3188874b8574SRichard Henderson 3189874b8574SRichard Henderson switch (op->opc) { 3190874b8574SRichard Henderson case INDEX_op_set_label: 3191874b8574SRichard Henderson ebb = op; 3192874b8574SRichard Henderson continue; 3193874b8574SRichard Henderson case INDEX_op_discard: 3194874b8574SRichard Henderson continue; 3195874b8574SRichard Henderson case INDEX_op_call: 3196874b8574SRichard Henderson nb_oargs = TCGOP_CALLO(op); 3197874b8574SRichard Henderson nb_iargs = TCGOP_CALLI(op); 3198874b8574SRichard Henderson break; 3199874b8574SRichard Henderson default: 3200874b8574SRichard Henderson def = &tcg_op_defs[op->opc]; 3201874b8574SRichard Henderson nb_oargs = def->nb_oargs; 3202874b8574SRichard Henderson nb_iargs = def->nb_iargs; 3203874b8574SRichard Henderson break; 3204874b8574SRichard Henderson } 3205874b8574SRichard Henderson 3206874b8574SRichard Henderson for (int i = 0; i < nb_oargs + nb_iargs; ++i) { 3207874b8574SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 3208874b8574SRichard Henderson 3209874b8574SRichard Henderson if (ts->kind != TEMP_TB) { 3210874b8574SRichard Henderson continue; 3211874b8574SRichard Henderson } 3212874b8574SRichard Henderson if (ts->state_ptr == NULL) { 3213874b8574SRichard Henderson ts->state_ptr = ebb; 3214874b8574SRichard Henderson } else if (ts->state_ptr != ebb) { 3215874b8574SRichard Henderson ts->state_ptr = multiple_ebb; 3216874b8574SRichard Henderson } 3217874b8574SRichard Henderson } 3218874b8574SRichard Henderson } 3219874b8574SRichard Henderson 3220874b8574SRichard Henderson /* 3221874b8574SRichard Henderson * For TEMP_TB that turned out not to be used beyond one EBB, 3222874b8574SRichard Henderson * reduce the liveness to TEMP_EBB. 3223874b8574SRichard Henderson */ 3224874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) { 3225874b8574SRichard Henderson TCGTemp *ts = &s->temps[i]; 3226874b8574SRichard Henderson if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { 3227874b8574SRichard Henderson ts->kind = TEMP_EBB; 3228874b8574SRichard Henderson } 3229874b8574SRichard Henderson } 3230874b8574SRichard Henderson } 3231874b8574SRichard Henderson 3232a1b3c48dSRichard Henderson /* Liveness analysis : update the opc_arg_life array to tell if a 3233c896fe29Sbellard given input arguments is dead. Instructions updating dead 3234c896fe29Sbellard temporaries are removed. */ 32359bbee4c0SRichard Henderson static void __attribute__((noinline)) 32369bbee4c0SRichard Henderson liveness_pass_1(TCGContext *s) 3237c896fe29Sbellard { 3238c70fbf0aSRichard Henderson int nb_globals = s->nb_globals; 32392616c808SRichard Henderson int nb_temps = s->nb_temps; 324015fa08f8SRichard Henderson TCGOp *op, *op_prev; 324125f49c5fSRichard Henderson TCGRegSet *prefs; 324225f49c5fSRichard Henderson int i; 324325f49c5fSRichard Henderson 324425f49c5fSRichard Henderson prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 324525f49c5fSRichard Henderson for (i = 0; i < nb_temps; ++i) { 324625f49c5fSRichard Henderson s->temps[i].state_ptr = prefs + i; 324725f49c5fSRichard Henderson } 3248c896fe29Sbellard 3249ae36a246SRichard Henderson /* ??? Should be redundant with the exit_tb that ends the TB. */ 32502616c808SRichard Henderson la_func_end(s, nb_globals, nb_temps); 3251c896fe29Sbellard 3252eae3eb3eSPaolo Bonzini QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 325325f49c5fSRichard Henderson int nb_iargs, nb_oargs; 3254c45cb8bbSRichard Henderson TCGOpcode opc_new, opc_new2; 3255c45cb8bbSRichard Henderson bool have_opc_new2; 3256a1b3c48dSRichard Henderson TCGLifeData arg_life = 0; 325725f49c5fSRichard Henderson TCGTemp *ts; 3258c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 3259c45cb8bbSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 3260c45cb8bbSRichard Henderson 3261c45cb8bbSRichard Henderson switch (opc) { 3262c896fe29Sbellard case INDEX_op_call: 3263c6e113f5Sbellard { 326439004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 326539004a71SRichard Henderson int call_flags = tcg_call_flags(op); 3266c6e113f5Sbellard 3267cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 3268cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 3269c6e113f5Sbellard 3270c45cb8bbSRichard Henderson /* pure functions can be removed if their result is unused */ 327178505279SAurelien Jarno if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 3272c6e113f5Sbellard for (i = 0; i < nb_oargs; i++) { 327325f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 327425f49c5fSRichard Henderson if (ts->state != TS_DEAD) { 3275c6e113f5Sbellard goto do_not_remove_call; 3276c6e113f5Sbellard } 32779c43b68dSAurelien Jarno } 3278c45cb8bbSRichard Henderson goto do_remove; 3279152c35aaSRichard Henderson } 3280c6e113f5Sbellard do_not_remove_call: 3281c896fe29Sbellard 328225f49c5fSRichard Henderson /* Output args are dead. */ 3283c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 328425f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 328525f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3286a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 32876b64b624SAurelien Jarno } 328825f49c5fSRichard Henderson if (ts->state & TS_MEM) { 3289a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 32909c43b68dSAurelien Jarno } 329125f49c5fSRichard Henderson ts->state = TS_DEAD; 329225f49c5fSRichard Henderson la_reset_pref(ts); 3293c896fe29Sbellard } 3294c896fe29Sbellard 329531fd884bSRichard Henderson /* Not used -- it will be tcg_target_call_oarg_reg(). */ 329631fd884bSRichard Henderson memset(op->output_pref, 0, sizeof(op->output_pref)); 329731fd884bSRichard Henderson 329878505279SAurelien Jarno if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 329978505279SAurelien Jarno TCG_CALL_NO_READ_GLOBALS))) { 3300f65a061cSRichard Henderson la_global_kill(s, nb_globals); 3301c70fbf0aSRichard Henderson } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 3302f65a061cSRichard Henderson la_global_sync(s, nb_globals); 3303b9c18f56Saurel32 } 3304c896fe29Sbellard 330525f49c5fSRichard Henderson /* Record arguments that die in this helper. */ 3306866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 330725f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 330839004a71SRichard Henderson if (ts->state & TS_DEAD) { 3309a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 3310c896fe29Sbellard } 3311c896fe29Sbellard } 331225f49c5fSRichard Henderson 331325f49c5fSRichard Henderson /* For all live registers, remove call-clobbered prefs. */ 331425f49c5fSRichard Henderson la_cross_call(s, nb_temps); 331525f49c5fSRichard Henderson 331639004a71SRichard Henderson /* 331739004a71SRichard Henderson * Input arguments are live for preceding opcodes. 331839004a71SRichard Henderson * 331939004a71SRichard Henderson * For those arguments that die, and will be allocated in 332039004a71SRichard Henderson * registers, clear the register set for that arg, to be 332139004a71SRichard Henderson * filled in below. For args that will be on the stack, 332239004a71SRichard Henderson * reset to any available reg. Process arguments in reverse 332339004a71SRichard Henderson * order so that if a temp is used more than once, the stack 332439004a71SRichard Henderson * reset to max happens before the register reset to 0. 332525f49c5fSRichard Henderson */ 332639004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; i--) { 332739004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 332839004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]); 332939004a71SRichard Henderson 333039004a71SRichard Henderson if (ts->state & TS_DEAD) { 333139004a71SRichard Henderson switch (loc->kind) { 333239004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 333339004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 333439004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 3335338b61e9SRichard Henderson if (arg_slot_reg_p(loc->arg_slot)) { 333639004a71SRichard Henderson *la_temp_pref(ts) = 0; 333739004a71SRichard Henderson break; 333839004a71SRichard Henderson } 333939004a71SRichard Henderson /* fall through */ 334039004a71SRichard Henderson default: 334139004a71SRichard Henderson *la_temp_pref(ts) = 334239004a71SRichard Henderson tcg_target_available_regs[ts->type]; 334339004a71SRichard Henderson break; 334439004a71SRichard Henderson } 334525f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 334625f49c5fSRichard Henderson } 334725f49c5fSRichard Henderson } 334825f49c5fSRichard Henderson 334939004a71SRichard Henderson /* 335039004a71SRichard Henderson * For each input argument, add its input register to prefs. 335139004a71SRichard Henderson * If a temp is used once, this produces a single set bit; 335239004a71SRichard Henderson * if a temp is used multiple times, this produces a set. 335339004a71SRichard Henderson */ 335439004a71SRichard Henderson for (i = 0; i < nb_iargs; i++) { 335539004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 335639004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]); 335739004a71SRichard Henderson 335839004a71SRichard Henderson switch (loc->kind) { 335939004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 336039004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 336139004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 3362338b61e9SRichard Henderson if (arg_slot_reg_p(loc->arg_slot)) { 336325f49c5fSRichard Henderson tcg_regset_set_reg(*la_temp_pref(ts), 336439004a71SRichard Henderson tcg_target_call_iarg_regs[loc->arg_slot]); 336539004a71SRichard Henderson } 336639004a71SRichard Henderson break; 336739004a71SRichard Henderson default: 336839004a71SRichard Henderson break; 3369c70fbf0aSRichard Henderson } 3370c19f47bfSAurelien Jarno } 3371c6e113f5Sbellard } 3372c896fe29Sbellard break; 3373765b842aSRichard Henderson case INDEX_op_insn_start: 3374c896fe29Sbellard break; 33755ff9d6a4Sbellard case INDEX_op_discard: 33765ff9d6a4Sbellard /* mark the temporary as dead */ 337725f49c5fSRichard Henderson ts = arg_temp(op->args[0]); 337825f49c5fSRichard Henderson ts->state = TS_DEAD; 337925f49c5fSRichard Henderson la_reset_pref(ts); 33805ff9d6a4Sbellard break; 33811305c451SRichard Henderson 33821305c451SRichard Henderson case INDEX_op_add2_i32: 3383c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i32; 3384f1fae40cSRichard Henderson goto do_addsub2; 33851305c451SRichard Henderson case INDEX_op_sub2_i32: 3386c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i32; 3387f1fae40cSRichard Henderson goto do_addsub2; 3388f1fae40cSRichard Henderson case INDEX_op_add2_i64: 3389c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i64; 3390f1fae40cSRichard Henderson goto do_addsub2; 3391f1fae40cSRichard Henderson case INDEX_op_sub2_i64: 3392c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i64; 3393f1fae40cSRichard Henderson do_addsub2: 33941305c451SRichard Henderson nb_iargs = 4; 33951305c451SRichard Henderson nb_oargs = 2; 33961305c451SRichard Henderson /* Test if the high part of the operation is dead, but not 33971305c451SRichard Henderson the low part. The result can be optimized to a simple 33981305c451SRichard Henderson add or sub. This happens often for x86_64 guest when the 33991305c451SRichard Henderson cpu mode is set to 32 bit. */ 3400b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 3401b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 34021305c451SRichard Henderson goto do_remove; 34031305c451SRichard Henderson } 3404c45cb8bbSRichard Henderson /* Replace the opcode and adjust the args in place, 3405c45cb8bbSRichard Henderson leaving 3 unused args at the end. */ 3406c45cb8bbSRichard Henderson op->opc = opc = opc_new; 3407efee3746SRichard Henderson op->args[1] = op->args[2]; 3408efee3746SRichard Henderson op->args[2] = op->args[4]; 34091305c451SRichard Henderson /* Fall through and mark the single-word operation live. */ 34101305c451SRichard Henderson nb_iargs = 2; 34111305c451SRichard Henderson nb_oargs = 1; 34121305c451SRichard Henderson } 34131305c451SRichard Henderson goto do_not_remove; 34141305c451SRichard Henderson 34151414968aSRichard Henderson case INDEX_op_mulu2_i32: 3416c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 3417c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i32; 3418c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 341903271524SRichard Henderson goto do_mul2; 3420f1fae40cSRichard Henderson case INDEX_op_muls2_i32: 3421c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 3422c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i32; 3423c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 3424f1fae40cSRichard Henderson goto do_mul2; 3425f1fae40cSRichard Henderson case INDEX_op_mulu2_i64: 3426c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 3427c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i64; 3428c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 342903271524SRichard Henderson goto do_mul2; 3430f1fae40cSRichard Henderson case INDEX_op_muls2_i64: 3431c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 3432c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i64; 3433c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 343403271524SRichard Henderson goto do_mul2; 3435f1fae40cSRichard Henderson do_mul2: 34361414968aSRichard Henderson nb_iargs = 2; 34371414968aSRichard Henderson nb_oargs = 2; 3438b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 3439b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 344003271524SRichard Henderson /* Both parts of the operation are dead. */ 34411414968aSRichard Henderson goto do_remove; 34421414968aSRichard Henderson } 344303271524SRichard Henderson /* The high part of the operation is dead; generate the low. */ 3444c45cb8bbSRichard Henderson op->opc = opc = opc_new; 3445efee3746SRichard Henderson op->args[1] = op->args[2]; 3446efee3746SRichard Henderson op->args[2] = op->args[3]; 3447b83eabeaSRichard Henderson } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 344803271524SRichard Henderson /* The low part of the operation is dead; generate the high. */ 3449c45cb8bbSRichard Henderson op->opc = opc = opc_new2; 3450efee3746SRichard Henderson op->args[0] = op->args[1]; 3451efee3746SRichard Henderson op->args[1] = op->args[2]; 3452efee3746SRichard Henderson op->args[2] = op->args[3]; 345303271524SRichard Henderson } else { 345403271524SRichard Henderson goto do_not_remove; 345503271524SRichard Henderson } 345603271524SRichard Henderson /* Mark the single-word operation live. */ 34571414968aSRichard Henderson nb_oargs = 1; 34581414968aSRichard Henderson goto do_not_remove; 34591414968aSRichard Henderson 3460c896fe29Sbellard default: 34611305c451SRichard Henderson /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 3462c896fe29Sbellard nb_iargs = def->nb_iargs; 3463c896fe29Sbellard nb_oargs = def->nb_oargs; 3464c896fe29Sbellard 3465c896fe29Sbellard /* Test if the operation can be removed because all 34665ff9d6a4Sbellard its outputs are dead. We assume that nb_oargs == 0 34675ff9d6a4Sbellard implies side effects */ 34685ff9d6a4Sbellard if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 3469c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 3470b83eabeaSRichard Henderson if (arg_temp(op->args[i])->state != TS_DEAD) { 3471c896fe29Sbellard goto do_not_remove; 3472c896fe29Sbellard } 34739c43b68dSAurelien Jarno } 3474152c35aaSRichard Henderson goto do_remove; 3475152c35aaSRichard Henderson } 3476152c35aaSRichard Henderson goto do_not_remove; 3477152c35aaSRichard Henderson 34781305c451SRichard Henderson do_remove: 34790c627cdcSRichard Henderson tcg_op_remove(s, op); 3480152c35aaSRichard Henderson break; 3481152c35aaSRichard Henderson 3482c896fe29Sbellard do_not_remove: 3483c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 348425f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 348525f49c5fSRichard Henderson 348625f49c5fSRichard Henderson /* Remember the preference of the uses that followed. */ 348731fd884bSRichard Henderson if (i < ARRAY_SIZE(op->output_pref)) { 348825f49c5fSRichard Henderson op->output_pref[i] = *la_temp_pref(ts); 348931fd884bSRichard Henderson } 349025f49c5fSRichard Henderson 349125f49c5fSRichard Henderson /* Output args are dead. */ 349225f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3493a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 34946b64b624SAurelien Jarno } 349525f49c5fSRichard Henderson if (ts->state & TS_MEM) { 3496a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 34979c43b68dSAurelien Jarno } 349825f49c5fSRichard Henderson ts->state = TS_DEAD; 349925f49c5fSRichard Henderson la_reset_pref(ts); 3500c896fe29Sbellard } 3501c896fe29Sbellard 350225f49c5fSRichard Henderson /* If end of basic block, update. */ 3503ae36a246SRichard Henderson if (def->flags & TCG_OPF_BB_EXIT) { 3504ae36a246SRichard Henderson la_func_end(s, nb_globals, nb_temps); 3505b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_COND_BRANCH) { 3506b4cb76e6SRichard Henderson la_bb_sync(s, nb_globals, nb_temps); 3507ae36a246SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 35082616c808SRichard Henderson la_bb_end(s, nb_globals, nb_temps); 35093d5c5f87SAurelien Jarno } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3510f65a061cSRichard Henderson la_global_sync(s, nb_globals); 351125f49c5fSRichard Henderson if (def->flags & TCG_OPF_CALL_CLOBBER) { 351225f49c5fSRichard Henderson la_cross_call(s, nb_temps); 351325f49c5fSRichard Henderson } 3514c896fe29Sbellard } 3515c896fe29Sbellard 351625f49c5fSRichard Henderson /* Record arguments that die in this opcode. */ 3517866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 351825f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 351925f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3520a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 3521c896fe29Sbellard } 3522c19f47bfSAurelien Jarno } 352325f49c5fSRichard Henderson 352425f49c5fSRichard Henderson /* Input arguments are live for preceding opcodes. */ 3525c19f47bfSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 352625f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 352725f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 352825f49c5fSRichard Henderson /* For operands that were dead, initially allow 352925f49c5fSRichard Henderson all regs for the type. */ 353025f49c5fSRichard Henderson *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 353125f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 353225f49c5fSRichard Henderson } 353325f49c5fSRichard Henderson } 353425f49c5fSRichard Henderson 353525f49c5fSRichard Henderson /* Incorporate constraints for this operand. */ 353625f49c5fSRichard Henderson switch (opc) { 353725f49c5fSRichard Henderson case INDEX_op_mov_i32: 353825f49c5fSRichard Henderson case INDEX_op_mov_i64: 353925f49c5fSRichard Henderson /* Note that these are TCG_OPF_NOT_PRESENT and do not 354025f49c5fSRichard Henderson have proper constraints. That said, special case 354125f49c5fSRichard Henderson moves to propagate preferences backward. */ 354225f49c5fSRichard Henderson if (IS_DEAD_ARG(1)) { 354325f49c5fSRichard Henderson *la_temp_pref(arg_temp(op->args[0])) 354425f49c5fSRichard Henderson = *la_temp_pref(arg_temp(op->args[1])); 354525f49c5fSRichard Henderson } 354625f49c5fSRichard Henderson break; 354725f49c5fSRichard Henderson 354825f49c5fSRichard Henderson default: 354925f49c5fSRichard Henderson for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 355025f49c5fSRichard Henderson const TCGArgConstraint *ct = &def->args_ct[i]; 355125f49c5fSRichard Henderson TCGRegSet set, *pset; 355225f49c5fSRichard Henderson 355325f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 355425f49c5fSRichard Henderson pset = la_temp_pref(ts); 355525f49c5fSRichard Henderson set = *pset; 355625f49c5fSRichard Henderson 35579be0d080SRichard Henderson set &= ct->regs; 3558bc2b17e6SRichard Henderson if (ct->ialias) { 355931fd884bSRichard Henderson set &= output_pref(op, ct->alias_index); 356025f49c5fSRichard Henderson } 356125f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 356225f49c5fSRichard Henderson if (set == 0) { 35639be0d080SRichard Henderson set = ct->regs; 356425f49c5fSRichard Henderson } 356525f49c5fSRichard Henderson *pset = set; 356625f49c5fSRichard Henderson } 356725f49c5fSRichard Henderson break; 3568c896fe29Sbellard } 3569c896fe29Sbellard break; 3570c896fe29Sbellard } 3571bee158cbSRichard Henderson op->life = arg_life; 3572c896fe29Sbellard } 35731ff0a2c5SEvgeny Voevodin } 3574c896fe29Sbellard 35755a18407fSRichard Henderson /* Liveness analysis: Convert indirect regs to direct temporaries. */ 35769bbee4c0SRichard Henderson static bool __attribute__((noinline)) 35779bbee4c0SRichard Henderson liveness_pass_2(TCGContext *s) 35785a18407fSRichard Henderson { 35795a18407fSRichard Henderson int nb_globals = s->nb_globals; 358015fa08f8SRichard Henderson int nb_temps, i; 35815a18407fSRichard Henderson bool changes = false; 358215fa08f8SRichard Henderson TCGOp *op, *op_next; 35835a18407fSRichard Henderson 35845a18407fSRichard Henderson /* Create a temporary for each indirect global. */ 35855a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 35865a18407fSRichard Henderson TCGTemp *its = &s->temps[i]; 35875a18407fSRichard Henderson if (its->indirect_reg) { 35885a18407fSRichard Henderson TCGTemp *dts = tcg_temp_alloc(s); 35895a18407fSRichard Henderson dts->type = its->type; 35905a18407fSRichard Henderson dts->base_type = its->base_type; 3591e1e64652SRichard Henderson dts->temp_subindex = its->temp_subindex; 3592c7482438SRichard Henderson dts->kind = TEMP_EBB; 3593b83eabeaSRichard Henderson its->state_ptr = dts; 3594b83eabeaSRichard Henderson } else { 3595b83eabeaSRichard Henderson its->state_ptr = NULL; 35965a18407fSRichard Henderson } 3597b83eabeaSRichard Henderson /* All globals begin dead. */ 3598b83eabeaSRichard Henderson its->state = TS_DEAD; 35995a18407fSRichard Henderson } 3600b83eabeaSRichard Henderson for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 3601b83eabeaSRichard Henderson TCGTemp *its = &s->temps[i]; 3602b83eabeaSRichard Henderson its->state_ptr = NULL; 3603b83eabeaSRichard Henderson its->state = TS_DEAD; 3604b83eabeaSRichard Henderson } 36055a18407fSRichard Henderson 360615fa08f8SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 36075a18407fSRichard Henderson TCGOpcode opc = op->opc; 36085a18407fSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 36095a18407fSRichard Henderson TCGLifeData arg_life = op->life; 36105a18407fSRichard Henderson int nb_iargs, nb_oargs, call_flags; 3611b83eabeaSRichard Henderson TCGTemp *arg_ts, *dir_ts; 36125a18407fSRichard Henderson 36135a18407fSRichard Henderson if (opc == INDEX_op_call) { 3614cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 3615cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 361690163900SRichard Henderson call_flags = tcg_call_flags(op); 36175a18407fSRichard Henderson } else { 36185a18407fSRichard Henderson nb_iargs = def->nb_iargs; 36195a18407fSRichard Henderson nb_oargs = def->nb_oargs; 36205a18407fSRichard Henderson 36215a18407fSRichard Henderson /* Set flags similar to how calls require. */ 3622b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 3623b4cb76e6SRichard Henderson /* Like reading globals: sync_globals */ 3624b4cb76e6SRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 3625b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 36265a18407fSRichard Henderson /* Like writing globals: save_globals */ 36275a18407fSRichard Henderson call_flags = 0; 36285a18407fSRichard Henderson } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 36295a18407fSRichard Henderson /* Like reading globals: sync_globals */ 36305a18407fSRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 36315a18407fSRichard Henderson } else { 36325a18407fSRichard Henderson /* No effect on globals. */ 36335a18407fSRichard Henderson call_flags = (TCG_CALL_NO_READ_GLOBALS | 36345a18407fSRichard Henderson TCG_CALL_NO_WRITE_GLOBALS); 36355a18407fSRichard Henderson } 36365a18407fSRichard Henderson } 36375a18407fSRichard Henderson 36385a18407fSRichard Henderson /* Make sure that input arguments are available. */ 36395a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3640b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3641b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3642b83eabeaSRichard Henderson if (dir_ts && arg_ts->state == TS_DEAD) { 3643b83eabeaSRichard Henderson TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 36445a18407fSRichard Henderson ? INDEX_op_ld_i32 36455a18407fSRichard Henderson : INDEX_op_ld_i64); 3646d4478943SPhilippe Mathieu-Daudé TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 36475a18407fSRichard Henderson 3648b83eabeaSRichard Henderson lop->args[0] = temp_arg(dir_ts); 3649b83eabeaSRichard Henderson lop->args[1] = temp_arg(arg_ts->mem_base); 3650b83eabeaSRichard Henderson lop->args[2] = arg_ts->mem_offset; 36515a18407fSRichard Henderson 36525a18407fSRichard Henderson /* Loaded, but synced with memory. */ 3653b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 36545a18407fSRichard Henderson } 36555a18407fSRichard Henderson } 36565a18407fSRichard Henderson 36575a18407fSRichard Henderson /* Perform input replacement, and mark inputs that became dead. 36585a18407fSRichard Henderson No action is required except keeping temp_state up to date 36595a18407fSRichard Henderson so that we reload when needed. */ 36605a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3661b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3662b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3663b83eabeaSRichard Henderson if (dir_ts) { 3664b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 36655a18407fSRichard Henderson changes = true; 36665a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 3667b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 36685a18407fSRichard Henderson } 36695a18407fSRichard Henderson } 36705a18407fSRichard Henderson } 36715a18407fSRichard Henderson 36725a18407fSRichard Henderson /* Liveness analysis should ensure that the following are 36735a18407fSRichard Henderson all correct, for call sites and basic block end points. */ 36745a18407fSRichard Henderson if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 36755a18407fSRichard Henderson /* Nothing to do */ 36765a18407fSRichard Henderson } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 36775a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 36785a18407fSRichard Henderson /* Liveness should see that globals are synced back, 36795a18407fSRichard Henderson that is, either TS_DEAD or TS_MEM. */ 3680b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 3681b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 3682b83eabeaSRichard Henderson || arg_ts->state != 0); 36835a18407fSRichard Henderson } 36845a18407fSRichard Henderson } else { 36855a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 36865a18407fSRichard Henderson /* Liveness should see that globals are saved back, 36875a18407fSRichard Henderson that is, TS_DEAD, waiting to be reloaded. */ 3688b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 3689b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 3690b83eabeaSRichard Henderson || arg_ts->state == TS_DEAD); 36915a18407fSRichard Henderson } 36925a18407fSRichard Henderson } 36935a18407fSRichard Henderson 36945a18407fSRichard Henderson /* Outputs become available. */ 369561f15c48SRichard Henderson if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 369661f15c48SRichard Henderson arg_ts = arg_temp(op->args[0]); 369761f15c48SRichard Henderson dir_ts = arg_ts->state_ptr; 369861f15c48SRichard Henderson if (dir_ts) { 369961f15c48SRichard Henderson op->args[0] = temp_arg(dir_ts); 370061f15c48SRichard Henderson changes = true; 370161f15c48SRichard Henderson 370261f15c48SRichard Henderson /* The output is now live and modified. */ 370361f15c48SRichard Henderson arg_ts->state = 0; 370461f15c48SRichard Henderson 370561f15c48SRichard Henderson if (NEED_SYNC_ARG(0)) { 370661f15c48SRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 370761f15c48SRichard Henderson ? INDEX_op_st_i32 370861f15c48SRichard Henderson : INDEX_op_st_i64); 3709d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 371061f15c48SRichard Henderson TCGTemp *out_ts = dir_ts; 371161f15c48SRichard Henderson 371261f15c48SRichard Henderson if (IS_DEAD_ARG(0)) { 371361f15c48SRichard Henderson out_ts = arg_temp(op->args[1]); 371461f15c48SRichard Henderson arg_ts->state = TS_DEAD; 371561f15c48SRichard Henderson tcg_op_remove(s, op); 371661f15c48SRichard Henderson } else { 371761f15c48SRichard Henderson arg_ts->state = TS_MEM; 371861f15c48SRichard Henderson } 371961f15c48SRichard Henderson 372061f15c48SRichard Henderson sop->args[0] = temp_arg(out_ts); 372161f15c48SRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 372261f15c48SRichard Henderson sop->args[2] = arg_ts->mem_offset; 372361f15c48SRichard Henderson } else { 372461f15c48SRichard Henderson tcg_debug_assert(!IS_DEAD_ARG(0)); 372561f15c48SRichard Henderson } 372661f15c48SRichard Henderson } 372761f15c48SRichard Henderson } else { 37285a18407fSRichard Henderson for (i = 0; i < nb_oargs; i++) { 3729b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3730b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3731b83eabeaSRichard Henderson if (!dir_ts) { 37325a18407fSRichard Henderson continue; 37335a18407fSRichard Henderson } 3734b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 37355a18407fSRichard Henderson changes = true; 37365a18407fSRichard Henderson 37375a18407fSRichard Henderson /* The output is now live and modified. */ 3738b83eabeaSRichard Henderson arg_ts->state = 0; 37395a18407fSRichard Henderson 37405a18407fSRichard Henderson /* Sync outputs upon their last write. */ 37415a18407fSRichard Henderson if (NEED_SYNC_ARG(i)) { 3742b83eabeaSRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 37435a18407fSRichard Henderson ? INDEX_op_st_i32 37445a18407fSRichard Henderson : INDEX_op_st_i64); 3745d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 37465a18407fSRichard Henderson 3747b83eabeaSRichard Henderson sop->args[0] = temp_arg(dir_ts); 3748b83eabeaSRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 3749b83eabeaSRichard Henderson sop->args[2] = arg_ts->mem_offset; 37505a18407fSRichard Henderson 3751b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 37525a18407fSRichard Henderson } 37535a18407fSRichard Henderson /* Drop outputs that are dead. */ 37545a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 3755b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 37565a18407fSRichard Henderson } 37575a18407fSRichard Henderson } 37585a18407fSRichard Henderson } 375961f15c48SRichard Henderson } 37605a18407fSRichard Henderson 37615a18407fSRichard Henderson return changes; 37625a18407fSRichard Henderson } 37635a18407fSRichard Henderson 37642272e4a7SRichard Henderson static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3765c896fe29Sbellard { 376631c96417SRichard Henderson intptr_t off; 3767273eb50cSRichard Henderson int size, align; 3768c1c09194SRichard Henderson 3769273eb50cSRichard Henderson /* When allocating an object, look at the full type. */ 3770273eb50cSRichard Henderson size = tcg_type_size(ts->base_type); 3771273eb50cSRichard Henderson switch (ts->base_type) { 3772c1c09194SRichard Henderson case TCG_TYPE_I32: 377331c96417SRichard Henderson align = 4; 3774c1c09194SRichard Henderson break; 3775c1c09194SRichard Henderson case TCG_TYPE_I64: 3776c1c09194SRichard Henderson case TCG_TYPE_V64: 377731c96417SRichard Henderson align = 8; 3778c1c09194SRichard Henderson break; 377943eef72fSRichard Henderson case TCG_TYPE_I128: 3780c1c09194SRichard Henderson case TCG_TYPE_V128: 3781c1c09194SRichard Henderson case TCG_TYPE_V256: 378243eef72fSRichard Henderson /* 378343eef72fSRichard Henderson * Note that we do not require aligned storage for V256, 378443eef72fSRichard Henderson * and that we provide alignment for I128 to match V128, 378543eef72fSRichard Henderson * even if that's above what the host ABI requires. 378643eef72fSRichard Henderson */ 378731c96417SRichard Henderson align = 16; 3788c1c09194SRichard Henderson break; 3789c1c09194SRichard Henderson default: 3790c1c09194SRichard Henderson g_assert_not_reached(); 3791b591dc59SBlue Swirl } 3792c1c09194SRichard Henderson 3793b9537d59SRichard Henderson /* 3794b9537d59SRichard Henderson * Assume the stack is sufficiently aligned. 3795b9537d59SRichard Henderson * This affects e.g. ARM NEON, where we have 8 byte stack alignment 3796b9537d59SRichard Henderson * and do not require 16 byte vector alignment. This seems slightly 3797b9537d59SRichard Henderson * easier than fully parameterizing the above switch statement. 3798b9537d59SRichard Henderson */ 3799b9537d59SRichard Henderson align = MIN(TCG_TARGET_STACK_ALIGN, align); 3800c1c09194SRichard Henderson off = ROUND_UP(s->current_frame_offset, align); 3801732d5897SRichard Henderson 3802732d5897SRichard Henderson /* If we've exhausted the stack frame, restart with a smaller TB. */ 3803732d5897SRichard Henderson if (off + size > s->frame_end) { 3804732d5897SRichard Henderson tcg_raise_tb_overflow(s); 3805732d5897SRichard Henderson } 3806c1c09194SRichard Henderson s->current_frame_offset = off + size; 38079defd1bdSRichard Henderson #if defined(__sparc__) 3808273eb50cSRichard Henderson off += TCG_TARGET_STACK_BIAS; 38099defd1bdSRichard Henderson #endif 3810273eb50cSRichard Henderson 3811273eb50cSRichard Henderson /* If the object was subdivided, assign memory to all the parts. */ 3812273eb50cSRichard Henderson if (ts->base_type != ts->type) { 3813273eb50cSRichard Henderson int part_size = tcg_type_size(ts->type); 3814273eb50cSRichard Henderson int part_count = size / part_size; 3815273eb50cSRichard Henderson 3816273eb50cSRichard Henderson /* 3817273eb50cSRichard Henderson * Each part is allocated sequentially in tcg_temp_new_internal. 3818273eb50cSRichard Henderson * Jump back to the first part by subtracting the current index. 3819273eb50cSRichard Henderson */ 3820273eb50cSRichard Henderson ts -= ts->temp_subindex; 3821273eb50cSRichard Henderson for (int i = 0; i < part_count; ++i) { 3822273eb50cSRichard Henderson ts[i].mem_offset = off + i * part_size; 3823273eb50cSRichard Henderson ts[i].mem_base = s->frame_temp; 3824273eb50cSRichard Henderson ts[i].mem_allocated = 1; 3825273eb50cSRichard Henderson } 3826273eb50cSRichard Henderson } else { 3827273eb50cSRichard Henderson ts->mem_offset = off; 3828b3a62939SRichard Henderson ts->mem_base = s->frame_temp; 3829c896fe29Sbellard ts->mem_allocated = 1; 3830c896fe29Sbellard } 3831273eb50cSRichard Henderson } 3832c896fe29Sbellard 3833098859f1SRichard Henderson /* Assign @reg to @ts, and update reg_to_temp[]. */ 3834098859f1SRichard Henderson static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 3835098859f1SRichard Henderson { 3836098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 3837098859f1SRichard Henderson TCGReg old = ts->reg; 3838098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[old] == ts); 3839098859f1SRichard Henderson if (old == reg) { 3840098859f1SRichard Henderson return; 3841098859f1SRichard Henderson } 3842098859f1SRichard Henderson s->reg_to_temp[old] = NULL; 3843098859f1SRichard Henderson } 3844098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3845098859f1SRichard Henderson s->reg_to_temp[reg] = ts; 3846098859f1SRichard Henderson ts->val_type = TEMP_VAL_REG; 3847098859f1SRichard Henderson ts->reg = reg; 3848098859f1SRichard Henderson } 3849098859f1SRichard Henderson 3850098859f1SRichard Henderson /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 3851098859f1SRichard Henderson static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 3852098859f1SRichard Henderson { 3853098859f1SRichard Henderson tcg_debug_assert(type != TEMP_VAL_REG); 3854098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 3855098859f1SRichard Henderson TCGReg reg = ts->reg; 3856098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == ts); 3857098859f1SRichard Henderson s->reg_to_temp[reg] = NULL; 3858098859f1SRichard Henderson } 3859098859f1SRichard Henderson ts->val_type = type; 3860098859f1SRichard Henderson } 3861098859f1SRichard Henderson 3862b722452aSRichard Henderson static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3863b3915dbbSRichard Henderson 386459d7c14eSRichard Henderson /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 386559d7c14eSRichard Henderson mark it free; otherwise mark it dead. */ 386659d7c14eSRichard Henderson static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3867c896fe29Sbellard { 3868c0522136SRichard Henderson TCGTempVal new_type; 3869c0522136SRichard Henderson 3870c0522136SRichard Henderson switch (ts->kind) { 3871c0522136SRichard Henderson case TEMP_FIXED: 387259d7c14eSRichard Henderson return; 3873c0522136SRichard Henderson case TEMP_GLOBAL: 3874f57c6915SRichard Henderson case TEMP_TB: 3875c0522136SRichard Henderson new_type = TEMP_VAL_MEM; 3876c0522136SRichard Henderson break; 3877c7482438SRichard Henderson case TEMP_EBB: 3878c0522136SRichard Henderson new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3879c0522136SRichard Henderson break; 3880c0522136SRichard Henderson case TEMP_CONST: 3881c0522136SRichard Henderson new_type = TEMP_VAL_CONST; 3882c0522136SRichard Henderson break; 3883c0522136SRichard Henderson default: 3884c0522136SRichard Henderson g_assert_not_reached(); 388559d7c14eSRichard Henderson } 3886098859f1SRichard Henderson set_temp_val_nonreg(s, ts, new_type); 388759d7c14eSRichard Henderson } 3888c896fe29Sbellard 388959d7c14eSRichard Henderson /* Mark a temporary as dead. */ 389059d7c14eSRichard Henderson static inline void temp_dead(TCGContext *s, TCGTemp *ts) 389159d7c14eSRichard Henderson { 389259d7c14eSRichard Henderson temp_free_or_dead(s, ts, 1); 389359d7c14eSRichard Henderson } 389459d7c14eSRichard Henderson 389559d7c14eSRichard Henderson /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 389659d7c14eSRichard Henderson registers needs to be allocated to store a constant. If 'free_or_dead' 389759d7c14eSRichard Henderson is non-zero, subsequently release the temporary; if it is positive, the 389859d7c14eSRichard Henderson temp is dead; if it is negative, the temp is free. */ 389998b4e186SRichard Henderson static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 390098b4e186SRichard Henderson TCGRegSet preferred_regs, int free_or_dead) 390159d7c14eSRichard Henderson { 3902c0522136SRichard Henderson if (!temp_readonly(ts) && !ts->mem_coherent) { 39037f6ceedfSAurelien Jarno if (!ts->mem_allocated) { 39042272e4a7SRichard Henderson temp_allocate_frame(s, ts); 390559d7c14eSRichard Henderson } 390659d7c14eSRichard Henderson switch (ts->val_type) { 390759d7c14eSRichard Henderson case TEMP_VAL_CONST: 390859d7c14eSRichard Henderson /* If we're going to free the temp immediately, then we won't 390959d7c14eSRichard Henderson require it later in a register, so attempt to store the 391059d7c14eSRichard Henderson constant to memory directly. */ 391159d7c14eSRichard Henderson if (free_or_dead 391259d7c14eSRichard Henderson && tcg_out_sti(s, ts->type, ts->val, 391359d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset)) { 391459d7c14eSRichard Henderson break; 391559d7c14eSRichard Henderson } 391659d7c14eSRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 391798b4e186SRichard Henderson allocated_regs, preferred_regs); 391859d7c14eSRichard Henderson /* fallthrough */ 391959d7c14eSRichard Henderson 392059d7c14eSRichard Henderson case TEMP_VAL_REG: 392159d7c14eSRichard Henderson tcg_out_st(s, ts->type, ts->reg, 392259d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset); 392359d7c14eSRichard Henderson break; 392459d7c14eSRichard Henderson 392559d7c14eSRichard Henderson case TEMP_VAL_MEM: 392659d7c14eSRichard Henderson break; 392759d7c14eSRichard Henderson 392859d7c14eSRichard Henderson case TEMP_VAL_DEAD: 392959d7c14eSRichard Henderson default: 3930732e89f4SRichard Henderson g_assert_not_reached(); 3931c896fe29Sbellard } 39327f6ceedfSAurelien Jarno ts->mem_coherent = 1; 39337f6ceedfSAurelien Jarno } 393459d7c14eSRichard Henderson if (free_or_dead) { 393559d7c14eSRichard Henderson temp_free_or_dead(s, ts, free_or_dead); 393659d7c14eSRichard Henderson } 393759d7c14eSRichard Henderson } 39387f6ceedfSAurelien Jarno 39397f6ceedfSAurelien Jarno /* free register 'reg' by spilling the corresponding temporary if necessary */ 3940b3915dbbSRichard Henderson static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 39417f6ceedfSAurelien Jarno { 3942f8b2f202SRichard Henderson TCGTemp *ts = s->reg_to_temp[reg]; 3943f8b2f202SRichard Henderson if (ts != NULL) { 394498b4e186SRichard Henderson temp_sync(s, ts, allocated_regs, 0, -1); 3945c896fe29Sbellard } 3946c896fe29Sbellard } 3947c896fe29Sbellard 3948b016486eSRichard Henderson /** 3949b016486eSRichard Henderson * tcg_reg_alloc: 3950b016486eSRichard Henderson * @required_regs: Set of registers in which we must allocate. 3951b016486eSRichard Henderson * @allocated_regs: Set of registers which must be avoided. 3952b016486eSRichard Henderson * @preferred_regs: Set of registers we should prefer. 3953b016486eSRichard Henderson * @rev: True if we search the registers in "indirect" order. 3954b016486eSRichard Henderson * 3955b016486eSRichard Henderson * The allocated register must be in @required_regs & ~@allocated_regs, 3956b016486eSRichard Henderson * but if we can put it in @preferred_regs we may save a move later. 3957b016486eSRichard Henderson */ 3958b016486eSRichard Henderson static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3959b016486eSRichard Henderson TCGRegSet allocated_regs, 3960b016486eSRichard Henderson TCGRegSet preferred_regs, bool rev) 3961c896fe29Sbellard { 3962b016486eSRichard Henderson int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3963b016486eSRichard Henderson TCGRegSet reg_ct[2]; 396491478cefSRichard Henderson const int *order; 3965c896fe29Sbellard 3966b016486eSRichard Henderson reg_ct[1] = required_regs & ~allocated_regs; 3967b016486eSRichard Henderson tcg_debug_assert(reg_ct[1] != 0); 3968b016486eSRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs; 3969b016486eSRichard Henderson 3970b016486eSRichard Henderson /* Skip the preferred_regs option if it cannot be satisfied, 3971b016486eSRichard Henderson or if the preference made no difference. */ 3972b016486eSRichard Henderson f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3973b016486eSRichard Henderson 397491478cefSRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3975c896fe29Sbellard 3976b016486eSRichard Henderson /* Try free registers, preferences first. */ 3977b016486eSRichard Henderson for (j = f; j < 2; j++) { 3978b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 3979b016486eSRichard Henderson 3980b016486eSRichard Henderson if (tcg_regset_single(set)) { 3981b016486eSRichard Henderson /* One register in the set. */ 3982b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 3983b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL) { 3984c896fe29Sbellard return reg; 3985c896fe29Sbellard } 3986b016486eSRichard Henderson } else { 398791478cefSRichard Henderson for (i = 0; i < n; i++) { 3988b016486eSRichard Henderson TCGReg reg = order[i]; 3989b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL && 3990b016486eSRichard Henderson tcg_regset_test_reg(set, reg)) { 3991b016486eSRichard Henderson return reg; 3992b016486eSRichard Henderson } 3993b016486eSRichard Henderson } 3994b016486eSRichard Henderson } 3995b016486eSRichard Henderson } 3996b016486eSRichard Henderson 3997b016486eSRichard Henderson /* We must spill something. */ 3998b016486eSRichard Henderson for (j = f; j < 2; j++) { 3999b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 4000b016486eSRichard Henderson 4001b016486eSRichard Henderson if (tcg_regset_single(set)) { 4002b016486eSRichard Henderson /* One register in the set. */ 4003b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 4004b3915dbbSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 4005c896fe29Sbellard return reg; 4006b016486eSRichard Henderson } else { 4007b016486eSRichard Henderson for (i = 0; i < n; i++) { 4008b016486eSRichard Henderson TCGReg reg = order[i]; 4009b016486eSRichard Henderson if (tcg_regset_test_reg(set, reg)) { 4010b016486eSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 4011b016486eSRichard Henderson return reg; 4012b016486eSRichard Henderson } 4013b016486eSRichard Henderson } 4014c896fe29Sbellard } 4015c896fe29Sbellard } 4016c896fe29Sbellard 4017732e89f4SRichard Henderson g_assert_not_reached(); 4018c896fe29Sbellard } 4019c896fe29Sbellard 402029f5e925SRichard Henderson static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 402129f5e925SRichard Henderson TCGRegSet allocated_regs, 402229f5e925SRichard Henderson TCGRegSet preferred_regs, bool rev) 402329f5e925SRichard Henderson { 402429f5e925SRichard Henderson int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 402529f5e925SRichard Henderson TCGRegSet reg_ct[2]; 402629f5e925SRichard Henderson const int *order; 402729f5e925SRichard Henderson 402829f5e925SRichard Henderson /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 402929f5e925SRichard Henderson reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 403029f5e925SRichard Henderson tcg_debug_assert(reg_ct[1] != 0); 403129f5e925SRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs; 403229f5e925SRichard Henderson 403329f5e925SRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 403429f5e925SRichard Henderson 403529f5e925SRichard Henderson /* 403629f5e925SRichard Henderson * Skip the preferred_regs option if it cannot be satisfied, 403729f5e925SRichard Henderson * or if the preference made no difference. 403829f5e925SRichard Henderson */ 403929f5e925SRichard Henderson k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 404029f5e925SRichard Henderson 404129f5e925SRichard Henderson /* 404229f5e925SRichard Henderson * Minimize the number of flushes by looking for 2 free registers first, 404329f5e925SRichard Henderson * then a single flush, then two flushes. 404429f5e925SRichard Henderson */ 404529f5e925SRichard Henderson for (fmin = 2; fmin >= 0; fmin--) { 404629f5e925SRichard Henderson for (j = k; j < 2; j++) { 404729f5e925SRichard Henderson TCGRegSet set = reg_ct[j]; 404829f5e925SRichard Henderson 404929f5e925SRichard Henderson for (i = 0; i < n; i++) { 405029f5e925SRichard Henderson TCGReg reg = order[i]; 405129f5e925SRichard Henderson 405229f5e925SRichard Henderson if (tcg_regset_test_reg(set, reg)) { 405329f5e925SRichard Henderson int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 405429f5e925SRichard Henderson if (f >= fmin) { 405529f5e925SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 405629f5e925SRichard Henderson tcg_reg_free(s, reg + 1, allocated_regs); 405729f5e925SRichard Henderson return reg; 405829f5e925SRichard Henderson } 405929f5e925SRichard Henderson } 406029f5e925SRichard Henderson } 406129f5e925SRichard Henderson } 406229f5e925SRichard Henderson } 4063732e89f4SRichard Henderson g_assert_not_reached(); 406429f5e925SRichard Henderson } 406529f5e925SRichard Henderson 406640ae5c62SRichard Henderson /* Make sure the temporary is in a register. If needed, allocate the register 406740ae5c62SRichard Henderson from DESIRED while avoiding ALLOCATED. */ 406840ae5c62SRichard Henderson static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 4069b722452aSRichard Henderson TCGRegSet allocated_regs, TCGRegSet preferred_regs) 407040ae5c62SRichard Henderson { 407140ae5c62SRichard Henderson TCGReg reg; 407240ae5c62SRichard Henderson 407340ae5c62SRichard Henderson switch (ts->val_type) { 407440ae5c62SRichard Henderson case TEMP_VAL_REG: 407540ae5c62SRichard Henderson return; 407640ae5c62SRichard Henderson case TEMP_VAL_CONST: 4077b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 4078b722452aSRichard Henderson preferred_regs, ts->indirect_base); 40790a6a8bc8SRichard Henderson if (ts->type <= TCG_TYPE_I64) { 408040ae5c62SRichard Henderson tcg_out_movi(s, ts->type, reg, ts->val); 40810a6a8bc8SRichard Henderson } else { 40824e186175SRichard Henderson uint64_t val = ts->val; 40834e186175SRichard Henderson MemOp vece = MO_64; 40844e186175SRichard Henderson 40854e186175SRichard Henderson /* 40864e186175SRichard Henderson * Find the minimal vector element that matches the constant. 40874e186175SRichard Henderson * The targets will, in general, have to do this search anyway, 40884e186175SRichard Henderson * do this generically. 40894e186175SRichard Henderson */ 40904e186175SRichard Henderson if (val == dup_const(MO_8, val)) { 40914e186175SRichard Henderson vece = MO_8; 40924e186175SRichard Henderson } else if (val == dup_const(MO_16, val)) { 40934e186175SRichard Henderson vece = MO_16; 40940b4286ddSRichard Henderson } else if (val == dup_const(MO_32, val)) { 40954e186175SRichard Henderson vece = MO_32; 40964e186175SRichard Henderson } 40974e186175SRichard Henderson 40984e186175SRichard Henderson tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 40990a6a8bc8SRichard Henderson } 410040ae5c62SRichard Henderson ts->mem_coherent = 0; 410140ae5c62SRichard Henderson break; 410240ae5c62SRichard Henderson case TEMP_VAL_MEM: 4103b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 4104b722452aSRichard Henderson preferred_regs, ts->indirect_base); 410540ae5c62SRichard Henderson tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 410640ae5c62SRichard Henderson ts->mem_coherent = 1; 410740ae5c62SRichard Henderson break; 410840ae5c62SRichard Henderson case TEMP_VAL_DEAD: 410940ae5c62SRichard Henderson default: 4110732e89f4SRichard Henderson g_assert_not_reached(); 411140ae5c62SRichard Henderson } 4112098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 411340ae5c62SRichard Henderson } 411440ae5c62SRichard Henderson 411559d7c14eSRichard Henderson /* Save a temporary to memory. 'allocated_regs' is used in case a 4116e8996ee0Sbellard temporary registers needs to be allocated to store a constant. */ 411759d7c14eSRichard Henderson static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 41181ad80729SAurelien Jarno { 41192c0366f0SAurelien Jarno /* The liveness analysis already ensures that globals are back 4120eabb7b91SAurelien Jarno in memory. Keep an tcg_debug_assert for safety. */ 4121e01fa97dSRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 41221ad80729SAurelien Jarno } 41231ad80729SAurelien Jarno 41249814dd27SDong Xu Wang /* save globals to their canonical location and assume they can be 4125641d5fbeSbellard modified be the following code. 'allocated_regs' is used in case a 4126641d5fbeSbellard temporary registers needs to be allocated to store a constant. */ 4127641d5fbeSbellard static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 4128641d5fbeSbellard { 4129ac3b8891SRichard Henderson int i, n; 4130641d5fbeSbellard 4131ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 4132b13eb728SRichard Henderson temp_save(s, &s->temps[i], allocated_regs); 4133641d5fbeSbellard } 4134e5097dc8Sbellard } 4135e5097dc8Sbellard 41363d5c5f87SAurelien Jarno /* sync globals to their canonical location and assume they can be 41373d5c5f87SAurelien Jarno read by the following code. 'allocated_regs' is used in case a 41383d5c5f87SAurelien Jarno temporary registers needs to be allocated to store a constant. */ 41393d5c5f87SAurelien Jarno static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 41403d5c5f87SAurelien Jarno { 4141ac3b8891SRichard Henderson int i, n; 41423d5c5f87SAurelien Jarno 4143ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 414412b9b11aSRichard Henderson TCGTemp *ts = &s->temps[i]; 414512b9b11aSRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG 4146ee17db83SRichard Henderson || ts->kind == TEMP_FIXED 414712b9b11aSRichard Henderson || ts->mem_coherent); 41483d5c5f87SAurelien Jarno } 41493d5c5f87SAurelien Jarno } 41503d5c5f87SAurelien Jarno 4151e5097dc8Sbellard /* at the end of a basic block, we assume all temporaries are dead and 4152e8996ee0Sbellard all globals are stored at their canonical location. */ 4153e8996ee0Sbellard static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 4154e5097dc8Sbellard { 4155e5097dc8Sbellard int i; 4156e5097dc8Sbellard 4157c896fe29Sbellard for (i = s->nb_globals; i < s->nb_temps; i++) { 4158b13eb728SRichard Henderson TCGTemp *ts = &s->temps[i]; 4159c0522136SRichard Henderson 4160c0522136SRichard Henderson switch (ts->kind) { 4161f57c6915SRichard Henderson case TEMP_TB: 4162b13eb728SRichard Henderson temp_save(s, ts, allocated_regs); 4163c0522136SRichard Henderson break; 4164c7482438SRichard Henderson case TEMP_EBB: 41652c0366f0SAurelien Jarno /* The liveness analysis already ensures that temps are dead. 4166eabb7b91SAurelien Jarno Keep an tcg_debug_assert for safety. */ 4167eabb7b91SAurelien Jarno tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 4168c0522136SRichard Henderson break; 4169c0522136SRichard Henderson case TEMP_CONST: 4170c0522136SRichard Henderson /* Similarly, we should have freed any allocated register. */ 4171c0522136SRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 4172c0522136SRichard Henderson break; 4173c0522136SRichard Henderson default: 4174c0522136SRichard Henderson g_assert_not_reached(); 4175c896fe29Sbellard } 4176641d5fbeSbellard } 4177e8996ee0Sbellard 4178e8996ee0Sbellard save_globals(s, allocated_regs); 4179c896fe29Sbellard } 4180c896fe29Sbellard 4181bab1671fSRichard Henderson /* 4182c7482438SRichard Henderson * At a conditional branch, we assume all temporaries are dead unless 4183c7482438SRichard Henderson * explicitly live-across-conditional-branch; all globals and local 4184c7482438SRichard Henderson * temps are synced to their location. 4185b4cb76e6SRichard Henderson */ 4186b4cb76e6SRichard Henderson static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 4187b4cb76e6SRichard Henderson { 4188b4cb76e6SRichard Henderson sync_globals(s, allocated_regs); 4189b4cb76e6SRichard Henderson 4190b4cb76e6SRichard Henderson for (int i = s->nb_globals; i < s->nb_temps; i++) { 4191b4cb76e6SRichard Henderson TCGTemp *ts = &s->temps[i]; 4192b4cb76e6SRichard Henderson /* 4193b4cb76e6SRichard Henderson * The liveness analysis already ensures that temps are dead. 4194b4cb76e6SRichard Henderson * Keep tcg_debug_asserts for safety. 4195b4cb76e6SRichard Henderson */ 4196c0522136SRichard Henderson switch (ts->kind) { 4197f57c6915SRichard Henderson case TEMP_TB: 4198b4cb76e6SRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 4199c0522136SRichard Henderson break; 4200c7482438SRichard Henderson case TEMP_EBB: 4201c0522136SRichard Henderson case TEMP_CONST: 4202c0522136SRichard Henderson break; 4203c0522136SRichard Henderson default: 4204c0522136SRichard Henderson g_assert_not_reached(); 4205b4cb76e6SRichard Henderson } 4206b4cb76e6SRichard Henderson } 4207b4cb76e6SRichard Henderson } 4208b4cb76e6SRichard Henderson 4209b4cb76e6SRichard Henderson /* 4210c58f4c97SRichard Henderson * Specialized code generation for INDEX_op_mov_* with a constant. 4211bab1671fSRichard Henderson */ 42120fe4fca4SPaolo Bonzini static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 4213ba87719cSRichard Henderson tcg_target_ulong val, TCGLifeData arg_life, 4214ba87719cSRichard Henderson TCGRegSet preferred_regs) 4215e8996ee0Sbellard { 4216d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4217e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 421859d7c14eSRichard Henderson 421959d7c14eSRichard Henderson /* The movi is not explicitly generated here. */ 4220098859f1SRichard Henderson set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 4221e8996ee0Sbellard ots->val = val; 422259d7c14eSRichard Henderson ots->mem_coherent = 0; 4223ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 4224ba87719cSRichard Henderson temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 422559d7c14eSRichard Henderson } else if (IS_DEAD_ARG(0)) { 4226f8bf00f1SRichard Henderson temp_dead(s, ots); 42274c4e1ab2SAurelien Jarno } 4228e8996ee0Sbellard } 4229e8996ee0Sbellard 4230bab1671fSRichard Henderson /* 4231bab1671fSRichard Henderson * Specialized code generation for INDEX_op_mov_*. 4232bab1671fSRichard Henderson */ 4233dd186292SRichard Henderson static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 4234c896fe29Sbellard { 4235dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 423669e3706dSRichard Henderson TCGRegSet allocated_regs, preferred_regs; 4237c896fe29Sbellard TCGTemp *ts, *ots; 4238450445d5SRichard Henderson TCGType otype, itype; 4239098859f1SRichard Henderson TCGReg oreg, ireg; 4240c896fe29Sbellard 4241d21369f5SRichard Henderson allocated_regs = s->reserved_regs; 424231fd884bSRichard Henderson preferred_regs = output_pref(op, 0); 424343439139SRichard Henderson ots = arg_temp(op->args[0]); 424443439139SRichard Henderson ts = arg_temp(op->args[1]); 4245450445d5SRichard Henderson 4246d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4247e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4248d63e3b6eSRichard Henderson 4249450445d5SRichard Henderson /* Note that otype != itype for no-op truncation. */ 4250450445d5SRichard Henderson otype = ots->type; 4251450445d5SRichard Henderson itype = ts->type; 4252c896fe29Sbellard 42530fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_CONST) { 42540fe4fca4SPaolo Bonzini /* propagate constant or generate sti */ 42550fe4fca4SPaolo Bonzini tcg_target_ulong val = ts->val; 42560fe4fca4SPaolo Bonzini if (IS_DEAD_ARG(1)) { 42570fe4fca4SPaolo Bonzini temp_dead(s, ts); 42580fe4fca4SPaolo Bonzini } 425969e3706dSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 42600fe4fca4SPaolo Bonzini return; 42610fe4fca4SPaolo Bonzini } 42620fe4fca4SPaolo Bonzini 42630fe4fca4SPaolo Bonzini /* If the source value is in memory we're going to be forced 42640fe4fca4SPaolo Bonzini to have it in a register in order to perform the copy. Copy 42650fe4fca4SPaolo Bonzini the SOURCE value into its own register first, that way we 42660fe4fca4SPaolo Bonzini don't have to reload SOURCE the next time it is used. */ 42670fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_MEM) { 426869e3706dSRichard Henderson temp_load(s, ts, tcg_target_available_regs[itype], 426969e3706dSRichard Henderson allocated_regs, preferred_regs); 4270c29c1d7eSAurelien Jarno } 42710fe4fca4SPaolo Bonzini tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 4272098859f1SRichard Henderson ireg = ts->reg; 4273098859f1SRichard Henderson 4274d63e3b6eSRichard Henderson if (IS_DEAD_ARG(0)) { 4275c29c1d7eSAurelien Jarno /* mov to a non-saved dead register makes no sense (even with 4276c29c1d7eSAurelien Jarno liveness analysis disabled). */ 4277eabb7b91SAurelien Jarno tcg_debug_assert(NEED_SYNC_ARG(0)); 4278c29c1d7eSAurelien Jarno if (!ots->mem_allocated) { 42792272e4a7SRichard Henderson temp_allocate_frame(s, ots); 4280c29c1d7eSAurelien Jarno } 4281098859f1SRichard Henderson tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 4282c29c1d7eSAurelien Jarno if (IS_DEAD_ARG(1)) { 4283f8bf00f1SRichard Henderson temp_dead(s, ts); 4284c29c1d7eSAurelien Jarno } 4285f8bf00f1SRichard Henderson temp_dead(s, ots); 4286098859f1SRichard Henderson return; 4287098859f1SRichard Henderson } 4288098859f1SRichard Henderson 4289ee17db83SRichard Henderson if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 4290098859f1SRichard Henderson /* 4291098859f1SRichard Henderson * The mov can be suppressed. Kill input first, so that it 4292098859f1SRichard Henderson * is unlinked from reg_to_temp, then set the output to the 4293098859f1SRichard Henderson * reg that we saved from the input. 4294098859f1SRichard Henderson */ 4295f8bf00f1SRichard Henderson temp_dead(s, ts); 4296098859f1SRichard Henderson oreg = ireg; 4297c29c1d7eSAurelien Jarno } else { 4298098859f1SRichard Henderson if (ots->val_type == TEMP_VAL_REG) { 4299098859f1SRichard Henderson oreg = ots->reg; 4300098859f1SRichard Henderson } else { 4301098859f1SRichard Henderson /* Make sure to not spill the input register during allocation. */ 4302098859f1SRichard Henderson oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 4303098859f1SRichard Henderson allocated_regs | ((TCGRegSet)1 << ireg), 4304098859f1SRichard Henderson preferred_regs, ots->indirect_base); 4305c29c1d7eSAurelien Jarno } 4306098859f1SRichard Henderson if (!tcg_out_mov(s, otype, oreg, ireg)) { 4307240c08d0SRichard Henderson /* 4308240c08d0SRichard Henderson * Cross register class move not supported. 4309240c08d0SRichard Henderson * Store the source register into the destination slot 4310240c08d0SRichard Henderson * and leave the destination temp as TEMP_VAL_MEM. 4311240c08d0SRichard Henderson */ 4312e01fa97dSRichard Henderson assert(!temp_readonly(ots)); 4313240c08d0SRichard Henderson if (!ts->mem_allocated) { 4314240c08d0SRichard Henderson temp_allocate_frame(s, ots); 4315240c08d0SRichard Henderson } 4316098859f1SRichard Henderson tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 4317098859f1SRichard Henderson set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 4318240c08d0SRichard Henderson ots->mem_coherent = 1; 4319240c08d0SRichard Henderson return; 432078113e83SRichard Henderson } 4321c29c1d7eSAurelien Jarno } 4322098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4323c896fe29Sbellard ots->mem_coherent = 0; 4324098859f1SRichard Henderson 4325ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 432698b4e186SRichard Henderson temp_sync(s, ots, allocated_regs, 0, 0); 4327c29c1d7eSAurelien Jarno } 4328ec7a869dSAurelien Jarno } 4329c896fe29Sbellard 4330bab1671fSRichard Henderson /* 4331bab1671fSRichard Henderson * Specialized code generation for INDEX_op_dup_vec. 4332bab1671fSRichard Henderson */ 4333bab1671fSRichard Henderson static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 4334bab1671fSRichard Henderson { 4335bab1671fSRichard Henderson const TCGLifeData arg_life = op->life; 4336bab1671fSRichard Henderson TCGRegSet dup_out_regs, dup_in_regs; 4337bab1671fSRichard Henderson TCGTemp *its, *ots; 4338bab1671fSRichard Henderson TCGType itype, vtype; 4339bab1671fSRichard Henderson unsigned vece; 434031c96417SRichard Henderson int lowpart_ofs; 4341bab1671fSRichard Henderson bool ok; 4342bab1671fSRichard Henderson 4343bab1671fSRichard Henderson ots = arg_temp(op->args[0]); 4344bab1671fSRichard Henderson its = arg_temp(op->args[1]); 4345bab1671fSRichard Henderson 4346bab1671fSRichard Henderson /* ENV should not be modified. */ 4347e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4348bab1671fSRichard Henderson 4349bab1671fSRichard Henderson itype = its->type; 4350bab1671fSRichard Henderson vece = TCGOP_VECE(op); 4351bab1671fSRichard Henderson vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4352bab1671fSRichard Henderson 4353bab1671fSRichard Henderson if (its->val_type == TEMP_VAL_CONST) { 4354bab1671fSRichard Henderson /* Propagate constant via movi -> dupi. */ 4355bab1671fSRichard Henderson tcg_target_ulong val = its->val; 4356bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 4357bab1671fSRichard Henderson temp_dead(s, its); 4358bab1671fSRichard Henderson } 435931fd884bSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); 4360bab1671fSRichard Henderson return; 4361bab1671fSRichard Henderson } 4362bab1671fSRichard Henderson 43639be0d080SRichard Henderson dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 43649be0d080SRichard Henderson dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 4365bab1671fSRichard Henderson 4366bab1671fSRichard Henderson /* Allocate the output register now. */ 4367bab1671fSRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 4368bab1671fSRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 4369098859f1SRichard Henderson TCGReg oreg; 4370bab1671fSRichard Henderson 4371bab1671fSRichard Henderson if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 4372bab1671fSRichard Henderson /* Make sure to not spill the input register. */ 4373bab1671fSRichard Henderson tcg_regset_set_reg(allocated_regs, its->reg); 4374bab1671fSRichard Henderson } 4375098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 437631fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base); 4377098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4378bab1671fSRichard Henderson } 4379bab1671fSRichard Henderson 4380bab1671fSRichard Henderson switch (its->val_type) { 4381bab1671fSRichard Henderson case TEMP_VAL_REG: 4382bab1671fSRichard Henderson /* 4383bab1671fSRichard Henderson * The dup constriaints must be broad, covering all possible VECE. 4384bab1671fSRichard Henderson * However, tcg_op_dup_vec() gets to see the VECE and we allow it 4385bab1671fSRichard Henderson * to fail, indicating that extra moves are required for that case. 4386bab1671fSRichard Henderson */ 4387bab1671fSRichard Henderson if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 4388bab1671fSRichard Henderson if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 4389bab1671fSRichard Henderson goto done; 4390bab1671fSRichard Henderson } 4391bab1671fSRichard Henderson /* Try again from memory or a vector input register. */ 4392bab1671fSRichard Henderson } 4393bab1671fSRichard Henderson if (!its->mem_coherent) { 4394bab1671fSRichard Henderson /* 4395bab1671fSRichard Henderson * The input register is not synced, and so an extra store 4396bab1671fSRichard Henderson * would be required to use memory. Attempt an integer-vector 4397bab1671fSRichard Henderson * register move first. We do not have a TCGRegSet for this. 4398bab1671fSRichard Henderson */ 4399bab1671fSRichard Henderson if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 4400bab1671fSRichard Henderson break; 4401bab1671fSRichard Henderson } 4402bab1671fSRichard Henderson /* Sync the temp back to its slot and load from there. */ 4403bab1671fSRichard Henderson temp_sync(s, its, s->reserved_regs, 0, 0); 4404bab1671fSRichard Henderson } 4405bab1671fSRichard Henderson /* fall through */ 4406bab1671fSRichard Henderson 4407bab1671fSRichard Henderson case TEMP_VAL_MEM: 440831c96417SRichard Henderson lowpart_ofs = 0; 440931c96417SRichard Henderson if (HOST_BIG_ENDIAN) { 441031c96417SRichard Henderson lowpart_ofs = tcg_type_size(itype) - (1 << vece); 441131c96417SRichard Henderson } 4412d6ecb4a9SRichard Henderson if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 441331c96417SRichard Henderson its->mem_offset + lowpart_ofs)) { 4414d6ecb4a9SRichard Henderson goto done; 4415d6ecb4a9SRichard Henderson } 4416098859f1SRichard Henderson /* Load the input into the destination vector register. */ 4417bab1671fSRichard Henderson tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 4418bab1671fSRichard Henderson break; 4419bab1671fSRichard Henderson 4420bab1671fSRichard Henderson default: 4421bab1671fSRichard Henderson g_assert_not_reached(); 4422bab1671fSRichard Henderson } 4423bab1671fSRichard Henderson 4424bab1671fSRichard Henderson /* We now have a vector input register, so dup must succeed. */ 4425bab1671fSRichard Henderson ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 4426bab1671fSRichard Henderson tcg_debug_assert(ok); 4427bab1671fSRichard Henderson 4428bab1671fSRichard Henderson done: 442936f5539cSRichard Henderson ots->mem_coherent = 0; 4430bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 4431bab1671fSRichard Henderson temp_dead(s, its); 4432bab1671fSRichard Henderson } 4433bab1671fSRichard Henderson if (NEED_SYNC_ARG(0)) { 4434bab1671fSRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, 0); 4435bab1671fSRichard Henderson } 4436bab1671fSRichard Henderson if (IS_DEAD_ARG(0)) { 4437bab1671fSRichard Henderson temp_dead(s, ots); 4438bab1671fSRichard Henderson } 4439bab1671fSRichard Henderson } 4440bab1671fSRichard Henderson 4441dd186292SRichard Henderson static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 4442c896fe29Sbellard { 4443dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 4444dd186292SRichard Henderson const TCGOpDef * const def = &tcg_op_defs[op->opc]; 444582790a87SRichard Henderson TCGRegSet i_allocated_regs; 444682790a87SRichard Henderson TCGRegSet o_allocated_regs; 4447b6638662SRichard Henderson int i, k, nb_iargs, nb_oargs; 4448b6638662SRichard Henderson TCGReg reg; 4449c896fe29Sbellard TCGArg arg; 4450c896fe29Sbellard const TCGArgConstraint *arg_ct; 4451c896fe29Sbellard TCGTemp *ts; 4452c896fe29Sbellard TCGArg new_args[TCG_MAX_OP_ARGS]; 4453c896fe29Sbellard int const_args[TCG_MAX_OP_ARGS]; 4454c896fe29Sbellard 4455c896fe29Sbellard nb_oargs = def->nb_oargs; 4456c896fe29Sbellard nb_iargs = def->nb_iargs; 4457c896fe29Sbellard 4458c896fe29Sbellard /* copy constants */ 4459c896fe29Sbellard memcpy(new_args + nb_oargs + nb_iargs, 4460dd186292SRichard Henderson op->args + nb_oargs + nb_iargs, 4461c896fe29Sbellard sizeof(TCGArg) * def->nb_cargs); 4462c896fe29Sbellard 4463d21369f5SRichard Henderson i_allocated_regs = s->reserved_regs; 4464d21369f5SRichard Henderson o_allocated_regs = s->reserved_regs; 446582790a87SRichard Henderson 4466c896fe29Sbellard /* satisfy input constraints */ 4467c896fe29Sbellard for (k = 0; k < nb_iargs; k++) { 446829f5e925SRichard Henderson TCGRegSet i_preferred_regs, i_required_regs; 446929f5e925SRichard Henderson bool allocate_new_reg, copyto_new_reg; 447029f5e925SRichard Henderson TCGTemp *ts2; 447129f5e925SRichard Henderson int i1, i2; 4472d62816f2SRichard Henderson 447366792f90SRichard Henderson i = def->args_ct[nb_oargs + k].sort_index; 4474dd186292SRichard Henderson arg = op->args[i]; 4475c896fe29Sbellard arg_ct = &def->args_ct[i]; 447643439139SRichard Henderson ts = arg_temp(arg); 447740ae5c62SRichard Henderson 447840ae5c62SRichard Henderson if (ts->val_type == TEMP_VAL_CONST 4479a4fbbd77SRichard Henderson && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 4480c896fe29Sbellard /* constant is OK for instruction */ 4481c896fe29Sbellard const_args[i] = 1; 4482c896fe29Sbellard new_args[i] = ts->val; 4483d62816f2SRichard Henderson continue; 4484c896fe29Sbellard } 448540ae5c62SRichard Henderson 44861c1824dcSRichard Henderson reg = ts->reg; 44871c1824dcSRichard Henderson i_preferred_regs = 0; 448829f5e925SRichard Henderson i_required_regs = arg_ct->regs; 44891c1824dcSRichard Henderson allocate_new_reg = false; 449029f5e925SRichard Henderson copyto_new_reg = false; 44911c1824dcSRichard Henderson 449229f5e925SRichard Henderson switch (arg_ct->pair) { 449329f5e925SRichard Henderson case 0: /* not paired */ 4494bc2b17e6SRichard Henderson if (arg_ct->ialias) { 449531fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 4496c0522136SRichard Henderson 4497c0522136SRichard Henderson /* 4498c0522136SRichard Henderson * If the input is readonly, then it cannot also be an 4499c0522136SRichard Henderson * output and aliased to itself. If the input is not 4500c0522136SRichard Henderson * dead after the instruction, we must allocate a new 4501c0522136SRichard Henderson * register and move it. 4502c0522136SRichard Henderson */ 4503c0522136SRichard Henderson if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 45041c1824dcSRichard Henderson allocate_new_reg = true; 45051c1824dcSRichard Henderson } else if (ts->val_type == TEMP_VAL_REG) { 4506c0522136SRichard Henderson /* 45071c1824dcSRichard Henderson * Check if the current register has already been 45081c1824dcSRichard Henderson * allocated for another input. 4509c0522136SRichard Henderson */ 451029f5e925SRichard Henderson allocate_new_reg = 451129f5e925SRichard Henderson tcg_regset_test_reg(i_allocated_regs, reg); 45127e1df267SAurelien Jarno } 45137e1df267SAurelien Jarno } 45141c1824dcSRichard Henderson if (!allocate_new_reg) { 451529f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs, 451629f5e925SRichard Henderson i_preferred_regs); 4517c896fe29Sbellard reg = ts->reg; 451829f5e925SRichard Henderson allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 45191c1824dcSRichard Henderson } 45201c1824dcSRichard Henderson if (allocate_new_reg) { 4521c0522136SRichard Henderson /* 4522c0522136SRichard Henderson * Allocate a new register matching the constraint 4523c0522136SRichard Henderson * and move the temporary register into it. 4524c0522136SRichard Henderson */ 4525d62816f2SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 4526d62816f2SRichard Henderson i_allocated_regs, 0); 452729f5e925SRichard Henderson reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 45281c1824dcSRichard Henderson i_preferred_regs, ts->indirect_base); 452929f5e925SRichard Henderson copyto_new_reg = true; 453029f5e925SRichard Henderson } 453129f5e925SRichard Henderson break; 453229f5e925SRichard Henderson 453329f5e925SRichard Henderson case 1: 453429f5e925SRichard Henderson /* First of an input pair; if i1 == i2, the second is an output. */ 453529f5e925SRichard Henderson i1 = i; 453629f5e925SRichard Henderson i2 = arg_ct->pair_index; 453729f5e925SRichard Henderson ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 453829f5e925SRichard Henderson 453929f5e925SRichard Henderson /* 454029f5e925SRichard Henderson * It is easier to default to allocating a new pair 454129f5e925SRichard Henderson * and to identify a few cases where it's not required. 454229f5e925SRichard Henderson */ 454329f5e925SRichard Henderson if (arg_ct->ialias) { 454431fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 454529f5e925SRichard Henderson if (IS_DEAD_ARG(i1) && 454629f5e925SRichard Henderson IS_DEAD_ARG(i2) && 454729f5e925SRichard Henderson !temp_readonly(ts) && 454829f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG && 454929f5e925SRichard Henderson ts->reg < TCG_TARGET_NB_REGS - 1 && 455029f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) && 455129f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) && 455229f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 455329f5e925SRichard Henderson (ts2 455429f5e925SRichard Henderson ? ts2->val_type == TEMP_VAL_REG && 455529f5e925SRichard Henderson ts2->reg == reg + 1 && 455629f5e925SRichard Henderson !temp_readonly(ts2) 455729f5e925SRichard Henderson : s->reg_to_temp[reg + 1] == NULL)) { 455829f5e925SRichard Henderson break; 455929f5e925SRichard Henderson } 456029f5e925SRichard Henderson } else { 456129f5e925SRichard Henderson /* Without aliasing, the pair must also be an input. */ 456229f5e925SRichard Henderson tcg_debug_assert(ts2); 456329f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG && 456429f5e925SRichard Henderson ts2->val_type == TEMP_VAL_REG && 456529f5e925SRichard Henderson ts2->reg == reg + 1 && 456629f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg)) { 456729f5e925SRichard Henderson break; 456829f5e925SRichard Henderson } 456929f5e925SRichard Henderson } 457029f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 457129f5e925SRichard Henderson 0, ts->indirect_base); 457229f5e925SRichard Henderson goto do_pair; 457329f5e925SRichard Henderson 457429f5e925SRichard Henderson case 2: /* pair second */ 457529f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1; 457629f5e925SRichard Henderson goto do_pair; 457729f5e925SRichard Henderson 457829f5e925SRichard Henderson case 3: /* ialias with second output, no first input */ 457929f5e925SRichard Henderson tcg_debug_assert(arg_ct->ialias); 458031fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 458129f5e925SRichard Henderson 458229f5e925SRichard Henderson if (IS_DEAD_ARG(i) && 458329f5e925SRichard Henderson !temp_readonly(ts) && 458429f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG && 458529f5e925SRichard Henderson reg > 0 && 458629f5e925SRichard Henderson s->reg_to_temp[reg - 1] == NULL && 458729f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) && 458829f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) && 458929f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 459029f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg - 1); 459129f5e925SRichard Henderson break; 459229f5e925SRichard Henderson } 459329f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 459429f5e925SRichard Henderson i_allocated_regs, 0, 459529f5e925SRichard Henderson ts->indirect_base); 459629f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg); 459729f5e925SRichard Henderson reg += 1; 459829f5e925SRichard Henderson goto do_pair; 459929f5e925SRichard Henderson 460029f5e925SRichard Henderson do_pair: 460129f5e925SRichard Henderson /* 460229f5e925SRichard Henderson * If an aliased input is not dead after the instruction, 460329f5e925SRichard Henderson * we must allocate a new register and move it. 460429f5e925SRichard Henderson */ 460529f5e925SRichard Henderson if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 460629f5e925SRichard Henderson TCGRegSet t_allocated_regs = i_allocated_regs; 460729f5e925SRichard Henderson 460829f5e925SRichard Henderson /* 460929f5e925SRichard Henderson * Because of the alias, and the continued life, make sure 461029f5e925SRichard Henderson * that the temp is somewhere *other* than the reg pair, 461129f5e925SRichard Henderson * and we get a copy in reg. 461229f5e925SRichard Henderson */ 461329f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg); 461429f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg + 1); 461529f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 461629f5e925SRichard Henderson /* If ts was already in reg, copy it somewhere else. */ 461729f5e925SRichard Henderson TCGReg nr; 461829f5e925SRichard Henderson bool ok; 461929f5e925SRichard Henderson 462029f5e925SRichard Henderson tcg_debug_assert(ts->kind != TEMP_FIXED); 462129f5e925SRichard Henderson nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 462229f5e925SRichard Henderson t_allocated_regs, 0, ts->indirect_base); 462329f5e925SRichard Henderson ok = tcg_out_mov(s, ts->type, nr, reg); 462429f5e925SRichard Henderson tcg_debug_assert(ok); 462529f5e925SRichard Henderson 462629f5e925SRichard Henderson set_temp_val_reg(s, ts, nr); 462729f5e925SRichard Henderson } else { 462829f5e925SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 462929f5e925SRichard Henderson t_allocated_regs, 0); 463029f5e925SRichard Henderson copyto_new_reg = true; 463129f5e925SRichard Henderson } 463229f5e925SRichard Henderson } else { 463329f5e925SRichard Henderson /* Preferably allocate to reg, otherwise copy. */ 463429f5e925SRichard Henderson i_required_regs = (TCGRegSet)1 << reg; 463529f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs, 463629f5e925SRichard Henderson i_preferred_regs); 463729f5e925SRichard Henderson copyto_new_reg = ts->reg != reg; 463829f5e925SRichard Henderson } 463929f5e925SRichard Henderson break; 464029f5e925SRichard Henderson 464129f5e925SRichard Henderson default: 464229f5e925SRichard Henderson g_assert_not_reached(); 464329f5e925SRichard Henderson } 464429f5e925SRichard Henderson 464529f5e925SRichard Henderson if (copyto_new_reg) { 464678113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4647240c08d0SRichard Henderson /* 4648240c08d0SRichard Henderson * Cross register class move not supported. Sync the 4649240c08d0SRichard Henderson * temp back to its slot and load from there. 4650240c08d0SRichard Henderson */ 4651240c08d0SRichard Henderson temp_sync(s, ts, i_allocated_regs, 0, 0); 4652240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 4653240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 465478113e83SRichard Henderson } 4655c896fe29Sbellard } 4656c896fe29Sbellard new_args[i] = reg; 4657c896fe29Sbellard const_args[i] = 0; 465882790a87SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg); 4659c896fe29Sbellard } 4660c896fe29Sbellard 4661c896fe29Sbellard /* mark dead temporaries and free the associated registers */ 4662866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4663866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 466443439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 4665c896fe29Sbellard } 4666c896fe29Sbellard } 4667c896fe29Sbellard 4668b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 4669b4cb76e6SRichard Henderson tcg_reg_alloc_cbranch(s, i_allocated_regs); 4670b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 467182790a87SRichard Henderson tcg_reg_alloc_bb_end(s, i_allocated_regs); 4672a52ad07eSAurelien Jarno } else { 4673c896fe29Sbellard if (def->flags & TCG_OPF_CALL_CLOBBER) { 4674b03cce8eSbellard /* XXX: permit generic clobber register list ? */ 4675c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4676c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 467782790a87SRichard Henderson tcg_reg_free(s, i, i_allocated_regs); 4678c896fe29Sbellard } 4679c896fe29Sbellard } 46803d5c5f87SAurelien Jarno } 46813d5c5f87SAurelien Jarno if (def->flags & TCG_OPF_SIDE_EFFECTS) { 46823d5c5f87SAurelien Jarno /* sync globals if the op has side effects and might trigger 46833d5c5f87SAurelien Jarno an exception. */ 468482790a87SRichard Henderson sync_globals(s, i_allocated_regs); 4685c896fe29Sbellard } 4686c896fe29Sbellard 4687c896fe29Sbellard /* satisfy the output constraints */ 4688c896fe29Sbellard for(k = 0; k < nb_oargs; k++) { 468966792f90SRichard Henderson i = def->args_ct[k].sort_index; 4690dd186292SRichard Henderson arg = op->args[i]; 4691c896fe29Sbellard arg_ct = &def->args_ct[i]; 469243439139SRichard Henderson ts = arg_temp(arg); 4693d63e3b6eSRichard Henderson 4694d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4695e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4696d63e3b6eSRichard Henderson 469729f5e925SRichard Henderson switch (arg_ct->pair) { 469829f5e925SRichard Henderson case 0: /* not paired */ 4699bc2b17e6SRichard Henderson if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 47005ff9d6a4Sbellard reg = new_args[arg_ct->alias_index]; 4701bc2b17e6SRichard Henderson } else if (arg_ct->newreg) { 47029be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, 470382790a87SRichard Henderson i_allocated_regs | o_allocated_regs, 470431fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 4705c896fe29Sbellard } else { 47069be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 470731fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 4708c896fe29Sbellard } 470929f5e925SRichard Henderson break; 471029f5e925SRichard Henderson 471129f5e925SRichard Henderson case 1: /* first of pair */ 471229f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 471329f5e925SRichard Henderson if (arg_ct->oalias) { 471429f5e925SRichard Henderson reg = new_args[arg_ct->alias_index]; 471529f5e925SRichard Henderson break; 471629f5e925SRichard Henderson } 471729f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 471831fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 471929f5e925SRichard Henderson break; 472029f5e925SRichard Henderson 472129f5e925SRichard Henderson case 2: /* second of pair */ 472229f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 472329f5e925SRichard Henderson if (arg_ct->oalias) { 472429f5e925SRichard Henderson reg = new_args[arg_ct->alias_index]; 472529f5e925SRichard Henderson } else { 472629f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1; 472729f5e925SRichard Henderson } 472829f5e925SRichard Henderson break; 472929f5e925SRichard Henderson 473029f5e925SRichard Henderson case 3: /* first of pair, aliasing with a second input */ 473129f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 473229f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] - 1; 473329f5e925SRichard Henderson break; 473429f5e925SRichard Henderson 473529f5e925SRichard Henderson default: 473629f5e925SRichard Henderson g_assert_not_reached(); 473729f5e925SRichard Henderson } 473882790a87SRichard Henderson tcg_regset_set_reg(o_allocated_regs, reg); 4739098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 4740c896fe29Sbellard ts->mem_coherent = 0; 4741c896fe29Sbellard new_args[i] = reg; 4742c896fe29Sbellard } 4743e8996ee0Sbellard } 4744c896fe29Sbellard 4745c896fe29Sbellard /* emit instruction */ 4746678155b2SRichard Henderson switch (op->opc) { 4747678155b2SRichard Henderson case INDEX_op_ext8s_i32: 4748678155b2SRichard Henderson tcg_out_ext8s(s, TCG_TYPE_I32, new_args[0], new_args[1]); 4749678155b2SRichard Henderson break; 4750678155b2SRichard Henderson case INDEX_op_ext8s_i64: 4751678155b2SRichard Henderson tcg_out_ext8s(s, TCG_TYPE_I64, new_args[0], new_args[1]); 4752678155b2SRichard Henderson break; 4753d0e66c89SRichard Henderson case INDEX_op_ext8u_i32: 4754d0e66c89SRichard Henderson case INDEX_op_ext8u_i64: 4755d0e66c89SRichard Henderson tcg_out_ext8u(s, new_args[0], new_args[1]); 4756d0e66c89SRichard Henderson break; 4757753e42eaSRichard Henderson case INDEX_op_ext16s_i32: 4758753e42eaSRichard Henderson tcg_out_ext16s(s, TCG_TYPE_I32, new_args[0], new_args[1]); 4759753e42eaSRichard Henderson break; 4760753e42eaSRichard Henderson case INDEX_op_ext16s_i64: 4761753e42eaSRichard Henderson tcg_out_ext16s(s, TCG_TYPE_I64, new_args[0], new_args[1]); 4762753e42eaSRichard Henderson break; 4763379afdffSRichard Henderson case INDEX_op_ext16u_i32: 4764379afdffSRichard Henderson case INDEX_op_ext16u_i64: 4765379afdffSRichard Henderson tcg_out_ext16u(s, new_args[0], new_args[1]); 4766379afdffSRichard Henderson break; 476752bf3398SRichard Henderson case INDEX_op_ext32s_i64: 476852bf3398SRichard Henderson tcg_out_ext32s(s, new_args[0], new_args[1]); 476952bf3398SRichard Henderson break; 47709ecf5f61SRichard Henderson case INDEX_op_ext32u_i64: 47719ecf5f61SRichard Henderson tcg_out_ext32u(s, new_args[0], new_args[1]); 47729ecf5f61SRichard Henderson break; 47739c6aa274SRichard Henderson case INDEX_op_ext_i32_i64: 47749c6aa274SRichard Henderson tcg_out_exts_i32_i64(s, new_args[0], new_args[1]); 47759c6aa274SRichard Henderson break; 4776b9bfe000SRichard Henderson case INDEX_op_extu_i32_i64: 4777b9bfe000SRichard Henderson tcg_out_extu_i32_i64(s, new_args[0], new_args[1]); 4778b9bfe000SRichard Henderson break; 4779b8b94ac6SRichard Henderson case INDEX_op_extrl_i64_i32: 4780b8b94ac6SRichard Henderson tcg_out_extrl_i64_i32(s, new_args[0], new_args[1]); 4781b8b94ac6SRichard Henderson break; 4782678155b2SRichard Henderson default: 4783d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 4784d2fd745fSRichard Henderson tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 4785d2fd745fSRichard Henderson new_args, const_args); 4786d2fd745fSRichard Henderson } else { 4787dd186292SRichard Henderson tcg_out_op(s, op->opc, new_args, const_args); 4788d2fd745fSRichard Henderson } 4789678155b2SRichard Henderson break; 4790678155b2SRichard Henderson } 4791c896fe29Sbellard 4792c896fe29Sbellard /* move the outputs in the correct register if needed */ 4793c896fe29Sbellard for(i = 0; i < nb_oargs; i++) { 479443439139SRichard Henderson ts = arg_temp(op->args[i]); 4795d63e3b6eSRichard Henderson 4796d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4797e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4798d63e3b6eSRichard Henderson 4799ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 480098b4e186SRichard Henderson temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 480159d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 4802f8bf00f1SRichard Henderson temp_dead(s, ts); 4803ec7a869dSAurelien Jarno } 4804c896fe29Sbellard } 4805c896fe29Sbellard } 4806c896fe29Sbellard 4807efe86b21SRichard Henderson static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 4808efe86b21SRichard Henderson { 4809efe86b21SRichard Henderson const TCGLifeData arg_life = op->life; 4810efe86b21SRichard Henderson TCGTemp *ots, *itsl, *itsh; 4811efe86b21SRichard Henderson TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4812efe86b21SRichard Henderson 4813efe86b21SRichard Henderson /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 4814efe86b21SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 4815efe86b21SRichard Henderson tcg_debug_assert(TCGOP_VECE(op) == MO_64); 4816efe86b21SRichard Henderson 4817efe86b21SRichard Henderson ots = arg_temp(op->args[0]); 4818efe86b21SRichard Henderson itsl = arg_temp(op->args[1]); 4819efe86b21SRichard Henderson itsh = arg_temp(op->args[2]); 4820efe86b21SRichard Henderson 4821efe86b21SRichard Henderson /* ENV should not be modified. */ 4822efe86b21SRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4823efe86b21SRichard Henderson 4824efe86b21SRichard Henderson /* Allocate the output register now. */ 4825efe86b21SRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 4826efe86b21SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 4827efe86b21SRichard Henderson TCGRegSet dup_out_regs = 4828efe86b21SRichard Henderson tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 4829098859f1SRichard Henderson TCGReg oreg; 4830efe86b21SRichard Henderson 4831efe86b21SRichard Henderson /* Make sure to not spill the input registers. */ 4832efe86b21SRichard Henderson if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 4833efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsl->reg); 4834efe86b21SRichard Henderson } 4835efe86b21SRichard Henderson if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 4836efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsh->reg); 4837efe86b21SRichard Henderson } 4838efe86b21SRichard Henderson 4839098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 484031fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base); 4841098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4842efe86b21SRichard Henderson } 4843efe86b21SRichard Henderson 4844efe86b21SRichard Henderson /* Promote dup2 of immediates to dupi_vec. */ 4845efe86b21SRichard Henderson if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 4846efe86b21SRichard Henderson uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 4847efe86b21SRichard Henderson MemOp vece = MO_64; 4848efe86b21SRichard Henderson 4849efe86b21SRichard Henderson if (val == dup_const(MO_8, val)) { 4850efe86b21SRichard Henderson vece = MO_8; 4851efe86b21SRichard Henderson } else if (val == dup_const(MO_16, val)) { 4852efe86b21SRichard Henderson vece = MO_16; 4853efe86b21SRichard Henderson } else if (val == dup_const(MO_32, val)) { 4854efe86b21SRichard Henderson vece = MO_32; 4855efe86b21SRichard Henderson } 4856efe86b21SRichard Henderson 4857efe86b21SRichard Henderson tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 4858efe86b21SRichard Henderson goto done; 4859efe86b21SRichard Henderson } 4860efe86b21SRichard Henderson 4861efe86b21SRichard Henderson /* If the two inputs form one 64-bit value, try dupm_vec. */ 4862aef85402SRichard Henderson if (itsl->temp_subindex == HOST_BIG_ENDIAN && 4863aef85402SRichard Henderson itsh->temp_subindex == !HOST_BIG_ENDIAN && 4864aef85402SRichard Henderson itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { 4865aef85402SRichard Henderson TCGTemp *its = itsl - HOST_BIG_ENDIAN; 4866aef85402SRichard Henderson 4867aef85402SRichard Henderson temp_sync(s, its + 0, s->reserved_regs, 0, 0); 4868aef85402SRichard Henderson temp_sync(s, its + 1, s->reserved_regs, 0, 0); 4869aef85402SRichard Henderson 4870efe86b21SRichard Henderson if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 4871efe86b21SRichard Henderson its->mem_base->reg, its->mem_offset)) { 4872efe86b21SRichard Henderson goto done; 4873efe86b21SRichard Henderson } 4874efe86b21SRichard Henderson } 4875efe86b21SRichard Henderson 4876efe86b21SRichard Henderson /* Fall back to generic expansion. */ 4877efe86b21SRichard Henderson return false; 4878efe86b21SRichard Henderson 4879efe86b21SRichard Henderson done: 488036f5539cSRichard Henderson ots->mem_coherent = 0; 4881efe86b21SRichard Henderson if (IS_DEAD_ARG(1)) { 4882efe86b21SRichard Henderson temp_dead(s, itsl); 4883efe86b21SRichard Henderson } 4884efe86b21SRichard Henderson if (IS_DEAD_ARG(2)) { 4885efe86b21SRichard Henderson temp_dead(s, itsh); 4886efe86b21SRichard Henderson } 4887efe86b21SRichard Henderson if (NEED_SYNC_ARG(0)) { 4888efe86b21SRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 4889efe86b21SRichard Henderson } else if (IS_DEAD_ARG(0)) { 4890efe86b21SRichard Henderson temp_dead(s, ots); 4891efe86b21SRichard Henderson } 4892efe86b21SRichard Henderson return true; 4893efe86b21SRichard Henderson } 4894efe86b21SRichard Henderson 489539004a71SRichard Henderson static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, 489639004a71SRichard Henderson TCGRegSet allocated_regs) 4897c896fe29Sbellard { 4898c896fe29Sbellard if (ts->val_type == TEMP_VAL_REG) { 4899c896fe29Sbellard if (ts->reg != reg) { 49004250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 490178113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4902240c08d0SRichard Henderson /* 4903240c08d0SRichard Henderson * Cross register class move not supported. Sync the 4904240c08d0SRichard Henderson * temp back to its slot and load from there. 4905240c08d0SRichard Henderson */ 4906240c08d0SRichard Henderson temp_sync(s, ts, allocated_regs, 0, 0); 4907240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 4908240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 490978113e83SRichard Henderson } 4910c896fe29Sbellard } 4911c896fe29Sbellard } else { 4912ccb1bb66SRichard Henderson TCGRegSet arg_set = 0; 491340ae5c62SRichard Henderson 49144250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 491540ae5c62SRichard Henderson tcg_regset_set_reg(arg_set, reg); 4916b722452aSRichard Henderson temp_load(s, ts, arg_set, allocated_regs, 0); 4917c896fe29Sbellard } 491839004a71SRichard Henderson } 491940ae5c62SRichard Henderson 4920d78e4a4fSRichard Henderson static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts, 492139004a71SRichard Henderson TCGRegSet allocated_regs) 492239004a71SRichard Henderson { 492339004a71SRichard Henderson /* 492439004a71SRichard Henderson * When the destination is on the stack, load up the temp and store. 492539004a71SRichard Henderson * If there are many call-saved registers, the temp might live to 492639004a71SRichard Henderson * see another use; otherwise it'll be discarded. 492739004a71SRichard Henderson */ 492839004a71SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); 492939004a71SRichard Henderson tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, 4930d78e4a4fSRichard Henderson arg_slot_stk_ofs(arg_slot)); 493139004a71SRichard Henderson } 493239004a71SRichard Henderson 493339004a71SRichard Henderson static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, 493439004a71SRichard Henderson TCGTemp *ts, TCGRegSet *allocated_regs) 493539004a71SRichard Henderson { 4936338b61e9SRichard Henderson if (arg_slot_reg_p(l->arg_slot)) { 493739004a71SRichard Henderson TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; 493839004a71SRichard Henderson load_arg_reg(s, reg, ts, *allocated_regs); 493939004a71SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg); 494039004a71SRichard Henderson } else { 4941d78e4a4fSRichard Henderson load_arg_stk(s, l->arg_slot, ts, *allocated_regs); 4942c896fe29Sbellard } 494339cf05d3Sbellard } 4944c896fe29Sbellard 4945d78e4a4fSRichard Henderson static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base, 4946313bdea8SRichard Henderson intptr_t ref_off, TCGRegSet *allocated_regs) 4947313bdea8SRichard Henderson { 4948313bdea8SRichard Henderson TCGReg reg; 4949313bdea8SRichard Henderson 4950d78e4a4fSRichard Henderson if (arg_slot_reg_p(arg_slot)) { 4951313bdea8SRichard Henderson reg = tcg_target_call_iarg_regs[arg_slot]; 4952313bdea8SRichard Henderson tcg_reg_free(s, reg, *allocated_regs); 4953313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4954313bdea8SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg); 4955313bdea8SRichard Henderson } else { 4956313bdea8SRichard Henderson reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR], 4957313bdea8SRichard Henderson *allocated_regs, 0, false); 4958313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4959313bdea8SRichard Henderson tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, 4960d78e4a4fSRichard Henderson arg_slot_stk_ofs(arg_slot)); 4961313bdea8SRichard Henderson } 4962313bdea8SRichard Henderson } 4963313bdea8SRichard Henderson 496439004a71SRichard Henderson static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 496539004a71SRichard Henderson { 496639004a71SRichard Henderson const int nb_oargs = TCGOP_CALLO(op); 496739004a71SRichard Henderson const int nb_iargs = TCGOP_CALLI(op); 496839004a71SRichard Henderson const TCGLifeData arg_life = op->life; 496939004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 497039004a71SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 497139004a71SRichard Henderson int i; 497239004a71SRichard Henderson 497339004a71SRichard Henderson /* 497439004a71SRichard Henderson * Move inputs into place in reverse order, 497539004a71SRichard Henderson * so that we place stacked arguments first. 497639004a71SRichard Henderson */ 497739004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; --i) { 497839004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 497939004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); 498039004a71SRichard Henderson 498139004a71SRichard Henderson switch (loc->kind) { 498239004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 498339004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 498439004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 498539004a71SRichard Henderson load_arg_normal(s, loc, ts, &allocated_regs); 498639004a71SRichard Henderson break; 4987313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 4988313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4989313bdea8SRichard Henderson load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, 4990d78e4a4fSRichard Henderson arg_slot_stk_ofs(loc->ref_slot), 4991313bdea8SRichard Henderson &allocated_regs); 4992313bdea8SRichard Henderson break; 4993313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 4994313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4995313bdea8SRichard Henderson break; 499639004a71SRichard Henderson default: 499739004a71SRichard Henderson g_assert_not_reached(); 499839004a71SRichard Henderson } 499939004a71SRichard Henderson } 500039004a71SRichard Henderson 500139004a71SRichard Henderson /* Mark dead temporaries and free the associated registers. */ 5002866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 5003866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 500443439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 5005c896fe29Sbellard } 5006c896fe29Sbellard } 5007c896fe29Sbellard 500839004a71SRichard Henderson /* Clobber call registers. */ 5009c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 5010c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 5011b3915dbbSRichard Henderson tcg_reg_free(s, i, allocated_regs); 5012c896fe29Sbellard } 5013c896fe29Sbellard } 5014c896fe29Sbellard 501539004a71SRichard Henderson /* 501639004a71SRichard Henderson * Save globals if they might be written by the helper, 501739004a71SRichard Henderson * sync them if they might be read. 501839004a71SRichard Henderson */ 501939004a71SRichard Henderson if (info->flags & TCG_CALL_NO_READ_GLOBALS) { 502078505279SAurelien Jarno /* Nothing to do */ 502139004a71SRichard Henderson } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { 502278505279SAurelien Jarno sync_globals(s, allocated_regs); 502378505279SAurelien Jarno } else { 5024e8996ee0Sbellard save_globals(s, allocated_regs); 5025b9c18f56Saurel32 } 5026c896fe29Sbellard 5027313bdea8SRichard Henderson /* 5028313bdea8SRichard Henderson * If the ABI passes a pointer to the returned struct as the first 5029313bdea8SRichard Henderson * argument, load that now. Pass a pointer to the output home slot. 5030313bdea8SRichard Henderson */ 5031313bdea8SRichard Henderson if (info->out_kind == TCG_CALL_RET_BY_REF) { 5032313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]); 5033313bdea8SRichard Henderson 5034313bdea8SRichard Henderson if (!ts->mem_allocated) { 5035313bdea8SRichard Henderson temp_allocate_frame(s, ts); 5036313bdea8SRichard Henderson } 5037313bdea8SRichard Henderson load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); 5038313bdea8SRichard Henderson } 5039313bdea8SRichard Henderson 5040cee44b03SRichard Henderson tcg_out_call(s, tcg_call_func(op), info); 5041c896fe29Sbellard 504239004a71SRichard Henderson /* Assign output registers and emit moves if needed. */ 504339004a71SRichard Henderson switch (info->out_kind) { 504439004a71SRichard Henderson case TCG_CALL_RET_NORMAL: 5045c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 504639004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 50475e3d0c19SRichard Henderson TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i); 5048d63e3b6eSRichard Henderson 5049d63e3b6eSRichard Henderson /* ENV should not be modified. */ 5050e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 5051d63e3b6eSRichard Henderson 5052098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 5053c896fe29Sbellard ts->mem_coherent = 0; 505439004a71SRichard Henderson } 505539004a71SRichard Henderson break; 5056313bdea8SRichard Henderson 5057c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC: 5058c6556aa0SRichard Henderson { 5059c6556aa0SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]); 5060c6556aa0SRichard Henderson 5061c6556aa0SRichard Henderson tcg_debug_assert(ts->base_type == TCG_TYPE_I128); 5062c6556aa0SRichard Henderson tcg_debug_assert(ts->temp_subindex == 0); 5063c6556aa0SRichard Henderson if (!ts->mem_allocated) { 5064c6556aa0SRichard Henderson temp_allocate_frame(s, ts); 5065c6556aa0SRichard Henderson } 5066c6556aa0SRichard Henderson tcg_out_st(s, TCG_TYPE_V128, 5067c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 5068c6556aa0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 5069c6556aa0SRichard Henderson } 5070c6556aa0SRichard Henderson /* fall through to mark all parts in memory */ 5071c6556aa0SRichard Henderson 5072313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF: 5073313bdea8SRichard Henderson /* The callee has performed a write through the reference. */ 5074313bdea8SRichard Henderson for (i = 0; i < nb_oargs; i++) { 5075313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 5076313bdea8SRichard Henderson ts->val_type = TEMP_VAL_MEM; 5077313bdea8SRichard Henderson } 5078313bdea8SRichard Henderson break; 5079313bdea8SRichard Henderson 508039004a71SRichard Henderson default: 508139004a71SRichard Henderson g_assert_not_reached(); 508239004a71SRichard Henderson } 508339004a71SRichard Henderson 508439004a71SRichard Henderson /* Flush or discard output registers as needed. */ 508539004a71SRichard Henderson for (i = 0; i < nb_oargs; i++) { 508639004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 5087ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 508839004a71SRichard Henderson temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); 508959d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 5090f8bf00f1SRichard Henderson temp_dead(s, ts); 5091c896fe29Sbellard } 5092c896fe29Sbellard } 50938c11ad25SAurelien Jarno } 5094c896fe29Sbellard 5095*8429a1caSRichard Henderson /* 5096*8429a1caSRichard Henderson * Similarly for qemu_ld/st slow path helpers. 5097*8429a1caSRichard Henderson * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously, 5098*8429a1caSRichard Henderson * using only the provided backend tcg_out_* functions. 5099*8429a1caSRichard Henderson */ 5100*8429a1caSRichard Henderson 5101*8429a1caSRichard Henderson static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot) 5102*8429a1caSRichard Henderson { 5103*8429a1caSRichard Henderson int ofs = arg_slot_stk_ofs(slot); 5104*8429a1caSRichard Henderson 5105*8429a1caSRichard Henderson /* 5106*8429a1caSRichard Henderson * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not 5107*8429a1caSRichard Henderson * require extension to uint64_t, adjust the address for uint32_t. 5108*8429a1caSRichard Henderson */ 5109*8429a1caSRichard Henderson if (HOST_BIG_ENDIAN && 5110*8429a1caSRichard Henderson TCG_TARGET_REG_BITS == 64 && 5111*8429a1caSRichard Henderson type == TCG_TYPE_I32) { 5112*8429a1caSRichard Henderson ofs += 4; 5113*8429a1caSRichard Henderson } 5114*8429a1caSRichard Henderson return ofs; 5115*8429a1caSRichard Henderson } 5116*8429a1caSRichard Henderson 5117*8429a1caSRichard Henderson static void tcg_out_helper_load_regs(TCGContext *s, 5118*8429a1caSRichard Henderson unsigned nmov, TCGMovExtend *mov, 5119*8429a1caSRichard Henderson unsigned ntmp, const int *tmp) 5120*8429a1caSRichard Henderson { 5121*8429a1caSRichard Henderson switch (nmov) { 5122*8429a1caSRichard Henderson default: 5123*8429a1caSRichard Henderson /* The backend must have provided enough temps for the worst case. */ 5124*8429a1caSRichard Henderson tcg_debug_assert(ntmp + 1 >= nmov); 5125*8429a1caSRichard Henderson 5126*8429a1caSRichard Henderson for (unsigned i = nmov - 1; i >= 2; --i) { 5127*8429a1caSRichard Henderson TCGReg dst = mov[i].dst; 5128*8429a1caSRichard Henderson 5129*8429a1caSRichard Henderson for (unsigned j = 0; j < i; ++j) { 5130*8429a1caSRichard Henderson if (dst == mov[j].src) { 5131*8429a1caSRichard Henderson /* 5132*8429a1caSRichard Henderson * Conflict. 5133*8429a1caSRichard Henderson * Copy the source to a temporary, recurse for the 5134*8429a1caSRichard Henderson * remaining moves, perform the extension from our 5135*8429a1caSRichard Henderson * scratch on the way out. 5136*8429a1caSRichard Henderson */ 5137*8429a1caSRichard Henderson TCGReg scratch = tmp[--ntmp]; 5138*8429a1caSRichard Henderson tcg_out_mov(s, mov[i].src_type, scratch, mov[i].src); 5139*8429a1caSRichard Henderson mov[i].src = scratch; 5140*8429a1caSRichard Henderson 5141*8429a1caSRichard Henderson tcg_out_helper_load_regs(s, i, mov, ntmp, tmp); 5142*8429a1caSRichard Henderson tcg_out_movext1(s, &mov[i]); 5143*8429a1caSRichard Henderson return; 5144*8429a1caSRichard Henderson } 5145*8429a1caSRichard Henderson } 5146*8429a1caSRichard Henderson 5147*8429a1caSRichard Henderson /* No conflicts: perform this move and continue. */ 5148*8429a1caSRichard Henderson tcg_out_movext1(s, &mov[i]); 5149*8429a1caSRichard Henderson } 5150*8429a1caSRichard Henderson /* fall through for the final two moves */ 5151*8429a1caSRichard Henderson 5152*8429a1caSRichard Henderson case 2: 5153*8429a1caSRichard Henderson tcg_out_movext2(s, mov, mov + 1, ntmp ? tmp[0] : -1); 5154*8429a1caSRichard Henderson return; 5155*8429a1caSRichard Henderson case 1: 5156*8429a1caSRichard Henderson tcg_out_movext1(s, mov); 5157*8429a1caSRichard Henderson return; 5158*8429a1caSRichard Henderson case 0: 5159*8429a1caSRichard Henderson g_assert_not_reached(); 5160*8429a1caSRichard Henderson } 5161*8429a1caSRichard Henderson } 5162*8429a1caSRichard Henderson 5163*8429a1caSRichard Henderson static void tcg_out_helper_load_slots(TCGContext *s, 5164*8429a1caSRichard Henderson unsigned nmov, TCGMovExtend *mov, 5165*8429a1caSRichard Henderson const TCGLdstHelperParam *parm) 5166*8429a1caSRichard Henderson { 5167*8429a1caSRichard Henderson unsigned i; 5168*8429a1caSRichard Henderson 5169*8429a1caSRichard Henderson /* 5170*8429a1caSRichard Henderson * Start from the end, storing to the stack first. 5171*8429a1caSRichard Henderson * This frees those registers, so we need not consider overlap. 5172*8429a1caSRichard Henderson */ 5173*8429a1caSRichard Henderson for (i = nmov; i-- > 0; ) { 5174*8429a1caSRichard Henderson unsigned slot = mov[i].dst; 5175*8429a1caSRichard Henderson 5176*8429a1caSRichard Henderson if (arg_slot_reg_p(slot)) { 5177*8429a1caSRichard Henderson goto found_reg; 5178*8429a1caSRichard Henderson } 5179*8429a1caSRichard Henderson 5180*8429a1caSRichard Henderson TCGReg src = mov[i].src; 5181*8429a1caSRichard Henderson TCGType dst_type = mov[i].dst_type; 5182*8429a1caSRichard Henderson MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64; 5183*8429a1caSRichard Henderson 5184*8429a1caSRichard Henderson /* The argument is going onto the stack; extend into scratch. */ 5185*8429a1caSRichard Henderson if ((mov[i].src_ext & MO_SIZE) != dst_mo) { 5186*8429a1caSRichard Henderson tcg_debug_assert(parm->ntmp != 0); 5187*8429a1caSRichard Henderson mov[i].dst = src = parm->tmp[0]; 5188*8429a1caSRichard Henderson tcg_out_movext1(s, &mov[i]); 5189*8429a1caSRichard Henderson } 5190*8429a1caSRichard Henderson 5191*8429a1caSRichard Henderson tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK, 5192*8429a1caSRichard Henderson tcg_out_helper_stk_ofs(dst_type, slot)); 5193*8429a1caSRichard Henderson } 5194*8429a1caSRichard Henderson return; 5195*8429a1caSRichard Henderson 5196*8429a1caSRichard Henderson found_reg: 5197*8429a1caSRichard Henderson /* 5198*8429a1caSRichard Henderson * The remaining arguments are in registers. 5199*8429a1caSRichard Henderson * Convert slot numbers to argument registers. 5200*8429a1caSRichard Henderson */ 5201*8429a1caSRichard Henderson nmov = i + 1; 5202*8429a1caSRichard Henderson for (i = 0; i < nmov; ++i) { 5203*8429a1caSRichard Henderson mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst]; 5204*8429a1caSRichard Henderson } 5205*8429a1caSRichard Henderson tcg_out_helper_load_regs(s, nmov, mov, parm->ntmp, parm->tmp); 5206*8429a1caSRichard Henderson } 5207*8429a1caSRichard Henderson 5208*8429a1caSRichard Henderson static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot, 5209*8429a1caSRichard Henderson TCGType type, tcg_target_long imm, 5210*8429a1caSRichard Henderson const TCGLdstHelperParam *parm) 5211*8429a1caSRichard Henderson { 5212*8429a1caSRichard Henderson if (arg_slot_reg_p(slot)) { 5213*8429a1caSRichard Henderson tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm); 5214*8429a1caSRichard Henderson } else { 5215*8429a1caSRichard Henderson int ofs = tcg_out_helper_stk_ofs(type, slot); 5216*8429a1caSRichard Henderson if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) { 5217*8429a1caSRichard Henderson tcg_debug_assert(parm->ntmp != 0); 5218*8429a1caSRichard Henderson tcg_out_movi(s, type, parm->tmp[0], imm); 5219*8429a1caSRichard Henderson tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs); 5220*8429a1caSRichard Henderson } 5221*8429a1caSRichard Henderson } 5222*8429a1caSRichard Henderson } 5223*8429a1caSRichard Henderson 5224*8429a1caSRichard Henderson static void tcg_out_helper_load_common_args(TCGContext *s, 5225*8429a1caSRichard Henderson const TCGLabelQemuLdst *ldst, 5226*8429a1caSRichard Henderson const TCGLdstHelperParam *parm, 5227*8429a1caSRichard Henderson const TCGHelperInfo *info, 5228*8429a1caSRichard Henderson unsigned next_arg) 5229*8429a1caSRichard Henderson { 5230*8429a1caSRichard Henderson TCGMovExtend ptr_mov = { 5231*8429a1caSRichard Henderson .dst_type = TCG_TYPE_PTR, 5232*8429a1caSRichard Henderson .src_type = TCG_TYPE_PTR, 5233*8429a1caSRichard Henderson .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64 5234*8429a1caSRichard Henderson }; 5235*8429a1caSRichard Henderson const TCGCallArgumentLoc *loc = &info->in[0]; 5236*8429a1caSRichard Henderson TCGType type; 5237*8429a1caSRichard Henderson unsigned slot; 5238*8429a1caSRichard Henderson tcg_target_ulong imm; 5239*8429a1caSRichard Henderson 5240*8429a1caSRichard Henderson /* 5241*8429a1caSRichard Henderson * Handle env, which is always first. 5242*8429a1caSRichard Henderson */ 5243*8429a1caSRichard Henderson ptr_mov.dst = loc->arg_slot; 5244*8429a1caSRichard Henderson ptr_mov.src = TCG_AREG0; 5245*8429a1caSRichard Henderson tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); 5246*8429a1caSRichard Henderson 5247*8429a1caSRichard Henderson /* 5248*8429a1caSRichard Henderson * Handle oi. 5249*8429a1caSRichard Henderson */ 5250*8429a1caSRichard Henderson imm = ldst->oi; 5251*8429a1caSRichard Henderson loc = &info->in[next_arg]; 5252*8429a1caSRichard Henderson type = TCG_TYPE_I32; 5253*8429a1caSRichard Henderson switch (loc->kind) { 5254*8429a1caSRichard Henderson case TCG_CALL_ARG_NORMAL: 5255*8429a1caSRichard Henderson break; 5256*8429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_U: 5257*8429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_S: 5258*8429a1caSRichard Henderson /* No extension required for MemOpIdx. */ 5259*8429a1caSRichard Henderson tcg_debug_assert(imm <= INT32_MAX); 5260*8429a1caSRichard Henderson type = TCG_TYPE_REG; 5261*8429a1caSRichard Henderson break; 5262*8429a1caSRichard Henderson default: 5263*8429a1caSRichard Henderson g_assert_not_reached(); 5264*8429a1caSRichard Henderson } 5265*8429a1caSRichard Henderson tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm); 5266*8429a1caSRichard Henderson next_arg++; 5267*8429a1caSRichard Henderson 5268*8429a1caSRichard Henderson /* 5269*8429a1caSRichard Henderson * Handle ra. 5270*8429a1caSRichard Henderson */ 5271*8429a1caSRichard Henderson loc = &info->in[next_arg]; 5272*8429a1caSRichard Henderson slot = loc->arg_slot; 5273*8429a1caSRichard Henderson if (parm->ra_gen) { 5274*8429a1caSRichard Henderson int arg_reg = -1; 5275*8429a1caSRichard Henderson TCGReg ra_reg; 5276*8429a1caSRichard Henderson 5277*8429a1caSRichard Henderson if (arg_slot_reg_p(slot)) { 5278*8429a1caSRichard Henderson arg_reg = tcg_target_call_iarg_regs[slot]; 5279*8429a1caSRichard Henderson } 5280*8429a1caSRichard Henderson ra_reg = parm->ra_gen(s, ldst, arg_reg); 5281*8429a1caSRichard Henderson 5282*8429a1caSRichard Henderson ptr_mov.dst = slot; 5283*8429a1caSRichard Henderson ptr_mov.src = ra_reg; 5284*8429a1caSRichard Henderson tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); 5285*8429a1caSRichard Henderson } else { 5286*8429a1caSRichard Henderson imm = (uintptr_t)ldst->raddr; 5287*8429a1caSRichard Henderson tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm); 5288*8429a1caSRichard Henderson } 5289*8429a1caSRichard Henderson } 5290*8429a1caSRichard Henderson 5291*8429a1caSRichard Henderson static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov, 5292*8429a1caSRichard Henderson const TCGCallArgumentLoc *loc, 5293*8429a1caSRichard Henderson TCGType dst_type, TCGType src_type, 5294*8429a1caSRichard Henderson TCGReg lo, TCGReg hi) 5295*8429a1caSRichard Henderson { 5296*8429a1caSRichard Henderson if (dst_type <= TCG_TYPE_REG) { 5297*8429a1caSRichard Henderson MemOp src_ext; 5298*8429a1caSRichard Henderson 5299*8429a1caSRichard Henderson switch (loc->kind) { 5300*8429a1caSRichard Henderson case TCG_CALL_ARG_NORMAL: 5301*8429a1caSRichard Henderson src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64; 5302*8429a1caSRichard Henderson break; 5303*8429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_U: 5304*8429a1caSRichard Henderson dst_type = TCG_TYPE_REG; 5305*8429a1caSRichard Henderson src_ext = MO_UL; 5306*8429a1caSRichard Henderson break; 5307*8429a1caSRichard Henderson case TCG_CALL_ARG_EXTEND_S: 5308*8429a1caSRichard Henderson dst_type = TCG_TYPE_REG; 5309*8429a1caSRichard Henderson src_ext = MO_SL; 5310*8429a1caSRichard Henderson break; 5311*8429a1caSRichard Henderson default: 5312*8429a1caSRichard Henderson g_assert_not_reached(); 5313*8429a1caSRichard Henderson } 5314*8429a1caSRichard Henderson 5315*8429a1caSRichard Henderson mov[0].dst = loc->arg_slot; 5316*8429a1caSRichard Henderson mov[0].dst_type = dst_type; 5317*8429a1caSRichard Henderson mov[0].src = lo; 5318*8429a1caSRichard Henderson mov[0].src_type = src_type; 5319*8429a1caSRichard Henderson mov[0].src_ext = src_ext; 5320*8429a1caSRichard Henderson return 1; 5321*8429a1caSRichard Henderson } 5322*8429a1caSRichard Henderson 5323*8429a1caSRichard Henderson assert(TCG_TARGET_REG_BITS == 32); 5324*8429a1caSRichard Henderson 5325*8429a1caSRichard Henderson mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot; 5326*8429a1caSRichard Henderson mov[0].src = lo; 5327*8429a1caSRichard Henderson mov[0].dst_type = TCG_TYPE_I32; 5328*8429a1caSRichard Henderson mov[0].src_type = TCG_TYPE_I32; 5329*8429a1caSRichard Henderson mov[0].src_ext = MO_32; 5330*8429a1caSRichard Henderson 5331*8429a1caSRichard Henderson mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot; 5332*8429a1caSRichard Henderson mov[1].src = hi; 5333*8429a1caSRichard Henderson mov[1].dst_type = TCG_TYPE_I32; 5334*8429a1caSRichard Henderson mov[1].src_type = TCG_TYPE_I32; 5335*8429a1caSRichard Henderson mov[1].src_ext = MO_32; 5336*8429a1caSRichard Henderson 5337*8429a1caSRichard Henderson return 2; 5338*8429a1caSRichard Henderson } 5339*8429a1caSRichard Henderson 5340*8429a1caSRichard Henderson static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, 5341*8429a1caSRichard Henderson const TCGLdstHelperParam *parm) 5342*8429a1caSRichard Henderson { 5343*8429a1caSRichard Henderson const TCGHelperInfo *info; 5344*8429a1caSRichard Henderson const TCGCallArgumentLoc *loc; 5345*8429a1caSRichard Henderson TCGMovExtend mov[2]; 5346*8429a1caSRichard Henderson unsigned next_arg, nmov; 5347*8429a1caSRichard Henderson MemOp mop = get_memop(ldst->oi); 5348*8429a1caSRichard Henderson 5349*8429a1caSRichard Henderson switch (mop & MO_SIZE) { 5350*8429a1caSRichard Henderson case MO_8: 5351*8429a1caSRichard Henderson case MO_16: 5352*8429a1caSRichard Henderson case MO_32: 5353*8429a1caSRichard Henderson info = &info_helper_ld32_mmu; 5354*8429a1caSRichard Henderson break; 5355*8429a1caSRichard Henderson case MO_64: 5356*8429a1caSRichard Henderson info = &info_helper_ld64_mmu; 5357*8429a1caSRichard Henderson break; 5358*8429a1caSRichard Henderson default: 5359*8429a1caSRichard Henderson g_assert_not_reached(); 5360*8429a1caSRichard Henderson } 5361*8429a1caSRichard Henderson 5362*8429a1caSRichard Henderson /* Defer env argument. */ 5363*8429a1caSRichard Henderson next_arg = 1; 5364*8429a1caSRichard Henderson 5365*8429a1caSRichard Henderson loc = &info->in[next_arg]; 5366*8429a1caSRichard Henderson nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_TL, TCG_TYPE_TL, 5367*8429a1caSRichard Henderson ldst->addrlo_reg, ldst->addrhi_reg); 5368*8429a1caSRichard Henderson next_arg += nmov; 5369*8429a1caSRichard Henderson 5370*8429a1caSRichard Henderson tcg_out_helper_load_slots(s, nmov, mov, parm); 5371*8429a1caSRichard Henderson 5372*8429a1caSRichard Henderson /* No special attention for 32 and 64-bit return values. */ 5373*8429a1caSRichard Henderson tcg_debug_assert(info->out_kind == TCG_CALL_RET_NORMAL); 5374*8429a1caSRichard Henderson 5375*8429a1caSRichard Henderson tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); 5376*8429a1caSRichard Henderson } 5377*8429a1caSRichard Henderson 5378*8429a1caSRichard Henderson static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst, 5379*8429a1caSRichard Henderson bool load_sign, 5380*8429a1caSRichard Henderson const TCGLdstHelperParam *parm) 5381*8429a1caSRichard Henderson { 5382*8429a1caSRichard Henderson TCGMovExtend mov[2]; 5383*8429a1caSRichard Henderson 5384*8429a1caSRichard Henderson if (ldst->type <= TCG_TYPE_REG) { 5385*8429a1caSRichard Henderson MemOp mop = get_memop(ldst->oi); 5386*8429a1caSRichard Henderson 5387*8429a1caSRichard Henderson mov[0].dst = ldst->datalo_reg; 5388*8429a1caSRichard Henderson mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0); 5389*8429a1caSRichard Henderson mov[0].dst_type = ldst->type; 5390*8429a1caSRichard Henderson mov[0].src_type = TCG_TYPE_REG; 5391*8429a1caSRichard Henderson 5392*8429a1caSRichard Henderson /* 5393*8429a1caSRichard Henderson * If load_sign, then we allowed the helper to perform the 5394*8429a1caSRichard Henderson * appropriate sign extension to tcg_target_ulong, and all 5395*8429a1caSRichard Henderson * we need now is a plain move. 5396*8429a1caSRichard Henderson * 5397*8429a1caSRichard Henderson * If they do not, then we expect the relevant extension 5398*8429a1caSRichard Henderson * instruction to be no more expensive than a move, and 5399*8429a1caSRichard Henderson * we thus save the icache etc by only using one of two 5400*8429a1caSRichard Henderson * helper functions. 5401*8429a1caSRichard Henderson */ 5402*8429a1caSRichard Henderson if (load_sign || !(mop & MO_SIGN)) { 5403*8429a1caSRichard Henderson if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) { 5404*8429a1caSRichard Henderson mov[0].src_ext = MO_32; 5405*8429a1caSRichard Henderson } else { 5406*8429a1caSRichard Henderson mov[0].src_ext = MO_64; 5407*8429a1caSRichard Henderson } 5408*8429a1caSRichard Henderson } else { 5409*8429a1caSRichard Henderson mov[0].src_ext = mop & MO_SSIZE; 5410*8429a1caSRichard Henderson } 5411*8429a1caSRichard Henderson tcg_out_movext1(s, mov); 5412*8429a1caSRichard Henderson } else { 5413*8429a1caSRichard Henderson assert(TCG_TARGET_REG_BITS == 32); 5414*8429a1caSRichard Henderson 5415*8429a1caSRichard Henderson mov[0].dst = ldst->datalo_reg; 5416*8429a1caSRichard Henderson mov[0].src = 5417*8429a1caSRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN); 5418*8429a1caSRichard Henderson mov[0].dst_type = TCG_TYPE_I32; 5419*8429a1caSRichard Henderson mov[0].src_type = TCG_TYPE_I32; 5420*8429a1caSRichard Henderson mov[0].src_ext = MO_32; 5421*8429a1caSRichard Henderson 5422*8429a1caSRichard Henderson mov[1].dst = ldst->datahi_reg; 5423*8429a1caSRichard Henderson mov[1].src = 5424*8429a1caSRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN); 5425*8429a1caSRichard Henderson mov[1].dst_type = TCG_TYPE_REG; 5426*8429a1caSRichard Henderson mov[1].src_type = TCG_TYPE_REG; 5427*8429a1caSRichard Henderson mov[1].src_ext = MO_32; 5428*8429a1caSRichard Henderson 5429*8429a1caSRichard Henderson tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); 5430*8429a1caSRichard Henderson } 5431*8429a1caSRichard Henderson } 5432*8429a1caSRichard Henderson 5433*8429a1caSRichard Henderson static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, 5434*8429a1caSRichard Henderson const TCGLdstHelperParam *parm) 5435*8429a1caSRichard Henderson { 5436*8429a1caSRichard Henderson const TCGHelperInfo *info; 5437*8429a1caSRichard Henderson const TCGCallArgumentLoc *loc; 5438*8429a1caSRichard Henderson TCGMovExtend mov[4]; 5439*8429a1caSRichard Henderson TCGType data_type; 5440*8429a1caSRichard Henderson unsigned next_arg, nmov, n; 5441*8429a1caSRichard Henderson MemOp mop = get_memop(ldst->oi); 5442*8429a1caSRichard Henderson 5443*8429a1caSRichard Henderson switch (mop & MO_SIZE) { 5444*8429a1caSRichard Henderson case MO_8: 5445*8429a1caSRichard Henderson case MO_16: 5446*8429a1caSRichard Henderson case MO_32: 5447*8429a1caSRichard Henderson info = &info_helper_st32_mmu; 5448*8429a1caSRichard Henderson data_type = TCG_TYPE_I32; 5449*8429a1caSRichard Henderson break; 5450*8429a1caSRichard Henderson case MO_64: 5451*8429a1caSRichard Henderson info = &info_helper_st64_mmu; 5452*8429a1caSRichard Henderson data_type = TCG_TYPE_I64; 5453*8429a1caSRichard Henderson break; 5454*8429a1caSRichard Henderson default: 5455*8429a1caSRichard Henderson g_assert_not_reached(); 5456*8429a1caSRichard Henderson } 5457*8429a1caSRichard Henderson 5458*8429a1caSRichard Henderson /* Defer env argument. */ 5459*8429a1caSRichard Henderson next_arg = 1; 5460*8429a1caSRichard Henderson nmov = 0; 5461*8429a1caSRichard Henderson 5462*8429a1caSRichard Henderson /* Handle addr argument. */ 5463*8429a1caSRichard Henderson loc = &info->in[next_arg]; 5464*8429a1caSRichard Henderson n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_TL, TCG_TYPE_TL, 5465*8429a1caSRichard Henderson ldst->addrlo_reg, ldst->addrhi_reg); 5466*8429a1caSRichard Henderson next_arg += n; 5467*8429a1caSRichard Henderson nmov += n; 5468*8429a1caSRichard Henderson 5469*8429a1caSRichard Henderson /* Handle data argument. */ 5470*8429a1caSRichard Henderson loc = &info->in[next_arg]; 5471*8429a1caSRichard Henderson n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, 5472*8429a1caSRichard Henderson ldst->datalo_reg, ldst->datahi_reg); 5473*8429a1caSRichard Henderson next_arg += n; 5474*8429a1caSRichard Henderson nmov += n; 5475*8429a1caSRichard Henderson tcg_debug_assert(nmov <= ARRAY_SIZE(mov)); 5476*8429a1caSRichard Henderson 5477*8429a1caSRichard Henderson tcg_out_helper_load_slots(s, nmov, mov, parm); 5478*8429a1caSRichard Henderson tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); 5479*8429a1caSRichard Henderson } 5480*8429a1caSRichard Henderson 5481c896fe29Sbellard #ifdef CONFIG_PROFILER 5482c896fe29Sbellard 5483c3fac113SEmilio G. Cota /* avoid copy/paste errors */ 5484c3fac113SEmilio G. Cota #define PROF_ADD(to, from, field) \ 5485c3fac113SEmilio G. Cota do { \ 5486d73415a3SStefan Hajnoczi (to)->field += qatomic_read(&((from)->field)); \ 5487c3fac113SEmilio G. Cota } while (0) 5488c896fe29Sbellard 5489c3fac113SEmilio G. Cota #define PROF_MAX(to, from, field) \ 5490c3fac113SEmilio G. Cota do { \ 5491d73415a3SStefan Hajnoczi typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 5492c3fac113SEmilio G. Cota if (val__ > (to)->field) { \ 5493c3fac113SEmilio G. Cota (to)->field = val__; \ 5494c3fac113SEmilio G. Cota } \ 5495c3fac113SEmilio G. Cota } while (0) 5496c3fac113SEmilio G. Cota 5497c3fac113SEmilio G. Cota /* Pass in a zero'ed @prof */ 5498c3fac113SEmilio G. Cota static inline 5499c3fac113SEmilio G. Cota void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 5500c896fe29Sbellard { 55010e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 5502c3fac113SEmilio G. Cota unsigned int i; 5503c3fac113SEmilio G. Cota 55043468b59eSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 5505d73415a3SStefan Hajnoczi TCGContext *s = qatomic_read(&tcg_ctxs[i]); 55063468b59eSEmilio G. Cota const TCGProfile *orig = &s->prof; 5507c3fac113SEmilio G. Cota 5508c3fac113SEmilio G. Cota if (counters) { 550972fd2efbSEmilio G. Cota PROF_ADD(prof, orig, cpu_exec_time); 5510c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count1); 5511c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count); 5512c3fac113SEmilio G. Cota PROF_ADD(prof, orig, op_count); 5513c3fac113SEmilio G. Cota PROF_MAX(prof, orig, op_count_max); 5514c3fac113SEmilio G. Cota PROF_ADD(prof, orig, temp_count); 5515c3fac113SEmilio G. Cota PROF_MAX(prof, orig, temp_count_max); 5516c3fac113SEmilio G. Cota PROF_ADD(prof, orig, del_op_count); 5517c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_in_len); 5518c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_out_len); 5519c3fac113SEmilio G. Cota PROF_ADD(prof, orig, search_out_len); 5520c3fac113SEmilio G. Cota PROF_ADD(prof, orig, interm_time); 5521c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_time); 5522c3fac113SEmilio G. Cota PROF_ADD(prof, orig, la_time); 5523c3fac113SEmilio G. Cota PROF_ADD(prof, orig, opt_time); 5524c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_count); 5525c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_time); 5526c3fac113SEmilio G. Cota } 5527c3fac113SEmilio G. Cota if (table) { 5528c896fe29Sbellard int i; 5529d70724ceSzhanghailiang 553015fc7daaSRichard Henderson for (i = 0; i < NB_OPS; i++) { 5531c3fac113SEmilio G. Cota PROF_ADD(prof, orig, table_op_count[i]); 5532c3fac113SEmilio G. Cota } 5533c3fac113SEmilio G. Cota } 5534c3fac113SEmilio G. Cota } 5535c3fac113SEmilio G. Cota } 5536c3fac113SEmilio G. Cota 5537c3fac113SEmilio G. Cota #undef PROF_ADD 5538c3fac113SEmilio G. Cota #undef PROF_MAX 5539c3fac113SEmilio G. Cota 5540c3fac113SEmilio G. Cota static void tcg_profile_snapshot_counters(TCGProfile *prof) 5541c3fac113SEmilio G. Cota { 5542c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, true, false); 5543c3fac113SEmilio G. Cota } 5544c3fac113SEmilio G. Cota 5545c3fac113SEmilio G. Cota static void tcg_profile_snapshot_table(TCGProfile *prof) 5546c3fac113SEmilio G. Cota { 5547c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, false, true); 5548c3fac113SEmilio G. Cota } 5549c3fac113SEmilio G. Cota 5550b6a7f3e0SDaniel P. Berrangé void tcg_dump_op_count(GString *buf) 5551c3fac113SEmilio G. Cota { 5552c3fac113SEmilio G. Cota TCGProfile prof = {}; 5553c3fac113SEmilio G. Cota int i; 5554c3fac113SEmilio G. Cota 5555c3fac113SEmilio G. Cota tcg_profile_snapshot_table(&prof); 5556c3fac113SEmilio G. Cota for (i = 0; i < NB_OPS; i++) { 5557b6a7f3e0SDaniel P. Berrangé g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name, 5558c3fac113SEmilio G. Cota prof.table_op_count[i]); 5559c896fe29Sbellard } 5560c896fe29Sbellard } 556172fd2efbSEmilio G. Cota 556272fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 556372fd2efbSEmilio G. Cota { 55640e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 556572fd2efbSEmilio G. Cota unsigned int i; 556672fd2efbSEmilio G. Cota int64_t ret = 0; 556772fd2efbSEmilio G. Cota 556872fd2efbSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 5569d73415a3SStefan Hajnoczi const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 557072fd2efbSEmilio G. Cota const TCGProfile *prof = &s->prof; 557172fd2efbSEmilio G. Cota 5572d73415a3SStefan Hajnoczi ret += qatomic_read(&prof->cpu_exec_time); 557372fd2efbSEmilio G. Cota } 557472fd2efbSEmilio G. Cota return ret; 557572fd2efbSEmilio G. Cota } 5576246ae24dSMax Filippov #else 5577b6a7f3e0SDaniel P. Berrangé void tcg_dump_op_count(GString *buf) 5578246ae24dSMax Filippov { 5579b6a7f3e0SDaniel P. Berrangé g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 5580246ae24dSMax Filippov } 558172fd2efbSEmilio G. Cota 558272fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 558372fd2efbSEmilio G. Cota { 558472fd2efbSEmilio G. Cota error_report("%s: TCG profiler not compiled", __func__); 558572fd2efbSEmilio G. Cota exit(EXIT_FAILURE); 558672fd2efbSEmilio G. Cota } 5587c896fe29Sbellard #endif 5588c896fe29Sbellard 5589c896fe29Sbellard 5590fbf59aadSRichard Henderson int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) 5591c896fe29Sbellard { 5592c3fac113SEmilio G. Cota #ifdef CONFIG_PROFILER 5593c3fac113SEmilio G. Cota TCGProfile *prof = &s->prof; 5594c3fac113SEmilio G. Cota #endif 559515fa08f8SRichard Henderson int i, num_insns; 559615fa08f8SRichard Henderson TCGOp *op; 5597c896fe29Sbellard 559804fe6400SRichard Henderson #ifdef CONFIG_PROFILER 559904fe6400SRichard Henderson { 5600c1f543b7SEmilio G. Cota int n = 0; 560104fe6400SRichard Henderson 560215fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 560315fa08f8SRichard Henderson n++; 560415fa08f8SRichard Henderson } 5605d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count, prof->op_count + n); 5606c3fac113SEmilio G. Cota if (n > prof->op_count_max) { 5607d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count_max, n); 560804fe6400SRichard Henderson } 560904fe6400SRichard Henderson 561004fe6400SRichard Henderson n = s->nb_temps; 5611d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count, prof->temp_count + n); 5612c3fac113SEmilio G. Cota if (n > prof->temp_count_max) { 5613d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count_max, n); 561404fe6400SRichard Henderson } 561504fe6400SRichard Henderson } 561604fe6400SRichard Henderson #endif 561704fe6400SRichard Henderson 5618c896fe29Sbellard #ifdef DEBUG_DISAS 5619d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 5620fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 5621c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 562278b54858SRichard Henderson if (logfile) { 562378b54858SRichard Henderson fprintf(logfile, "OP:\n"); 5624b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false); 562578b54858SRichard Henderson fprintf(logfile, "\n"); 5626fc59d2d8SRobert Foley qemu_log_unlock(logfile); 5627c896fe29Sbellard } 562878b54858SRichard Henderson } 5629c896fe29Sbellard #endif 5630c896fe29Sbellard 5631bef16ab4SRichard Henderson #ifdef CONFIG_DEBUG_TCG 5632bef16ab4SRichard Henderson /* Ensure all labels referenced have been emitted. */ 5633bef16ab4SRichard Henderson { 5634bef16ab4SRichard Henderson TCGLabel *l; 5635bef16ab4SRichard Henderson bool error = false; 5636bef16ab4SRichard Henderson 5637bef16ab4SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 5638f85b1fc4SRichard Henderson if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { 5639bef16ab4SRichard Henderson qemu_log_mask(CPU_LOG_TB_OP, 5640bef16ab4SRichard Henderson "$L%d referenced but not present.\n", l->id); 5641bef16ab4SRichard Henderson error = true; 5642bef16ab4SRichard Henderson } 5643bef16ab4SRichard Henderson } 5644bef16ab4SRichard Henderson assert(!error); 5645bef16ab4SRichard Henderson } 5646bef16ab4SRichard Henderson #endif 5647bef16ab4SRichard Henderson 5648c5cc28ffSAurelien Jarno #ifdef CONFIG_PROFILER 5649d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 5650c5cc28ffSAurelien Jarno #endif 5651c5cc28ffSAurelien Jarno 56528f2e8c07SKirill Batuzov #ifdef USE_TCG_OPTIMIZATIONS 5653c45cb8bbSRichard Henderson tcg_optimize(s); 56548f2e8c07SKirill Batuzov #endif 56558f2e8c07SKirill Batuzov 5656a23a9ec6Sbellard #ifdef CONFIG_PROFILER 5657d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 5658d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 5659a23a9ec6Sbellard #endif 5660c5cc28ffSAurelien Jarno 5661b4fc67c7SRichard Henderson reachable_code_pass(s); 5662874b8574SRichard Henderson liveness_pass_0(s); 5663b83eabeaSRichard Henderson liveness_pass_1(s); 56645a18407fSRichard Henderson 56655a18407fSRichard Henderson if (s->nb_indirects > 0) { 56665a18407fSRichard Henderson #ifdef DEBUG_DISAS 56675a18407fSRichard Henderson if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 5668fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 5669c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 567078b54858SRichard Henderson if (logfile) { 567178b54858SRichard Henderson fprintf(logfile, "OP before indirect lowering:\n"); 5672b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false); 567378b54858SRichard Henderson fprintf(logfile, "\n"); 5674fc59d2d8SRobert Foley qemu_log_unlock(logfile); 56755a18407fSRichard Henderson } 567678b54858SRichard Henderson } 56775a18407fSRichard Henderson #endif 56785a18407fSRichard Henderson /* Replace indirect temps with direct temps. */ 5679b83eabeaSRichard Henderson if (liveness_pass_2(s)) { 56805a18407fSRichard Henderson /* If changes were made, re-run liveness. */ 5681b83eabeaSRichard Henderson liveness_pass_1(s); 56825a18407fSRichard Henderson } 56835a18407fSRichard Henderson } 5684c5cc28ffSAurelien Jarno 5685a23a9ec6Sbellard #ifdef CONFIG_PROFILER 5686d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 5687a23a9ec6Sbellard #endif 5688c896fe29Sbellard 5689c896fe29Sbellard #ifdef DEBUG_DISAS 5690d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 5691fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 5692c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 569378b54858SRichard Henderson if (logfile) { 569478b54858SRichard Henderson fprintf(logfile, "OP after optimization and liveness analysis:\n"); 5695b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, true); 569678b54858SRichard Henderson fprintf(logfile, "\n"); 5697fc59d2d8SRobert Foley qemu_log_unlock(logfile); 5698c896fe29Sbellard } 569978b54858SRichard Henderson } 5700c896fe29Sbellard #endif 5701c896fe29Sbellard 570235abb009SRichard Henderson /* Initialize goto_tb jump offsets. */ 57033a50f424SRichard Henderson tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; 57043a50f424SRichard Henderson tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; 57059da6079bSRichard Henderson tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; 57069da6079bSRichard Henderson tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; 570735abb009SRichard Henderson 5708c896fe29Sbellard tcg_reg_alloc_start(s); 5709c896fe29Sbellard 5710db0c51a3SRichard Henderson /* 5711db0c51a3SRichard Henderson * Reset the buffer pointers when restarting after overflow. 5712db0c51a3SRichard Henderson * TODO: Move this into translate-all.c with the rest of the 5713db0c51a3SRichard Henderson * buffer management. Having only this done here is confusing. 5714db0c51a3SRichard Henderson */ 5715db0c51a3SRichard Henderson s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 5716db0c51a3SRichard Henderson s->code_ptr = s->code_buf; 5717c896fe29Sbellard 5718659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 57196001f772SLaurent Vivier QSIMPLEQ_INIT(&s->ldst_labels); 5720659ef5cbSRichard Henderson #endif 572157a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 572257a26946SRichard Henderson s->pool_labels = NULL; 572357a26946SRichard Henderson #endif 57249ecefc84SRichard Henderson 5725fca8a500SRichard Henderson num_insns = -1; 572615fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 5727c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 5728b3db8758Sblueswir1 5729c896fe29Sbellard #ifdef CONFIG_PROFILER 5730d73415a3SStefan Hajnoczi qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 5731c896fe29Sbellard #endif 5732c45cb8bbSRichard Henderson 5733c896fe29Sbellard switch (opc) { 5734c896fe29Sbellard case INDEX_op_mov_i32: 5735c896fe29Sbellard case INDEX_op_mov_i64: 5736d2fd745fSRichard Henderson case INDEX_op_mov_vec: 5737dd186292SRichard Henderson tcg_reg_alloc_mov(s, op); 5738c896fe29Sbellard break; 5739bab1671fSRichard Henderson case INDEX_op_dup_vec: 5740bab1671fSRichard Henderson tcg_reg_alloc_dup(s, op); 5741bab1671fSRichard Henderson break; 5742765b842aSRichard Henderson case INDEX_op_insn_start: 5743fca8a500SRichard Henderson if (num_insns >= 0) { 57449f754620SRichard Henderson size_t off = tcg_current_code_size(s); 57459f754620SRichard Henderson s->gen_insn_end_off[num_insns] = off; 57469f754620SRichard Henderson /* Assert that we do not overflow our stored offset. */ 57479f754620SRichard Henderson assert(s->gen_insn_end_off[num_insns] == off); 5748fca8a500SRichard Henderson } 5749fca8a500SRichard Henderson num_insns++; 5750bad729e2SRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 5751bad729e2SRichard Henderson target_ulong a; 5752bad729e2SRichard Henderson #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 5753efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 5754bad729e2SRichard Henderson #else 5755efee3746SRichard Henderson a = op->args[i]; 5756bad729e2SRichard Henderson #endif 5757fca8a500SRichard Henderson s->gen_insn_data[num_insns][i] = a; 5758bad729e2SRichard Henderson } 5759c896fe29Sbellard break; 57605ff9d6a4Sbellard case INDEX_op_discard: 576143439139SRichard Henderson temp_dead(s, arg_temp(op->args[0])); 57625ff9d6a4Sbellard break; 5763c896fe29Sbellard case INDEX_op_set_label: 5764e8996ee0Sbellard tcg_reg_alloc_bb_end(s, s->reserved_regs); 576592ab8e7dSRichard Henderson tcg_out_label(s, arg_label(op->args[0])); 5766c896fe29Sbellard break; 5767c896fe29Sbellard case INDEX_op_call: 5768dd186292SRichard Henderson tcg_reg_alloc_call(s, op); 5769c45cb8bbSRichard Henderson break; 5770b55a8d9dSRichard Henderson case INDEX_op_exit_tb: 5771b55a8d9dSRichard Henderson tcg_out_exit_tb(s, op->args[0]); 5772b55a8d9dSRichard Henderson break; 5773cf7d6b8eSRichard Henderson case INDEX_op_goto_tb: 5774cf7d6b8eSRichard Henderson tcg_out_goto_tb(s, op->args[0]); 5775cf7d6b8eSRichard Henderson break; 5776efe86b21SRichard Henderson case INDEX_op_dup2_vec: 5777efe86b21SRichard Henderson if (tcg_reg_alloc_dup2(s, op)) { 5778efe86b21SRichard Henderson break; 5779efe86b21SRichard Henderson } 5780efe86b21SRichard Henderson /* fall through */ 5781c896fe29Sbellard default: 578225c4d9ccSRichard Henderson /* Sanity check that we've not introduced any unhandled opcodes. */ 5783be0f34b5SRichard Henderson tcg_debug_assert(tcg_op_supported(opc)); 5784c896fe29Sbellard /* Note: in order to speed up the code, it would be much 5785c896fe29Sbellard faster to have specialized register allocator functions for 5786c896fe29Sbellard some common argument patterns */ 5787dd186292SRichard Henderson tcg_reg_alloc_op(s, op); 5788c896fe29Sbellard break; 5789c896fe29Sbellard } 5790b125f9dcSRichard Henderson /* Test for (pending) buffer overflow. The assumption is that any 5791b125f9dcSRichard Henderson one operation beginning below the high water mark cannot overrun 5792b125f9dcSRichard Henderson the buffer completely. Thus we can test for overflow after 5793b125f9dcSRichard Henderson generating code without having to check during generation. */ 5794644da9b3SJohn Clarke if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 5795b125f9dcSRichard Henderson return -1; 5796b125f9dcSRichard Henderson } 57976e6c4efeSRichard Henderson /* Test for TB overflow, as seen by gen_insn_end_off. */ 57986e6c4efeSRichard Henderson if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 57996e6c4efeSRichard Henderson return -2; 58006e6c4efeSRichard Henderson } 5801c896fe29Sbellard } 5802fca8a500SRichard Henderson tcg_debug_assert(num_insns >= 0); 5803fca8a500SRichard Henderson s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 5804c45cb8bbSRichard Henderson 5805b76f0d8cSYeongkyoon Lee /* Generate TB finalization at the end of block */ 5806659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 5807aeee05f5SRichard Henderson i = tcg_out_ldst_finalize(s); 5808aeee05f5SRichard Henderson if (i < 0) { 5809aeee05f5SRichard Henderson return i; 581023dceda6SRichard Henderson } 5811659ef5cbSRichard Henderson #endif 581257a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 58131768987bSRichard Henderson i = tcg_out_pool_finalize(s); 58141768987bSRichard Henderson if (i < 0) { 58151768987bSRichard Henderson return i; 581657a26946SRichard Henderson } 581757a26946SRichard Henderson #endif 58187ecd02a0SRichard Henderson if (!tcg_resolve_relocs(s)) { 58197ecd02a0SRichard Henderson return -2; 58207ecd02a0SRichard Henderson } 5821c896fe29Sbellard 5822df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 5823c896fe29Sbellard /* flush instruction cache */ 5824db0c51a3SRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 5825db0c51a3SRichard Henderson (uintptr_t)s->code_buf, 58261da8de39SRichard Henderson tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 5827df5d2b16SRichard Henderson #endif 58282aeabc08SStefan Weil 58291813e175SRichard Henderson return tcg_current_code_size(s); 5830c896fe29Sbellard } 5831c896fe29Sbellard 5832a23a9ec6Sbellard #ifdef CONFIG_PROFILER 58333a841ab5SDaniel P. Berrangé void tcg_dump_info(GString *buf) 5834a23a9ec6Sbellard { 5835c3fac113SEmilio G. Cota TCGProfile prof = {}; 5836c3fac113SEmilio G. Cota const TCGProfile *s; 5837c3fac113SEmilio G. Cota int64_t tb_count; 5838c3fac113SEmilio G. Cota int64_t tb_div_count; 5839c3fac113SEmilio G. Cota int64_t tot; 5840c3fac113SEmilio G. Cota 5841c3fac113SEmilio G. Cota tcg_profile_snapshot_counters(&prof); 5842c3fac113SEmilio G. Cota s = &prof; 5843c3fac113SEmilio G. Cota tb_count = s->tb_count; 5844c3fac113SEmilio G. Cota tb_div_count = tb_count ? tb_count : 1; 5845c3fac113SEmilio G. Cota tot = s->interm_time + s->code_time; 5846a23a9ec6Sbellard 58473a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "JIT cycles %" PRId64 58483a841ab5SDaniel P. Berrangé " (%0.3f s at 2.4 GHz)\n", 5849a23a9ec6Sbellard tot, tot / 2.4e9); 58503a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "translated TBs %" PRId64 58513a841ab5SDaniel P. Berrangé " (aborted=%" PRId64 " %0.1f%%)\n", 5852fca8a500SRichard Henderson tb_count, s->tb_count1 - tb_count, 5853fca8a500SRichard Henderson (double)(s->tb_count1 - s->tb_count) 5854fca8a500SRichard Henderson / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 58553a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n", 5856fca8a500SRichard Henderson (double)s->op_count / tb_div_count, s->op_count_max); 58573a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "deleted ops/TB %0.2f\n", 5858fca8a500SRichard Henderson (double)s->del_op_count / tb_div_count); 58593a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n", 58603a841ab5SDaniel P. Berrangé (double)s->temp_count / tb_div_count, 58613a841ab5SDaniel P. Berrangé s->temp_count_max); 58623a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg host code/TB %0.1f\n", 5863fca8a500SRichard Henderson (double)s->code_out_len / tb_div_count); 58643a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg search data/TB %0.1f\n", 5865fca8a500SRichard Henderson (double)s->search_out_len / tb_div_count); 5866a23a9ec6Sbellard 58673a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/op %0.1f\n", 5868a23a9ec6Sbellard s->op_count ? (double)tot / s->op_count : 0); 58693a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/in byte %0.1f\n", 5870a23a9ec6Sbellard s->code_in_len ? (double)tot / s->code_in_len : 0); 58713a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/out byte %0.1f\n", 5872a23a9ec6Sbellard s->code_out_len ? (double)tot / s->code_out_len : 0); 58733a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/search byte %0.1f\n", 58743a841ab5SDaniel P. Berrangé s->search_out_len ? 58753a841ab5SDaniel P. Berrangé (double)tot / s->search_out_len : 0); 5876fca8a500SRichard Henderson if (tot == 0) { 5877a23a9ec6Sbellard tot = 1; 5878fca8a500SRichard Henderson } 58793a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " gen_interm time %0.1f%%\n", 5880a23a9ec6Sbellard (double)s->interm_time / tot * 100.0); 58813a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " gen_code time %0.1f%%\n", 5882a23a9ec6Sbellard (double)s->code_time / tot * 100.0); 58833a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "optim./code time %0.1f%%\n", 58843a841ab5SDaniel P. Berrangé (double)s->opt_time / (s->code_time ? 58853a841ab5SDaniel P. Berrangé s->code_time : 1) 5886c5cc28ffSAurelien Jarno * 100.0); 58873a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "liveness/code time %0.1f%%\n", 58883a841ab5SDaniel P. Berrangé (double)s->la_time / (s->code_time ? 58893a841ab5SDaniel P. Berrangé s->code_time : 1) * 100.0); 58903a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n", 5891a23a9ec6Sbellard s->restore_count); 58923a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " avg cycles %0.1f\n", 58933a841ab5SDaniel P. Berrangé s->restore_count ? 58943a841ab5SDaniel P. Berrangé (double)s->restore_time / s->restore_count : 0); 5895a23a9ec6Sbellard } 5896a23a9ec6Sbellard #else 58973a841ab5SDaniel P. Berrangé void tcg_dump_info(GString *buf) 5898a23a9ec6Sbellard { 58993a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 5900a23a9ec6Sbellard } 5901a23a9ec6Sbellard #endif 5902813da627SRichard Henderson 5903813da627SRichard Henderson #ifdef ELF_HOST_MACHINE 59045872bbf2SRichard Henderson /* In order to use this feature, the backend needs to do three things: 59055872bbf2SRichard Henderson 59065872bbf2SRichard Henderson (1) Define ELF_HOST_MACHINE to indicate both what value to 59075872bbf2SRichard Henderson put into the ELF image and to indicate support for the feature. 59085872bbf2SRichard Henderson 59095872bbf2SRichard Henderson (2) Define tcg_register_jit. This should create a buffer containing 59105872bbf2SRichard Henderson the contents of a .debug_frame section that describes the post- 59115872bbf2SRichard Henderson prologue unwind info for the tcg machine. 59125872bbf2SRichard Henderson 59135872bbf2SRichard Henderson (3) Call tcg_register_jit_int, with the constructed .debug_frame. 59145872bbf2SRichard Henderson */ 5915813da627SRichard Henderson 5916813da627SRichard Henderson /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 5917813da627SRichard Henderson typedef enum { 5918813da627SRichard Henderson JIT_NOACTION = 0, 5919813da627SRichard Henderson JIT_REGISTER_FN, 5920813da627SRichard Henderson JIT_UNREGISTER_FN 5921813da627SRichard Henderson } jit_actions_t; 5922813da627SRichard Henderson 5923813da627SRichard Henderson struct jit_code_entry { 5924813da627SRichard Henderson struct jit_code_entry *next_entry; 5925813da627SRichard Henderson struct jit_code_entry *prev_entry; 5926813da627SRichard Henderson const void *symfile_addr; 5927813da627SRichard Henderson uint64_t symfile_size; 5928813da627SRichard Henderson }; 5929813da627SRichard Henderson 5930813da627SRichard Henderson struct jit_descriptor { 5931813da627SRichard Henderson uint32_t version; 5932813da627SRichard Henderson uint32_t action_flag; 5933813da627SRichard Henderson struct jit_code_entry *relevant_entry; 5934813da627SRichard Henderson struct jit_code_entry *first_entry; 5935813da627SRichard Henderson }; 5936813da627SRichard Henderson 5937813da627SRichard Henderson void __jit_debug_register_code(void) __attribute__((noinline)); 5938813da627SRichard Henderson void __jit_debug_register_code(void) 5939813da627SRichard Henderson { 5940813da627SRichard Henderson asm(""); 5941813da627SRichard Henderson } 5942813da627SRichard Henderson 5943813da627SRichard Henderson /* Must statically initialize the version, because GDB may check 5944813da627SRichard Henderson the version before we can set it. */ 5945813da627SRichard Henderson struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 5946813da627SRichard Henderson 5947813da627SRichard Henderson /* End GDB interface. */ 5948813da627SRichard Henderson 5949813da627SRichard Henderson static int find_string(const char *strtab, const char *str) 5950813da627SRichard Henderson { 5951813da627SRichard Henderson const char *p = strtab + 1; 5952813da627SRichard Henderson 5953813da627SRichard Henderson while (1) { 5954813da627SRichard Henderson if (strcmp(p, str) == 0) { 5955813da627SRichard Henderson return p - strtab; 5956813da627SRichard Henderson } 5957813da627SRichard Henderson p += strlen(p) + 1; 5958813da627SRichard Henderson } 5959813da627SRichard Henderson } 5960813da627SRichard Henderson 5961755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 59622c90784aSRichard Henderson const void *debug_frame, 59632c90784aSRichard Henderson size_t debug_frame_size) 5964813da627SRichard Henderson { 59655872bbf2SRichard Henderson struct __attribute__((packed)) DebugInfo { 59665872bbf2SRichard Henderson uint32_t len; 59675872bbf2SRichard Henderson uint16_t version; 59685872bbf2SRichard Henderson uint32_t abbrev; 59695872bbf2SRichard Henderson uint8_t ptr_size; 59705872bbf2SRichard Henderson uint8_t cu_die; 59715872bbf2SRichard Henderson uint16_t cu_lang; 59725872bbf2SRichard Henderson uintptr_t cu_low_pc; 59735872bbf2SRichard Henderson uintptr_t cu_high_pc; 59745872bbf2SRichard Henderson uint8_t fn_die; 59755872bbf2SRichard Henderson char fn_name[16]; 59765872bbf2SRichard Henderson uintptr_t fn_low_pc; 59775872bbf2SRichard Henderson uintptr_t fn_high_pc; 59785872bbf2SRichard Henderson uint8_t cu_eoc; 59795872bbf2SRichard Henderson }; 5980813da627SRichard Henderson 5981813da627SRichard Henderson struct ElfImage { 5982813da627SRichard Henderson ElfW(Ehdr) ehdr; 5983813da627SRichard Henderson ElfW(Phdr) phdr; 59845872bbf2SRichard Henderson ElfW(Shdr) shdr[7]; 59855872bbf2SRichard Henderson ElfW(Sym) sym[2]; 59865872bbf2SRichard Henderson struct DebugInfo di; 59875872bbf2SRichard Henderson uint8_t da[24]; 59885872bbf2SRichard Henderson char str[80]; 59895872bbf2SRichard Henderson }; 59905872bbf2SRichard Henderson 59915872bbf2SRichard Henderson struct ElfImage *img; 59925872bbf2SRichard Henderson 59935872bbf2SRichard Henderson static const struct ElfImage img_template = { 59945872bbf2SRichard Henderson .ehdr = { 59955872bbf2SRichard Henderson .e_ident[EI_MAG0] = ELFMAG0, 59965872bbf2SRichard Henderson .e_ident[EI_MAG1] = ELFMAG1, 59975872bbf2SRichard Henderson .e_ident[EI_MAG2] = ELFMAG2, 59985872bbf2SRichard Henderson .e_ident[EI_MAG3] = ELFMAG3, 59995872bbf2SRichard Henderson .e_ident[EI_CLASS] = ELF_CLASS, 60005872bbf2SRichard Henderson .e_ident[EI_DATA] = ELF_DATA, 60015872bbf2SRichard Henderson .e_ident[EI_VERSION] = EV_CURRENT, 60025872bbf2SRichard Henderson .e_type = ET_EXEC, 60035872bbf2SRichard Henderson .e_machine = ELF_HOST_MACHINE, 60045872bbf2SRichard Henderson .e_version = EV_CURRENT, 60055872bbf2SRichard Henderson .e_phoff = offsetof(struct ElfImage, phdr), 60065872bbf2SRichard Henderson .e_shoff = offsetof(struct ElfImage, shdr), 60075872bbf2SRichard Henderson .e_ehsize = sizeof(ElfW(Shdr)), 60085872bbf2SRichard Henderson .e_phentsize = sizeof(ElfW(Phdr)), 60095872bbf2SRichard Henderson .e_phnum = 1, 60105872bbf2SRichard Henderson .e_shentsize = sizeof(ElfW(Shdr)), 60115872bbf2SRichard Henderson .e_shnum = ARRAY_SIZE(img->shdr), 60125872bbf2SRichard Henderson .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 6013abbb3eaeSRichard Henderson #ifdef ELF_HOST_FLAGS 6014abbb3eaeSRichard Henderson .e_flags = ELF_HOST_FLAGS, 6015abbb3eaeSRichard Henderson #endif 6016abbb3eaeSRichard Henderson #ifdef ELF_OSABI 6017abbb3eaeSRichard Henderson .e_ident[EI_OSABI] = ELF_OSABI, 6018abbb3eaeSRichard Henderson #endif 60195872bbf2SRichard Henderson }, 60205872bbf2SRichard Henderson .phdr = { 60215872bbf2SRichard Henderson .p_type = PT_LOAD, 60225872bbf2SRichard Henderson .p_flags = PF_X, 60235872bbf2SRichard Henderson }, 60245872bbf2SRichard Henderson .shdr = { 60255872bbf2SRichard Henderson [0] = { .sh_type = SHT_NULL }, 60265872bbf2SRichard Henderson /* Trick: The contents of code_gen_buffer are not present in 60275872bbf2SRichard Henderson this fake ELF file; that got allocated elsewhere. Therefore 60285872bbf2SRichard Henderson we mark .text as SHT_NOBITS (similar to .bss) so that readers 60295872bbf2SRichard Henderson will not look for contents. We can record any address. */ 60305872bbf2SRichard Henderson [1] = { /* .text */ 60315872bbf2SRichard Henderson .sh_type = SHT_NOBITS, 60325872bbf2SRichard Henderson .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 60335872bbf2SRichard Henderson }, 60345872bbf2SRichard Henderson [2] = { /* .debug_info */ 60355872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 60365872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, di), 60375872bbf2SRichard Henderson .sh_size = sizeof(struct DebugInfo), 60385872bbf2SRichard Henderson }, 60395872bbf2SRichard Henderson [3] = { /* .debug_abbrev */ 60405872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 60415872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, da), 60425872bbf2SRichard Henderson .sh_size = sizeof(img->da), 60435872bbf2SRichard Henderson }, 60445872bbf2SRichard Henderson [4] = { /* .debug_frame */ 60455872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 60465872bbf2SRichard Henderson .sh_offset = sizeof(struct ElfImage), 60475872bbf2SRichard Henderson }, 60485872bbf2SRichard Henderson [5] = { /* .symtab */ 60495872bbf2SRichard Henderson .sh_type = SHT_SYMTAB, 60505872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, sym), 60515872bbf2SRichard Henderson .sh_size = sizeof(img->sym), 60525872bbf2SRichard Henderson .sh_info = 1, 60535872bbf2SRichard Henderson .sh_link = ARRAY_SIZE(img->shdr) - 1, 60545872bbf2SRichard Henderson .sh_entsize = sizeof(ElfW(Sym)), 60555872bbf2SRichard Henderson }, 60565872bbf2SRichard Henderson [6] = { /* .strtab */ 60575872bbf2SRichard Henderson .sh_type = SHT_STRTAB, 60585872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, str), 60595872bbf2SRichard Henderson .sh_size = sizeof(img->str), 60605872bbf2SRichard Henderson } 60615872bbf2SRichard Henderson }, 60625872bbf2SRichard Henderson .sym = { 60635872bbf2SRichard Henderson [1] = { /* code_gen_buffer */ 60645872bbf2SRichard Henderson .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 60655872bbf2SRichard Henderson .st_shndx = 1, 60665872bbf2SRichard Henderson } 60675872bbf2SRichard Henderson }, 60685872bbf2SRichard Henderson .di = { 60695872bbf2SRichard Henderson .len = sizeof(struct DebugInfo) - 4, 60705872bbf2SRichard Henderson .version = 2, 60715872bbf2SRichard Henderson .ptr_size = sizeof(void *), 60725872bbf2SRichard Henderson .cu_die = 1, 60735872bbf2SRichard Henderson .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 60745872bbf2SRichard Henderson .fn_die = 2, 60755872bbf2SRichard Henderson .fn_name = "code_gen_buffer" 60765872bbf2SRichard Henderson }, 60775872bbf2SRichard Henderson .da = { 60785872bbf2SRichard Henderson 1, /* abbrev number (the cu) */ 60795872bbf2SRichard Henderson 0x11, 1, /* DW_TAG_compile_unit, has children */ 60805872bbf2SRichard Henderson 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 60815872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 60825872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 60835872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 60845872bbf2SRichard Henderson 2, /* abbrev number (the fn) */ 60855872bbf2SRichard Henderson 0x2e, 0, /* DW_TAG_subprogram, no children */ 60865872bbf2SRichard Henderson 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 60875872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 60885872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 60895872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 60905872bbf2SRichard Henderson 0 /* no more abbrev */ 60915872bbf2SRichard Henderson }, 60925872bbf2SRichard Henderson .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 60935872bbf2SRichard Henderson ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 6094813da627SRichard Henderson }; 6095813da627SRichard Henderson 6096813da627SRichard Henderson /* We only need a single jit entry; statically allocate it. */ 6097813da627SRichard Henderson static struct jit_code_entry one_entry; 6098813da627SRichard Henderson 60995872bbf2SRichard Henderson uintptr_t buf = (uintptr_t)buf_ptr; 6100813da627SRichard Henderson size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 61012c90784aSRichard Henderson DebugFrameHeader *dfh; 6102813da627SRichard Henderson 61035872bbf2SRichard Henderson img = g_malloc(img_size); 61045872bbf2SRichard Henderson *img = img_template; 6105813da627SRichard Henderson 61065872bbf2SRichard Henderson img->phdr.p_vaddr = buf; 61075872bbf2SRichard Henderson img->phdr.p_paddr = buf; 61085872bbf2SRichard Henderson img->phdr.p_memsz = buf_size; 6109813da627SRichard Henderson 61105872bbf2SRichard Henderson img->shdr[1].sh_name = find_string(img->str, ".text"); 61115872bbf2SRichard Henderson img->shdr[1].sh_addr = buf; 61125872bbf2SRichard Henderson img->shdr[1].sh_size = buf_size; 6113813da627SRichard Henderson 61145872bbf2SRichard Henderson img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 61155872bbf2SRichard Henderson img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 61165872bbf2SRichard Henderson 61175872bbf2SRichard Henderson img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 61185872bbf2SRichard Henderson img->shdr[4].sh_size = debug_frame_size; 61195872bbf2SRichard Henderson 61205872bbf2SRichard Henderson img->shdr[5].sh_name = find_string(img->str, ".symtab"); 61215872bbf2SRichard Henderson img->shdr[6].sh_name = find_string(img->str, ".strtab"); 61225872bbf2SRichard Henderson 61235872bbf2SRichard Henderson img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 61245872bbf2SRichard Henderson img->sym[1].st_value = buf; 61255872bbf2SRichard Henderson img->sym[1].st_size = buf_size; 61265872bbf2SRichard Henderson 61275872bbf2SRichard Henderson img->di.cu_low_pc = buf; 612845aba097SRichard Henderson img->di.cu_high_pc = buf + buf_size; 61295872bbf2SRichard Henderson img->di.fn_low_pc = buf; 613045aba097SRichard Henderson img->di.fn_high_pc = buf + buf_size; 6131813da627SRichard Henderson 61322c90784aSRichard Henderson dfh = (DebugFrameHeader *)(img + 1); 61332c90784aSRichard Henderson memcpy(dfh, debug_frame, debug_frame_size); 61342c90784aSRichard Henderson dfh->fde.func_start = buf; 61352c90784aSRichard Henderson dfh->fde.func_len = buf_size; 61362c90784aSRichard Henderson 6137813da627SRichard Henderson #ifdef DEBUG_JIT 6138813da627SRichard Henderson /* Enable this block to be able to debug the ELF image file creation. 6139813da627SRichard Henderson One can use readelf, objdump, or other inspection utilities. */ 6140813da627SRichard Henderson { 6141eb6b2edfSBin Meng g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 6142eb6b2edfSBin Meng FILE *f = fopen(jit, "w+b"); 6143813da627SRichard Henderson if (f) { 61445872bbf2SRichard Henderson if (fwrite(img, img_size, 1, f) != img_size) { 6145813da627SRichard Henderson /* Avoid stupid unused return value warning for fwrite. */ 6146813da627SRichard Henderson } 6147813da627SRichard Henderson fclose(f); 6148813da627SRichard Henderson } 6149813da627SRichard Henderson } 6150813da627SRichard Henderson #endif 6151813da627SRichard Henderson 6152813da627SRichard Henderson one_entry.symfile_addr = img; 6153813da627SRichard Henderson one_entry.symfile_size = img_size; 6154813da627SRichard Henderson 6155813da627SRichard Henderson __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 6156813da627SRichard Henderson __jit_debug_descriptor.relevant_entry = &one_entry; 6157813da627SRichard Henderson __jit_debug_descriptor.first_entry = &one_entry; 6158813da627SRichard Henderson __jit_debug_register_code(); 6159813da627SRichard Henderson } 6160813da627SRichard Henderson #else 61615872bbf2SRichard Henderson /* No support for the feature. Provide the entry point expected by exec.c, 61625872bbf2SRichard Henderson and implement the internal function we declared earlier. */ 6163813da627SRichard Henderson 6164755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 61652c90784aSRichard Henderson const void *debug_frame, 61662c90784aSRichard Henderson size_t debug_frame_size) 6167813da627SRichard Henderson { 6168813da627SRichard Henderson } 6169813da627SRichard Henderson 6170755bf9e5SRichard Henderson void tcg_register_jit(const void *buf, size_t buf_size) 6171813da627SRichard Henderson { 6172813da627SRichard Henderson } 6173813da627SRichard Henderson #endif /* ELF_HOST_MACHINE */ 6174db432672SRichard Henderson 6175db432672SRichard Henderson #if !TCG_TARGET_MAYBE_vec 6176db432672SRichard Henderson void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 6177db432672SRichard Henderson { 6178db432672SRichard Henderson g_assert_not_reached(); 6179db432672SRichard Henderson } 6180db432672SRichard Henderson #endif 6181