1c896fe29Sbellard /* 2c896fe29Sbellard * Tiny Code Generator for QEMU 3c896fe29Sbellard * 4c896fe29Sbellard * Copyright (c) 2008 Fabrice Bellard 5c896fe29Sbellard * 6c896fe29Sbellard * Permission is hereby granted, free of charge, to any person obtaining a copy 7c896fe29Sbellard * of this software and associated documentation files (the "Software"), to deal 8c896fe29Sbellard * in the Software without restriction, including without limitation the rights 9c896fe29Sbellard * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10c896fe29Sbellard * copies of the Software, and to permit persons to whom the Software is 11c896fe29Sbellard * furnished to do so, subject to the following conditions: 12c896fe29Sbellard * 13c896fe29Sbellard * The above copyright notice and this permission notice shall be included in 14c896fe29Sbellard * all copies or substantial portions of the Software. 15c896fe29Sbellard * 16c896fe29Sbellard * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17c896fe29Sbellard * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18c896fe29Sbellard * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19c896fe29Sbellard * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20c896fe29Sbellard * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21c896fe29Sbellard * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22c896fe29Sbellard * THE SOFTWARE. 23c896fe29Sbellard */ 24c896fe29Sbellard 25c896fe29Sbellard /* define it to use liveness analysis (better code) */ 268f2e8c07SKirill Batuzov #define USE_TCG_OPTIMIZATIONS 27c896fe29Sbellard 28757e725bSPeter Maydell #include "qemu/osdep.h" 29cca82982Saurel32 30813da627SRichard Henderson /* Define to jump the ELF file used to communicate with GDB. */ 31813da627SRichard Henderson #undef DEBUG_JIT 32813da627SRichard Henderson 3372fd2efbSEmilio G. Cota #include "qemu/error-report.h" 34f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 351de7afc9SPaolo Bonzini #include "qemu/host-utils.h" 36d4c51a0aSMarkus Armbruster #include "qemu/qemu-print.h" 37084cfca1SRichard Henderson #include "qemu/cacheflush.h" 38ad768e6fSPeter Maydell #include "qemu/cacheinfo.h" 39533206f0SRichard W.M. Jones #include "qemu/timer.h" 40c896fe29Sbellard 41c5d3c498SStefan Weil /* Note: the long term plan is to reduce the dependencies on the QEMU 42c896fe29Sbellard CPU definitions. Currently they are used for qemu_ld/st 43c896fe29Sbellard instructions */ 44c896fe29Sbellard #define NO_CPU_IO_DEFS 45c896fe29Sbellard 4663c91552SPaolo Bonzini #include "exec/exec-all.h" 47dcb32f1dSPhilippe Mathieu-Daudé #include "tcg/tcg-op.h" 48813da627SRichard Henderson 49edee2579SRichard Henderson #if UINTPTR_MAX == UINT32_MAX 50813da627SRichard Henderson # define ELF_CLASS ELFCLASS32 51edee2579SRichard Henderson #else 52edee2579SRichard Henderson # define ELF_CLASS ELFCLASS64 53813da627SRichard Henderson #endif 54e03b5686SMarc-André Lureau #if HOST_BIG_ENDIAN 55813da627SRichard Henderson # define ELF_DATA ELFDATA2MSB 56813da627SRichard Henderson #else 57813da627SRichard Henderson # define ELF_DATA ELFDATA2LSB 58813da627SRichard Henderson #endif 59813da627SRichard Henderson 60c896fe29Sbellard #include "elf.h" 61508127e2SPaolo Bonzini #include "exec/log.h" 62d2ba8026SRichard Henderson #include "tcg/tcg-ldst.h" 635ff7258cSRichard Henderson #include "tcg-internal.h" 645584e2dbSIlya Leoshkevich #include "accel/tcg/perf.h" 65c896fe29Sbellard 66139c1837SPaolo Bonzini /* Forward declarations for functions declared in tcg-target.c.inc and 67ce151109SPeter Maydell used here. */ 68e4d58b41SRichard Henderson static void tcg_target_init(TCGContext *s); 69e4d58b41SRichard Henderson static void tcg_target_qemu_prologue(TCGContext *s); 706ac17786SRichard Henderson static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 712ba7fae2SRichard Henderson intptr_t value, intptr_t addend); 72c896fe29Sbellard 73497a22ebSRichard Henderson /* The CIE and FDE header definitions will be common to all hosts. */ 74497a22ebSRichard Henderson typedef struct { 75497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 76497a22ebSRichard Henderson uint32_t id; 77497a22ebSRichard Henderson uint8_t version; 78497a22ebSRichard Henderson char augmentation[1]; 79497a22ebSRichard Henderson uint8_t code_align; 80497a22ebSRichard Henderson uint8_t data_align; 81497a22ebSRichard Henderson uint8_t return_column; 82497a22ebSRichard Henderson } DebugFrameCIE; 83497a22ebSRichard Henderson 84497a22ebSRichard Henderson typedef struct QEMU_PACKED { 85497a22ebSRichard Henderson uint32_t len __attribute__((aligned((sizeof(void *))))); 86497a22ebSRichard Henderson uint32_t cie_offset; 87edee2579SRichard Henderson uintptr_t func_start; 88edee2579SRichard Henderson uintptr_t func_len; 89497a22ebSRichard Henderson } DebugFrameFDEHeader; 90497a22ebSRichard Henderson 912c90784aSRichard Henderson typedef struct QEMU_PACKED { 922c90784aSRichard Henderson DebugFrameCIE cie; 932c90784aSRichard Henderson DebugFrameFDEHeader fde; 942c90784aSRichard Henderson } DebugFrameHeader; 952c90784aSRichard Henderson 96755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 972c90784aSRichard Henderson const void *debug_frame, 982c90784aSRichard Henderson size_t debug_frame_size) 99813da627SRichard Henderson __attribute__((unused)); 100813da627SRichard Henderson 101139c1837SPaolo Bonzini /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 1022a534affSRichard Henderson static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 103a05b5b9bSRichard Henderson intptr_t arg2); 10478113e83SRichard Henderson static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 105c0ad3001SStefan Weil static void tcg_out_movi(TCGContext *s, TCGType type, 1062a534affSRichard Henderson TCGReg ret, tcg_target_long arg); 107313bdea8SRichard Henderson static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); 108b55a8d9dSRichard Henderson static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); 109cf7d6b8eSRichard Henderson static void tcg_out_goto_tb(TCGContext *s, int which); 1105e8892dbSMiroslav Rezanina static void tcg_out_op(TCGContext *s, TCGOpcode opc, 1115e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1125e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 113d2fd745fSRichard Henderson #if TCG_TARGET_MAYBE_vec 114e7632cfaSRichard Henderson static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 115e7632cfaSRichard Henderson TCGReg dst, TCGReg src); 116d6ecb4a9SRichard Henderson static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 117d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset); 1184e186175SRichard Henderson static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1194e186175SRichard Henderson TCGReg dst, int64_t arg); 1205e8892dbSMiroslav Rezanina static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1215e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1225e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1235e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]); 124d2fd745fSRichard Henderson #else 125e7632cfaSRichard Henderson static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 126e7632cfaSRichard Henderson TCGReg dst, TCGReg src) 127e7632cfaSRichard Henderson { 128e7632cfaSRichard Henderson g_assert_not_reached(); 129e7632cfaSRichard Henderson } 130d6ecb4a9SRichard Henderson static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 131d6ecb4a9SRichard Henderson TCGReg dst, TCGReg base, intptr_t offset) 132d6ecb4a9SRichard Henderson { 133d6ecb4a9SRichard Henderson g_assert_not_reached(); 134d6ecb4a9SRichard Henderson } 1354e186175SRichard Henderson static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1364e186175SRichard Henderson TCGReg dst, int64_t arg) 137e7632cfaSRichard Henderson { 138e7632cfaSRichard Henderson g_assert_not_reached(); 139e7632cfaSRichard Henderson } 1405e8892dbSMiroslav Rezanina static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 1415e8892dbSMiroslav Rezanina unsigned vecl, unsigned vece, 1425e8892dbSMiroslav Rezanina const TCGArg args[TCG_MAX_OP_ARGS], 1435e8892dbSMiroslav Rezanina const int const_args[TCG_MAX_OP_ARGS]) 144d2fd745fSRichard Henderson { 145d2fd745fSRichard Henderson g_assert_not_reached(); 146d2fd745fSRichard Henderson } 147d2fd745fSRichard Henderson #endif 1482a534affSRichard Henderson static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 149a05b5b9bSRichard Henderson intptr_t arg2); 15059d7c14eSRichard Henderson static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 15159d7c14eSRichard Henderson TCGReg base, intptr_t ofs); 1527b7d8b2dSRichard Henderson static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 153cee44b03SRichard Henderson const TCGHelperInfo *info); 1545e3d0c19SRichard Henderson static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); 155a4fbbd77SRichard Henderson static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 156659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 157aeee05f5SRichard Henderson static int tcg_out_ldst_finalize(TCGContext *s); 158659ef5cbSRichard Henderson #endif 159c896fe29Sbellard 16042eb6dfcSRichard Henderson TCGContext tcg_init_ctx; 16142eb6dfcSRichard Henderson __thread TCGContext *tcg_ctx; 16242eb6dfcSRichard Henderson 1635ff7258cSRichard Henderson TCGContext **tcg_ctxs; 1640e2d61cfSRichard Henderson unsigned int tcg_cur_ctxs; 1650e2d61cfSRichard Henderson unsigned int tcg_max_ctxs; 1661c2adb95SRichard Henderson TCGv_env cpu_env = 0; 167c8bc1168SRichard Henderson const void *tcg_code_gen_epilogue; 168db0c51a3SRichard Henderson uintptr_t tcg_splitwx_diff; 169df2cce29SEmilio G. Cota 170b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 171b91ccb31SRichard Henderson tcg_prologue_fn *tcg_qemu_tb_exec; 172b91ccb31SRichard Henderson #endif 173b91ccb31SRichard Henderson 174d2fd745fSRichard Henderson static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 175b1d8e52eSblueswir1 static TCGRegSet tcg_target_call_clobber_regs; 176c896fe29Sbellard 1771813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE == 1 1784196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 179c896fe29Sbellard { 180c896fe29Sbellard *s->code_ptr++ = v; 181c896fe29Sbellard } 182c896fe29Sbellard 1834196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 1844196dca6SPeter Maydell uint8_t v) 1855c53bb81SPeter Maydell { 1861813e175SRichard Henderson *p = v; 1875c53bb81SPeter Maydell } 1881813e175SRichard Henderson #endif 1895c53bb81SPeter Maydell 1901813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 2 1914196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 192c896fe29Sbellard { 1931813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 1941813e175SRichard Henderson *s->code_ptr++ = v; 1951813e175SRichard Henderson } else { 1961813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 1974387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 1981813e175SRichard Henderson s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 1991813e175SRichard Henderson } 200c896fe29Sbellard } 201c896fe29Sbellard 2024196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 2034196dca6SPeter Maydell uint16_t v) 2045c53bb81SPeter Maydell { 2051813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 2061813e175SRichard Henderson *p = v; 2071813e175SRichard Henderson } else { 2085c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2095c53bb81SPeter Maydell } 2101813e175SRichard Henderson } 2111813e175SRichard Henderson #endif 2125c53bb81SPeter Maydell 2131813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 4 2144196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 215c896fe29Sbellard { 2161813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2171813e175SRichard Henderson *s->code_ptr++ = v; 2181813e175SRichard Henderson } else { 2191813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2204387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2211813e175SRichard Henderson s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 2221813e175SRichard Henderson } 223c896fe29Sbellard } 224c896fe29Sbellard 2254196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 2264196dca6SPeter Maydell uint32_t v) 2275c53bb81SPeter Maydell { 2281813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 2291813e175SRichard Henderson *p = v; 2301813e175SRichard Henderson } else { 2315c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2325c53bb81SPeter Maydell } 2331813e175SRichard Henderson } 2341813e175SRichard Henderson #endif 2355c53bb81SPeter Maydell 2361813e175SRichard Henderson #if TCG_TARGET_INSN_UNIT_SIZE <= 8 2374196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 238ac26eb69SRichard Henderson { 2391813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2401813e175SRichard Henderson *s->code_ptr++ = v; 2411813e175SRichard Henderson } else { 2421813e175SRichard Henderson tcg_insn_unit *p = s->code_ptr; 2434387345aSPeter Maydell memcpy(p, &v, sizeof(v)); 2441813e175SRichard Henderson s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 2451813e175SRichard Henderson } 246ac26eb69SRichard Henderson } 247ac26eb69SRichard Henderson 2484196dca6SPeter Maydell static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 2494196dca6SPeter Maydell uint64_t v) 2505c53bb81SPeter Maydell { 2511813e175SRichard Henderson if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 2521813e175SRichard Henderson *p = v; 2531813e175SRichard Henderson } else { 2545c53bb81SPeter Maydell memcpy(p, &v, sizeof(v)); 2555c53bb81SPeter Maydell } 2561813e175SRichard Henderson } 2571813e175SRichard Henderson #endif 2585c53bb81SPeter Maydell 259c896fe29Sbellard /* label relocation processing */ 260c896fe29Sbellard 2611813e175SRichard Henderson static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 262bec16311SRichard Henderson TCGLabel *l, intptr_t addend) 263c896fe29Sbellard { 2647ecd02a0SRichard Henderson TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 265c896fe29Sbellard 266c896fe29Sbellard r->type = type; 267c896fe29Sbellard r->ptr = code_ptr; 268c896fe29Sbellard r->addend = addend; 2697ecd02a0SRichard Henderson QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 270c896fe29Sbellard } 271c896fe29Sbellard 27292ab8e7dSRichard Henderson static void tcg_out_label(TCGContext *s, TCGLabel *l) 273c896fe29Sbellard { 274eabb7b91SAurelien Jarno tcg_debug_assert(!l->has_value); 275c896fe29Sbellard l->has_value = 1; 27692ab8e7dSRichard Henderson l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 277c896fe29Sbellard } 278c896fe29Sbellard 27942a268c2SRichard Henderson TCGLabel *gen_new_label(void) 280c896fe29Sbellard { 281b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 28251e3972cSRichard Henderson TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 283c896fe29Sbellard 2847ecd02a0SRichard Henderson memset(l, 0, sizeof(TCGLabel)); 2857ecd02a0SRichard Henderson l->id = s->nb_labels++; 286f85b1fc4SRichard Henderson QSIMPLEQ_INIT(&l->branches); 2877ecd02a0SRichard Henderson QSIMPLEQ_INIT(&l->relocs); 2887ecd02a0SRichard Henderson 289bef16ab4SRichard Henderson QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 29042a268c2SRichard Henderson 29142a268c2SRichard Henderson return l; 292c896fe29Sbellard } 293c896fe29Sbellard 2947ecd02a0SRichard Henderson static bool tcg_resolve_relocs(TCGContext *s) 2957ecd02a0SRichard Henderson { 2967ecd02a0SRichard Henderson TCGLabel *l; 2977ecd02a0SRichard Henderson 2987ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 2997ecd02a0SRichard Henderson TCGRelocation *r; 3007ecd02a0SRichard Henderson uintptr_t value = l->u.value; 3017ecd02a0SRichard Henderson 3027ecd02a0SRichard Henderson QSIMPLEQ_FOREACH(r, &l->relocs, next) { 3037ecd02a0SRichard Henderson if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 3047ecd02a0SRichard Henderson return false; 3057ecd02a0SRichard Henderson } 3067ecd02a0SRichard Henderson } 3077ecd02a0SRichard Henderson } 3087ecd02a0SRichard Henderson return true; 3097ecd02a0SRichard Henderson } 3107ecd02a0SRichard Henderson 3119f754620SRichard Henderson static void set_jmp_reset_offset(TCGContext *s, int which) 3129f754620SRichard Henderson { 313f14bed3fSRichard Henderson /* 314f14bed3fSRichard Henderson * We will check for overflow at the end of the opcode loop in 315f14bed3fSRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 316f14bed3fSRichard Henderson */ 317b7e4afbdSRichard Henderson s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); 3189f754620SRichard Henderson } 3199f754620SRichard Henderson 320b52a2c03SRichard Henderson static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) 321b52a2c03SRichard Henderson { 322b52a2c03SRichard Henderson /* 323b52a2c03SRichard Henderson * We will check for overflow at the end of the opcode loop in 324b52a2c03SRichard Henderson * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 325b52a2c03SRichard Henderson */ 3269da6079bSRichard Henderson s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); 327b52a2c03SRichard Henderson } 328b52a2c03SRichard Henderson 329becc452aSRichard Henderson static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) 330becc452aSRichard Henderson { 331becc452aSRichard Henderson /* 332becc452aSRichard Henderson * Return the read-execute version of the pointer, for the benefit 333becc452aSRichard Henderson * of any pc-relative addressing mode. 334becc452aSRichard Henderson */ 3359da6079bSRichard Henderson return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); 336becc452aSRichard Henderson } 337becc452aSRichard Henderson 338db6b7d0cSRichard Henderson /* Signal overflow, starting over with fewer guest insns. */ 3398905770bSMarc-André Lureau static G_NORETURN 3408905770bSMarc-André Lureau void tcg_raise_tb_overflow(TCGContext *s) 341db6b7d0cSRichard Henderson { 342db6b7d0cSRichard Henderson siglongjmp(s->jmp_trans, -2); 343db6b7d0cSRichard Henderson } 344db6b7d0cSRichard Henderson 3454c22e840SRichard Henderson #define C_PFX1(P, A) P##A 3464c22e840SRichard Henderson #define C_PFX2(P, A, B) P##A##_##B 3474c22e840SRichard Henderson #define C_PFX3(P, A, B, C) P##A##_##B##_##C 3484c22e840SRichard Henderson #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 3494c22e840SRichard Henderson #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 3504c22e840SRichard Henderson #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 3514c22e840SRichard Henderson 3524c22e840SRichard Henderson /* Define an enumeration for the various combinations. */ 3534c22e840SRichard Henderson 3544c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 3554c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 3564c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 3574c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 3584c22e840SRichard Henderson 3594c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 3604c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 3614c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 3624c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 3634c22e840SRichard Henderson 3644c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 3654c22e840SRichard Henderson 3664c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 3674c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 3684c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 3694c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 3704c22e840SRichard Henderson 3714c22e840SRichard Henderson typedef enum { 3724c22e840SRichard Henderson #include "tcg-target-con-set.h" 3734c22e840SRichard Henderson } TCGConstraintSetIndex; 3744c22e840SRichard Henderson 3754c22e840SRichard Henderson static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 3764c22e840SRichard Henderson 3774c22e840SRichard Henderson #undef C_O0_I1 3784c22e840SRichard Henderson #undef C_O0_I2 3794c22e840SRichard Henderson #undef C_O0_I3 3804c22e840SRichard Henderson #undef C_O0_I4 3814c22e840SRichard Henderson #undef C_O1_I1 3824c22e840SRichard Henderson #undef C_O1_I2 3834c22e840SRichard Henderson #undef C_O1_I3 3844c22e840SRichard Henderson #undef C_O1_I4 3854c22e840SRichard Henderson #undef C_N1_I2 3864c22e840SRichard Henderson #undef C_O2_I1 3874c22e840SRichard Henderson #undef C_O2_I2 3884c22e840SRichard Henderson #undef C_O2_I3 3894c22e840SRichard Henderson #undef C_O2_I4 3904c22e840SRichard Henderson 3914c22e840SRichard Henderson /* Put all of the constraint sets into an array, indexed by the enum. */ 3924c22e840SRichard Henderson 3934c22e840SRichard Henderson #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 3944c22e840SRichard Henderson #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 3954c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 3964c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 3974c22e840SRichard Henderson 3984c22e840SRichard Henderson #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 3994c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 4004c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 4014c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 4024c22e840SRichard Henderson 4034c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 4044c22e840SRichard Henderson 4054c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 4064c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 4074c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 4084c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 4094c22e840SRichard Henderson 4104c22e840SRichard Henderson static const TCGTargetOpDef constraint_sets[] = { 4114c22e840SRichard Henderson #include "tcg-target-con-set.h" 4124c22e840SRichard Henderson }; 4134c22e840SRichard Henderson 4144c22e840SRichard Henderson 4154c22e840SRichard Henderson #undef C_O0_I1 4164c22e840SRichard Henderson #undef C_O0_I2 4174c22e840SRichard Henderson #undef C_O0_I3 4184c22e840SRichard Henderson #undef C_O0_I4 4194c22e840SRichard Henderson #undef C_O1_I1 4204c22e840SRichard Henderson #undef C_O1_I2 4214c22e840SRichard Henderson #undef C_O1_I3 4224c22e840SRichard Henderson #undef C_O1_I4 4234c22e840SRichard Henderson #undef C_N1_I2 4244c22e840SRichard Henderson #undef C_O2_I1 4254c22e840SRichard Henderson #undef C_O2_I2 4264c22e840SRichard Henderson #undef C_O2_I3 4274c22e840SRichard Henderson #undef C_O2_I4 4284c22e840SRichard Henderson 4294c22e840SRichard Henderson /* Expand the enumerator to be returned from tcg_target_op_def(). */ 4304c22e840SRichard Henderson 4314c22e840SRichard Henderson #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 4324c22e840SRichard Henderson #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 4334c22e840SRichard Henderson #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 4344c22e840SRichard Henderson #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 4354c22e840SRichard Henderson 4364c22e840SRichard Henderson #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 4374c22e840SRichard Henderson #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 4384c22e840SRichard Henderson #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 4394c22e840SRichard Henderson #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 4404c22e840SRichard Henderson 4414c22e840SRichard Henderson #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 4424c22e840SRichard Henderson 4434c22e840SRichard Henderson #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 4444c22e840SRichard Henderson #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 4454c22e840SRichard Henderson #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 4464c22e840SRichard Henderson #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 4474c22e840SRichard Henderson 448139c1837SPaolo Bonzini #include "tcg-target.c.inc" 449c896fe29Sbellard 45038b47b19SEmilio G. Cota static void alloc_tcg_plugin_context(TCGContext *s) 45138b47b19SEmilio G. Cota { 45238b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 45338b47b19SEmilio G. Cota s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 45438b47b19SEmilio G. Cota s->plugin_tb->insns = 45538b47b19SEmilio G. Cota g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 45638b47b19SEmilio G. Cota #endif 45738b47b19SEmilio G. Cota } 45838b47b19SEmilio G. Cota 459e8feb96fSEmilio G. Cota /* 4603468b59eSEmilio G. Cota * All TCG threads except the parent (i.e. the one that called tcg_context_init 4613468b59eSEmilio G. Cota * and registered the target's TCG globals) must register with this function 4623468b59eSEmilio G. Cota * before initiating translation. 4633468b59eSEmilio G. Cota * 4643468b59eSEmilio G. Cota * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 4653468b59eSEmilio G. Cota * of tcg_region_init() for the reasoning behind this. 4663468b59eSEmilio G. Cota * 4673468b59eSEmilio G. Cota * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 4683468b59eSEmilio G. Cota * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 4693468b59eSEmilio G. Cota * is not used anymore for translation once this function is called. 4703468b59eSEmilio G. Cota * 4713468b59eSEmilio G. Cota * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 4723468b59eSEmilio G. Cota * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 4733468b59eSEmilio G. Cota */ 4743468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 4753468b59eSEmilio G. Cota void tcg_register_thread(void) 4763468b59eSEmilio G. Cota { 4773468b59eSEmilio G. Cota tcg_ctx = &tcg_init_ctx; 4783468b59eSEmilio G. Cota } 4793468b59eSEmilio G. Cota #else 4803468b59eSEmilio G. Cota void tcg_register_thread(void) 4813468b59eSEmilio G. Cota { 4823468b59eSEmilio G. Cota TCGContext *s = g_malloc(sizeof(*s)); 4833468b59eSEmilio G. Cota unsigned int i, n; 4843468b59eSEmilio G. Cota 4853468b59eSEmilio G. Cota *s = tcg_init_ctx; 4863468b59eSEmilio G. Cota 4873468b59eSEmilio G. Cota /* Relink mem_base. */ 4883468b59eSEmilio G. Cota for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 4893468b59eSEmilio G. Cota if (tcg_init_ctx.temps[i].mem_base) { 4903468b59eSEmilio G. Cota ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 4913468b59eSEmilio G. Cota tcg_debug_assert(b >= 0 && b < n); 4923468b59eSEmilio G. Cota s->temps[i].mem_base = &s->temps[b]; 4933468b59eSEmilio G. Cota } 4943468b59eSEmilio G. Cota } 4953468b59eSEmilio G. Cota 4963468b59eSEmilio G. Cota /* Claim an entry in tcg_ctxs */ 4970e2d61cfSRichard Henderson n = qatomic_fetch_inc(&tcg_cur_ctxs); 4980e2d61cfSRichard Henderson g_assert(n < tcg_max_ctxs); 499d73415a3SStefan Hajnoczi qatomic_set(&tcg_ctxs[n], s); 5003468b59eSEmilio G. Cota 50138b47b19SEmilio G. Cota if (n > 0) { 50238b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 503bf042e8eSRichard Henderson tcg_region_initial_alloc(s); 50438b47b19SEmilio G. Cota } 50538b47b19SEmilio G. Cota 5063468b59eSEmilio G. Cota tcg_ctx = s; 5073468b59eSEmilio G. Cota } 5083468b59eSEmilio G. Cota #endif /* !CONFIG_USER_ONLY */ 5093468b59eSEmilio G. Cota 510c896fe29Sbellard /* pool based memory allocation */ 511c896fe29Sbellard void *tcg_malloc_internal(TCGContext *s, int size) 512c896fe29Sbellard { 513c896fe29Sbellard TCGPool *p; 514c896fe29Sbellard int pool_size; 515c896fe29Sbellard 516c896fe29Sbellard if (size > TCG_POOL_CHUNK_SIZE) { 517c896fe29Sbellard /* big malloc: insert a new pool (XXX: could optimize) */ 5187267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + size); 519c896fe29Sbellard p->size = size; 5204055299eSKirill Batuzov p->next = s->pool_first_large; 5214055299eSKirill Batuzov s->pool_first_large = p; 5224055299eSKirill Batuzov return p->data; 523c896fe29Sbellard } else { 524c896fe29Sbellard p = s->pool_current; 525c896fe29Sbellard if (!p) { 526c896fe29Sbellard p = s->pool_first; 527c896fe29Sbellard if (!p) 528c896fe29Sbellard goto new_pool; 529c896fe29Sbellard } else { 530c896fe29Sbellard if (!p->next) { 531c896fe29Sbellard new_pool: 532c896fe29Sbellard pool_size = TCG_POOL_CHUNK_SIZE; 5337267c094SAnthony Liguori p = g_malloc(sizeof(TCGPool) + pool_size); 534c896fe29Sbellard p->size = pool_size; 535c896fe29Sbellard p->next = NULL; 536a813e36fSRichard Henderson if (s->pool_current) { 537c896fe29Sbellard s->pool_current->next = p; 538a813e36fSRichard Henderson } else { 539c896fe29Sbellard s->pool_first = p; 540a813e36fSRichard Henderson } 541c896fe29Sbellard } else { 542c896fe29Sbellard p = p->next; 543c896fe29Sbellard } 544c896fe29Sbellard } 545c896fe29Sbellard } 546c896fe29Sbellard s->pool_current = p; 547c896fe29Sbellard s->pool_cur = p->data + size; 548c896fe29Sbellard s->pool_end = p->data + p->size; 549c896fe29Sbellard return p->data; 550c896fe29Sbellard } 551c896fe29Sbellard 552c896fe29Sbellard void tcg_pool_reset(TCGContext *s) 553c896fe29Sbellard { 5544055299eSKirill Batuzov TCGPool *p, *t; 5554055299eSKirill Batuzov for (p = s->pool_first_large; p; p = t) { 5564055299eSKirill Batuzov t = p->next; 5574055299eSKirill Batuzov g_free(p); 5584055299eSKirill Batuzov } 5594055299eSKirill Batuzov s->pool_first_large = NULL; 560c896fe29Sbellard s->pool_cur = s->pool_end = NULL; 561c896fe29Sbellard s->pool_current = NULL; 562c896fe29Sbellard } 563c896fe29Sbellard 5642ef6175aSRichard Henderson #include "exec/helper-proto.h" 5652ef6175aSRichard Henderson 56639004a71SRichard Henderson static TCGHelperInfo all_helpers[] = { 5672ef6175aSRichard Henderson #include "exec/helper-tcg.h" 568100b5e01SRichard Henderson }; 569619205fdSEmilio G. Cota static GHashTable *helper_table; 570100b5e01SRichard Henderson 57122f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 572c6ef8c7bSPhilippe Mathieu-Daudé static ffi_type *typecode_to_ffi(int argmask) 573c6ef8c7bSPhilippe Mathieu-Daudé { 574e9709e17SRichard Henderson /* 575e9709e17SRichard Henderson * libffi does not support __int128_t, so we have forced Int128 576e9709e17SRichard Henderson * to use the structure definition instead of the builtin type. 577e9709e17SRichard Henderson */ 578e9709e17SRichard Henderson static ffi_type *ffi_type_i128_elements[3] = { 579e9709e17SRichard Henderson &ffi_type_uint64, 580e9709e17SRichard Henderson &ffi_type_uint64, 581e9709e17SRichard Henderson NULL 582e9709e17SRichard Henderson }; 583e9709e17SRichard Henderson static ffi_type ffi_type_i128 = { 584e9709e17SRichard Henderson .size = 16, 585e9709e17SRichard Henderson .alignment = __alignof__(Int128), 586e9709e17SRichard Henderson .type = FFI_TYPE_STRUCT, 587e9709e17SRichard Henderson .elements = ffi_type_i128_elements, 588e9709e17SRichard Henderson }; 589e9709e17SRichard Henderson 590c6ef8c7bSPhilippe Mathieu-Daudé switch (argmask) { 591c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_void: 592c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_void; 593c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i32: 594c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint32; 595c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s32: 596c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint32; 597c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_i64: 598c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_uint64; 599c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_s64: 600c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_sint64; 601c6ef8c7bSPhilippe Mathieu-Daudé case dh_typecode_ptr: 602c6ef8c7bSPhilippe Mathieu-Daudé return &ffi_type_pointer; 603e9709e17SRichard Henderson case dh_typecode_i128: 604e9709e17SRichard Henderson return &ffi_type_i128; 605c6ef8c7bSPhilippe Mathieu-Daudé } 606c6ef8c7bSPhilippe Mathieu-Daudé g_assert_not_reached(); 607c6ef8c7bSPhilippe Mathieu-Daudé } 6080c22e176SPhilippe Mathieu-Daudé 6090c22e176SPhilippe Mathieu-Daudé static void init_ffi_layouts(void) 6100c22e176SPhilippe Mathieu-Daudé { 6110c22e176SPhilippe Mathieu-Daudé /* g_direct_hash/equal for direct comparisons on uint32_t. */ 612f9c4bb80SRichard Henderson GHashTable *ffi_table = g_hash_table_new(NULL, NULL); 613f9c4bb80SRichard Henderson 6140c22e176SPhilippe Mathieu-Daudé for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 615f9c4bb80SRichard Henderson TCGHelperInfo *info = &all_helpers[i]; 616f9c4bb80SRichard Henderson unsigned typemask = info->typemask; 6170c22e176SPhilippe Mathieu-Daudé gpointer hash = (gpointer)(uintptr_t)typemask; 6180c22e176SPhilippe Mathieu-Daudé struct { 6190c22e176SPhilippe Mathieu-Daudé ffi_cif cif; 6200c22e176SPhilippe Mathieu-Daudé ffi_type *args[]; 6210c22e176SPhilippe Mathieu-Daudé } *ca; 6220c22e176SPhilippe Mathieu-Daudé ffi_status status; 6230c22e176SPhilippe Mathieu-Daudé int nargs; 624f9c4bb80SRichard Henderson ffi_cif *cif; 6250c22e176SPhilippe Mathieu-Daudé 626f9c4bb80SRichard Henderson cif = g_hash_table_lookup(ffi_table, hash); 627f9c4bb80SRichard Henderson if (cif) { 628f9c4bb80SRichard Henderson info->cif = cif; 6290c22e176SPhilippe Mathieu-Daudé continue; 6300c22e176SPhilippe Mathieu-Daudé } 6310c22e176SPhilippe Mathieu-Daudé 6320c22e176SPhilippe Mathieu-Daudé /* Ignoring the return type, find the last non-zero field. */ 6330c22e176SPhilippe Mathieu-Daudé nargs = 32 - clz32(typemask >> 3); 6340c22e176SPhilippe Mathieu-Daudé nargs = DIV_ROUND_UP(nargs, 3); 635e9709e17SRichard Henderson assert(nargs <= MAX_CALL_IARGS); 6360c22e176SPhilippe Mathieu-Daudé 6370c22e176SPhilippe Mathieu-Daudé ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 6380c22e176SPhilippe Mathieu-Daudé ca->cif.rtype = typecode_to_ffi(typemask & 7); 6390c22e176SPhilippe Mathieu-Daudé ca->cif.nargs = nargs; 6400c22e176SPhilippe Mathieu-Daudé 6410c22e176SPhilippe Mathieu-Daudé if (nargs != 0) { 6420c22e176SPhilippe Mathieu-Daudé ca->cif.arg_types = ca->args; 6430c22e176SPhilippe Mathieu-Daudé for (int j = 0; j < nargs; ++j) { 6440c22e176SPhilippe Mathieu-Daudé int typecode = extract32(typemask, (j + 1) * 3, 3); 6450c22e176SPhilippe Mathieu-Daudé ca->args[j] = typecode_to_ffi(typecode); 6460c22e176SPhilippe Mathieu-Daudé } 6470c22e176SPhilippe Mathieu-Daudé } 6480c22e176SPhilippe Mathieu-Daudé 6490c22e176SPhilippe Mathieu-Daudé status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 6500c22e176SPhilippe Mathieu-Daudé ca->cif.rtype, ca->cif.arg_types); 6510c22e176SPhilippe Mathieu-Daudé assert(status == FFI_OK); 6520c22e176SPhilippe Mathieu-Daudé 653f9c4bb80SRichard Henderson cif = &ca->cif; 654f9c4bb80SRichard Henderson info->cif = cif; 655f9c4bb80SRichard Henderson g_hash_table_insert(ffi_table, hash, (gpointer)cif); 6560c22e176SPhilippe Mathieu-Daudé } 657f9c4bb80SRichard Henderson 658f9c4bb80SRichard Henderson g_hash_table_destroy(ffi_table); 6590c22e176SPhilippe Mathieu-Daudé } 6600c22e176SPhilippe Mathieu-Daudé #endif /* CONFIG_TCG_INTERPRETER */ 66122f15579SRichard Henderson 66239004a71SRichard Henderson typedef struct TCGCumulativeArgs { 66339004a71SRichard Henderson int arg_idx; /* tcg_gen_callN args[] */ 66439004a71SRichard Henderson int info_in_idx; /* TCGHelperInfo in[] */ 66539004a71SRichard Henderson int arg_slot; /* regs+stack slot */ 66639004a71SRichard Henderson int ref_slot; /* stack slots for references */ 66739004a71SRichard Henderson } TCGCumulativeArgs; 66839004a71SRichard Henderson 66939004a71SRichard Henderson static void layout_arg_even(TCGCumulativeArgs *cum) 67039004a71SRichard Henderson { 67139004a71SRichard Henderson cum->arg_slot += cum->arg_slot & 1; 67239004a71SRichard Henderson } 67339004a71SRichard Henderson 67439004a71SRichard Henderson static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, 67539004a71SRichard Henderson TCGCallArgumentKind kind) 67639004a71SRichard Henderson { 67739004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 67839004a71SRichard Henderson 67939004a71SRichard Henderson *loc = (TCGCallArgumentLoc){ 68039004a71SRichard Henderson .kind = kind, 68139004a71SRichard Henderson .arg_idx = cum->arg_idx, 68239004a71SRichard Henderson .arg_slot = cum->arg_slot, 68339004a71SRichard Henderson }; 68439004a71SRichard Henderson cum->info_in_idx++; 68539004a71SRichard Henderson cum->arg_slot++; 68639004a71SRichard Henderson } 68739004a71SRichard Henderson 68839004a71SRichard Henderson static void layout_arg_normal_n(TCGCumulativeArgs *cum, 68939004a71SRichard Henderson TCGHelperInfo *info, int n) 69039004a71SRichard Henderson { 69139004a71SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 69239004a71SRichard Henderson 69339004a71SRichard Henderson for (int i = 0; i < n; ++i) { 69439004a71SRichard Henderson /* Layout all using the same arg_idx, adjusting the subindex. */ 69539004a71SRichard Henderson loc[i] = (TCGCallArgumentLoc){ 69639004a71SRichard Henderson .kind = TCG_CALL_ARG_NORMAL, 69739004a71SRichard Henderson .arg_idx = cum->arg_idx, 69839004a71SRichard Henderson .tmp_subindex = i, 69939004a71SRichard Henderson .arg_slot = cum->arg_slot + i, 70039004a71SRichard Henderson }; 70139004a71SRichard Henderson } 70239004a71SRichard Henderson cum->info_in_idx += n; 70339004a71SRichard Henderson cum->arg_slot += n; 70439004a71SRichard Henderson } 70539004a71SRichard Henderson 706313bdea8SRichard Henderson static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info) 707313bdea8SRichard Henderson { 708313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 709313bdea8SRichard Henderson int n = 128 / TCG_TARGET_REG_BITS; 710313bdea8SRichard Henderson 711313bdea8SRichard Henderson /* The first subindex carries the pointer. */ 712313bdea8SRichard Henderson layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF); 713313bdea8SRichard Henderson 714313bdea8SRichard Henderson /* 715313bdea8SRichard Henderson * The callee is allowed to clobber memory associated with 716313bdea8SRichard Henderson * structure pass by-reference. Therefore we must make copies. 717313bdea8SRichard Henderson * Allocate space from "ref_slot", which will be adjusted to 718313bdea8SRichard Henderson * follow the parameters on the stack. 719313bdea8SRichard Henderson */ 720313bdea8SRichard Henderson loc[0].ref_slot = cum->ref_slot; 721313bdea8SRichard Henderson 722313bdea8SRichard Henderson /* 723313bdea8SRichard Henderson * Subsequent words also go into the reference slot, but 724313bdea8SRichard Henderson * do not accumulate into the regular arguments. 725313bdea8SRichard Henderson */ 726313bdea8SRichard Henderson for (int i = 1; i < n; ++i) { 727313bdea8SRichard Henderson loc[i] = (TCGCallArgumentLoc){ 728313bdea8SRichard Henderson .kind = TCG_CALL_ARG_BY_REF_N, 729313bdea8SRichard Henderson .arg_idx = cum->arg_idx, 730313bdea8SRichard Henderson .tmp_subindex = i, 731313bdea8SRichard Henderson .ref_slot = cum->ref_slot + i, 732313bdea8SRichard Henderson }; 733313bdea8SRichard Henderson } 734313bdea8SRichard Henderson cum->info_in_idx += n; 735313bdea8SRichard Henderson cum->ref_slot += n; 736313bdea8SRichard Henderson } 737313bdea8SRichard Henderson 73839004a71SRichard Henderson static void init_call_layout(TCGHelperInfo *info) 73939004a71SRichard Henderson { 74039004a71SRichard Henderson int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); 74139004a71SRichard Henderson int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 74239004a71SRichard Henderson unsigned typemask = info->typemask; 74339004a71SRichard Henderson unsigned typecode; 74439004a71SRichard Henderson TCGCumulativeArgs cum = { }; 74539004a71SRichard Henderson 74639004a71SRichard Henderson /* 74739004a71SRichard Henderson * Parse and place any function return value. 74839004a71SRichard Henderson */ 74939004a71SRichard Henderson typecode = typemask & 7; 75039004a71SRichard Henderson switch (typecode) { 75139004a71SRichard Henderson case dh_typecode_void: 75239004a71SRichard Henderson info->nr_out = 0; 75339004a71SRichard Henderson break; 75439004a71SRichard Henderson case dh_typecode_i32: 75539004a71SRichard Henderson case dh_typecode_s32: 75639004a71SRichard Henderson case dh_typecode_ptr: 75739004a71SRichard Henderson info->nr_out = 1; 75839004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL; 75939004a71SRichard Henderson break; 76039004a71SRichard Henderson case dh_typecode_i64: 76139004a71SRichard Henderson case dh_typecode_s64: 76239004a71SRichard Henderson info->nr_out = 64 / TCG_TARGET_REG_BITS; 76339004a71SRichard Henderson info->out_kind = TCG_CALL_RET_NORMAL; 7645e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */ 7655e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 766466d3759SRichard Henderson break; 767466d3759SRichard Henderson case dh_typecode_i128: 768466d3759SRichard Henderson info->nr_out = 128 / TCG_TARGET_REG_BITS; 7695427a9a7SRichard Henderson info->out_kind = TCG_TARGET_CALL_RET_I128; 7705427a9a7SRichard Henderson switch (TCG_TARGET_CALL_RET_I128) { 771466d3759SRichard Henderson case TCG_CALL_RET_NORMAL: 7725e3d0c19SRichard Henderson /* Query the last register now to trigger any assert early. */ 7735e3d0c19SRichard Henderson tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 774466d3759SRichard Henderson break; 775c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC: 776c6556aa0SRichard Henderson /* Query the single register now to trigger any assert early. */ 777c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0); 778c6556aa0SRichard Henderson break; 779313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF: 780313bdea8SRichard Henderson /* 781313bdea8SRichard Henderson * Allocate the first argument to the output. 782313bdea8SRichard Henderson * We don't need to store this anywhere, just make it 783313bdea8SRichard Henderson * unavailable for use in the input loop below. 784313bdea8SRichard Henderson */ 785313bdea8SRichard Henderson cum.arg_slot = 1; 786313bdea8SRichard Henderson break; 787466d3759SRichard Henderson default: 788466d3759SRichard Henderson qemu_build_not_reached(); 789466d3759SRichard Henderson } 79039004a71SRichard Henderson break; 79139004a71SRichard Henderson default: 79239004a71SRichard Henderson g_assert_not_reached(); 79339004a71SRichard Henderson } 79439004a71SRichard Henderson 79539004a71SRichard Henderson /* 79639004a71SRichard Henderson * Parse and place function arguments. 79739004a71SRichard Henderson */ 79839004a71SRichard Henderson for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { 79939004a71SRichard Henderson TCGCallArgumentKind kind; 80039004a71SRichard Henderson TCGType type; 80139004a71SRichard Henderson 80239004a71SRichard Henderson typecode = typemask & 7; 80339004a71SRichard Henderson switch (typecode) { 80439004a71SRichard Henderson case dh_typecode_i32: 80539004a71SRichard Henderson case dh_typecode_s32: 80639004a71SRichard Henderson type = TCG_TYPE_I32; 80739004a71SRichard Henderson break; 80839004a71SRichard Henderson case dh_typecode_i64: 80939004a71SRichard Henderson case dh_typecode_s64: 81039004a71SRichard Henderson type = TCG_TYPE_I64; 81139004a71SRichard Henderson break; 81239004a71SRichard Henderson case dh_typecode_ptr: 81339004a71SRichard Henderson type = TCG_TYPE_PTR; 81439004a71SRichard Henderson break; 815466d3759SRichard Henderson case dh_typecode_i128: 816466d3759SRichard Henderson type = TCG_TYPE_I128; 817466d3759SRichard Henderson break; 81839004a71SRichard Henderson default: 81939004a71SRichard Henderson g_assert_not_reached(); 82039004a71SRichard Henderson } 82139004a71SRichard Henderson 82239004a71SRichard Henderson switch (type) { 82339004a71SRichard Henderson case TCG_TYPE_I32: 82439004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I32) { 82539004a71SRichard Henderson case TCG_CALL_ARG_EVEN: 82639004a71SRichard Henderson layout_arg_even(&cum); 82739004a71SRichard Henderson /* fall through */ 82839004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 82939004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 83039004a71SRichard Henderson break; 83139004a71SRichard Henderson case TCG_CALL_ARG_EXTEND: 83239004a71SRichard Henderson kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); 83339004a71SRichard Henderson layout_arg_1(&cum, info, kind); 83439004a71SRichard Henderson break; 83539004a71SRichard Henderson default: 83639004a71SRichard Henderson qemu_build_not_reached(); 83739004a71SRichard Henderson } 83839004a71SRichard Henderson break; 83939004a71SRichard Henderson 84039004a71SRichard Henderson case TCG_TYPE_I64: 84139004a71SRichard Henderson switch (TCG_TARGET_CALL_ARG_I64) { 84239004a71SRichard Henderson case TCG_CALL_ARG_EVEN: 84339004a71SRichard Henderson layout_arg_even(&cum); 84439004a71SRichard Henderson /* fall through */ 84539004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 84639004a71SRichard Henderson if (TCG_TARGET_REG_BITS == 32) { 84739004a71SRichard Henderson layout_arg_normal_n(&cum, info, 2); 84839004a71SRichard Henderson } else { 84939004a71SRichard Henderson layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 85039004a71SRichard Henderson } 85139004a71SRichard Henderson break; 85239004a71SRichard Henderson default: 85339004a71SRichard Henderson qemu_build_not_reached(); 85439004a71SRichard Henderson } 85539004a71SRichard Henderson break; 85639004a71SRichard Henderson 857466d3759SRichard Henderson case TCG_TYPE_I128: 8585427a9a7SRichard Henderson switch (TCG_TARGET_CALL_ARG_I128) { 859466d3759SRichard Henderson case TCG_CALL_ARG_EVEN: 860466d3759SRichard Henderson layout_arg_even(&cum); 861466d3759SRichard Henderson /* fall through */ 862466d3759SRichard Henderson case TCG_CALL_ARG_NORMAL: 863466d3759SRichard Henderson layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS); 864466d3759SRichard Henderson break; 865313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 866313bdea8SRichard Henderson layout_arg_by_ref(&cum, info); 867313bdea8SRichard Henderson break; 868466d3759SRichard Henderson default: 869466d3759SRichard Henderson qemu_build_not_reached(); 870466d3759SRichard Henderson } 871466d3759SRichard Henderson break; 872466d3759SRichard Henderson 87339004a71SRichard Henderson default: 87439004a71SRichard Henderson g_assert_not_reached(); 87539004a71SRichard Henderson } 87639004a71SRichard Henderson } 87739004a71SRichard Henderson info->nr_in = cum.info_in_idx; 87839004a71SRichard Henderson 87939004a71SRichard Henderson /* Validate that we didn't overrun the input array. */ 88039004a71SRichard Henderson assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); 88139004a71SRichard Henderson /* Validate the backend has enough argument space. */ 88239004a71SRichard Henderson assert(cum.arg_slot <= max_reg_slots + max_stk_slots); 883313bdea8SRichard Henderson 884313bdea8SRichard Henderson /* 885313bdea8SRichard Henderson * Relocate the "ref_slot" area to the end of the parameters. 886313bdea8SRichard Henderson * Minimizing this stack offset helps code size for x86, 887313bdea8SRichard Henderson * which has a signed 8-bit offset encoding. 888313bdea8SRichard Henderson */ 889313bdea8SRichard Henderson if (cum.ref_slot != 0) { 890313bdea8SRichard Henderson int ref_base = 0; 891313bdea8SRichard Henderson 892313bdea8SRichard Henderson if (cum.arg_slot > max_reg_slots) { 893313bdea8SRichard Henderson int align = __alignof(Int128) / sizeof(tcg_target_long); 894313bdea8SRichard Henderson 895313bdea8SRichard Henderson ref_base = cum.arg_slot - max_reg_slots; 896313bdea8SRichard Henderson if (align > 1) { 897313bdea8SRichard Henderson ref_base = ROUND_UP(ref_base, align); 898313bdea8SRichard Henderson } 899313bdea8SRichard Henderson } 900313bdea8SRichard Henderson assert(ref_base + cum.ref_slot <= max_stk_slots); 901313bdea8SRichard Henderson 902313bdea8SRichard Henderson if (ref_base != 0) { 903313bdea8SRichard Henderson for (int i = cum.info_in_idx - 1; i >= 0; --i) { 904313bdea8SRichard Henderson TCGCallArgumentLoc *loc = &info->in[i]; 905313bdea8SRichard Henderson switch (loc->kind) { 906313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 907313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 908313bdea8SRichard Henderson loc->ref_slot += ref_base; 909313bdea8SRichard Henderson break; 910313bdea8SRichard Henderson default: 911313bdea8SRichard Henderson break; 912313bdea8SRichard Henderson } 913313bdea8SRichard Henderson } 914313bdea8SRichard Henderson } 915313bdea8SRichard Henderson } 91639004a71SRichard Henderson } 91739004a71SRichard Henderson 91891478cefSRichard Henderson static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 919f69d277eSRichard Henderson static void process_op_defs(TCGContext *s); 9201c2adb95SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 9211c2adb95SRichard Henderson TCGReg reg, const char *name); 92291478cefSRichard Henderson 92343b972b7SRichard Henderson static void tcg_context_init(unsigned max_cpus) 924c896fe29Sbellard { 925a76aabd3SRichard Henderson TCGContext *s = &tcg_init_ctx; 926100b5e01SRichard Henderson int op, total_args, n, i; 927c896fe29Sbellard TCGOpDef *def; 928c896fe29Sbellard TCGArgConstraint *args_ct; 9291c2adb95SRichard Henderson TCGTemp *ts; 930c896fe29Sbellard 931c896fe29Sbellard memset(s, 0, sizeof(*s)); 932c896fe29Sbellard s->nb_globals = 0; 933c896fe29Sbellard 934c896fe29Sbellard /* Count total number of arguments and allocate the corresponding 935c896fe29Sbellard space */ 936c896fe29Sbellard total_args = 0; 937c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 938c896fe29Sbellard def = &tcg_op_defs[op]; 939c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 940c896fe29Sbellard total_args += n; 941c896fe29Sbellard } 942c896fe29Sbellard 943bc2b17e6SRichard Henderson args_ct = g_new0(TCGArgConstraint, total_args); 944c896fe29Sbellard 945c896fe29Sbellard for(op = 0; op < NB_OPS; op++) { 946c896fe29Sbellard def = &tcg_op_defs[op]; 947c896fe29Sbellard def->args_ct = args_ct; 948c896fe29Sbellard n = def->nb_iargs + def->nb_oargs; 949c896fe29Sbellard args_ct += n; 950c896fe29Sbellard } 951c896fe29Sbellard 9525cd8f621SRichard Henderson /* Register helpers. */ 95384fd9dd3SRichard Henderson /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 954619205fdSEmilio G. Cota helper_table = g_hash_table_new(NULL, NULL); 95584fd9dd3SRichard Henderson 956100b5e01SRichard Henderson for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 95739004a71SRichard Henderson init_call_layout(&all_helpers[i]); 95884fd9dd3SRichard Henderson g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 95972866e82SRichard Henderson (gpointer)&all_helpers[i]); 960100b5e01SRichard Henderson } 9615cd8f621SRichard Henderson 96222f15579SRichard Henderson #ifdef CONFIG_TCG_INTERPRETER 9630c22e176SPhilippe Mathieu-Daudé init_ffi_layouts(); 96422f15579SRichard Henderson #endif 96522f15579SRichard Henderson 966c896fe29Sbellard tcg_target_init(s); 967f69d277eSRichard Henderson process_op_defs(s); 96891478cefSRichard Henderson 96991478cefSRichard Henderson /* Reverse the order of the saved registers, assuming they're all at 97091478cefSRichard Henderson the start of tcg_target_reg_alloc_order. */ 97191478cefSRichard Henderson for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 97291478cefSRichard Henderson int r = tcg_target_reg_alloc_order[n]; 97391478cefSRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 97491478cefSRichard Henderson break; 97591478cefSRichard Henderson } 97691478cefSRichard Henderson } 97791478cefSRichard Henderson for (i = 0; i < n; ++i) { 97891478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 97991478cefSRichard Henderson } 98091478cefSRichard Henderson for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 98191478cefSRichard Henderson indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 98291478cefSRichard Henderson } 983b1311c4aSEmilio G. Cota 98438b47b19SEmilio G. Cota alloc_tcg_plugin_context(s); 98538b47b19SEmilio G. Cota 986b1311c4aSEmilio G. Cota tcg_ctx = s; 9873468b59eSEmilio G. Cota /* 9883468b59eSEmilio G. Cota * In user-mode we simply share the init context among threads, since we 9893468b59eSEmilio G. Cota * use a single region. See the documentation tcg_region_init() for the 9903468b59eSEmilio G. Cota * reasoning behind this. 9913468b59eSEmilio G. Cota * In softmmu we will have at most max_cpus TCG threads. 9923468b59eSEmilio G. Cota */ 9933468b59eSEmilio G. Cota #ifdef CONFIG_USER_ONLY 994df2cce29SEmilio G. Cota tcg_ctxs = &tcg_ctx; 9950e2d61cfSRichard Henderson tcg_cur_ctxs = 1; 9960e2d61cfSRichard Henderson tcg_max_ctxs = 1; 9973468b59eSEmilio G. Cota #else 9980e2d61cfSRichard Henderson tcg_max_ctxs = max_cpus; 9990e2d61cfSRichard Henderson tcg_ctxs = g_new0(TCGContext *, max_cpus); 10003468b59eSEmilio G. Cota #endif 10011c2adb95SRichard Henderson 10021c2adb95SRichard Henderson tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 10031c2adb95SRichard Henderson ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 10041c2adb95SRichard Henderson cpu_env = temp_tcgv_ptr(ts); 10059002ec79SRichard Henderson } 1006b03cce8eSbellard 100743b972b7SRichard Henderson void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 1008a76aabd3SRichard Henderson { 100943b972b7SRichard Henderson tcg_context_init(max_cpus); 101043b972b7SRichard Henderson tcg_region_init(tb_size, splitwx, max_cpus); 1011a76aabd3SRichard Henderson } 1012a76aabd3SRichard Henderson 10136e3b2bfdSEmilio G. Cota /* 10146e3b2bfdSEmilio G. Cota * Allocate TBs right before their corresponding translated code, making 10156e3b2bfdSEmilio G. Cota * sure that TBs and code are on different cache lines. 10166e3b2bfdSEmilio G. Cota */ 10176e3b2bfdSEmilio G. Cota TranslationBlock *tcg_tb_alloc(TCGContext *s) 10186e3b2bfdSEmilio G. Cota { 10196e3b2bfdSEmilio G. Cota uintptr_t align = qemu_icache_linesize; 10206e3b2bfdSEmilio G. Cota TranslationBlock *tb; 10216e3b2bfdSEmilio G. Cota void *next; 10226e3b2bfdSEmilio G. Cota 1023e8feb96fSEmilio G. Cota retry: 10246e3b2bfdSEmilio G. Cota tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 10256e3b2bfdSEmilio G. Cota next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 10266e3b2bfdSEmilio G. Cota 10276e3b2bfdSEmilio G. Cota if (unlikely(next > s->code_gen_highwater)) { 1028e8feb96fSEmilio G. Cota if (tcg_region_alloc(s)) { 10296e3b2bfdSEmilio G. Cota return NULL; 10306e3b2bfdSEmilio G. Cota } 1031e8feb96fSEmilio G. Cota goto retry; 1032e8feb96fSEmilio G. Cota } 1033d73415a3SStefan Hajnoczi qatomic_set(&s->code_gen_ptr, next); 103457a26946SRichard Henderson s->data_gen_ptr = NULL; 10356e3b2bfdSEmilio G. Cota return tb; 10366e3b2bfdSEmilio G. Cota } 10376e3b2bfdSEmilio G. Cota 10389002ec79SRichard Henderson void tcg_prologue_init(TCGContext *s) 10399002ec79SRichard Henderson { 1040b0a0794aSRichard Henderson size_t prologue_size; 10418163b749SRichard Henderson 1042b0a0794aSRichard Henderson s->code_ptr = s->code_gen_ptr; 1043b0a0794aSRichard Henderson s->code_buf = s->code_gen_ptr; 10445b38ee31SRichard Henderson s->data_gen_ptr = NULL; 1045b91ccb31SRichard Henderson 1046b91ccb31SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 1047b0a0794aSRichard Henderson tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 1048b91ccb31SRichard Henderson #endif 10498163b749SRichard Henderson 10505b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 10515b38ee31SRichard Henderson s->pool_labels = NULL; 10525b38ee31SRichard Henderson #endif 10535b38ee31SRichard Henderson 1054653b87ebSRoman Bolshakov qemu_thread_jit_write(); 10558163b749SRichard Henderson /* Generate the prologue. */ 1056b03cce8eSbellard tcg_target_qemu_prologue(s); 10575b38ee31SRichard Henderson 10585b38ee31SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 10595b38ee31SRichard Henderson /* Allow the prologue to put e.g. guest_base into a pool entry. */ 10605b38ee31SRichard Henderson { 10611768987bSRichard Henderson int result = tcg_out_pool_finalize(s); 10621768987bSRichard Henderson tcg_debug_assert(result == 0); 10635b38ee31SRichard Henderson } 10645b38ee31SRichard Henderson #endif 10655b38ee31SRichard Henderson 1066b0a0794aSRichard Henderson prologue_size = tcg_current_code_size(s); 10675584e2dbSIlya Leoshkevich perf_report_prologue(s->code_gen_ptr, prologue_size); 1068b0a0794aSRichard Henderson 1069df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 1070b0a0794aSRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 1071b0a0794aSRichard Henderson (uintptr_t)s->code_buf, prologue_size); 1072df5d2b16SRichard Henderson #endif 10738163b749SRichard Henderson 1074d6b64b2bSRichard Henderson #ifdef DEBUG_DISAS 1075d6b64b2bSRichard Henderson if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 1076c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 107778b54858SRichard Henderson if (logfile) { 107878b54858SRichard Henderson fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 10795b38ee31SRichard Henderson if (s->data_gen_ptr) { 1080b0a0794aSRichard Henderson size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 10815b38ee31SRichard Henderson size_t data_size = prologue_size - code_size; 10825b38ee31SRichard Henderson size_t i; 10835b38ee31SRichard Henderson 108478b54858SRichard Henderson disas(logfile, s->code_gen_ptr, code_size); 10855b38ee31SRichard Henderson 10865b38ee31SRichard Henderson for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 10875b38ee31SRichard Henderson if (sizeof(tcg_target_ulong) == 8) { 108878b54858SRichard Henderson fprintf(logfile, 108978b54858SRichard Henderson "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 10905b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 10915b38ee31SRichard Henderson *(uint64_t *)(s->data_gen_ptr + i)); 10925b38ee31SRichard Henderson } else { 109378b54858SRichard Henderson fprintf(logfile, 109478b54858SRichard Henderson "0x%08" PRIxPTR ": .long 0x%08x\n", 10955b38ee31SRichard Henderson (uintptr_t)s->data_gen_ptr + i, 10965b38ee31SRichard Henderson *(uint32_t *)(s->data_gen_ptr + i)); 10975b38ee31SRichard Henderson } 10985b38ee31SRichard Henderson } 10995b38ee31SRichard Henderson } else { 110078b54858SRichard Henderson disas(logfile, s->code_gen_ptr, prologue_size); 11015b38ee31SRichard Henderson } 110278b54858SRichard Henderson fprintf(logfile, "\n"); 1103fc59d2d8SRobert Foley qemu_log_unlock(logfile); 1104d6b64b2bSRichard Henderson } 110578b54858SRichard Henderson } 1106d6b64b2bSRichard Henderson #endif 1107cedbcb01SEmilio G. Cota 11086eea0434SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 11096eea0434SRichard Henderson /* 11106eea0434SRichard Henderson * Assert that goto_ptr is implemented completely, setting an epilogue. 11116eea0434SRichard Henderson * For tci, we use NULL as the signal to return from the interpreter, 11126eea0434SRichard Henderson * so skip this check. 11136eea0434SRichard Henderson */ 11148b5c2b62SRichard Henderson tcg_debug_assert(tcg_code_gen_epilogue != NULL); 11156eea0434SRichard Henderson #endif 1116d1c74ab3SRichard Henderson 1117d1c74ab3SRichard Henderson tcg_region_prologue_set(s); 1118c896fe29Sbellard } 1119c896fe29Sbellard 1120c896fe29Sbellard void tcg_func_start(TCGContext *s) 1121c896fe29Sbellard { 1122c896fe29Sbellard tcg_pool_reset(s); 1123c896fe29Sbellard s->nb_temps = s->nb_globals; 11240ec9eabcSRichard Henderson 11250ec9eabcSRichard Henderson /* No temps have been previously allocated for size or locality. */ 11260ec9eabcSRichard Henderson memset(s->free_temps, 0, sizeof(s->free_temps)); 11270ec9eabcSRichard Henderson 1128c0522136SRichard Henderson /* No constant temps have been previously allocated. */ 1129c0522136SRichard Henderson for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 1130c0522136SRichard Henderson if (s->const_table[i]) { 1131c0522136SRichard Henderson g_hash_table_remove_all(s->const_table[i]); 1132c0522136SRichard Henderson } 1133c0522136SRichard Henderson } 1134c0522136SRichard Henderson 1135abebf925SRichard Henderson s->nb_ops = 0; 1136c896fe29Sbellard s->nb_labels = 0; 1137c896fe29Sbellard s->current_frame_offset = s->frame_start; 1138c896fe29Sbellard 11390a209d4bSRichard Henderson #ifdef CONFIG_DEBUG_TCG 11400a209d4bSRichard Henderson s->goto_tb_issue_mask = 0; 11410a209d4bSRichard Henderson #endif 11420a209d4bSRichard Henderson 114315fa08f8SRichard Henderson QTAILQ_INIT(&s->ops); 114415fa08f8SRichard Henderson QTAILQ_INIT(&s->free_ops); 1145bef16ab4SRichard Henderson QSIMPLEQ_INIT(&s->labels); 1146c896fe29Sbellard } 1147c896fe29Sbellard 1148ae30e866SRichard Henderson static TCGTemp *tcg_temp_alloc(TCGContext *s) 11497ca4b752SRichard Henderson { 11507ca4b752SRichard Henderson int n = s->nb_temps++; 1151ae30e866SRichard Henderson 1152ae30e866SRichard Henderson if (n >= TCG_MAX_TEMPS) { 1153db6b7d0cSRichard Henderson tcg_raise_tb_overflow(s); 1154ae30e866SRichard Henderson } 11557ca4b752SRichard Henderson return memset(&s->temps[n], 0, sizeof(TCGTemp)); 11567ca4b752SRichard Henderson } 11577ca4b752SRichard Henderson 1158ae30e866SRichard Henderson static TCGTemp *tcg_global_alloc(TCGContext *s) 11597ca4b752SRichard Henderson { 1160fa477d25SRichard Henderson TCGTemp *ts; 1161fa477d25SRichard Henderson 11627ca4b752SRichard Henderson tcg_debug_assert(s->nb_globals == s->nb_temps); 1163ae30e866SRichard Henderson tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 11647ca4b752SRichard Henderson s->nb_globals++; 1165fa477d25SRichard Henderson ts = tcg_temp_alloc(s); 1166ee17db83SRichard Henderson ts->kind = TEMP_GLOBAL; 1167fa477d25SRichard Henderson 1168fa477d25SRichard Henderson return ts; 1169c896fe29Sbellard } 1170c896fe29Sbellard 1171085272b3SRichard Henderson static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1172b6638662SRichard Henderson TCGReg reg, const char *name) 1173c896fe29Sbellard { 1174c896fe29Sbellard TCGTemp *ts; 1175c896fe29Sbellard 1176b3a62939SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 1177c896fe29Sbellard tcg_abort(); 1178b3a62939SRichard Henderson } 11797ca4b752SRichard Henderson 11807ca4b752SRichard Henderson ts = tcg_global_alloc(s); 1181c896fe29Sbellard ts->base_type = type; 1182c896fe29Sbellard ts->type = type; 1183ee17db83SRichard Henderson ts->kind = TEMP_FIXED; 1184c896fe29Sbellard ts->reg = reg; 1185c896fe29Sbellard ts->name = name; 1186c896fe29Sbellard tcg_regset_set_reg(s->reserved_regs, reg); 11877ca4b752SRichard Henderson 1188085272b3SRichard Henderson return ts; 1189a7812ae4Spbrook } 1190a7812ae4Spbrook 1191b6638662SRichard Henderson void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 1192a7812ae4Spbrook { 1193b3a62939SRichard Henderson s->frame_start = start; 1194b3a62939SRichard Henderson s->frame_end = start + size; 1195085272b3SRichard Henderson s->frame_temp 1196085272b3SRichard Henderson = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 1197b3a62939SRichard Henderson } 1198a7812ae4Spbrook 1199085272b3SRichard Henderson TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 1200e1ccc054SRichard Henderson intptr_t offset, const char *name) 1201c896fe29Sbellard { 1202b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1203dc41aa7dSRichard Henderson TCGTemp *base_ts = tcgv_ptr_temp(base); 12047ca4b752SRichard Henderson TCGTemp *ts = tcg_global_alloc(s); 1205aef85402SRichard Henderson int indirect_reg = 0; 1206c896fe29Sbellard 1207c0522136SRichard Henderson switch (base_ts->kind) { 1208c0522136SRichard Henderson case TEMP_FIXED: 1209c0522136SRichard Henderson break; 1210c0522136SRichard Henderson case TEMP_GLOBAL: 12115a18407fSRichard Henderson /* We do not support double-indirect registers. */ 12125a18407fSRichard Henderson tcg_debug_assert(!base_ts->indirect_reg); 1213b3915dbbSRichard Henderson base_ts->indirect_base = 1; 12145a18407fSRichard Henderson s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 12155a18407fSRichard Henderson ? 2 : 1); 12165a18407fSRichard Henderson indirect_reg = 1; 1217c0522136SRichard Henderson break; 1218c0522136SRichard Henderson default: 1219c0522136SRichard Henderson g_assert_not_reached(); 1220b3915dbbSRichard Henderson } 1221b3915dbbSRichard Henderson 12227ca4b752SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 12237ca4b752SRichard Henderson TCGTemp *ts2 = tcg_global_alloc(s); 1224c896fe29Sbellard char buf[64]; 12257ca4b752SRichard Henderson 12267ca4b752SRichard Henderson ts->base_type = TCG_TYPE_I64; 1227c896fe29Sbellard ts->type = TCG_TYPE_I32; 1228b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 1229c896fe29Sbellard ts->mem_allocated = 1; 1230b3a62939SRichard Henderson ts->mem_base = base_ts; 1231aef85402SRichard Henderson ts->mem_offset = offset; 1232c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 1233c896fe29Sbellard pstrcat(buf, sizeof(buf), "_0"); 1234c896fe29Sbellard ts->name = strdup(buf); 1235c896fe29Sbellard 12367ca4b752SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 12377ca4b752SRichard Henderson ts2->base_type = TCG_TYPE_I64; 12387ca4b752SRichard Henderson ts2->type = TCG_TYPE_I32; 1239b3915dbbSRichard Henderson ts2->indirect_reg = indirect_reg; 12407ca4b752SRichard Henderson ts2->mem_allocated = 1; 12417ca4b752SRichard Henderson ts2->mem_base = base_ts; 1242aef85402SRichard Henderson ts2->mem_offset = offset + 4; 1243fac87bd2SRichard Henderson ts2->temp_subindex = 1; 1244c896fe29Sbellard pstrcpy(buf, sizeof(buf), name); 1245c896fe29Sbellard pstrcat(buf, sizeof(buf), "_1"); 1246120c1084SRichard Henderson ts2->name = strdup(buf); 12477ca4b752SRichard Henderson } else { 1248c896fe29Sbellard ts->base_type = type; 1249c896fe29Sbellard ts->type = type; 1250b3915dbbSRichard Henderson ts->indirect_reg = indirect_reg; 1251c896fe29Sbellard ts->mem_allocated = 1; 1252b3a62939SRichard Henderson ts->mem_base = base_ts; 1253c896fe29Sbellard ts->mem_offset = offset; 1254c896fe29Sbellard ts->name = name; 1255c896fe29Sbellard } 1256085272b3SRichard Henderson return ts; 1257c896fe29Sbellard } 1258c896fe29Sbellard 1259bbf989bfSRichard Henderson TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) 1260c896fe29Sbellard { 1261b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1262c896fe29Sbellard TCGTemp *ts; 1263e1c08b00SRichard Henderson int n; 1264c896fe29Sbellard 1265e1c08b00SRichard Henderson if (kind == TEMP_EBB) { 1266e1c08b00SRichard Henderson int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); 1267e1c08b00SRichard Henderson 12680ec9eabcSRichard Henderson if (idx < TCG_MAX_TEMPS) { 12690ec9eabcSRichard Henderson /* There is already an available temp with the right type. */ 1270e1c08b00SRichard Henderson clear_bit(idx, s->free_temps[type].l); 12710ec9eabcSRichard Henderson 1272e8996ee0Sbellard ts = &s->temps[idx]; 1273e8996ee0Sbellard ts->temp_allocated = 1; 12747ca4b752SRichard Henderson tcg_debug_assert(ts->base_type == type); 1275ee17db83SRichard Henderson tcg_debug_assert(ts->kind == kind); 1276*2f2e911dSRichard Henderson return ts; 1277e1c08b00SRichard Henderson } 1278e8996ee0Sbellard } else { 1279e1c08b00SRichard Henderson tcg_debug_assert(kind == TEMP_TB); 1280e1c08b00SRichard Henderson } 128143eef72fSRichard Henderson 128243eef72fSRichard Henderson switch (type) { 128343eef72fSRichard Henderson case TCG_TYPE_I32: 128443eef72fSRichard Henderson case TCG_TYPE_V64: 128543eef72fSRichard Henderson case TCG_TYPE_V128: 128643eef72fSRichard Henderson case TCG_TYPE_V256: 128743eef72fSRichard Henderson n = 1; 128843eef72fSRichard Henderson break; 128943eef72fSRichard Henderson case TCG_TYPE_I64: 129043eef72fSRichard Henderson n = 64 / TCG_TARGET_REG_BITS; 129143eef72fSRichard Henderson break; 129243eef72fSRichard Henderson case TCG_TYPE_I128: 129343eef72fSRichard Henderson n = 128 / TCG_TARGET_REG_BITS; 129443eef72fSRichard Henderson break; 129543eef72fSRichard Henderson default: 129643eef72fSRichard Henderson g_assert_not_reached(); 129743eef72fSRichard Henderson } 129843eef72fSRichard Henderson 12997ca4b752SRichard Henderson ts = tcg_temp_alloc(s); 130043eef72fSRichard Henderson ts->base_type = type; 130143eef72fSRichard Henderson ts->temp_allocated = 1; 130243eef72fSRichard Henderson ts->kind = kind; 130343eef72fSRichard Henderson 130443eef72fSRichard Henderson if (n == 1) { 130543eef72fSRichard Henderson ts->type = type; 130643eef72fSRichard Henderson } else { 130743eef72fSRichard Henderson ts->type = TCG_TYPE_REG; 130843eef72fSRichard Henderson 1309e1c08b00SRichard Henderson for (int i = 1; i < n; ++i) { 13107ca4b752SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 13117ca4b752SRichard Henderson 131243eef72fSRichard Henderson tcg_debug_assert(ts2 == ts + i); 131343eef72fSRichard Henderson ts2->base_type = type; 131443eef72fSRichard Henderson ts2->type = TCG_TYPE_REG; 13157ca4b752SRichard Henderson ts2->temp_allocated = 1; 131643eef72fSRichard Henderson ts2->temp_subindex = i; 1317ee17db83SRichard Henderson ts2->kind = kind; 131843eef72fSRichard Henderson } 1319c896fe29Sbellard } 1320085272b3SRichard Henderson return ts; 1321c896fe29Sbellard } 1322c896fe29Sbellard 1323d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec(TCGType type) 1324d2fd745fSRichard Henderson { 1325d2fd745fSRichard Henderson TCGTemp *t; 1326d2fd745fSRichard Henderson 1327d2fd745fSRichard Henderson #ifdef CONFIG_DEBUG_TCG 1328d2fd745fSRichard Henderson switch (type) { 1329d2fd745fSRichard Henderson case TCG_TYPE_V64: 1330d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v64); 1331d2fd745fSRichard Henderson break; 1332d2fd745fSRichard Henderson case TCG_TYPE_V128: 1333d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v128); 1334d2fd745fSRichard Henderson break; 1335d2fd745fSRichard Henderson case TCG_TYPE_V256: 1336d2fd745fSRichard Henderson assert(TCG_TARGET_HAS_v256); 1337d2fd745fSRichard Henderson break; 1338d2fd745fSRichard Henderson default: 1339d2fd745fSRichard Henderson g_assert_not_reached(); 1340d2fd745fSRichard Henderson } 1341d2fd745fSRichard Henderson #endif 1342d2fd745fSRichard Henderson 1343bbf989bfSRichard Henderson t = tcg_temp_new_internal(type, TEMP_EBB); 1344d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1345d2fd745fSRichard Henderson } 1346d2fd745fSRichard Henderson 1347d2fd745fSRichard Henderson /* Create a new temp of the same type as an existing temp. */ 1348d2fd745fSRichard Henderson TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1349d2fd745fSRichard Henderson { 1350d2fd745fSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 1351d2fd745fSRichard Henderson 1352d2fd745fSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 1353d2fd745fSRichard Henderson 1354bbf989bfSRichard Henderson t = tcg_temp_new_internal(t->base_type, TEMP_EBB); 1355d2fd745fSRichard Henderson return temp_tcgv_vec(t); 1356d2fd745fSRichard Henderson } 1357d2fd745fSRichard Henderson 13585bfa8034SRichard Henderson void tcg_temp_free_internal(TCGTemp *ts) 1359c896fe29Sbellard { 1360b1311c4aSEmilio G. Cota TCGContext *s = tcg_ctx; 1361c896fe29Sbellard 1362c7482438SRichard Henderson switch (ts->kind) { 1363c7482438SRichard Henderson case TEMP_CONST: 1364f57c6915SRichard Henderson case TEMP_TB: 1365*2f2e911dSRichard Henderson /* Silently ignore free. */ 1366c7482438SRichard Henderson break; 1367*2f2e911dSRichard Henderson case TEMP_EBB: 1368eabb7b91SAurelien Jarno tcg_debug_assert(ts->temp_allocated != 0); 1369e8996ee0Sbellard ts->temp_allocated = 0; 1370*2f2e911dSRichard Henderson set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); 1371*2f2e911dSRichard Henderson break; 1372*2f2e911dSRichard Henderson default: 1373*2f2e911dSRichard Henderson /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */ 1374*2f2e911dSRichard Henderson g_assert_not_reached(); 1375e1c08b00SRichard Henderson } 1376e8996ee0Sbellard } 1377e8996ee0Sbellard 1378c0522136SRichard Henderson TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1379c0522136SRichard Henderson { 1380c0522136SRichard Henderson TCGContext *s = tcg_ctx; 1381c0522136SRichard Henderson GHashTable *h = s->const_table[type]; 1382c0522136SRichard Henderson TCGTemp *ts; 1383c0522136SRichard Henderson 1384c0522136SRichard Henderson if (h == NULL) { 1385c0522136SRichard Henderson h = g_hash_table_new(g_int64_hash, g_int64_equal); 1386c0522136SRichard Henderson s->const_table[type] = h; 1387c0522136SRichard Henderson } 1388c0522136SRichard Henderson 1389c0522136SRichard Henderson ts = g_hash_table_lookup(h, &val); 1390c0522136SRichard Henderson if (ts == NULL) { 1391aef85402SRichard Henderson int64_t *val_ptr; 1392aef85402SRichard Henderson 1393c0522136SRichard Henderson ts = tcg_temp_alloc(s); 1394c0522136SRichard Henderson 1395c0522136SRichard Henderson if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1396c0522136SRichard Henderson TCGTemp *ts2 = tcg_temp_alloc(s); 1397c0522136SRichard Henderson 1398aef85402SRichard Henderson tcg_debug_assert(ts2 == ts + 1); 1399aef85402SRichard Henderson 1400c0522136SRichard Henderson ts->base_type = TCG_TYPE_I64; 1401c0522136SRichard Henderson ts->type = TCG_TYPE_I32; 1402c0522136SRichard Henderson ts->kind = TEMP_CONST; 1403c0522136SRichard Henderson ts->temp_allocated = 1; 1404c0522136SRichard Henderson 1405c0522136SRichard Henderson ts2->base_type = TCG_TYPE_I64; 1406c0522136SRichard Henderson ts2->type = TCG_TYPE_I32; 1407c0522136SRichard Henderson ts2->kind = TEMP_CONST; 1408c0522136SRichard Henderson ts2->temp_allocated = 1; 1409fac87bd2SRichard Henderson ts2->temp_subindex = 1; 1410aef85402SRichard Henderson 1411aef85402SRichard Henderson /* 1412aef85402SRichard Henderson * Retain the full value of the 64-bit constant in the low 1413aef85402SRichard Henderson * part, so that the hash table works. Actual uses will 1414aef85402SRichard Henderson * truncate the value to the low part. 1415aef85402SRichard Henderson */ 1416aef85402SRichard Henderson ts[HOST_BIG_ENDIAN].val = val; 1417aef85402SRichard Henderson ts[!HOST_BIG_ENDIAN].val = val >> 32; 1418aef85402SRichard Henderson val_ptr = &ts[HOST_BIG_ENDIAN].val; 1419c0522136SRichard Henderson } else { 1420c0522136SRichard Henderson ts->base_type = type; 1421c0522136SRichard Henderson ts->type = type; 1422c0522136SRichard Henderson ts->kind = TEMP_CONST; 1423c0522136SRichard Henderson ts->temp_allocated = 1; 1424c0522136SRichard Henderson ts->val = val; 1425aef85402SRichard Henderson val_ptr = &ts->val; 1426c0522136SRichard Henderson } 1427aef85402SRichard Henderson g_hash_table_insert(h, val_ptr, ts); 1428c0522136SRichard Henderson } 1429c0522136SRichard Henderson 1430c0522136SRichard Henderson return ts; 1431c0522136SRichard Henderson } 1432c0522136SRichard Henderson 1433c0522136SRichard Henderson TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1434c0522136SRichard Henderson { 1435c0522136SRichard Henderson val = dup_const(vece, val); 1436c0522136SRichard Henderson return temp_tcgv_vec(tcg_constant_internal(type, val)); 1437c0522136SRichard Henderson } 1438c0522136SRichard Henderson 143988d4005bSRichard Henderson TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 144088d4005bSRichard Henderson { 144188d4005bSRichard Henderson TCGTemp *t = tcgv_vec_temp(match); 144288d4005bSRichard Henderson 144388d4005bSRichard Henderson tcg_debug_assert(t->temp_allocated != 0); 144488d4005bSRichard Henderson return tcg_constant_vec(t->base_type, vece, val); 144588d4005bSRichard Henderson } 144688d4005bSRichard Henderson 1447a7812ae4Spbrook TCGv_i32 tcg_const_i32(int32_t val) 1448a7812ae4Spbrook { 1449a7812ae4Spbrook TCGv_i32 t0; 1450a7812ae4Spbrook t0 = tcg_temp_new_i32(); 1451e8996ee0Sbellard tcg_gen_movi_i32(t0, val); 1452e8996ee0Sbellard return t0; 1453c896fe29Sbellard } 1454c896fe29Sbellard 1455a7812ae4Spbrook TCGv_i64 tcg_const_i64(int64_t val) 1456c896fe29Sbellard { 1457a7812ae4Spbrook TCGv_i64 t0; 1458a7812ae4Spbrook t0 = tcg_temp_new_i64(); 1459e8996ee0Sbellard tcg_gen_movi_i64(t0, val); 1460e8996ee0Sbellard return t0; 1461c896fe29Sbellard } 1462c896fe29Sbellard 1463be0f34b5SRichard Henderson /* Return true if OP may appear in the opcode stream. 1464be0f34b5SRichard Henderson Test the runtime variable that controls each opcode. */ 1465be0f34b5SRichard Henderson bool tcg_op_supported(TCGOpcode op) 1466be0f34b5SRichard Henderson { 1467d2fd745fSRichard Henderson const bool have_vec 1468d2fd745fSRichard Henderson = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1469d2fd745fSRichard Henderson 1470be0f34b5SRichard Henderson switch (op) { 1471be0f34b5SRichard Henderson case INDEX_op_discard: 1472be0f34b5SRichard Henderson case INDEX_op_set_label: 1473be0f34b5SRichard Henderson case INDEX_op_call: 1474be0f34b5SRichard Henderson case INDEX_op_br: 1475be0f34b5SRichard Henderson case INDEX_op_mb: 1476be0f34b5SRichard Henderson case INDEX_op_insn_start: 1477be0f34b5SRichard Henderson case INDEX_op_exit_tb: 1478be0f34b5SRichard Henderson case INDEX_op_goto_tb: 1479f4e01e30SRichard Henderson case INDEX_op_goto_ptr: 1480be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i32: 1481be0f34b5SRichard Henderson case INDEX_op_qemu_st_i32: 1482be0f34b5SRichard Henderson case INDEX_op_qemu_ld_i64: 1483be0f34b5SRichard Henderson case INDEX_op_qemu_st_i64: 1484be0f34b5SRichard Henderson return true; 1485be0f34b5SRichard Henderson 148607ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 148707ce0b05SRichard Henderson return TCG_TARGET_HAS_qemu_st8_i32; 148807ce0b05SRichard Henderson 1489be0f34b5SRichard Henderson case INDEX_op_mov_i32: 1490be0f34b5SRichard Henderson case INDEX_op_setcond_i32: 1491be0f34b5SRichard Henderson case INDEX_op_brcond_i32: 1492be0f34b5SRichard Henderson case INDEX_op_ld8u_i32: 1493be0f34b5SRichard Henderson case INDEX_op_ld8s_i32: 1494be0f34b5SRichard Henderson case INDEX_op_ld16u_i32: 1495be0f34b5SRichard Henderson case INDEX_op_ld16s_i32: 1496be0f34b5SRichard Henderson case INDEX_op_ld_i32: 1497be0f34b5SRichard Henderson case INDEX_op_st8_i32: 1498be0f34b5SRichard Henderson case INDEX_op_st16_i32: 1499be0f34b5SRichard Henderson case INDEX_op_st_i32: 1500be0f34b5SRichard Henderson case INDEX_op_add_i32: 1501be0f34b5SRichard Henderson case INDEX_op_sub_i32: 1502be0f34b5SRichard Henderson case INDEX_op_mul_i32: 1503be0f34b5SRichard Henderson case INDEX_op_and_i32: 1504be0f34b5SRichard Henderson case INDEX_op_or_i32: 1505be0f34b5SRichard Henderson case INDEX_op_xor_i32: 1506be0f34b5SRichard Henderson case INDEX_op_shl_i32: 1507be0f34b5SRichard Henderson case INDEX_op_shr_i32: 1508be0f34b5SRichard Henderson case INDEX_op_sar_i32: 1509be0f34b5SRichard Henderson return true; 1510be0f34b5SRichard Henderson 1511be0f34b5SRichard Henderson case INDEX_op_movcond_i32: 1512be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i32; 1513be0f34b5SRichard Henderson case INDEX_op_div_i32: 1514be0f34b5SRichard Henderson case INDEX_op_divu_i32: 1515be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i32; 1516be0f34b5SRichard Henderson case INDEX_op_rem_i32: 1517be0f34b5SRichard Henderson case INDEX_op_remu_i32: 1518be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i32; 1519be0f34b5SRichard Henderson case INDEX_op_div2_i32: 1520be0f34b5SRichard Henderson case INDEX_op_divu2_i32: 1521be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i32; 1522be0f34b5SRichard Henderson case INDEX_op_rotl_i32: 1523be0f34b5SRichard Henderson case INDEX_op_rotr_i32: 1524be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i32; 1525be0f34b5SRichard Henderson case INDEX_op_deposit_i32: 1526be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i32; 1527be0f34b5SRichard Henderson case INDEX_op_extract_i32: 1528be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i32; 1529be0f34b5SRichard Henderson case INDEX_op_sextract_i32: 1530be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i32; 1531fce1296fSRichard Henderson case INDEX_op_extract2_i32: 1532fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i32; 1533be0f34b5SRichard Henderson case INDEX_op_add2_i32: 1534be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i32; 1535be0f34b5SRichard Henderson case INDEX_op_sub2_i32: 1536be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i32; 1537be0f34b5SRichard Henderson case INDEX_op_mulu2_i32: 1538be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i32; 1539be0f34b5SRichard Henderson case INDEX_op_muls2_i32: 1540be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i32; 1541be0f34b5SRichard Henderson case INDEX_op_muluh_i32: 1542be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i32; 1543be0f34b5SRichard Henderson case INDEX_op_mulsh_i32: 1544be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i32; 1545be0f34b5SRichard Henderson case INDEX_op_ext8s_i32: 1546be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i32; 1547be0f34b5SRichard Henderson case INDEX_op_ext16s_i32: 1548be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i32; 1549be0f34b5SRichard Henderson case INDEX_op_ext8u_i32: 1550be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i32; 1551be0f34b5SRichard Henderson case INDEX_op_ext16u_i32: 1552be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i32; 1553be0f34b5SRichard Henderson case INDEX_op_bswap16_i32: 1554be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i32; 1555be0f34b5SRichard Henderson case INDEX_op_bswap32_i32: 1556be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i32; 1557be0f34b5SRichard Henderson case INDEX_op_not_i32: 1558be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i32; 1559be0f34b5SRichard Henderson case INDEX_op_neg_i32: 1560be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i32; 1561be0f34b5SRichard Henderson case INDEX_op_andc_i32: 1562be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i32; 1563be0f34b5SRichard Henderson case INDEX_op_orc_i32: 1564be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i32; 1565be0f34b5SRichard Henderson case INDEX_op_eqv_i32: 1566be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i32; 1567be0f34b5SRichard Henderson case INDEX_op_nand_i32: 1568be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i32; 1569be0f34b5SRichard Henderson case INDEX_op_nor_i32: 1570be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i32; 1571be0f34b5SRichard Henderson case INDEX_op_clz_i32: 1572be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i32; 1573be0f34b5SRichard Henderson case INDEX_op_ctz_i32: 1574be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i32; 1575be0f34b5SRichard Henderson case INDEX_op_ctpop_i32: 1576be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i32; 1577be0f34b5SRichard Henderson 1578be0f34b5SRichard Henderson case INDEX_op_brcond2_i32: 1579be0f34b5SRichard Henderson case INDEX_op_setcond2_i32: 1580be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 32; 1581be0f34b5SRichard Henderson 1582be0f34b5SRichard Henderson case INDEX_op_mov_i64: 1583be0f34b5SRichard Henderson case INDEX_op_setcond_i64: 1584be0f34b5SRichard Henderson case INDEX_op_brcond_i64: 1585be0f34b5SRichard Henderson case INDEX_op_ld8u_i64: 1586be0f34b5SRichard Henderson case INDEX_op_ld8s_i64: 1587be0f34b5SRichard Henderson case INDEX_op_ld16u_i64: 1588be0f34b5SRichard Henderson case INDEX_op_ld16s_i64: 1589be0f34b5SRichard Henderson case INDEX_op_ld32u_i64: 1590be0f34b5SRichard Henderson case INDEX_op_ld32s_i64: 1591be0f34b5SRichard Henderson case INDEX_op_ld_i64: 1592be0f34b5SRichard Henderson case INDEX_op_st8_i64: 1593be0f34b5SRichard Henderson case INDEX_op_st16_i64: 1594be0f34b5SRichard Henderson case INDEX_op_st32_i64: 1595be0f34b5SRichard Henderson case INDEX_op_st_i64: 1596be0f34b5SRichard Henderson case INDEX_op_add_i64: 1597be0f34b5SRichard Henderson case INDEX_op_sub_i64: 1598be0f34b5SRichard Henderson case INDEX_op_mul_i64: 1599be0f34b5SRichard Henderson case INDEX_op_and_i64: 1600be0f34b5SRichard Henderson case INDEX_op_or_i64: 1601be0f34b5SRichard Henderson case INDEX_op_xor_i64: 1602be0f34b5SRichard Henderson case INDEX_op_shl_i64: 1603be0f34b5SRichard Henderson case INDEX_op_shr_i64: 1604be0f34b5SRichard Henderson case INDEX_op_sar_i64: 1605be0f34b5SRichard Henderson case INDEX_op_ext_i32_i64: 1606be0f34b5SRichard Henderson case INDEX_op_extu_i32_i64: 1607be0f34b5SRichard Henderson return TCG_TARGET_REG_BITS == 64; 1608be0f34b5SRichard Henderson 1609be0f34b5SRichard Henderson case INDEX_op_movcond_i64: 1610be0f34b5SRichard Henderson return TCG_TARGET_HAS_movcond_i64; 1611be0f34b5SRichard Henderson case INDEX_op_div_i64: 1612be0f34b5SRichard Henderson case INDEX_op_divu_i64: 1613be0f34b5SRichard Henderson return TCG_TARGET_HAS_div_i64; 1614be0f34b5SRichard Henderson case INDEX_op_rem_i64: 1615be0f34b5SRichard Henderson case INDEX_op_remu_i64: 1616be0f34b5SRichard Henderson return TCG_TARGET_HAS_rem_i64; 1617be0f34b5SRichard Henderson case INDEX_op_div2_i64: 1618be0f34b5SRichard Henderson case INDEX_op_divu2_i64: 1619be0f34b5SRichard Henderson return TCG_TARGET_HAS_div2_i64; 1620be0f34b5SRichard Henderson case INDEX_op_rotl_i64: 1621be0f34b5SRichard Henderson case INDEX_op_rotr_i64: 1622be0f34b5SRichard Henderson return TCG_TARGET_HAS_rot_i64; 1623be0f34b5SRichard Henderson case INDEX_op_deposit_i64: 1624be0f34b5SRichard Henderson return TCG_TARGET_HAS_deposit_i64; 1625be0f34b5SRichard Henderson case INDEX_op_extract_i64: 1626be0f34b5SRichard Henderson return TCG_TARGET_HAS_extract_i64; 1627be0f34b5SRichard Henderson case INDEX_op_sextract_i64: 1628be0f34b5SRichard Henderson return TCG_TARGET_HAS_sextract_i64; 1629fce1296fSRichard Henderson case INDEX_op_extract2_i64: 1630fce1296fSRichard Henderson return TCG_TARGET_HAS_extract2_i64; 1631be0f34b5SRichard Henderson case INDEX_op_extrl_i64_i32: 1632be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrl_i64_i32; 1633be0f34b5SRichard Henderson case INDEX_op_extrh_i64_i32: 1634be0f34b5SRichard Henderson return TCG_TARGET_HAS_extrh_i64_i32; 1635be0f34b5SRichard Henderson case INDEX_op_ext8s_i64: 1636be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8s_i64; 1637be0f34b5SRichard Henderson case INDEX_op_ext16s_i64: 1638be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16s_i64; 1639be0f34b5SRichard Henderson case INDEX_op_ext32s_i64: 1640be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32s_i64; 1641be0f34b5SRichard Henderson case INDEX_op_ext8u_i64: 1642be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext8u_i64; 1643be0f34b5SRichard Henderson case INDEX_op_ext16u_i64: 1644be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext16u_i64; 1645be0f34b5SRichard Henderson case INDEX_op_ext32u_i64: 1646be0f34b5SRichard Henderson return TCG_TARGET_HAS_ext32u_i64; 1647be0f34b5SRichard Henderson case INDEX_op_bswap16_i64: 1648be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap16_i64; 1649be0f34b5SRichard Henderson case INDEX_op_bswap32_i64: 1650be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap32_i64; 1651be0f34b5SRichard Henderson case INDEX_op_bswap64_i64: 1652be0f34b5SRichard Henderson return TCG_TARGET_HAS_bswap64_i64; 1653be0f34b5SRichard Henderson case INDEX_op_not_i64: 1654be0f34b5SRichard Henderson return TCG_TARGET_HAS_not_i64; 1655be0f34b5SRichard Henderson case INDEX_op_neg_i64: 1656be0f34b5SRichard Henderson return TCG_TARGET_HAS_neg_i64; 1657be0f34b5SRichard Henderson case INDEX_op_andc_i64: 1658be0f34b5SRichard Henderson return TCG_TARGET_HAS_andc_i64; 1659be0f34b5SRichard Henderson case INDEX_op_orc_i64: 1660be0f34b5SRichard Henderson return TCG_TARGET_HAS_orc_i64; 1661be0f34b5SRichard Henderson case INDEX_op_eqv_i64: 1662be0f34b5SRichard Henderson return TCG_TARGET_HAS_eqv_i64; 1663be0f34b5SRichard Henderson case INDEX_op_nand_i64: 1664be0f34b5SRichard Henderson return TCG_TARGET_HAS_nand_i64; 1665be0f34b5SRichard Henderson case INDEX_op_nor_i64: 1666be0f34b5SRichard Henderson return TCG_TARGET_HAS_nor_i64; 1667be0f34b5SRichard Henderson case INDEX_op_clz_i64: 1668be0f34b5SRichard Henderson return TCG_TARGET_HAS_clz_i64; 1669be0f34b5SRichard Henderson case INDEX_op_ctz_i64: 1670be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctz_i64; 1671be0f34b5SRichard Henderson case INDEX_op_ctpop_i64: 1672be0f34b5SRichard Henderson return TCG_TARGET_HAS_ctpop_i64; 1673be0f34b5SRichard Henderson case INDEX_op_add2_i64: 1674be0f34b5SRichard Henderson return TCG_TARGET_HAS_add2_i64; 1675be0f34b5SRichard Henderson case INDEX_op_sub2_i64: 1676be0f34b5SRichard Henderson return TCG_TARGET_HAS_sub2_i64; 1677be0f34b5SRichard Henderson case INDEX_op_mulu2_i64: 1678be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulu2_i64; 1679be0f34b5SRichard Henderson case INDEX_op_muls2_i64: 1680be0f34b5SRichard Henderson return TCG_TARGET_HAS_muls2_i64; 1681be0f34b5SRichard Henderson case INDEX_op_muluh_i64: 1682be0f34b5SRichard Henderson return TCG_TARGET_HAS_muluh_i64; 1683be0f34b5SRichard Henderson case INDEX_op_mulsh_i64: 1684be0f34b5SRichard Henderson return TCG_TARGET_HAS_mulsh_i64; 1685be0f34b5SRichard Henderson 1686d2fd745fSRichard Henderson case INDEX_op_mov_vec: 1687d2fd745fSRichard Henderson case INDEX_op_dup_vec: 168837ee55a0SRichard Henderson case INDEX_op_dupm_vec: 1689d2fd745fSRichard Henderson case INDEX_op_ld_vec: 1690d2fd745fSRichard Henderson case INDEX_op_st_vec: 1691d2fd745fSRichard Henderson case INDEX_op_add_vec: 1692d2fd745fSRichard Henderson case INDEX_op_sub_vec: 1693d2fd745fSRichard Henderson case INDEX_op_and_vec: 1694d2fd745fSRichard Henderson case INDEX_op_or_vec: 1695d2fd745fSRichard Henderson case INDEX_op_xor_vec: 1696212be173SRichard Henderson case INDEX_op_cmp_vec: 1697d2fd745fSRichard Henderson return have_vec; 1698d2fd745fSRichard Henderson case INDEX_op_dup2_vec: 1699d2fd745fSRichard Henderson return have_vec && TCG_TARGET_REG_BITS == 32; 1700d2fd745fSRichard Henderson case INDEX_op_not_vec: 1701d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_not_vec; 1702d2fd745fSRichard Henderson case INDEX_op_neg_vec: 1703d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_neg_vec; 1704bcefc902SRichard Henderson case INDEX_op_abs_vec: 1705bcefc902SRichard Henderson return have_vec && TCG_TARGET_HAS_abs_vec; 1706d2fd745fSRichard Henderson case INDEX_op_andc_vec: 1707d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_andc_vec; 1708d2fd745fSRichard Henderson case INDEX_op_orc_vec: 1709d2fd745fSRichard Henderson return have_vec && TCG_TARGET_HAS_orc_vec; 1710ed523473SRichard Henderson case INDEX_op_nand_vec: 1711ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nand_vec; 1712ed523473SRichard Henderson case INDEX_op_nor_vec: 1713ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_nor_vec; 1714ed523473SRichard Henderson case INDEX_op_eqv_vec: 1715ed523473SRichard Henderson return have_vec && TCG_TARGET_HAS_eqv_vec; 17163774030aSRichard Henderson case INDEX_op_mul_vec: 17173774030aSRichard Henderson return have_vec && TCG_TARGET_HAS_mul_vec; 1718d0ec9796SRichard Henderson case INDEX_op_shli_vec: 1719d0ec9796SRichard Henderson case INDEX_op_shri_vec: 1720d0ec9796SRichard Henderson case INDEX_op_sari_vec: 1721d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shi_vec; 1722d0ec9796SRichard Henderson case INDEX_op_shls_vec: 1723d0ec9796SRichard Henderson case INDEX_op_shrs_vec: 1724d0ec9796SRichard Henderson case INDEX_op_sars_vec: 1725d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shs_vec; 1726d0ec9796SRichard Henderson case INDEX_op_shlv_vec: 1727d0ec9796SRichard Henderson case INDEX_op_shrv_vec: 1728d0ec9796SRichard Henderson case INDEX_op_sarv_vec: 1729d0ec9796SRichard Henderson return have_vec && TCG_TARGET_HAS_shv_vec; 1730b0f7e744SRichard Henderson case INDEX_op_rotli_vec: 1731b0f7e744SRichard Henderson return have_vec && TCG_TARGET_HAS_roti_vec; 173223850a74SRichard Henderson case INDEX_op_rotls_vec: 173323850a74SRichard Henderson return have_vec && TCG_TARGET_HAS_rots_vec; 17345d0ceda9SRichard Henderson case INDEX_op_rotlv_vec: 17355d0ceda9SRichard Henderson case INDEX_op_rotrv_vec: 17365d0ceda9SRichard Henderson return have_vec && TCG_TARGET_HAS_rotv_vec; 17378afaf050SRichard Henderson case INDEX_op_ssadd_vec: 17388afaf050SRichard Henderson case INDEX_op_usadd_vec: 17398afaf050SRichard Henderson case INDEX_op_sssub_vec: 17408afaf050SRichard Henderson case INDEX_op_ussub_vec: 17418afaf050SRichard Henderson return have_vec && TCG_TARGET_HAS_sat_vec; 1742dd0a0fcdSRichard Henderson case INDEX_op_smin_vec: 1743dd0a0fcdSRichard Henderson case INDEX_op_umin_vec: 1744dd0a0fcdSRichard Henderson case INDEX_op_smax_vec: 1745dd0a0fcdSRichard Henderson case INDEX_op_umax_vec: 1746dd0a0fcdSRichard Henderson return have_vec && TCG_TARGET_HAS_minmax_vec; 174738dc1294SRichard Henderson case INDEX_op_bitsel_vec: 174838dc1294SRichard Henderson return have_vec && TCG_TARGET_HAS_bitsel_vec; 1749f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 1750f75da298SRichard Henderson return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1751d2fd745fSRichard Henderson 1752db432672SRichard Henderson default: 1753db432672SRichard Henderson tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1754db432672SRichard Henderson return true; 1755be0f34b5SRichard Henderson } 1756be0f34b5SRichard Henderson } 1757be0f34b5SRichard Henderson 175839004a71SRichard Henderson static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); 175939004a71SRichard Henderson 1760ae8b75dcSRichard Henderson void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1761c896fe29Sbellard { 17623e92aa34SRichard Henderson const TCGHelperInfo *info; 176339004a71SRichard Henderson TCGv_i64 extend_free[MAX_CALL_IARGS]; 176439004a71SRichard Henderson int n_extend = 0; 176575e8b9b7SRichard Henderson TCGOp *op; 176639004a71SRichard Henderson int i, n, pi = 0, total_args; 1767afb49896SRichard Henderson 1768619205fdSEmilio G. Cota info = g_hash_table_lookup(helper_table, (gpointer)func); 176939004a71SRichard Henderson total_args = info->nr_out + info->nr_in + 2; 177039004a71SRichard Henderson op = tcg_op_alloc(INDEX_op_call, total_args); 17712bece2c8SRichard Henderson 177238b47b19SEmilio G. Cota #ifdef CONFIG_PLUGIN 177317083f6fSEmilio Cota /* Flag helpers that may affect guest state */ 177417083f6fSEmilio Cota if (tcg_ctx->plugin_insn && 177517083f6fSEmilio Cota !(info->flags & TCG_CALL_PLUGIN) && 177617083f6fSEmilio Cota !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { 177738b47b19SEmilio G. Cota tcg_ctx->plugin_insn->calls_helpers = true; 177838b47b19SEmilio G. Cota } 177938b47b19SEmilio G. Cota #endif 178038b47b19SEmilio G. Cota 178139004a71SRichard Henderson TCGOP_CALLO(op) = n = info->nr_out; 178239004a71SRichard Henderson switch (n) { 178339004a71SRichard Henderson case 0: 178439004a71SRichard Henderson tcg_debug_assert(ret == NULL); 178539004a71SRichard Henderson break; 178639004a71SRichard Henderson case 1: 178739004a71SRichard Henderson tcg_debug_assert(ret != NULL); 178839004a71SRichard Henderson op->args[pi++] = temp_arg(ret); 178939004a71SRichard Henderson break; 179039004a71SRichard Henderson case 2: 1791466d3759SRichard Henderson case 4: 179239004a71SRichard Henderson tcg_debug_assert(ret != NULL); 1793466d3759SRichard Henderson tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); 179439004a71SRichard Henderson tcg_debug_assert(ret->temp_subindex == 0); 1795466d3759SRichard Henderson for (i = 0; i < n; ++i) { 1796466d3759SRichard Henderson op->args[pi++] = temp_arg(ret + i); 1797466d3759SRichard Henderson } 179839004a71SRichard Henderson break; 179939004a71SRichard Henderson default: 180039004a71SRichard Henderson g_assert_not_reached(); 180139004a71SRichard Henderson } 18027319d83aSRichard Henderson 180339004a71SRichard Henderson TCGOP_CALLI(op) = n = info->nr_in; 180439004a71SRichard Henderson for (i = 0; i < n; i++) { 180539004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 180639004a71SRichard Henderson TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; 180739004a71SRichard Henderson 180839004a71SRichard Henderson switch (loc->kind) { 180939004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 1810313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 1811313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 181239004a71SRichard Henderson op->args[pi++] = temp_arg(ts); 181339004a71SRichard Henderson break; 181439004a71SRichard Henderson 181539004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 181639004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 181739004a71SRichard Henderson { 18185dd48602SRichard Henderson TCGv_i64 temp = tcg_temp_ebb_new_i64(); 181939004a71SRichard Henderson TCGv_i32 orig = temp_tcgv_i32(ts); 182039004a71SRichard Henderson 182139004a71SRichard Henderson if (loc->kind == TCG_CALL_ARG_EXTEND_S) { 182218cf3d07SRichard Henderson tcg_gen_ext_i32_i64(temp, orig); 18232bece2c8SRichard Henderson } else { 182418cf3d07SRichard Henderson tcg_gen_extu_i32_i64(temp, orig); 18252bece2c8SRichard Henderson } 182639004a71SRichard Henderson op->args[pi++] = tcgv_i64_arg(temp); 182739004a71SRichard Henderson extend_free[n_extend++] = temp; 18282bece2c8SRichard Henderson } 182939004a71SRichard Henderson break; 18302bece2c8SRichard Henderson 1831e2a9dd6bSRichard Henderson default: 1832e2a9dd6bSRichard Henderson g_assert_not_reached(); 1833e2a9dd6bSRichard Henderson } 1834c896fe29Sbellard } 183575e8b9b7SRichard Henderson op->args[pi++] = (uintptr_t)func; 18363e92aa34SRichard Henderson op->args[pi++] = (uintptr_t)info; 183739004a71SRichard Henderson tcg_debug_assert(pi == total_args); 1838a7812ae4Spbrook 183939004a71SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 18402bece2c8SRichard Henderson 184139004a71SRichard Henderson tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); 184239004a71SRichard Henderson for (i = 0; i < n_extend; ++i) { 184339004a71SRichard Henderson tcg_temp_free_i64(extend_free[i]); 1844eb8b0224SRichard Henderson } 1845a7812ae4Spbrook } 1846c896fe29Sbellard 18478fcd3692Sblueswir1 static void tcg_reg_alloc_start(TCGContext *s) 1848c896fe29Sbellard { 1849ac3b8891SRichard Henderson int i, n; 1850ac3b8891SRichard Henderson 1851ee17db83SRichard Henderson for (i = 0, n = s->nb_temps; i < n; i++) { 1852ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 1853ee17db83SRichard Henderson TCGTempVal val = TEMP_VAL_MEM; 1854ee17db83SRichard Henderson 1855ee17db83SRichard Henderson switch (ts->kind) { 1856c0522136SRichard Henderson case TEMP_CONST: 1857c0522136SRichard Henderson val = TEMP_VAL_CONST; 1858c0522136SRichard Henderson break; 1859ee17db83SRichard Henderson case TEMP_FIXED: 1860ee17db83SRichard Henderson val = TEMP_VAL_REG; 1861ee17db83SRichard Henderson break; 1862ee17db83SRichard Henderson case TEMP_GLOBAL: 1863ee17db83SRichard Henderson break; 1864c7482438SRichard Henderson case TEMP_EBB: 1865ee17db83SRichard Henderson val = TEMP_VAL_DEAD; 1866ee17db83SRichard Henderson /* fall through */ 1867f57c6915SRichard Henderson case TEMP_TB: 1868e8996ee0Sbellard ts->mem_allocated = 0; 1869ee17db83SRichard Henderson break; 1870ee17db83SRichard Henderson default: 1871ee17db83SRichard Henderson g_assert_not_reached(); 1872ee17db83SRichard Henderson } 1873ee17db83SRichard Henderson ts->val_type = val; 1874e8996ee0Sbellard } 1875f8b2f202SRichard Henderson 1876f8b2f202SRichard Henderson memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1877c896fe29Sbellard } 1878c896fe29Sbellard 1879f8b2f202SRichard Henderson static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1880f8b2f202SRichard Henderson TCGTemp *ts) 1881c896fe29Sbellard { 18821807f4c4SRichard Henderson int idx = temp_idx(ts); 1883ac56dd48Spbrook 1884ee17db83SRichard Henderson switch (ts->kind) { 1885ee17db83SRichard Henderson case TEMP_FIXED: 1886ee17db83SRichard Henderson case TEMP_GLOBAL: 1887ac56dd48Spbrook pstrcpy(buf, buf_size, ts->name); 1888ee17db83SRichard Henderson break; 1889f57c6915SRichard Henderson case TEMP_TB: 1890641d5fbeSbellard snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1891ee17db83SRichard Henderson break; 1892c7482438SRichard Henderson case TEMP_EBB: 1893ac56dd48Spbrook snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1894ee17db83SRichard Henderson break; 1895c0522136SRichard Henderson case TEMP_CONST: 1896c0522136SRichard Henderson switch (ts->type) { 1897c0522136SRichard Henderson case TCG_TYPE_I32: 1898c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1899c0522136SRichard Henderson break; 1900c0522136SRichard Henderson #if TCG_TARGET_REG_BITS > 32 1901c0522136SRichard Henderson case TCG_TYPE_I64: 1902c0522136SRichard Henderson snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1903c0522136SRichard Henderson break; 1904c0522136SRichard Henderson #endif 1905c0522136SRichard Henderson case TCG_TYPE_V64: 1906c0522136SRichard Henderson case TCG_TYPE_V128: 1907c0522136SRichard Henderson case TCG_TYPE_V256: 1908c0522136SRichard Henderson snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1909c0522136SRichard Henderson 64 << (ts->type - TCG_TYPE_V64), ts->val); 1910c0522136SRichard Henderson break; 1911c0522136SRichard Henderson default: 1912c0522136SRichard Henderson g_assert_not_reached(); 1913c0522136SRichard Henderson } 1914c0522136SRichard Henderson break; 1915c896fe29Sbellard } 1916c896fe29Sbellard return buf; 1917c896fe29Sbellard } 1918c896fe29Sbellard 191943439139SRichard Henderson static char *tcg_get_arg_str(TCGContext *s, char *buf, 192043439139SRichard Henderson int buf_size, TCGArg arg) 1921f8b2f202SRichard Henderson { 192243439139SRichard Henderson return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1923f8b2f202SRichard Henderson } 1924f8b2f202SRichard Henderson 1925f48f3edeSblueswir1 static const char * const cond_name[] = 1926f48f3edeSblueswir1 { 19270aed257fSRichard Henderson [TCG_COND_NEVER] = "never", 19280aed257fSRichard Henderson [TCG_COND_ALWAYS] = "always", 1929f48f3edeSblueswir1 [TCG_COND_EQ] = "eq", 1930f48f3edeSblueswir1 [TCG_COND_NE] = "ne", 1931f48f3edeSblueswir1 [TCG_COND_LT] = "lt", 1932f48f3edeSblueswir1 [TCG_COND_GE] = "ge", 1933f48f3edeSblueswir1 [TCG_COND_LE] = "le", 1934f48f3edeSblueswir1 [TCG_COND_GT] = "gt", 1935f48f3edeSblueswir1 [TCG_COND_LTU] = "ltu", 1936f48f3edeSblueswir1 [TCG_COND_GEU] = "geu", 1937f48f3edeSblueswir1 [TCG_COND_LEU] = "leu", 1938f48f3edeSblueswir1 [TCG_COND_GTU] = "gtu" 1939f48f3edeSblueswir1 }; 1940f48f3edeSblueswir1 1941f713d6adSRichard Henderson static const char * const ldst_name[] = 1942f713d6adSRichard Henderson { 1943f713d6adSRichard Henderson [MO_UB] = "ub", 1944f713d6adSRichard Henderson [MO_SB] = "sb", 1945f713d6adSRichard Henderson [MO_LEUW] = "leuw", 1946f713d6adSRichard Henderson [MO_LESW] = "lesw", 1947f713d6adSRichard Henderson [MO_LEUL] = "leul", 1948f713d6adSRichard Henderson [MO_LESL] = "lesl", 1949fc313c64SFrédéric Pétrot [MO_LEUQ] = "leq", 1950f713d6adSRichard Henderson [MO_BEUW] = "beuw", 1951f713d6adSRichard Henderson [MO_BESW] = "besw", 1952f713d6adSRichard Henderson [MO_BEUL] = "beul", 1953f713d6adSRichard Henderson [MO_BESL] = "besl", 1954fc313c64SFrédéric Pétrot [MO_BEUQ] = "beq", 1955f713d6adSRichard Henderson }; 1956f713d6adSRichard Henderson 19571f00b27fSSergey Sorokin static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 195852bf9771Stony.nguyen@bt.com #ifdef TARGET_ALIGNED_ONLY 19591f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "un+", 19601f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "", 19611f00b27fSSergey Sorokin #else 19621f00b27fSSergey Sorokin [MO_UNALN >> MO_ASHIFT] = "", 19631f00b27fSSergey Sorokin [MO_ALIGN >> MO_ASHIFT] = "al+", 19641f00b27fSSergey Sorokin #endif 19651f00b27fSSergey Sorokin [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 19661f00b27fSSergey Sorokin [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 19671f00b27fSSergey Sorokin [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 19681f00b27fSSergey Sorokin [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 19691f00b27fSSergey Sorokin [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 19701f00b27fSSergey Sorokin [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 19711f00b27fSSergey Sorokin }; 19721f00b27fSSergey Sorokin 1973587195bdSRichard Henderson static const char bswap_flag_name[][6] = { 1974587195bdSRichard Henderson [TCG_BSWAP_IZ] = "iz", 1975587195bdSRichard Henderson [TCG_BSWAP_OZ] = "oz", 1976587195bdSRichard Henderson [TCG_BSWAP_OS] = "os", 1977587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1978587195bdSRichard Henderson [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1979587195bdSRichard Henderson }; 1980587195bdSRichard Henderson 1981b016486eSRichard Henderson static inline bool tcg_regset_single(TCGRegSet d) 1982b016486eSRichard Henderson { 1983b016486eSRichard Henderson return (d & (d - 1)) == 0; 1984b016486eSRichard Henderson } 1985b016486eSRichard Henderson 1986b016486eSRichard Henderson static inline TCGReg tcg_regset_first(TCGRegSet d) 1987b016486eSRichard Henderson { 1988b016486eSRichard Henderson if (TCG_TARGET_NB_REGS <= 32) { 1989b016486eSRichard Henderson return ctz32(d); 1990b016486eSRichard Henderson } else { 1991b016486eSRichard Henderson return ctz64(d); 1992b016486eSRichard Henderson } 1993b016486eSRichard Henderson } 1994b016486eSRichard Henderson 1995b7a83ff8SRichard Henderson /* Return only the number of characters output -- no error return. */ 1996b7a83ff8SRichard Henderson #define ne_fprintf(...) \ 1997b7a83ff8SRichard Henderson ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 1998b7a83ff8SRichard Henderson 1999b7a83ff8SRichard Henderson static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 2000c896fe29Sbellard { 2001c896fe29Sbellard char buf[128]; 2002c45cb8bbSRichard Henderson TCGOp *op; 2003c896fe29Sbellard 200415fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 2005c45cb8bbSRichard Henderson int i, k, nb_oargs, nb_iargs, nb_cargs; 2006c45cb8bbSRichard Henderson const TCGOpDef *def; 2007c45cb8bbSRichard Henderson TCGOpcode c; 2008bdfb460eSRichard Henderson int col = 0; 2009c45cb8bbSRichard Henderson 2010c45cb8bbSRichard Henderson c = op->opc; 2011c896fe29Sbellard def = &tcg_op_defs[c]; 2012c45cb8bbSRichard Henderson 2013765b842aSRichard Henderson if (c == INDEX_op_insn_start) { 2014b016486eSRichard Henderson nb_oargs = 0; 2015b7a83ff8SRichard Henderson col += ne_fprintf(f, "\n ----"); 20169aef40edSRichard Henderson 20179aef40edSRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 20189aef40edSRichard Henderson target_ulong a; 20197e4597d7Sbellard #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 2020efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 20217e4597d7Sbellard #else 2022efee3746SRichard Henderson a = op->args[i]; 20237e4597d7Sbellard #endif 2024b7a83ff8SRichard Henderson col += ne_fprintf(f, " " TARGET_FMT_lx, a); 2025eeacee4dSBlue Swirl } 20267e4597d7Sbellard } else if (c == INDEX_op_call) { 20273e92aa34SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 2028fa52e660SRichard Henderson void *func = tcg_call_func(op); 20293e92aa34SRichard Henderson 2030c896fe29Sbellard /* variable number of arguments */ 2031cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 2032cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 2033c896fe29Sbellard nb_cargs = def->nb_cargs; 2034b03cce8eSbellard 2035b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name); 20363e92aa34SRichard Henderson 20373e92aa34SRichard Henderson /* 20383e92aa34SRichard Henderson * Print the function name from TCGHelperInfo, if available. 20393e92aa34SRichard Henderson * Note that plugins have a template function for the info, 20403e92aa34SRichard Henderson * but the actual function pointer comes from the plugin. 20413e92aa34SRichard Henderson */ 20423e92aa34SRichard Henderson if (func == info->func) { 2043b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s", info->name); 20443e92aa34SRichard Henderson } else { 2045b7a83ff8SRichard Henderson col += ne_fprintf(f, "plugin(%p)", func); 20463e92aa34SRichard Henderson } 20473e92aa34SRichard Henderson 2048b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 2049b03cce8eSbellard for (i = 0; i < nb_oargs; i++) { 2050b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 2051efee3746SRichard Henderson op->args[i])); 2052b03cce8eSbellard } 2053cf066674SRichard Henderson for (i = 0; i < nb_iargs; i++) { 2054efee3746SRichard Henderson TCGArg arg = op->args[nb_oargs + i]; 205539004a71SRichard Henderson const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 2056b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", t); 2057e8996ee0Sbellard } 2058b03cce8eSbellard } else { 2059b7a83ff8SRichard Henderson col += ne_fprintf(f, " %s ", def->name); 2060c45cb8bbSRichard Henderson 2061c896fe29Sbellard nb_oargs = def->nb_oargs; 2062c896fe29Sbellard nb_iargs = def->nb_iargs; 2063c896fe29Sbellard nb_cargs = def->nb_cargs; 2064c896fe29Sbellard 2065d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 2066b7a83ff8SRichard Henderson col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op), 2067d2fd745fSRichard Henderson 8 << TCGOP_VECE(op)); 2068d2fd745fSRichard Henderson } 2069d2fd745fSRichard Henderson 2070c896fe29Sbellard k = 0; 2071c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 2072b7a83ff8SRichard Henderson const char *sep = k ? "," : ""; 2073b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep, 2074b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf), 2075efee3746SRichard Henderson op->args[k++])); 2076c896fe29Sbellard } 2077c896fe29Sbellard for (i = 0; i < nb_iargs; i++) { 2078b7a83ff8SRichard Henderson const char *sep = k ? "," : ""; 2079b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s%s", sep, 2080b7a83ff8SRichard Henderson tcg_get_arg_str(s, buf, sizeof(buf), 2081efee3746SRichard Henderson op->args[k++])); 2082c896fe29Sbellard } 2083be210acbSRichard Henderson switch (c) { 2084be210acbSRichard Henderson case INDEX_op_brcond_i32: 2085ffc5ea09SRichard Henderson case INDEX_op_setcond_i32: 2086ffc5ea09SRichard Henderson case INDEX_op_movcond_i32: 2087be210acbSRichard Henderson case INDEX_op_brcond2_i32: 2088be210acbSRichard Henderson case INDEX_op_setcond2_i32: 2089ffc5ea09SRichard Henderson case INDEX_op_brcond_i64: 2090be210acbSRichard Henderson case INDEX_op_setcond_i64: 2091ffc5ea09SRichard Henderson case INDEX_op_movcond_i64: 2092212be173SRichard Henderson case INDEX_op_cmp_vec: 2093f75da298SRichard Henderson case INDEX_op_cmpsel_vec: 2094efee3746SRichard Henderson if (op->args[k] < ARRAY_SIZE(cond_name) 2095efee3746SRichard Henderson && cond_name[op->args[k]]) { 2096b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 2097eeacee4dSBlue Swirl } else { 2098b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 2099eeacee4dSBlue Swirl } 2100f48f3edeSblueswir1 i = 1; 2101be210acbSRichard Henderson break; 2102f713d6adSRichard Henderson case INDEX_op_qemu_ld_i32: 2103f713d6adSRichard Henderson case INDEX_op_qemu_st_i32: 210407ce0b05SRichard Henderson case INDEX_op_qemu_st8_i32: 2105f713d6adSRichard Henderson case INDEX_op_qemu_ld_i64: 2106f713d6adSRichard Henderson case INDEX_op_qemu_st_i64: 210759227d5dSRichard Henderson { 21089002ffcbSRichard Henderson MemOpIdx oi = op->args[k++]; 210914776ab5STony Nguyen MemOp op = get_memop(oi); 211059227d5dSRichard Henderson unsigned ix = get_mmuidx(oi); 211159227d5dSRichard Henderson 211259c4b7e8SRichard Henderson if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 2113b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%x,%u", op, ix); 211459c4b7e8SRichard Henderson } else { 21151f00b27fSSergey Sorokin const char *s_al, *s_op; 21161f00b27fSSergey Sorokin s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 211759c4b7e8SRichard Henderson s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 2118b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix); 2119f713d6adSRichard Henderson } 2120f713d6adSRichard Henderson i = 1; 212159227d5dSRichard Henderson } 2122f713d6adSRichard Henderson break; 2123587195bdSRichard Henderson case INDEX_op_bswap16_i32: 2124587195bdSRichard Henderson case INDEX_op_bswap16_i64: 2125587195bdSRichard Henderson case INDEX_op_bswap32_i32: 2126587195bdSRichard Henderson case INDEX_op_bswap32_i64: 2127587195bdSRichard Henderson case INDEX_op_bswap64_i64: 2128587195bdSRichard Henderson { 2129587195bdSRichard Henderson TCGArg flags = op->args[k]; 2130587195bdSRichard Henderson const char *name = NULL; 2131587195bdSRichard Henderson 2132587195bdSRichard Henderson if (flags < ARRAY_SIZE(bswap_flag_name)) { 2133587195bdSRichard Henderson name = bswap_flag_name[flags]; 2134587195bdSRichard Henderson } 2135587195bdSRichard Henderson if (name) { 2136b7a83ff8SRichard Henderson col += ne_fprintf(f, ",%s", name); 2137587195bdSRichard Henderson } else { 2138b7a83ff8SRichard Henderson col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 2139587195bdSRichard Henderson } 2140587195bdSRichard Henderson i = k = 1; 2141587195bdSRichard Henderson } 2142587195bdSRichard Henderson break; 2143be210acbSRichard Henderson default: 2144f48f3edeSblueswir1 i = 0; 2145be210acbSRichard Henderson break; 2146be210acbSRichard Henderson } 214751e3972cSRichard Henderson switch (c) { 214851e3972cSRichard Henderson case INDEX_op_set_label: 214951e3972cSRichard Henderson case INDEX_op_br: 215051e3972cSRichard Henderson case INDEX_op_brcond_i32: 215151e3972cSRichard Henderson case INDEX_op_brcond_i64: 215251e3972cSRichard Henderson case INDEX_op_brcond2_i32: 2153b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$L%d", k ? "," : "", 2154efee3746SRichard Henderson arg_label(op->args[k])->id); 215551e3972cSRichard Henderson i++, k++; 215651e3972cSRichard Henderson break; 21573470867bSRichard Henderson case INDEX_op_mb: 21583470867bSRichard Henderson { 21593470867bSRichard Henderson TCGBar membar = op->args[k]; 21603470867bSRichard Henderson const char *b_op, *m_op; 21613470867bSRichard Henderson 21623470867bSRichard Henderson switch (membar & TCG_BAR_SC) { 21633470867bSRichard Henderson case 0: 21643470867bSRichard Henderson b_op = "none"; 21653470867bSRichard Henderson break; 21663470867bSRichard Henderson case TCG_BAR_LDAQ: 21673470867bSRichard Henderson b_op = "acq"; 21683470867bSRichard Henderson break; 21693470867bSRichard Henderson case TCG_BAR_STRL: 21703470867bSRichard Henderson b_op = "rel"; 21713470867bSRichard Henderson break; 21723470867bSRichard Henderson case TCG_BAR_SC: 21733470867bSRichard Henderson b_op = "seq"; 21743470867bSRichard Henderson break; 21753470867bSRichard Henderson default: 21763470867bSRichard Henderson g_assert_not_reached(); 21773470867bSRichard Henderson } 21783470867bSRichard Henderson 21793470867bSRichard Henderson switch (membar & TCG_MO_ALL) { 21803470867bSRichard Henderson case 0: 21813470867bSRichard Henderson m_op = "none"; 21823470867bSRichard Henderson break; 21833470867bSRichard Henderson case TCG_MO_LD_LD: 21843470867bSRichard Henderson m_op = "rr"; 21853470867bSRichard Henderson break; 21863470867bSRichard Henderson case TCG_MO_LD_ST: 21873470867bSRichard Henderson m_op = "rw"; 21883470867bSRichard Henderson break; 21893470867bSRichard Henderson case TCG_MO_ST_LD: 21903470867bSRichard Henderson m_op = "wr"; 21913470867bSRichard Henderson break; 21923470867bSRichard Henderson case TCG_MO_ST_ST: 21933470867bSRichard Henderson m_op = "ww"; 21943470867bSRichard Henderson break; 21953470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST: 21963470867bSRichard Henderson m_op = "rr+rw"; 21973470867bSRichard Henderson break; 21983470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD: 21993470867bSRichard Henderson m_op = "rr+wr"; 22003470867bSRichard Henderson break; 22013470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_ST: 22023470867bSRichard Henderson m_op = "rr+ww"; 22033470867bSRichard Henderson break; 22043470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD: 22053470867bSRichard Henderson m_op = "rw+wr"; 22063470867bSRichard Henderson break; 22073470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_ST: 22083470867bSRichard Henderson m_op = "rw+ww"; 22093470867bSRichard Henderson break; 22103470867bSRichard Henderson case TCG_MO_ST_LD | TCG_MO_ST_ST: 22113470867bSRichard Henderson m_op = "wr+ww"; 22123470867bSRichard Henderson break; 22133470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD: 22143470867bSRichard Henderson m_op = "rr+rw+wr"; 22153470867bSRichard Henderson break; 22163470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST: 22173470867bSRichard Henderson m_op = "rr+rw+ww"; 22183470867bSRichard Henderson break; 22193470867bSRichard Henderson case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST: 22203470867bSRichard Henderson m_op = "rr+wr+ww"; 22213470867bSRichard Henderson break; 22223470867bSRichard Henderson case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST: 22233470867bSRichard Henderson m_op = "rw+wr+ww"; 22243470867bSRichard Henderson break; 22253470867bSRichard Henderson case TCG_MO_ALL: 22263470867bSRichard Henderson m_op = "all"; 22273470867bSRichard Henderson break; 22283470867bSRichard Henderson default: 22293470867bSRichard Henderson g_assert_not_reached(); 22303470867bSRichard Henderson } 22313470867bSRichard Henderson 22323470867bSRichard Henderson col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op); 22333470867bSRichard Henderson i++, k++; 22343470867bSRichard Henderson } 22353470867bSRichard Henderson break; 223651e3972cSRichard Henderson default: 223751e3972cSRichard Henderson break; 2238eeacee4dSBlue Swirl } 223951e3972cSRichard Henderson for (; i < nb_cargs; i++, k++) { 2240b7a83ff8SRichard Henderson col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 2241b7a83ff8SRichard Henderson op->args[k]); 2242bdfb460eSRichard Henderson } 2243bdfb460eSRichard Henderson } 2244bdfb460eSRichard Henderson 22451894f69aSRichard Henderson if (have_prefs || op->life) { 22461894f69aSRichard Henderson for (; col < 40; ++col) { 2247b7a83ff8SRichard Henderson putc(' ', f); 2248bdfb460eSRichard Henderson } 22491894f69aSRichard Henderson } 22501894f69aSRichard Henderson 22511894f69aSRichard Henderson if (op->life) { 22521894f69aSRichard Henderson unsigned life = op->life; 2253bdfb460eSRichard Henderson 2254bdfb460eSRichard Henderson if (life & (SYNC_ARG * 3)) { 2255b7a83ff8SRichard Henderson ne_fprintf(f, " sync:"); 2256bdfb460eSRichard Henderson for (i = 0; i < 2; ++i) { 2257bdfb460eSRichard Henderson if (life & (SYNC_ARG << i)) { 2258b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i); 2259bdfb460eSRichard Henderson } 2260bdfb460eSRichard Henderson } 2261bdfb460eSRichard Henderson } 2262bdfb460eSRichard Henderson life /= DEAD_ARG; 2263bdfb460eSRichard Henderson if (life) { 2264b7a83ff8SRichard Henderson ne_fprintf(f, " dead:"); 2265bdfb460eSRichard Henderson for (i = 0; life; ++i, life >>= 1) { 2266bdfb460eSRichard Henderson if (life & 1) { 2267b7a83ff8SRichard Henderson ne_fprintf(f, " %d", i); 2268bdfb460eSRichard Henderson } 2269bdfb460eSRichard Henderson } 2270c896fe29Sbellard } 2271b03cce8eSbellard } 22721894f69aSRichard Henderson 22731894f69aSRichard Henderson if (have_prefs) { 22741894f69aSRichard Henderson for (i = 0; i < nb_oargs; ++i) { 227531fd884bSRichard Henderson TCGRegSet set = output_pref(op, i); 22761894f69aSRichard Henderson 22771894f69aSRichard Henderson if (i == 0) { 2278b7a83ff8SRichard Henderson ne_fprintf(f, " pref="); 22791894f69aSRichard Henderson } else { 2280b7a83ff8SRichard Henderson ne_fprintf(f, ","); 22811894f69aSRichard Henderson } 22821894f69aSRichard Henderson if (set == 0) { 2283b7a83ff8SRichard Henderson ne_fprintf(f, "none"); 22841894f69aSRichard Henderson } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 2285b7a83ff8SRichard Henderson ne_fprintf(f, "all"); 22861894f69aSRichard Henderson #ifdef CONFIG_DEBUG_TCG 22871894f69aSRichard Henderson } else if (tcg_regset_single(set)) { 22881894f69aSRichard Henderson TCGReg reg = tcg_regset_first(set); 2289b7a83ff8SRichard Henderson ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 22901894f69aSRichard Henderson #endif 22911894f69aSRichard Henderson } else if (TCG_TARGET_NB_REGS <= 32) { 2292b7a83ff8SRichard Henderson ne_fprintf(f, "0x%x", (uint32_t)set); 22931894f69aSRichard Henderson } else { 2294b7a83ff8SRichard Henderson ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 22951894f69aSRichard Henderson } 22961894f69aSRichard Henderson } 22971894f69aSRichard Henderson } 22981894f69aSRichard Henderson 2299b7a83ff8SRichard Henderson putc('\n', f); 2300c896fe29Sbellard } 2301c896fe29Sbellard } 2302c896fe29Sbellard 2303c896fe29Sbellard /* we give more priority to constraints with less registers */ 2304c896fe29Sbellard static int get_constraint_priority(const TCGOpDef *def, int k) 2305c896fe29Sbellard { 230674a11790SRichard Henderson const TCGArgConstraint *arg_ct = &def->args_ct[k]; 230729f5e925SRichard Henderson int n = ctpop64(arg_ct->regs); 2308c896fe29Sbellard 230929f5e925SRichard Henderson /* 231029f5e925SRichard Henderson * Sort constraints of a single register first, which includes output 231129f5e925SRichard Henderson * aliases (which must exactly match the input already allocated). 231229f5e925SRichard Henderson */ 231329f5e925SRichard Henderson if (n == 1 || arg_ct->oalias) { 231429f5e925SRichard Henderson return INT_MAX; 2315c896fe29Sbellard } 231629f5e925SRichard Henderson 231729f5e925SRichard Henderson /* 231829f5e925SRichard Henderson * Sort register pairs next, first then second immediately after. 231929f5e925SRichard Henderson * Arbitrarily sort multiple pairs by the index of the first reg; 232029f5e925SRichard Henderson * there shouldn't be many pairs. 232129f5e925SRichard Henderson */ 232229f5e925SRichard Henderson switch (arg_ct->pair) { 232329f5e925SRichard Henderson case 1: 232429f5e925SRichard Henderson case 3: 232529f5e925SRichard Henderson return (k + 1) * 2; 232629f5e925SRichard Henderson case 2: 232729f5e925SRichard Henderson return (arg_ct->pair_index + 1) * 2 - 1; 232829f5e925SRichard Henderson } 232929f5e925SRichard Henderson 233029f5e925SRichard Henderson /* Finally, sort by decreasing register count. */ 233129f5e925SRichard Henderson assert(n > 1); 233229f5e925SRichard Henderson return -n; 2333c896fe29Sbellard } 2334c896fe29Sbellard 2335c896fe29Sbellard /* sort from highest priority to lowest */ 2336c896fe29Sbellard static void sort_constraints(TCGOpDef *def, int start, int n) 2337c896fe29Sbellard { 233866792f90SRichard Henderson int i, j; 233966792f90SRichard Henderson TCGArgConstraint *a = def->args_ct; 2340c896fe29Sbellard 234166792f90SRichard Henderson for (i = 0; i < n; i++) { 234266792f90SRichard Henderson a[start + i].sort_index = start + i; 234366792f90SRichard Henderson } 234466792f90SRichard Henderson if (n <= 1) { 2345c896fe29Sbellard return; 234666792f90SRichard Henderson } 2347c896fe29Sbellard for (i = 0; i < n - 1; i++) { 2348c896fe29Sbellard for (j = i + 1; j < n; j++) { 234966792f90SRichard Henderson int p1 = get_constraint_priority(def, a[start + i].sort_index); 235066792f90SRichard Henderson int p2 = get_constraint_priority(def, a[start + j].sort_index); 2351c896fe29Sbellard if (p1 < p2) { 235266792f90SRichard Henderson int tmp = a[start + i].sort_index; 235366792f90SRichard Henderson a[start + i].sort_index = a[start + j].sort_index; 235466792f90SRichard Henderson a[start + j].sort_index = tmp; 2355c896fe29Sbellard } 2356c896fe29Sbellard } 2357c896fe29Sbellard } 2358c896fe29Sbellard } 2359c896fe29Sbellard 2360f69d277eSRichard Henderson static void process_op_defs(TCGContext *s) 2361c896fe29Sbellard { 2362a9751609SRichard Henderson TCGOpcode op; 2363c896fe29Sbellard 2364f69d277eSRichard Henderson for (op = 0; op < NB_OPS; op++) { 2365f69d277eSRichard Henderson TCGOpDef *def = &tcg_op_defs[op]; 2366f69d277eSRichard Henderson const TCGTargetOpDef *tdefs; 236729f5e925SRichard Henderson bool saw_alias_pair = false; 236829f5e925SRichard Henderson int i, o, i2, o2, nb_args; 2369f69d277eSRichard Henderson 2370f69d277eSRichard Henderson if (def->flags & TCG_OPF_NOT_PRESENT) { 2371f69d277eSRichard Henderson continue; 2372f69d277eSRichard Henderson } 2373f69d277eSRichard Henderson 2374c896fe29Sbellard nb_args = def->nb_iargs + def->nb_oargs; 2375f69d277eSRichard Henderson if (nb_args == 0) { 2376f69d277eSRichard Henderson continue; 2377f69d277eSRichard Henderson } 2378f69d277eSRichard Henderson 23794c22e840SRichard Henderson /* 23804c22e840SRichard Henderson * Macro magic should make it impossible, but double-check that 23814c22e840SRichard Henderson * the array index is in range. Since the signness of an enum 23824c22e840SRichard Henderson * is implementation defined, force the result to unsigned. 23834c22e840SRichard Henderson */ 23844c22e840SRichard Henderson unsigned con_set = tcg_target_op_def(op); 23854c22e840SRichard Henderson tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 23864c22e840SRichard Henderson tdefs = &constraint_sets[con_set]; 2387f69d277eSRichard Henderson 2388c896fe29Sbellard for (i = 0; i < nb_args; i++) { 2389f69d277eSRichard Henderson const char *ct_str = tdefs->args_ct_str[i]; 23908940ea0dSPhilippe Mathieu-Daudé bool input_p = i >= def->nb_oargs; 23918940ea0dSPhilippe Mathieu-Daudé 2392f69d277eSRichard Henderson /* Incomplete TCGTargetOpDef entry. */ 2393eabb7b91SAurelien Jarno tcg_debug_assert(ct_str != NULL); 2394f69d277eSRichard Henderson 239517280ff4SRichard Henderson switch (*ct_str) { 239617280ff4SRichard Henderson case '0' ... '9': 23978940ea0dSPhilippe Mathieu-Daudé o = *ct_str - '0'; 23988940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(input_p); 23998940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(o < def->nb_oargs); 24008940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(def->args_ct[o].regs != 0); 24018940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!def->args_ct[o].oalias); 24028940ea0dSPhilippe Mathieu-Daudé def->args_ct[i] = def->args_ct[o]; 2403bc2b17e6SRichard Henderson /* The output sets oalias. */ 24048940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].oalias = 1; 24058940ea0dSPhilippe Mathieu-Daudé def->args_ct[o].alias_index = i; 2406bc2b17e6SRichard Henderson /* The input sets ialias. */ 24078940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].ialias = 1; 24088940ea0dSPhilippe Mathieu-Daudé def->args_ct[i].alias_index = o; 240929f5e925SRichard Henderson if (def->args_ct[i].pair) { 241029f5e925SRichard Henderson saw_alias_pair = true; 241129f5e925SRichard Henderson } 24128940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(ct_str[1] == '\0'); 24138940ea0dSPhilippe Mathieu-Daudé continue; 24148940ea0dSPhilippe Mathieu-Daudé 241582790a87SRichard Henderson case '&': 24168940ea0dSPhilippe Mathieu-Daudé tcg_debug_assert(!input_p); 2417bc2b17e6SRichard Henderson def->args_ct[i].newreg = true; 241882790a87SRichard Henderson ct_str++; 241982790a87SRichard Henderson break; 242029f5e925SRichard Henderson 242129f5e925SRichard Henderson case 'p': /* plus */ 242229f5e925SRichard Henderson /* Allocate to the register after the previous. */ 242329f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 242429f5e925SRichard Henderson o = i - 1; 242529f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair); 242629f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct); 242729f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){ 242829f5e925SRichard Henderson .pair = 2, 242929f5e925SRichard Henderson .pair_index = o, 243029f5e925SRichard Henderson .regs = def->args_ct[o].regs << 1, 243129f5e925SRichard Henderson }; 243229f5e925SRichard Henderson def->args_ct[o].pair = 1; 243329f5e925SRichard Henderson def->args_ct[o].pair_index = i; 243429f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0'); 243529f5e925SRichard Henderson continue; 243629f5e925SRichard Henderson 243729f5e925SRichard Henderson case 'm': /* minus */ 243829f5e925SRichard Henderson /* Allocate to the register before the previous. */ 243929f5e925SRichard Henderson tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 244029f5e925SRichard Henderson o = i - 1; 244129f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].pair); 244229f5e925SRichard Henderson tcg_debug_assert(!def->args_ct[o].ct); 244329f5e925SRichard Henderson def->args_ct[i] = (TCGArgConstraint){ 244429f5e925SRichard Henderson .pair = 1, 244529f5e925SRichard Henderson .pair_index = o, 244629f5e925SRichard Henderson .regs = def->args_ct[o].regs >> 1, 244729f5e925SRichard Henderson }; 244829f5e925SRichard Henderson def->args_ct[o].pair = 2; 244929f5e925SRichard Henderson def->args_ct[o].pair_index = i; 245029f5e925SRichard Henderson tcg_debug_assert(ct_str[1] == '\0'); 245129f5e925SRichard Henderson continue; 24528940ea0dSPhilippe Mathieu-Daudé } 24538940ea0dSPhilippe Mathieu-Daudé 24548940ea0dSPhilippe Mathieu-Daudé do { 24558940ea0dSPhilippe Mathieu-Daudé switch (*ct_str) { 2456c896fe29Sbellard case 'i': 2457c896fe29Sbellard def->args_ct[i].ct |= TCG_CT_CONST; 2458c896fe29Sbellard break; 2459358b4923SRichard Henderson 2460358b4923SRichard Henderson /* Include all of the target-specific constraints. */ 2461358b4923SRichard Henderson 2462358b4923SRichard Henderson #undef CONST 2463358b4923SRichard Henderson #define CONST(CASE, MASK) \ 24648940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].ct |= MASK; break; 2465358b4923SRichard Henderson #define REGS(CASE, MASK) \ 24668940ea0dSPhilippe Mathieu-Daudé case CASE: def->args_ct[i].regs |= MASK; break; 2467358b4923SRichard Henderson 2468358b4923SRichard Henderson #include "tcg-target-con-str.h" 2469358b4923SRichard Henderson 2470358b4923SRichard Henderson #undef REGS 2471358b4923SRichard Henderson #undef CONST 2472c896fe29Sbellard default: 24738940ea0dSPhilippe Mathieu-Daudé case '0' ... '9': 24748940ea0dSPhilippe Mathieu-Daudé case '&': 247529f5e925SRichard Henderson case 'p': 247629f5e925SRichard Henderson case 'm': 2477358b4923SRichard Henderson /* Typo in TCGTargetOpDef constraint. */ 2478358b4923SRichard Henderson g_assert_not_reached(); 2479358b4923SRichard Henderson } 24808940ea0dSPhilippe Mathieu-Daudé } while (*++ct_str != '\0'); 2481c896fe29Sbellard } 2482c896fe29Sbellard 2483c68aaa18SStefan Weil /* TCGTargetOpDef entry with too much information? */ 2484eabb7b91SAurelien Jarno tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2485c68aaa18SStefan Weil 248629f5e925SRichard Henderson /* 248729f5e925SRichard Henderson * Fix up output pairs that are aliased with inputs. 248829f5e925SRichard Henderson * When we created the alias, we copied pair from the output. 248929f5e925SRichard Henderson * There are three cases: 249029f5e925SRichard Henderson * (1a) Pairs of inputs alias pairs of outputs. 249129f5e925SRichard Henderson * (1b) One input aliases the first of a pair of outputs. 249229f5e925SRichard Henderson * (2) One input aliases the second of a pair of outputs. 249329f5e925SRichard Henderson * 249429f5e925SRichard Henderson * Case 1a is handled by making sure that the pair_index'es are 249529f5e925SRichard Henderson * properly updated so that they appear the same as a pair of inputs. 249629f5e925SRichard Henderson * 249729f5e925SRichard Henderson * Case 1b is handled by setting the pair_index of the input to 249829f5e925SRichard Henderson * itself, simply so it doesn't point to an unrelated argument. 249929f5e925SRichard Henderson * Since we don't encounter the "second" during the input allocation 250029f5e925SRichard Henderson * phase, nothing happens with the second half of the input pair. 250129f5e925SRichard Henderson * 250229f5e925SRichard Henderson * Case 2 is handled by setting the second input to pair=3, the 250329f5e925SRichard Henderson * first output to pair=3, and the pair_index'es to match. 250429f5e925SRichard Henderson */ 250529f5e925SRichard Henderson if (saw_alias_pair) { 250629f5e925SRichard Henderson for (i = def->nb_oargs; i < nb_args; i++) { 250729f5e925SRichard Henderson /* 250829f5e925SRichard Henderson * Since [0-9pm] must be alone in the constraint string, 250929f5e925SRichard Henderson * the only way they can both be set is if the pair comes 251029f5e925SRichard Henderson * from the output alias. 251129f5e925SRichard Henderson */ 251229f5e925SRichard Henderson if (!def->args_ct[i].ialias) { 251329f5e925SRichard Henderson continue; 251429f5e925SRichard Henderson } 251529f5e925SRichard Henderson switch (def->args_ct[i].pair) { 251629f5e925SRichard Henderson case 0: 251729f5e925SRichard Henderson break; 251829f5e925SRichard Henderson case 1: 251929f5e925SRichard Henderson o = def->args_ct[i].alias_index; 252029f5e925SRichard Henderson o2 = def->args_ct[o].pair_index; 252129f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 1); 252229f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 2); 252329f5e925SRichard Henderson if (def->args_ct[o2].oalias) { 252429f5e925SRichard Henderson /* Case 1a */ 252529f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index; 252629f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 2); 252729f5e925SRichard Henderson def->args_ct[i2].pair_index = i; 252829f5e925SRichard Henderson def->args_ct[i].pair_index = i2; 252929f5e925SRichard Henderson } else { 253029f5e925SRichard Henderson /* Case 1b */ 253129f5e925SRichard Henderson def->args_ct[i].pair_index = i; 253229f5e925SRichard Henderson } 253329f5e925SRichard Henderson break; 253429f5e925SRichard Henderson case 2: 253529f5e925SRichard Henderson o = def->args_ct[i].alias_index; 253629f5e925SRichard Henderson o2 = def->args_ct[o].pair_index; 253729f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o].pair == 2); 253829f5e925SRichard Henderson tcg_debug_assert(def->args_ct[o2].pair == 1); 253929f5e925SRichard Henderson if (def->args_ct[o2].oalias) { 254029f5e925SRichard Henderson /* Case 1a */ 254129f5e925SRichard Henderson i2 = def->args_ct[o2].alias_index; 254229f5e925SRichard Henderson tcg_debug_assert(def->args_ct[i2].pair == 1); 254329f5e925SRichard Henderson def->args_ct[i2].pair_index = i; 254429f5e925SRichard Henderson def->args_ct[i].pair_index = i2; 254529f5e925SRichard Henderson } else { 254629f5e925SRichard Henderson /* Case 2 */ 254729f5e925SRichard Henderson def->args_ct[i].pair = 3; 254829f5e925SRichard Henderson def->args_ct[o2].pair = 3; 254929f5e925SRichard Henderson def->args_ct[i].pair_index = o2; 255029f5e925SRichard Henderson def->args_ct[o2].pair_index = i; 255129f5e925SRichard Henderson } 255229f5e925SRichard Henderson break; 255329f5e925SRichard Henderson default: 255429f5e925SRichard Henderson g_assert_not_reached(); 255529f5e925SRichard Henderson } 255629f5e925SRichard Henderson } 255729f5e925SRichard Henderson } 255829f5e925SRichard Henderson 2559c896fe29Sbellard /* sort the constraints (XXX: this is just an heuristic) */ 2560c896fe29Sbellard sort_constraints(def, 0, def->nb_oargs); 2561c896fe29Sbellard sort_constraints(def, def->nb_oargs, def->nb_iargs); 2562c896fe29Sbellard } 2563c896fe29Sbellard } 2564c896fe29Sbellard 2565f85b1fc4SRichard Henderson static void remove_label_use(TCGOp *op, int idx) 2566f85b1fc4SRichard Henderson { 2567f85b1fc4SRichard Henderson TCGLabel *label = arg_label(op->args[idx]); 2568f85b1fc4SRichard Henderson TCGLabelUse *use; 2569f85b1fc4SRichard Henderson 2570f85b1fc4SRichard Henderson QSIMPLEQ_FOREACH(use, &label->branches, next) { 2571f85b1fc4SRichard Henderson if (use->op == op) { 2572f85b1fc4SRichard Henderson QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); 2573f85b1fc4SRichard Henderson return; 2574f85b1fc4SRichard Henderson } 2575f85b1fc4SRichard Henderson } 2576f85b1fc4SRichard Henderson g_assert_not_reached(); 2577f85b1fc4SRichard Henderson } 2578f85b1fc4SRichard Henderson 25790c627cdcSRichard Henderson void tcg_op_remove(TCGContext *s, TCGOp *op) 25800c627cdcSRichard Henderson { 2581d88a117eSRichard Henderson switch (op->opc) { 2582d88a117eSRichard Henderson case INDEX_op_br: 2583f85b1fc4SRichard Henderson remove_label_use(op, 0); 2584d88a117eSRichard Henderson break; 2585d88a117eSRichard Henderson case INDEX_op_brcond_i32: 2586d88a117eSRichard Henderson case INDEX_op_brcond_i64: 2587f85b1fc4SRichard Henderson remove_label_use(op, 3); 2588d88a117eSRichard Henderson break; 2589d88a117eSRichard Henderson case INDEX_op_brcond2_i32: 2590f85b1fc4SRichard Henderson remove_label_use(op, 5); 2591d88a117eSRichard Henderson break; 2592d88a117eSRichard Henderson default: 2593d88a117eSRichard Henderson break; 2594d88a117eSRichard Henderson } 2595d88a117eSRichard Henderson 259615fa08f8SRichard Henderson QTAILQ_REMOVE(&s->ops, op, link); 259715fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2598abebf925SRichard Henderson s->nb_ops--; 25990c627cdcSRichard Henderson 26000c627cdcSRichard Henderson #ifdef CONFIG_PROFILER 2601d73415a3SStefan Hajnoczi qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 26020c627cdcSRichard Henderson #endif 26030c627cdcSRichard Henderson } 26040c627cdcSRichard Henderson 2605a80cdd31SRichard Henderson void tcg_remove_ops_after(TCGOp *op) 2606a80cdd31SRichard Henderson { 2607a80cdd31SRichard Henderson TCGContext *s = tcg_ctx; 2608a80cdd31SRichard Henderson 2609a80cdd31SRichard Henderson while (true) { 2610a80cdd31SRichard Henderson TCGOp *last = tcg_last_op(); 2611a80cdd31SRichard Henderson if (last == op) { 2612a80cdd31SRichard Henderson return; 2613a80cdd31SRichard Henderson } 2614a80cdd31SRichard Henderson tcg_op_remove(s, last); 2615a80cdd31SRichard Henderson } 2616a80cdd31SRichard Henderson } 2617a80cdd31SRichard Henderson 2618d4478943SPhilippe Mathieu-Daudé static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) 261915fa08f8SRichard Henderson { 262015fa08f8SRichard Henderson TCGContext *s = tcg_ctx; 2621cb10bc63SRichard Henderson TCGOp *op = NULL; 262215fa08f8SRichard Henderson 2623cb10bc63SRichard Henderson if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { 2624cb10bc63SRichard Henderson QTAILQ_FOREACH(op, &s->free_ops, link) { 2625cb10bc63SRichard Henderson if (nargs <= op->nargs) { 262615fa08f8SRichard Henderson QTAILQ_REMOVE(&s->free_ops, op, link); 2627cb10bc63SRichard Henderson nargs = op->nargs; 2628cb10bc63SRichard Henderson goto found; 262915fa08f8SRichard Henderson } 2630cb10bc63SRichard Henderson } 2631cb10bc63SRichard Henderson } 2632cb10bc63SRichard Henderson 2633cb10bc63SRichard Henderson /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ 2634cb10bc63SRichard Henderson nargs = MAX(4, nargs); 2635cb10bc63SRichard Henderson op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); 2636cb10bc63SRichard Henderson 2637cb10bc63SRichard Henderson found: 263815fa08f8SRichard Henderson memset(op, 0, offsetof(TCGOp, link)); 263915fa08f8SRichard Henderson op->opc = opc; 2640cb10bc63SRichard Henderson op->nargs = nargs; 264115fa08f8SRichard Henderson 2642cb10bc63SRichard Henderson /* Check for bitfield overflow. */ 2643cb10bc63SRichard Henderson tcg_debug_assert(op->nargs == nargs); 2644cb10bc63SRichard Henderson 2645cb10bc63SRichard Henderson s->nb_ops++; 264615fa08f8SRichard Henderson return op; 264715fa08f8SRichard Henderson } 264815fa08f8SRichard Henderson 2649d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) 265015fa08f8SRichard Henderson { 2651d4478943SPhilippe Mathieu-Daudé TCGOp *op = tcg_op_alloc(opc, nargs); 265215fa08f8SRichard Henderson QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 265315fa08f8SRichard Henderson return op; 265415fa08f8SRichard Henderson } 265515fa08f8SRichard Henderson 2656d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 2657d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs) 26585a18407fSRichard Henderson { 2659d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs); 266015fa08f8SRichard Henderson QTAILQ_INSERT_BEFORE(old_op, new_op, link); 26615a18407fSRichard Henderson return new_op; 26625a18407fSRichard Henderson } 26635a18407fSRichard Henderson 2664d4478943SPhilippe Mathieu-Daudé TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 2665d4478943SPhilippe Mathieu-Daudé TCGOpcode opc, unsigned nargs) 26665a18407fSRichard Henderson { 2667d4478943SPhilippe Mathieu-Daudé TCGOp *new_op = tcg_op_alloc(opc, nargs); 266815fa08f8SRichard Henderson QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 26695a18407fSRichard Henderson return new_op; 26705a18407fSRichard Henderson } 26715a18407fSRichard Henderson 2672968f305eSRichard Henderson static void move_label_uses(TCGLabel *to, TCGLabel *from) 2673968f305eSRichard Henderson { 2674968f305eSRichard Henderson TCGLabelUse *u; 2675968f305eSRichard Henderson 2676968f305eSRichard Henderson QSIMPLEQ_FOREACH(u, &from->branches, next) { 2677968f305eSRichard Henderson TCGOp *op = u->op; 2678968f305eSRichard Henderson switch (op->opc) { 2679968f305eSRichard Henderson case INDEX_op_br: 2680968f305eSRichard Henderson op->args[0] = label_arg(to); 2681968f305eSRichard Henderson break; 2682968f305eSRichard Henderson case INDEX_op_brcond_i32: 2683968f305eSRichard Henderson case INDEX_op_brcond_i64: 2684968f305eSRichard Henderson op->args[3] = label_arg(to); 2685968f305eSRichard Henderson break; 2686968f305eSRichard Henderson case INDEX_op_brcond2_i32: 2687968f305eSRichard Henderson op->args[5] = label_arg(to); 2688968f305eSRichard Henderson break; 2689968f305eSRichard Henderson default: 2690968f305eSRichard Henderson g_assert_not_reached(); 2691968f305eSRichard Henderson } 2692968f305eSRichard Henderson } 2693968f305eSRichard Henderson 2694968f305eSRichard Henderson QSIMPLEQ_CONCAT(&to->branches, &from->branches); 2695968f305eSRichard Henderson } 2696968f305eSRichard Henderson 2697b4fc67c7SRichard Henderson /* Reachable analysis : remove unreachable code. */ 26989bbee4c0SRichard Henderson static void __attribute__((noinline)) 26999bbee4c0SRichard Henderson reachable_code_pass(TCGContext *s) 2700b4fc67c7SRichard Henderson { 27014d89d0bbSRichard Henderson TCGOp *op, *op_next, *op_prev; 2702b4fc67c7SRichard Henderson bool dead = false; 2703b4fc67c7SRichard Henderson 2704b4fc67c7SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2705b4fc67c7SRichard Henderson bool remove = dead; 2706b4fc67c7SRichard Henderson TCGLabel *label; 2707b4fc67c7SRichard Henderson 2708b4fc67c7SRichard Henderson switch (op->opc) { 2709b4fc67c7SRichard Henderson case INDEX_op_set_label: 2710b4fc67c7SRichard Henderson label = arg_label(op->args[0]); 27114d89d0bbSRichard Henderson 27124d89d0bbSRichard Henderson /* 2713968f305eSRichard Henderson * Note that the first op in the TB is always a load, 2714968f305eSRichard Henderson * so there is always something before a label. 2715968f305eSRichard Henderson */ 2716968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link); 2717968f305eSRichard Henderson 2718968f305eSRichard Henderson /* 2719968f305eSRichard Henderson * If we find two sequential labels, move all branches to 2720968f305eSRichard Henderson * reference the second label and remove the first label. 2721968f305eSRichard Henderson * Do this before branch to next optimization, so that the 2722968f305eSRichard Henderson * middle label is out of the way. 2723968f305eSRichard Henderson */ 2724968f305eSRichard Henderson if (op_prev->opc == INDEX_op_set_label) { 2725968f305eSRichard Henderson move_label_uses(label, arg_label(op_prev->args[0])); 2726968f305eSRichard Henderson tcg_op_remove(s, op_prev); 2727968f305eSRichard Henderson op_prev = QTAILQ_PREV(op, link); 2728968f305eSRichard Henderson } 2729968f305eSRichard Henderson 2730968f305eSRichard Henderson /* 27314d89d0bbSRichard Henderson * Optimization can fold conditional branches to unconditional. 27324d89d0bbSRichard Henderson * If we find a label which is preceded by an unconditional 27334d89d0bbSRichard Henderson * branch to next, remove the branch. We couldn't do this when 27344d89d0bbSRichard Henderson * processing the branch because any dead code between the branch 27354d89d0bbSRichard Henderson * and label had not yet been removed. 27364d89d0bbSRichard Henderson */ 27374d89d0bbSRichard Henderson if (op_prev->opc == INDEX_op_br && 27384d89d0bbSRichard Henderson label == arg_label(op_prev->args[0])) { 27394d89d0bbSRichard Henderson tcg_op_remove(s, op_prev); 27404d89d0bbSRichard Henderson /* Fall through means insns become live again. */ 27414d89d0bbSRichard Henderson dead = false; 27424d89d0bbSRichard Henderson } 27434d89d0bbSRichard Henderson 2744f85b1fc4SRichard Henderson if (QSIMPLEQ_EMPTY(&label->branches)) { 2745b4fc67c7SRichard Henderson /* 2746b4fc67c7SRichard Henderson * While there is an occasional backward branch, virtually 2747b4fc67c7SRichard Henderson * all branches generated by the translators are forward. 2748b4fc67c7SRichard Henderson * Which means that generally we will have already removed 2749b4fc67c7SRichard Henderson * all references to the label that will be, and there is 2750b4fc67c7SRichard Henderson * little to be gained by iterating. 2751b4fc67c7SRichard Henderson */ 2752b4fc67c7SRichard Henderson remove = true; 2753b4fc67c7SRichard Henderson } else { 2754b4fc67c7SRichard Henderson /* Once we see a label, insns become live again. */ 2755b4fc67c7SRichard Henderson dead = false; 2756b4fc67c7SRichard Henderson remove = false; 2757b4fc67c7SRichard Henderson } 2758b4fc67c7SRichard Henderson break; 2759b4fc67c7SRichard Henderson 2760b4fc67c7SRichard Henderson case INDEX_op_br: 2761b4fc67c7SRichard Henderson case INDEX_op_exit_tb: 2762b4fc67c7SRichard Henderson case INDEX_op_goto_ptr: 2763b4fc67c7SRichard Henderson /* Unconditional branches; everything following is dead. */ 2764b4fc67c7SRichard Henderson dead = true; 2765b4fc67c7SRichard Henderson break; 2766b4fc67c7SRichard Henderson 2767b4fc67c7SRichard Henderson case INDEX_op_call: 2768b4fc67c7SRichard Henderson /* Notice noreturn helper calls, raising exceptions. */ 276990163900SRichard Henderson if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2770b4fc67c7SRichard Henderson dead = true; 2771b4fc67c7SRichard Henderson } 2772b4fc67c7SRichard Henderson break; 2773b4fc67c7SRichard Henderson 2774b4fc67c7SRichard Henderson case INDEX_op_insn_start: 2775b4fc67c7SRichard Henderson /* Never remove -- we need to keep these for unwind. */ 2776b4fc67c7SRichard Henderson remove = false; 2777b4fc67c7SRichard Henderson break; 2778b4fc67c7SRichard Henderson 2779b4fc67c7SRichard Henderson default: 2780b4fc67c7SRichard Henderson break; 2781b4fc67c7SRichard Henderson } 2782b4fc67c7SRichard Henderson 2783b4fc67c7SRichard Henderson if (remove) { 2784b4fc67c7SRichard Henderson tcg_op_remove(s, op); 2785b4fc67c7SRichard Henderson } 2786b4fc67c7SRichard Henderson } 2787b4fc67c7SRichard Henderson } 2788b4fc67c7SRichard Henderson 2789c70fbf0aSRichard Henderson #define TS_DEAD 1 2790c70fbf0aSRichard Henderson #define TS_MEM 2 2791c70fbf0aSRichard Henderson 27925a18407fSRichard Henderson #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 27935a18407fSRichard Henderson #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 27945a18407fSRichard Henderson 279525f49c5fSRichard Henderson /* For liveness_pass_1, the register preferences for a given temp. */ 279625f49c5fSRichard Henderson static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 279725f49c5fSRichard Henderson { 279825f49c5fSRichard Henderson return ts->state_ptr; 279925f49c5fSRichard Henderson } 280025f49c5fSRichard Henderson 280125f49c5fSRichard Henderson /* For liveness_pass_1, reset the preferences for a given temp to the 280225f49c5fSRichard Henderson * maximal regset for its type. 280325f49c5fSRichard Henderson */ 280425f49c5fSRichard Henderson static inline void la_reset_pref(TCGTemp *ts) 280525f49c5fSRichard Henderson { 280625f49c5fSRichard Henderson *la_temp_pref(ts) 280725f49c5fSRichard Henderson = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 280825f49c5fSRichard Henderson } 280925f49c5fSRichard Henderson 28109c43b68dSAurelien Jarno /* liveness analysis: end of function: all temps are dead, and globals 28119c43b68dSAurelien Jarno should be in memory. */ 28122616c808SRichard Henderson static void la_func_end(TCGContext *s, int ng, int nt) 2813c896fe29Sbellard { 2814b83eabeaSRichard Henderson int i; 2815b83eabeaSRichard Henderson 2816b83eabeaSRichard Henderson for (i = 0; i < ng; ++i) { 2817b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 281825f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 2819b83eabeaSRichard Henderson } 2820b83eabeaSRichard Henderson for (i = ng; i < nt; ++i) { 2821b83eabeaSRichard Henderson s->temps[i].state = TS_DEAD; 282225f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 2823b83eabeaSRichard Henderson } 2824c896fe29Sbellard } 2825c896fe29Sbellard 28269c43b68dSAurelien Jarno /* liveness analysis: end of basic block: all temps are dead, globals 28279c43b68dSAurelien Jarno and local temps should be in memory. */ 28282616c808SRichard Henderson static void la_bb_end(TCGContext *s, int ng, int nt) 2829641d5fbeSbellard { 2830b83eabeaSRichard Henderson int i; 2831641d5fbeSbellard 2832ee17db83SRichard Henderson for (i = 0; i < nt; ++i) { 2833ee17db83SRichard Henderson TCGTemp *ts = &s->temps[i]; 2834ee17db83SRichard Henderson int state; 2835ee17db83SRichard Henderson 2836ee17db83SRichard Henderson switch (ts->kind) { 2837ee17db83SRichard Henderson case TEMP_FIXED: 2838ee17db83SRichard Henderson case TEMP_GLOBAL: 2839f57c6915SRichard Henderson case TEMP_TB: 2840ee17db83SRichard Henderson state = TS_DEAD | TS_MEM; 2841ee17db83SRichard Henderson break; 2842c7482438SRichard Henderson case TEMP_EBB: 2843c0522136SRichard Henderson case TEMP_CONST: 2844ee17db83SRichard Henderson state = TS_DEAD; 2845ee17db83SRichard Henderson break; 2846ee17db83SRichard Henderson default: 2847ee17db83SRichard Henderson g_assert_not_reached(); 2848c70fbf0aSRichard Henderson } 2849ee17db83SRichard Henderson ts->state = state; 2850ee17db83SRichard Henderson la_reset_pref(ts); 2851641d5fbeSbellard } 2852641d5fbeSbellard } 2853641d5fbeSbellard 2854f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory. */ 2855f65a061cSRichard Henderson static void la_global_sync(TCGContext *s, int ng) 2856f65a061cSRichard Henderson { 2857f65a061cSRichard Henderson int i; 2858f65a061cSRichard Henderson 2859f65a061cSRichard Henderson for (i = 0; i < ng; ++i) { 286025f49c5fSRichard Henderson int state = s->temps[i].state; 286125f49c5fSRichard Henderson s->temps[i].state = state | TS_MEM; 286225f49c5fSRichard Henderson if (state == TS_DEAD) { 286325f49c5fSRichard Henderson /* If the global was previously dead, reset prefs. */ 286425f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 286525f49c5fSRichard Henderson } 2866f65a061cSRichard Henderson } 2867f65a061cSRichard Henderson } 2868f65a061cSRichard Henderson 2869b4cb76e6SRichard Henderson /* 2870c7482438SRichard Henderson * liveness analysis: conditional branch: all temps are dead unless 2871c7482438SRichard Henderson * explicitly live-across-conditional-branch, globals and local temps 2872c7482438SRichard Henderson * should be synced. 2873b4cb76e6SRichard Henderson */ 2874b4cb76e6SRichard Henderson static void la_bb_sync(TCGContext *s, int ng, int nt) 2875b4cb76e6SRichard Henderson { 2876b4cb76e6SRichard Henderson la_global_sync(s, ng); 2877b4cb76e6SRichard Henderson 2878b4cb76e6SRichard Henderson for (int i = ng; i < nt; ++i) { 2879c0522136SRichard Henderson TCGTemp *ts = &s->temps[i]; 2880c0522136SRichard Henderson int state; 2881c0522136SRichard Henderson 2882c0522136SRichard Henderson switch (ts->kind) { 2883f57c6915SRichard Henderson case TEMP_TB: 2884c0522136SRichard Henderson state = ts->state; 2885c0522136SRichard Henderson ts->state = state | TS_MEM; 2886b4cb76e6SRichard Henderson if (state != TS_DEAD) { 2887b4cb76e6SRichard Henderson continue; 2888b4cb76e6SRichard Henderson } 2889c0522136SRichard Henderson break; 2890c7482438SRichard Henderson case TEMP_EBB: 2891c0522136SRichard Henderson case TEMP_CONST: 2892c0522136SRichard Henderson continue; 2893c0522136SRichard Henderson default: 2894c0522136SRichard Henderson g_assert_not_reached(); 2895b4cb76e6SRichard Henderson } 2896b4cb76e6SRichard Henderson la_reset_pref(&s->temps[i]); 2897b4cb76e6SRichard Henderson } 2898b4cb76e6SRichard Henderson } 2899b4cb76e6SRichard Henderson 2900f65a061cSRichard Henderson /* liveness analysis: sync globals back to memory and kill. */ 2901f65a061cSRichard Henderson static void la_global_kill(TCGContext *s, int ng) 2902f65a061cSRichard Henderson { 2903f65a061cSRichard Henderson int i; 2904f65a061cSRichard Henderson 2905f65a061cSRichard Henderson for (i = 0; i < ng; i++) { 2906f65a061cSRichard Henderson s->temps[i].state = TS_DEAD | TS_MEM; 290725f49c5fSRichard Henderson la_reset_pref(&s->temps[i]); 290825f49c5fSRichard Henderson } 290925f49c5fSRichard Henderson } 291025f49c5fSRichard Henderson 291125f49c5fSRichard Henderson /* liveness analysis: note live globals crossing calls. */ 291225f49c5fSRichard Henderson static void la_cross_call(TCGContext *s, int nt) 291325f49c5fSRichard Henderson { 291425f49c5fSRichard Henderson TCGRegSet mask = ~tcg_target_call_clobber_regs; 291525f49c5fSRichard Henderson int i; 291625f49c5fSRichard Henderson 291725f49c5fSRichard Henderson for (i = 0; i < nt; i++) { 291825f49c5fSRichard Henderson TCGTemp *ts = &s->temps[i]; 291925f49c5fSRichard Henderson if (!(ts->state & TS_DEAD)) { 292025f49c5fSRichard Henderson TCGRegSet *pset = la_temp_pref(ts); 292125f49c5fSRichard Henderson TCGRegSet set = *pset; 292225f49c5fSRichard Henderson 292325f49c5fSRichard Henderson set &= mask; 292425f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 292525f49c5fSRichard Henderson if (set == 0) { 292625f49c5fSRichard Henderson set = tcg_target_available_regs[ts->type] & mask; 292725f49c5fSRichard Henderson } 292825f49c5fSRichard Henderson *pset = set; 292925f49c5fSRichard Henderson } 2930f65a061cSRichard Henderson } 2931f65a061cSRichard Henderson } 2932f65a061cSRichard Henderson 2933874b8574SRichard Henderson /* 2934874b8574SRichard Henderson * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce 2935874b8574SRichard Henderson * to TEMP_EBB, if possible. 2936874b8574SRichard Henderson */ 2937874b8574SRichard Henderson static void __attribute__((noinline)) 2938874b8574SRichard Henderson liveness_pass_0(TCGContext *s) 2939874b8574SRichard Henderson { 2940874b8574SRichard Henderson void * const multiple_ebb = (void *)(uintptr_t)-1; 2941874b8574SRichard Henderson int nb_temps = s->nb_temps; 2942874b8574SRichard Henderson TCGOp *op, *ebb; 2943874b8574SRichard Henderson 2944874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) { 2945874b8574SRichard Henderson s->temps[i].state_ptr = NULL; 2946874b8574SRichard Henderson } 2947874b8574SRichard Henderson 2948874b8574SRichard Henderson /* 2949874b8574SRichard Henderson * Represent each EBB by the op at which it begins. In the case of 2950874b8574SRichard Henderson * the first EBB, this is the first op, otherwise it is a label. 2951874b8574SRichard Henderson * Collect the uses of each TEMP_TB: NULL for unused, EBB for use 2952874b8574SRichard Henderson * within a single EBB, else MULTIPLE_EBB. 2953874b8574SRichard Henderson */ 2954874b8574SRichard Henderson ebb = QTAILQ_FIRST(&s->ops); 2955874b8574SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 2956874b8574SRichard Henderson const TCGOpDef *def; 2957874b8574SRichard Henderson int nb_oargs, nb_iargs; 2958874b8574SRichard Henderson 2959874b8574SRichard Henderson switch (op->opc) { 2960874b8574SRichard Henderson case INDEX_op_set_label: 2961874b8574SRichard Henderson ebb = op; 2962874b8574SRichard Henderson continue; 2963874b8574SRichard Henderson case INDEX_op_discard: 2964874b8574SRichard Henderson continue; 2965874b8574SRichard Henderson case INDEX_op_call: 2966874b8574SRichard Henderson nb_oargs = TCGOP_CALLO(op); 2967874b8574SRichard Henderson nb_iargs = TCGOP_CALLI(op); 2968874b8574SRichard Henderson break; 2969874b8574SRichard Henderson default: 2970874b8574SRichard Henderson def = &tcg_op_defs[op->opc]; 2971874b8574SRichard Henderson nb_oargs = def->nb_oargs; 2972874b8574SRichard Henderson nb_iargs = def->nb_iargs; 2973874b8574SRichard Henderson break; 2974874b8574SRichard Henderson } 2975874b8574SRichard Henderson 2976874b8574SRichard Henderson for (int i = 0; i < nb_oargs + nb_iargs; ++i) { 2977874b8574SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 2978874b8574SRichard Henderson 2979874b8574SRichard Henderson if (ts->kind != TEMP_TB) { 2980874b8574SRichard Henderson continue; 2981874b8574SRichard Henderson } 2982874b8574SRichard Henderson if (ts->state_ptr == NULL) { 2983874b8574SRichard Henderson ts->state_ptr = ebb; 2984874b8574SRichard Henderson } else if (ts->state_ptr != ebb) { 2985874b8574SRichard Henderson ts->state_ptr = multiple_ebb; 2986874b8574SRichard Henderson } 2987874b8574SRichard Henderson } 2988874b8574SRichard Henderson } 2989874b8574SRichard Henderson 2990874b8574SRichard Henderson /* 2991874b8574SRichard Henderson * For TEMP_TB that turned out not to be used beyond one EBB, 2992874b8574SRichard Henderson * reduce the liveness to TEMP_EBB. 2993874b8574SRichard Henderson */ 2994874b8574SRichard Henderson for (int i = s->nb_globals; i < nb_temps; ++i) { 2995874b8574SRichard Henderson TCGTemp *ts = &s->temps[i]; 2996874b8574SRichard Henderson if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { 2997874b8574SRichard Henderson ts->kind = TEMP_EBB; 2998874b8574SRichard Henderson } 2999874b8574SRichard Henderson } 3000874b8574SRichard Henderson } 3001874b8574SRichard Henderson 3002a1b3c48dSRichard Henderson /* Liveness analysis : update the opc_arg_life array to tell if a 3003c896fe29Sbellard given input arguments is dead. Instructions updating dead 3004c896fe29Sbellard temporaries are removed. */ 30059bbee4c0SRichard Henderson static void __attribute__((noinline)) 30069bbee4c0SRichard Henderson liveness_pass_1(TCGContext *s) 3007c896fe29Sbellard { 3008c70fbf0aSRichard Henderson int nb_globals = s->nb_globals; 30092616c808SRichard Henderson int nb_temps = s->nb_temps; 301015fa08f8SRichard Henderson TCGOp *op, *op_prev; 301125f49c5fSRichard Henderson TCGRegSet *prefs; 301225f49c5fSRichard Henderson int i; 301325f49c5fSRichard Henderson 301425f49c5fSRichard Henderson prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 301525f49c5fSRichard Henderson for (i = 0; i < nb_temps; ++i) { 301625f49c5fSRichard Henderson s->temps[i].state_ptr = prefs + i; 301725f49c5fSRichard Henderson } 3018c896fe29Sbellard 3019ae36a246SRichard Henderson /* ??? Should be redundant with the exit_tb that ends the TB. */ 30202616c808SRichard Henderson la_func_end(s, nb_globals, nb_temps); 3021c896fe29Sbellard 3022eae3eb3eSPaolo Bonzini QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 302325f49c5fSRichard Henderson int nb_iargs, nb_oargs; 3024c45cb8bbSRichard Henderson TCGOpcode opc_new, opc_new2; 3025c45cb8bbSRichard Henderson bool have_opc_new2; 3026a1b3c48dSRichard Henderson TCGLifeData arg_life = 0; 302725f49c5fSRichard Henderson TCGTemp *ts; 3028c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 3029c45cb8bbSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 3030c45cb8bbSRichard Henderson 3031c45cb8bbSRichard Henderson switch (opc) { 3032c896fe29Sbellard case INDEX_op_call: 3033c6e113f5Sbellard { 303439004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 303539004a71SRichard Henderson int call_flags = tcg_call_flags(op); 3036c6e113f5Sbellard 3037cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 3038cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 3039c6e113f5Sbellard 3040c45cb8bbSRichard Henderson /* pure functions can be removed if their result is unused */ 304178505279SAurelien Jarno if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 3042c6e113f5Sbellard for (i = 0; i < nb_oargs; i++) { 304325f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 304425f49c5fSRichard Henderson if (ts->state != TS_DEAD) { 3045c6e113f5Sbellard goto do_not_remove_call; 3046c6e113f5Sbellard } 30479c43b68dSAurelien Jarno } 3048c45cb8bbSRichard Henderson goto do_remove; 3049152c35aaSRichard Henderson } 3050c6e113f5Sbellard do_not_remove_call: 3051c896fe29Sbellard 305225f49c5fSRichard Henderson /* Output args are dead. */ 3053c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 305425f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 305525f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3056a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 30576b64b624SAurelien Jarno } 305825f49c5fSRichard Henderson if (ts->state & TS_MEM) { 3059a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 30609c43b68dSAurelien Jarno } 306125f49c5fSRichard Henderson ts->state = TS_DEAD; 306225f49c5fSRichard Henderson la_reset_pref(ts); 3063c896fe29Sbellard } 3064c896fe29Sbellard 306531fd884bSRichard Henderson /* Not used -- it will be tcg_target_call_oarg_reg(). */ 306631fd884bSRichard Henderson memset(op->output_pref, 0, sizeof(op->output_pref)); 306731fd884bSRichard Henderson 306878505279SAurelien Jarno if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 306978505279SAurelien Jarno TCG_CALL_NO_READ_GLOBALS))) { 3070f65a061cSRichard Henderson la_global_kill(s, nb_globals); 3071c70fbf0aSRichard Henderson } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 3072f65a061cSRichard Henderson la_global_sync(s, nb_globals); 3073b9c18f56Saurel32 } 3074c896fe29Sbellard 307525f49c5fSRichard Henderson /* Record arguments that die in this helper. */ 3076866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 307725f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 307839004a71SRichard Henderson if (ts->state & TS_DEAD) { 3079a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 3080c896fe29Sbellard } 3081c896fe29Sbellard } 308225f49c5fSRichard Henderson 308325f49c5fSRichard Henderson /* For all live registers, remove call-clobbered prefs. */ 308425f49c5fSRichard Henderson la_cross_call(s, nb_temps); 308525f49c5fSRichard Henderson 308639004a71SRichard Henderson /* 308739004a71SRichard Henderson * Input arguments are live for preceding opcodes. 308839004a71SRichard Henderson * 308939004a71SRichard Henderson * For those arguments that die, and will be allocated in 309039004a71SRichard Henderson * registers, clear the register set for that arg, to be 309139004a71SRichard Henderson * filled in below. For args that will be on the stack, 309239004a71SRichard Henderson * reset to any available reg. Process arguments in reverse 309339004a71SRichard Henderson * order so that if a temp is used more than once, the stack 309439004a71SRichard Henderson * reset to max happens before the register reset to 0. 309525f49c5fSRichard Henderson */ 309639004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; i--) { 309739004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 309839004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]); 309939004a71SRichard Henderson 310039004a71SRichard Henderson if (ts->state & TS_DEAD) { 310139004a71SRichard Henderson switch (loc->kind) { 310239004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 310339004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 310439004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 310539004a71SRichard Henderson if (REG_P(loc)) { 310639004a71SRichard Henderson *la_temp_pref(ts) = 0; 310739004a71SRichard Henderson break; 310839004a71SRichard Henderson } 310939004a71SRichard Henderson /* fall through */ 311039004a71SRichard Henderson default: 311139004a71SRichard Henderson *la_temp_pref(ts) = 311239004a71SRichard Henderson tcg_target_available_regs[ts->type]; 311339004a71SRichard Henderson break; 311439004a71SRichard Henderson } 311525f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 311625f49c5fSRichard Henderson } 311725f49c5fSRichard Henderson } 311825f49c5fSRichard Henderson 311939004a71SRichard Henderson /* 312039004a71SRichard Henderson * For each input argument, add its input register to prefs. 312139004a71SRichard Henderson * If a temp is used once, this produces a single set bit; 312239004a71SRichard Henderson * if a temp is used multiple times, this produces a set. 312339004a71SRichard Henderson */ 312439004a71SRichard Henderson for (i = 0; i < nb_iargs; i++) { 312539004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 312639004a71SRichard Henderson ts = arg_temp(op->args[nb_oargs + i]); 312739004a71SRichard Henderson 312839004a71SRichard Henderson switch (loc->kind) { 312939004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 313039004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 313139004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 313239004a71SRichard Henderson if (REG_P(loc)) { 313325f49c5fSRichard Henderson tcg_regset_set_reg(*la_temp_pref(ts), 313439004a71SRichard Henderson tcg_target_call_iarg_regs[loc->arg_slot]); 313539004a71SRichard Henderson } 313639004a71SRichard Henderson break; 313739004a71SRichard Henderson default: 313839004a71SRichard Henderson break; 3139c70fbf0aSRichard Henderson } 3140c19f47bfSAurelien Jarno } 3141c6e113f5Sbellard } 3142c896fe29Sbellard break; 3143765b842aSRichard Henderson case INDEX_op_insn_start: 3144c896fe29Sbellard break; 31455ff9d6a4Sbellard case INDEX_op_discard: 31465ff9d6a4Sbellard /* mark the temporary as dead */ 314725f49c5fSRichard Henderson ts = arg_temp(op->args[0]); 314825f49c5fSRichard Henderson ts->state = TS_DEAD; 314925f49c5fSRichard Henderson la_reset_pref(ts); 31505ff9d6a4Sbellard break; 31511305c451SRichard Henderson 31521305c451SRichard Henderson case INDEX_op_add2_i32: 3153c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i32; 3154f1fae40cSRichard Henderson goto do_addsub2; 31551305c451SRichard Henderson case INDEX_op_sub2_i32: 3156c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i32; 3157f1fae40cSRichard Henderson goto do_addsub2; 3158f1fae40cSRichard Henderson case INDEX_op_add2_i64: 3159c45cb8bbSRichard Henderson opc_new = INDEX_op_add_i64; 3160f1fae40cSRichard Henderson goto do_addsub2; 3161f1fae40cSRichard Henderson case INDEX_op_sub2_i64: 3162c45cb8bbSRichard Henderson opc_new = INDEX_op_sub_i64; 3163f1fae40cSRichard Henderson do_addsub2: 31641305c451SRichard Henderson nb_iargs = 4; 31651305c451SRichard Henderson nb_oargs = 2; 31661305c451SRichard Henderson /* Test if the high part of the operation is dead, but not 31671305c451SRichard Henderson the low part. The result can be optimized to a simple 31681305c451SRichard Henderson add or sub. This happens often for x86_64 guest when the 31691305c451SRichard Henderson cpu mode is set to 32 bit. */ 3170b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 3171b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 31721305c451SRichard Henderson goto do_remove; 31731305c451SRichard Henderson } 3174c45cb8bbSRichard Henderson /* Replace the opcode and adjust the args in place, 3175c45cb8bbSRichard Henderson leaving 3 unused args at the end. */ 3176c45cb8bbSRichard Henderson op->opc = opc = opc_new; 3177efee3746SRichard Henderson op->args[1] = op->args[2]; 3178efee3746SRichard Henderson op->args[2] = op->args[4]; 31791305c451SRichard Henderson /* Fall through and mark the single-word operation live. */ 31801305c451SRichard Henderson nb_iargs = 2; 31811305c451SRichard Henderson nb_oargs = 1; 31821305c451SRichard Henderson } 31831305c451SRichard Henderson goto do_not_remove; 31841305c451SRichard Henderson 31851414968aSRichard Henderson case INDEX_op_mulu2_i32: 3186c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 3187c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i32; 3188c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 318903271524SRichard Henderson goto do_mul2; 3190f1fae40cSRichard Henderson case INDEX_op_muls2_i32: 3191c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i32; 3192c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i32; 3193c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 3194f1fae40cSRichard Henderson goto do_mul2; 3195f1fae40cSRichard Henderson case INDEX_op_mulu2_i64: 3196c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 3197c45cb8bbSRichard Henderson opc_new2 = INDEX_op_muluh_i64; 3198c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 319903271524SRichard Henderson goto do_mul2; 3200f1fae40cSRichard Henderson case INDEX_op_muls2_i64: 3201c45cb8bbSRichard Henderson opc_new = INDEX_op_mul_i64; 3202c45cb8bbSRichard Henderson opc_new2 = INDEX_op_mulsh_i64; 3203c45cb8bbSRichard Henderson have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 320403271524SRichard Henderson goto do_mul2; 3205f1fae40cSRichard Henderson do_mul2: 32061414968aSRichard Henderson nb_iargs = 2; 32071414968aSRichard Henderson nb_oargs = 2; 3208b83eabeaSRichard Henderson if (arg_temp(op->args[1])->state == TS_DEAD) { 3209b83eabeaSRichard Henderson if (arg_temp(op->args[0])->state == TS_DEAD) { 321003271524SRichard Henderson /* Both parts of the operation are dead. */ 32111414968aSRichard Henderson goto do_remove; 32121414968aSRichard Henderson } 321303271524SRichard Henderson /* The high part of the operation is dead; generate the low. */ 3214c45cb8bbSRichard Henderson op->opc = opc = opc_new; 3215efee3746SRichard Henderson op->args[1] = op->args[2]; 3216efee3746SRichard Henderson op->args[2] = op->args[3]; 3217b83eabeaSRichard Henderson } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 321803271524SRichard Henderson /* The low part of the operation is dead; generate the high. */ 3219c45cb8bbSRichard Henderson op->opc = opc = opc_new2; 3220efee3746SRichard Henderson op->args[0] = op->args[1]; 3221efee3746SRichard Henderson op->args[1] = op->args[2]; 3222efee3746SRichard Henderson op->args[2] = op->args[3]; 322303271524SRichard Henderson } else { 322403271524SRichard Henderson goto do_not_remove; 322503271524SRichard Henderson } 322603271524SRichard Henderson /* Mark the single-word operation live. */ 32271414968aSRichard Henderson nb_oargs = 1; 32281414968aSRichard Henderson goto do_not_remove; 32291414968aSRichard Henderson 3230c896fe29Sbellard default: 32311305c451SRichard Henderson /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 3232c896fe29Sbellard nb_iargs = def->nb_iargs; 3233c896fe29Sbellard nb_oargs = def->nb_oargs; 3234c896fe29Sbellard 3235c896fe29Sbellard /* Test if the operation can be removed because all 32365ff9d6a4Sbellard its outputs are dead. We assume that nb_oargs == 0 32375ff9d6a4Sbellard implies side effects */ 32385ff9d6a4Sbellard if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 3239c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 3240b83eabeaSRichard Henderson if (arg_temp(op->args[i])->state != TS_DEAD) { 3241c896fe29Sbellard goto do_not_remove; 3242c896fe29Sbellard } 32439c43b68dSAurelien Jarno } 3244152c35aaSRichard Henderson goto do_remove; 3245152c35aaSRichard Henderson } 3246152c35aaSRichard Henderson goto do_not_remove; 3247152c35aaSRichard Henderson 32481305c451SRichard Henderson do_remove: 32490c627cdcSRichard Henderson tcg_op_remove(s, op); 3250152c35aaSRichard Henderson break; 3251152c35aaSRichard Henderson 3252c896fe29Sbellard do_not_remove: 3253c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 325425f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 325525f49c5fSRichard Henderson 325625f49c5fSRichard Henderson /* Remember the preference of the uses that followed. */ 325731fd884bSRichard Henderson if (i < ARRAY_SIZE(op->output_pref)) { 325825f49c5fSRichard Henderson op->output_pref[i] = *la_temp_pref(ts); 325931fd884bSRichard Henderson } 326025f49c5fSRichard Henderson 326125f49c5fSRichard Henderson /* Output args are dead. */ 326225f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3263a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 32646b64b624SAurelien Jarno } 326525f49c5fSRichard Henderson if (ts->state & TS_MEM) { 3266a1b3c48dSRichard Henderson arg_life |= SYNC_ARG << i; 32679c43b68dSAurelien Jarno } 326825f49c5fSRichard Henderson ts->state = TS_DEAD; 326925f49c5fSRichard Henderson la_reset_pref(ts); 3270c896fe29Sbellard } 3271c896fe29Sbellard 327225f49c5fSRichard Henderson /* If end of basic block, update. */ 3273ae36a246SRichard Henderson if (def->flags & TCG_OPF_BB_EXIT) { 3274ae36a246SRichard Henderson la_func_end(s, nb_globals, nb_temps); 3275b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_COND_BRANCH) { 3276b4cb76e6SRichard Henderson la_bb_sync(s, nb_globals, nb_temps); 3277ae36a246SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 32782616c808SRichard Henderson la_bb_end(s, nb_globals, nb_temps); 32793d5c5f87SAurelien Jarno } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3280f65a061cSRichard Henderson la_global_sync(s, nb_globals); 328125f49c5fSRichard Henderson if (def->flags & TCG_OPF_CALL_CLOBBER) { 328225f49c5fSRichard Henderson la_cross_call(s, nb_temps); 328325f49c5fSRichard Henderson } 3284c896fe29Sbellard } 3285c896fe29Sbellard 328625f49c5fSRichard Henderson /* Record arguments that die in this opcode. */ 3287866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 328825f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 328925f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 3290a1b3c48dSRichard Henderson arg_life |= DEAD_ARG << i; 3291c896fe29Sbellard } 3292c19f47bfSAurelien Jarno } 329325f49c5fSRichard Henderson 329425f49c5fSRichard Henderson /* Input arguments are live for preceding opcodes. */ 3295c19f47bfSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 329625f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 329725f49c5fSRichard Henderson if (ts->state & TS_DEAD) { 329825f49c5fSRichard Henderson /* For operands that were dead, initially allow 329925f49c5fSRichard Henderson all regs for the type. */ 330025f49c5fSRichard Henderson *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 330125f49c5fSRichard Henderson ts->state &= ~TS_DEAD; 330225f49c5fSRichard Henderson } 330325f49c5fSRichard Henderson } 330425f49c5fSRichard Henderson 330525f49c5fSRichard Henderson /* Incorporate constraints for this operand. */ 330625f49c5fSRichard Henderson switch (opc) { 330725f49c5fSRichard Henderson case INDEX_op_mov_i32: 330825f49c5fSRichard Henderson case INDEX_op_mov_i64: 330925f49c5fSRichard Henderson /* Note that these are TCG_OPF_NOT_PRESENT and do not 331025f49c5fSRichard Henderson have proper constraints. That said, special case 331125f49c5fSRichard Henderson moves to propagate preferences backward. */ 331225f49c5fSRichard Henderson if (IS_DEAD_ARG(1)) { 331325f49c5fSRichard Henderson *la_temp_pref(arg_temp(op->args[0])) 331425f49c5fSRichard Henderson = *la_temp_pref(arg_temp(op->args[1])); 331525f49c5fSRichard Henderson } 331625f49c5fSRichard Henderson break; 331725f49c5fSRichard Henderson 331825f49c5fSRichard Henderson default: 331925f49c5fSRichard Henderson for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 332025f49c5fSRichard Henderson const TCGArgConstraint *ct = &def->args_ct[i]; 332125f49c5fSRichard Henderson TCGRegSet set, *pset; 332225f49c5fSRichard Henderson 332325f49c5fSRichard Henderson ts = arg_temp(op->args[i]); 332425f49c5fSRichard Henderson pset = la_temp_pref(ts); 332525f49c5fSRichard Henderson set = *pset; 332625f49c5fSRichard Henderson 33279be0d080SRichard Henderson set &= ct->regs; 3328bc2b17e6SRichard Henderson if (ct->ialias) { 332931fd884bSRichard Henderson set &= output_pref(op, ct->alias_index); 333025f49c5fSRichard Henderson } 333125f49c5fSRichard Henderson /* If the combination is not possible, restart. */ 333225f49c5fSRichard Henderson if (set == 0) { 33339be0d080SRichard Henderson set = ct->regs; 333425f49c5fSRichard Henderson } 333525f49c5fSRichard Henderson *pset = set; 333625f49c5fSRichard Henderson } 333725f49c5fSRichard Henderson break; 3338c896fe29Sbellard } 3339c896fe29Sbellard break; 3340c896fe29Sbellard } 3341bee158cbSRichard Henderson op->life = arg_life; 3342c896fe29Sbellard } 33431ff0a2c5SEvgeny Voevodin } 3344c896fe29Sbellard 33455a18407fSRichard Henderson /* Liveness analysis: Convert indirect regs to direct temporaries. */ 33469bbee4c0SRichard Henderson static bool __attribute__((noinline)) 33479bbee4c0SRichard Henderson liveness_pass_2(TCGContext *s) 33485a18407fSRichard Henderson { 33495a18407fSRichard Henderson int nb_globals = s->nb_globals; 335015fa08f8SRichard Henderson int nb_temps, i; 33515a18407fSRichard Henderson bool changes = false; 335215fa08f8SRichard Henderson TCGOp *op, *op_next; 33535a18407fSRichard Henderson 33545a18407fSRichard Henderson /* Create a temporary for each indirect global. */ 33555a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 33565a18407fSRichard Henderson TCGTemp *its = &s->temps[i]; 33575a18407fSRichard Henderson if (its->indirect_reg) { 33585a18407fSRichard Henderson TCGTemp *dts = tcg_temp_alloc(s); 33595a18407fSRichard Henderson dts->type = its->type; 33605a18407fSRichard Henderson dts->base_type = its->base_type; 3361e1e64652SRichard Henderson dts->temp_subindex = its->temp_subindex; 3362c7482438SRichard Henderson dts->kind = TEMP_EBB; 3363b83eabeaSRichard Henderson its->state_ptr = dts; 3364b83eabeaSRichard Henderson } else { 3365b83eabeaSRichard Henderson its->state_ptr = NULL; 33665a18407fSRichard Henderson } 3367b83eabeaSRichard Henderson /* All globals begin dead. */ 3368b83eabeaSRichard Henderson its->state = TS_DEAD; 33695a18407fSRichard Henderson } 3370b83eabeaSRichard Henderson for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 3371b83eabeaSRichard Henderson TCGTemp *its = &s->temps[i]; 3372b83eabeaSRichard Henderson its->state_ptr = NULL; 3373b83eabeaSRichard Henderson its->state = TS_DEAD; 3374b83eabeaSRichard Henderson } 33755a18407fSRichard Henderson 337615fa08f8SRichard Henderson QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 33775a18407fSRichard Henderson TCGOpcode opc = op->opc; 33785a18407fSRichard Henderson const TCGOpDef *def = &tcg_op_defs[opc]; 33795a18407fSRichard Henderson TCGLifeData arg_life = op->life; 33805a18407fSRichard Henderson int nb_iargs, nb_oargs, call_flags; 3381b83eabeaSRichard Henderson TCGTemp *arg_ts, *dir_ts; 33825a18407fSRichard Henderson 33835a18407fSRichard Henderson if (opc == INDEX_op_call) { 3384cd9090aaSRichard Henderson nb_oargs = TCGOP_CALLO(op); 3385cd9090aaSRichard Henderson nb_iargs = TCGOP_CALLI(op); 338690163900SRichard Henderson call_flags = tcg_call_flags(op); 33875a18407fSRichard Henderson } else { 33885a18407fSRichard Henderson nb_iargs = def->nb_iargs; 33895a18407fSRichard Henderson nb_oargs = def->nb_oargs; 33905a18407fSRichard Henderson 33915a18407fSRichard Henderson /* Set flags similar to how calls require. */ 3392b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 3393b4cb76e6SRichard Henderson /* Like reading globals: sync_globals */ 3394b4cb76e6SRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 3395b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 33965a18407fSRichard Henderson /* Like writing globals: save_globals */ 33975a18407fSRichard Henderson call_flags = 0; 33985a18407fSRichard Henderson } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 33995a18407fSRichard Henderson /* Like reading globals: sync_globals */ 34005a18407fSRichard Henderson call_flags = TCG_CALL_NO_WRITE_GLOBALS; 34015a18407fSRichard Henderson } else { 34025a18407fSRichard Henderson /* No effect on globals. */ 34035a18407fSRichard Henderson call_flags = (TCG_CALL_NO_READ_GLOBALS | 34045a18407fSRichard Henderson TCG_CALL_NO_WRITE_GLOBALS); 34055a18407fSRichard Henderson } 34065a18407fSRichard Henderson } 34075a18407fSRichard Henderson 34085a18407fSRichard Henderson /* Make sure that input arguments are available. */ 34095a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3410b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3411b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3412b83eabeaSRichard Henderson if (dir_ts && arg_ts->state == TS_DEAD) { 3413b83eabeaSRichard Henderson TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 34145a18407fSRichard Henderson ? INDEX_op_ld_i32 34155a18407fSRichard Henderson : INDEX_op_ld_i64); 3416d4478943SPhilippe Mathieu-Daudé TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 34175a18407fSRichard Henderson 3418b83eabeaSRichard Henderson lop->args[0] = temp_arg(dir_ts); 3419b83eabeaSRichard Henderson lop->args[1] = temp_arg(arg_ts->mem_base); 3420b83eabeaSRichard Henderson lop->args[2] = arg_ts->mem_offset; 34215a18407fSRichard Henderson 34225a18407fSRichard Henderson /* Loaded, but synced with memory. */ 3423b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 34245a18407fSRichard Henderson } 34255a18407fSRichard Henderson } 34265a18407fSRichard Henderson 34275a18407fSRichard Henderson /* Perform input replacement, and mark inputs that became dead. 34285a18407fSRichard Henderson No action is required except keeping temp_state up to date 34295a18407fSRichard Henderson so that we reload when needed. */ 34305a18407fSRichard Henderson for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3431b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3432b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3433b83eabeaSRichard Henderson if (dir_ts) { 3434b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 34355a18407fSRichard Henderson changes = true; 34365a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 3437b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 34385a18407fSRichard Henderson } 34395a18407fSRichard Henderson } 34405a18407fSRichard Henderson } 34415a18407fSRichard Henderson 34425a18407fSRichard Henderson /* Liveness analysis should ensure that the following are 34435a18407fSRichard Henderson all correct, for call sites and basic block end points. */ 34445a18407fSRichard Henderson if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 34455a18407fSRichard Henderson /* Nothing to do */ 34465a18407fSRichard Henderson } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 34475a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 34485a18407fSRichard Henderson /* Liveness should see that globals are synced back, 34495a18407fSRichard Henderson that is, either TS_DEAD or TS_MEM. */ 3450b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 3451b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 3452b83eabeaSRichard Henderson || arg_ts->state != 0); 34535a18407fSRichard Henderson } 34545a18407fSRichard Henderson } else { 34555a18407fSRichard Henderson for (i = 0; i < nb_globals; ++i) { 34565a18407fSRichard Henderson /* Liveness should see that globals are saved back, 34575a18407fSRichard Henderson that is, TS_DEAD, waiting to be reloaded. */ 3458b83eabeaSRichard Henderson arg_ts = &s->temps[i]; 3459b83eabeaSRichard Henderson tcg_debug_assert(arg_ts->state_ptr == 0 3460b83eabeaSRichard Henderson || arg_ts->state == TS_DEAD); 34615a18407fSRichard Henderson } 34625a18407fSRichard Henderson } 34635a18407fSRichard Henderson 34645a18407fSRichard Henderson /* Outputs become available. */ 346561f15c48SRichard Henderson if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 346661f15c48SRichard Henderson arg_ts = arg_temp(op->args[0]); 346761f15c48SRichard Henderson dir_ts = arg_ts->state_ptr; 346861f15c48SRichard Henderson if (dir_ts) { 346961f15c48SRichard Henderson op->args[0] = temp_arg(dir_ts); 347061f15c48SRichard Henderson changes = true; 347161f15c48SRichard Henderson 347261f15c48SRichard Henderson /* The output is now live and modified. */ 347361f15c48SRichard Henderson arg_ts->state = 0; 347461f15c48SRichard Henderson 347561f15c48SRichard Henderson if (NEED_SYNC_ARG(0)) { 347661f15c48SRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 347761f15c48SRichard Henderson ? INDEX_op_st_i32 347861f15c48SRichard Henderson : INDEX_op_st_i64); 3479d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 348061f15c48SRichard Henderson TCGTemp *out_ts = dir_ts; 348161f15c48SRichard Henderson 348261f15c48SRichard Henderson if (IS_DEAD_ARG(0)) { 348361f15c48SRichard Henderson out_ts = arg_temp(op->args[1]); 348461f15c48SRichard Henderson arg_ts->state = TS_DEAD; 348561f15c48SRichard Henderson tcg_op_remove(s, op); 348661f15c48SRichard Henderson } else { 348761f15c48SRichard Henderson arg_ts->state = TS_MEM; 348861f15c48SRichard Henderson } 348961f15c48SRichard Henderson 349061f15c48SRichard Henderson sop->args[0] = temp_arg(out_ts); 349161f15c48SRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 349261f15c48SRichard Henderson sop->args[2] = arg_ts->mem_offset; 349361f15c48SRichard Henderson } else { 349461f15c48SRichard Henderson tcg_debug_assert(!IS_DEAD_ARG(0)); 349561f15c48SRichard Henderson } 349661f15c48SRichard Henderson } 349761f15c48SRichard Henderson } else { 34985a18407fSRichard Henderson for (i = 0; i < nb_oargs; i++) { 3499b83eabeaSRichard Henderson arg_ts = arg_temp(op->args[i]); 3500b83eabeaSRichard Henderson dir_ts = arg_ts->state_ptr; 3501b83eabeaSRichard Henderson if (!dir_ts) { 35025a18407fSRichard Henderson continue; 35035a18407fSRichard Henderson } 3504b83eabeaSRichard Henderson op->args[i] = temp_arg(dir_ts); 35055a18407fSRichard Henderson changes = true; 35065a18407fSRichard Henderson 35075a18407fSRichard Henderson /* The output is now live and modified. */ 3508b83eabeaSRichard Henderson arg_ts->state = 0; 35095a18407fSRichard Henderson 35105a18407fSRichard Henderson /* Sync outputs upon their last write. */ 35115a18407fSRichard Henderson if (NEED_SYNC_ARG(i)) { 3512b83eabeaSRichard Henderson TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 35135a18407fSRichard Henderson ? INDEX_op_st_i32 35145a18407fSRichard Henderson : INDEX_op_st_i64); 3515d4478943SPhilippe Mathieu-Daudé TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 35165a18407fSRichard Henderson 3517b83eabeaSRichard Henderson sop->args[0] = temp_arg(dir_ts); 3518b83eabeaSRichard Henderson sop->args[1] = temp_arg(arg_ts->mem_base); 3519b83eabeaSRichard Henderson sop->args[2] = arg_ts->mem_offset; 35205a18407fSRichard Henderson 3521b83eabeaSRichard Henderson arg_ts->state = TS_MEM; 35225a18407fSRichard Henderson } 35235a18407fSRichard Henderson /* Drop outputs that are dead. */ 35245a18407fSRichard Henderson if (IS_DEAD_ARG(i)) { 3525b83eabeaSRichard Henderson arg_ts->state = TS_DEAD; 35265a18407fSRichard Henderson } 35275a18407fSRichard Henderson } 35285a18407fSRichard Henderson } 352961f15c48SRichard Henderson } 35305a18407fSRichard Henderson 35315a18407fSRichard Henderson return changes; 35325a18407fSRichard Henderson } 35335a18407fSRichard Henderson 35342272e4a7SRichard Henderson static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3535c896fe29Sbellard { 353631c96417SRichard Henderson intptr_t off; 3537273eb50cSRichard Henderson int size, align; 3538c1c09194SRichard Henderson 3539273eb50cSRichard Henderson /* When allocating an object, look at the full type. */ 3540273eb50cSRichard Henderson size = tcg_type_size(ts->base_type); 3541273eb50cSRichard Henderson switch (ts->base_type) { 3542c1c09194SRichard Henderson case TCG_TYPE_I32: 354331c96417SRichard Henderson align = 4; 3544c1c09194SRichard Henderson break; 3545c1c09194SRichard Henderson case TCG_TYPE_I64: 3546c1c09194SRichard Henderson case TCG_TYPE_V64: 354731c96417SRichard Henderson align = 8; 3548c1c09194SRichard Henderson break; 354943eef72fSRichard Henderson case TCG_TYPE_I128: 3550c1c09194SRichard Henderson case TCG_TYPE_V128: 3551c1c09194SRichard Henderson case TCG_TYPE_V256: 355243eef72fSRichard Henderson /* 355343eef72fSRichard Henderson * Note that we do not require aligned storage for V256, 355443eef72fSRichard Henderson * and that we provide alignment for I128 to match V128, 355543eef72fSRichard Henderson * even if that's above what the host ABI requires. 355643eef72fSRichard Henderson */ 355731c96417SRichard Henderson align = 16; 3558c1c09194SRichard Henderson break; 3559c1c09194SRichard Henderson default: 3560c1c09194SRichard Henderson g_assert_not_reached(); 3561b591dc59SBlue Swirl } 3562c1c09194SRichard Henderson 3563b9537d59SRichard Henderson /* 3564b9537d59SRichard Henderson * Assume the stack is sufficiently aligned. 3565b9537d59SRichard Henderson * This affects e.g. ARM NEON, where we have 8 byte stack alignment 3566b9537d59SRichard Henderson * and do not require 16 byte vector alignment. This seems slightly 3567b9537d59SRichard Henderson * easier than fully parameterizing the above switch statement. 3568b9537d59SRichard Henderson */ 3569b9537d59SRichard Henderson align = MIN(TCG_TARGET_STACK_ALIGN, align); 3570c1c09194SRichard Henderson off = ROUND_UP(s->current_frame_offset, align); 3571732d5897SRichard Henderson 3572732d5897SRichard Henderson /* If we've exhausted the stack frame, restart with a smaller TB. */ 3573732d5897SRichard Henderson if (off + size > s->frame_end) { 3574732d5897SRichard Henderson tcg_raise_tb_overflow(s); 3575732d5897SRichard Henderson } 3576c1c09194SRichard Henderson s->current_frame_offset = off + size; 35779defd1bdSRichard Henderson #if defined(__sparc__) 3578273eb50cSRichard Henderson off += TCG_TARGET_STACK_BIAS; 35799defd1bdSRichard Henderson #endif 3580273eb50cSRichard Henderson 3581273eb50cSRichard Henderson /* If the object was subdivided, assign memory to all the parts. */ 3582273eb50cSRichard Henderson if (ts->base_type != ts->type) { 3583273eb50cSRichard Henderson int part_size = tcg_type_size(ts->type); 3584273eb50cSRichard Henderson int part_count = size / part_size; 3585273eb50cSRichard Henderson 3586273eb50cSRichard Henderson /* 3587273eb50cSRichard Henderson * Each part is allocated sequentially in tcg_temp_new_internal. 3588273eb50cSRichard Henderson * Jump back to the first part by subtracting the current index. 3589273eb50cSRichard Henderson */ 3590273eb50cSRichard Henderson ts -= ts->temp_subindex; 3591273eb50cSRichard Henderson for (int i = 0; i < part_count; ++i) { 3592273eb50cSRichard Henderson ts[i].mem_offset = off + i * part_size; 3593273eb50cSRichard Henderson ts[i].mem_base = s->frame_temp; 3594273eb50cSRichard Henderson ts[i].mem_allocated = 1; 3595273eb50cSRichard Henderson } 3596273eb50cSRichard Henderson } else { 3597273eb50cSRichard Henderson ts->mem_offset = off; 3598b3a62939SRichard Henderson ts->mem_base = s->frame_temp; 3599c896fe29Sbellard ts->mem_allocated = 1; 3600c896fe29Sbellard } 3601273eb50cSRichard Henderson } 3602c896fe29Sbellard 3603098859f1SRichard Henderson /* Assign @reg to @ts, and update reg_to_temp[]. */ 3604098859f1SRichard Henderson static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 3605098859f1SRichard Henderson { 3606098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 3607098859f1SRichard Henderson TCGReg old = ts->reg; 3608098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[old] == ts); 3609098859f1SRichard Henderson if (old == reg) { 3610098859f1SRichard Henderson return; 3611098859f1SRichard Henderson } 3612098859f1SRichard Henderson s->reg_to_temp[old] = NULL; 3613098859f1SRichard Henderson } 3614098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3615098859f1SRichard Henderson s->reg_to_temp[reg] = ts; 3616098859f1SRichard Henderson ts->val_type = TEMP_VAL_REG; 3617098859f1SRichard Henderson ts->reg = reg; 3618098859f1SRichard Henderson } 3619098859f1SRichard Henderson 3620098859f1SRichard Henderson /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 3621098859f1SRichard Henderson static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 3622098859f1SRichard Henderson { 3623098859f1SRichard Henderson tcg_debug_assert(type != TEMP_VAL_REG); 3624098859f1SRichard Henderson if (ts->val_type == TEMP_VAL_REG) { 3625098859f1SRichard Henderson TCGReg reg = ts->reg; 3626098859f1SRichard Henderson tcg_debug_assert(s->reg_to_temp[reg] == ts); 3627098859f1SRichard Henderson s->reg_to_temp[reg] = NULL; 3628098859f1SRichard Henderson } 3629098859f1SRichard Henderson ts->val_type = type; 3630098859f1SRichard Henderson } 3631098859f1SRichard Henderson 3632b722452aSRichard Henderson static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3633b3915dbbSRichard Henderson 363459d7c14eSRichard Henderson /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 363559d7c14eSRichard Henderson mark it free; otherwise mark it dead. */ 363659d7c14eSRichard Henderson static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3637c896fe29Sbellard { 3638c0522136SRichard Henderson TCGTempVal new_type; 3639c0522136SRichard Henderson 3640c0522136SRichard Henderson switch (ts->kind) { 3641c0522136SRichard Henderson case TEMP_FIXED: 364259d7c14eSRichard Henderson return; 3643c0522136SRichard Henderson case TEMP_GLOBAL: 3644f57c6915SRichard Henderson case TEMP_TB: 3645c0522136SRichard Henderson new_type = TEMP_VAL_MEM; 3646c0522136SRichard Henderson break; 3647c7482438SRichard Henderson case TEMP_EBB: 3648c0522136SRichard Henderson new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3649c0522136SRichard Henderson break; 3650c0522136SRichard Henderson case TEMP_CONST: 3651c0522136SRichard Henderson new_type = TEMP_VAL_CONST; 3652c0522136SRichard Henderson break; 3653c0522136SRichard Henderson default: 3654c0522136SRichard Henderson g_assert_not_reached(); 365559d7c14eSRichard Henderson } 3656098859f1SRichard Henderson set_temp_val_nonreg(s, ts, new_type); 365759d7c14eSRichard Henderson } 3658c896fe29Sbellard 365959d7c14eSRichard Henderson /* Mark a temporary as dead. */ 366059d7c14eSRichard Henderson static inline void temp_dead(TCGContext *s, TCGTemp *ts) 366159d7c14eSRichard Henderson { 366259d7c14eSRichard Henderson temp_free_or_dead(s, ts, 1); 366359d7c14eSRichard Henderson } 366459d7c14eSRichard Henderson 366559d7c14eSRichard Henderson /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 366659d7c14eSRichard Henderson registers needs to be allocated to store a constant. If 'free_or_dead' 366759d7c14eSRichard Henderson is non-zero, subsequently release the temporary; if it is positive, the 366859d7c14eSRichard Henderson temp is dead; if it is negative, the temp is free. */ 366998b4e186SRichard Henderson static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 367098b4e186SRichard Henderson TCGRegSet preferred_regs, int free_or_dead) 367159d7c14eSRichard Henderson { 3672c0522136SRichard Henderson if (!temp_readonly(ts) && !ts->mem_coherent) { 36737f6ceedfSAurelien Jarno if (!ts->mem_allocated) { 36742272e4a7SRichard Henderson temp_allocate_frame(s, ts); 367559d7c14eSRichard Henderson } 367659d7c14eSRichard Henderson switch (ts->val_type) { 367759d7c14eSRichard Henderson case TEMP_VAL_CONST: 367859d7c14eSRichard Henderson /* If we're going to free the temp immediately, then we won't 367959d7c14eSRichard Henderson require it later in a register, so attempt to store the 368059d7c14eSRichard Henderson constant to memory directly. */ 368159d7c14eSRichard Henderson if (free_or_dead 368259d7c14eSRichard Henderson && tcg_out_sti(s, ts->type, ts->val, 368359d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset)) { 368459d7c14eSRichard Henderson break; 368559d7c14eSRichard Henderson } 368659d7c14eSRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 368798b4e186SRichard Henderson allocated_regs, preferred_regs); 368859d7c14eSRichard Henderson /* fallthrough */ 368959d7c14eSRichard Henderson 369059d7c14eSRichard Henderson case TEMP_VAL_REG: 369159d7c14eSRichard Henderson tcg_out_st(s, ts->type, ts->reg, 369259d7c14eSRichard Henderson ts->mem_base->reg, ts->mem_offset); 369359d7c14eSRichard Henderson break; 369459d7c14eSRichard Henderson 369559d7c14eSRichard Henderson case TEMP_VAL_MEM: 369659d7c14eSRichard Henderson break; 369759d7c14eSRichard Henderson 369859d7c14eSRichard Henderson case TEMP_VAL_DEAD: 369959d7c14eSRichard Henderson default: 370059d7c14eSRichard Henderson tcg_abort(); 3701c896fe29Sbellard } 37027f6ceedfSAurelien Jarno ts->mem_coherent = 1; 37037f6ceedfSAurelien Jarno } 370459d7c14eSRichard Henderson if (free_or_dead) { 370559d7c14eSRichard Henderson temp_free_or_dead(s, ts, free_or_dead); 370659d7c14eSRichard Henderson } 370759d7c14eSRichard Henderson } 37087f6ceedfSAurelien Jarno 37097f6ceedfSAurelien Jarno /* free register 'reg' by spilling the corresponding temporary if necessary */ 3710b3915dbbSRichard Henderson static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 37117f6ceedfSAurelien Jarno { 3712f8b2f202SRichard Henderson TCGTemp *ts = s->reg_to_temp[reg]; 3713f8b2f202SRichard Henderson if (ts != NULL) { 371498b4e186SRichard Henderson temp_sync(s, ts, allocated_regs, 0, -1); 3715c896fe29Sbellard } 3716c896fe29Sbellard } 3717c896fe29Sbellard 3718b016486eSRichard Henderson /** 3719b016486eSRichard Henderson * tcg_reg_alloc: 3720b016486eSRichard Henderson * @required_regs: Set of registers in which we must allocate. 3721b016486eSRichard Henderson * @allocated_regs: Set of registers which must be avoided. 3722b016486eSRichard Henderson * @preferred_regs: Set of registers we should prefer. 3723b016486eSRichard Henderson * @rev: True if we search the registers in "indirect" order. 3724b016486eSRichard Henderson * 3725b016486eSRichard Henderson * The allocated register must be in @required_regs & ~@allocated_regs, 3726b016486eSRichard Henderson * but if we can put it in @preferred_regs we may save a move later. 3727b016486eSRichard Henderson */ 3728b016486eSRichard Henderson static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3729b016486eSRichard Henderson TCGRegSet allocated_regs, 3730b016486eSRichard Henderson TCGRegSet preferred_regs, bool rev) 3731c896fe29Sbellard { 3732b016486eSRichard Henderson int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3733b016486eSRichard Henderson TCGRegSet reg_ct[2]; 373491478cefSRichard Henderson const int *order; 3735c896fe29Sbellard 3736b016486eSRichard Henderson reg_ct[1] = required_regs & ~allocated_regs; 3737b016486eSRichard Henderson tcg_debug_assert(reg_ct[1] != 0); 3738b016486eSRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs; 3739b016486eSRichard Henderson 3740b016486eSRichard Henderson /* Skip the preferred_regs option if it cannot be satisfied, 3741b016486eSRichard Henderson or if the preference made no difference. */ 3742b016486eSRichard Henderson f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3743b016486eSRichard Henderson 374491478cefSRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3745c896fe29Sbellard 3746b016486eSRichard Henderson /* Try free registers, preferences first. */ 3747b016486eSRichard Henderson for (j = f; j < 2; j++) { 3748b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 3749b016486eSRichard Henderson 3750b016486eSRichard Henderson if (tcg_regset_single(set)) { 3751b016486eSRichard Henderson /* One register in the set. */ 3752b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 3753b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL) { 3754c896fe29Sbellard return reg; 3755c896fe29Sbellard } 3756b016486eSRichard Henderson } else { 375791478cefSRichard Henderson for (i = 0; i < n; i++) { 3758b016486eSRichard Henderson TCGReg reg = order[i]; 3759b016486eSRichard Henderson if (s->reg_to_temp[reg] == NULL && 3760b016486eSRichard Henderson tcg_regset_test_reg(set, reg)) { 3761b016486eSRichard Henderson return reg; 3762b016486eSRichard Henderson } 3763b016486eSRichard Henderson } 3764b016486eSRichard Henderson } 3765b016486eSRichard Henderson } 3766b016486eSRichard Henderson 3767b016486eSRichard Henderson /* We must spill something. */ 3768b016486eSRichard Henderson for (j = f; j < 2; j++) { 3769b016486eSRichard Henderson TCGRegSet set = reg_ct[j]; 3770b016486eSRichard Henderson 3771b016486eSRichard Henderson if (tcg_regset_single(set)) { 3772b016486eSRichard Henderson /* One register in the set. */ 3773b016486eSRichard Henderson TCGReg reg = tcg_regset_first(set); 3774b3915dbbSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 3775c896fe29Sbellard return reg; 3776b016486eSRichard Henderson } else { 3777b016486eSRichard Henderson for (i = 0; i < n; i++) { 3778b016486eSRichard Henderson TCGReg reg = order[i]; 3779b016486eSRichard Henderson if (tcg_regset_test_reg(set, reg)) { 3780b016486eSRichard Henderson tcg_reg_free(s, reg, allocated_regs); 3781b016486eSRichard Henderson return reg; 3782b016486eSRichard Henderson } 3783b016486eSRichard Henderson } 3784c896fe29Sbellard } 3785c896fe29Sbellard } 3786c896fe29Sbellard 3787c896fe29Sbellard tcg_abort(); 3788c896fe29Sbellard } 3789c896fe29Sbellard 379029f5e925SRichard Henderson static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 379129f5e925SRichard Henderson TCGRegSet allocated_regs, 379229f5e925SRichard Henderson TCGRegSet preferred_regs, bool rev) 379329f5e925SRichard Henderson { 379429f5e925SRichard Henderson int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 379529f5e925SRichard Henderson TCGRegSet reg_ct[2]; 379629f5e925SRichard Henderson const int *order; 379729f5e925SRichard Henderson 379829f5e925SRichard Henderson /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 379929f5e925SRichard Henderson reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 380029f5e925SRichard Henderson tcg_debug_assert(reg_ct[1] != 0); 380129f5e925SRichard Henderson reg_ct[0] = reg_ct[1] & preferred_regs; 380229f5e925SRichard Henderson 380329f5e925SRichard Henderson order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 380429f5e925SRichard Henderson 380529f5e925SRichard Henderson /* 380629f5e925SRichard Henderson * Skip the preferred_regs option if it cannot be satisfied, 380729f5e925SRichard Henderson * or if the preference made no difference. 380829f5e925SRichard Henderson */ 380929f5e925SRichard Henderson k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 381029f5e925SRichard Henderson 381129f5e925SRichard Henderson /* 381229f5e925SRichard Henderson * Minimize the number of flushes by looking for 2 free registers first, 381329f5e925SRichard Henderson * then a single flush, then two flushes. 381429f5e925SRichard Henderson */ 381529f5e925SRichard Henderson for (fmin = 2; fmin >= 0; fmin--) { 381629f5e925SRichard Henderson for (j = k; j < 2; j++) { 381729f5e925SRichard Henderson TCGRegSet set = reg_ct[j]; 381829f5e925SRichard Henderson 381929f5e925SRichard Henderson for (i = 0; i < n; i++) { 382029f5e925SRichard Henderson TCGReg reg = order[i]; 382129f5e925SRichard Henderson 382229f5e925SRichard Henderson if (tcg_regset_test_reg(set, reg)) { 382329f5e925SRichard Henderson int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 382429f5e925SRichard Henderson if (f >= fmin) { 382529f5e925SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 382629f5e925SRichard Henderson tcg_reg_free(s, reg + 1, allocated_regs); 382729f5e925SRichard Henderson return reg; 382829f5e925SRichard Henderson } 382929f5e925SRichard Henderson } 383029f5e925SRichard Henderson } 383129f5e925SRichard Henderson } 383229f5e925SRichard Henderson } 383329f5e925SRichard Henderson tcg_abort(); 383429f5e925SRichard Henderson } 383529f5e925SRichard Henderson 383640ae5c62SRichard Henderson /* Make sure the temporary is in a register. If needed, allocate the register 383740ae5c62SRichard Henderson from DESIRED while avoiding ALLOCATED. */ 383840ae5c62SRichard Henderson static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3839b722452aSRichard Henderson TCGRegSet allocated_regs, TCGRegSet preferred_regs) 384040ae5c62SRichard Henderson { 384140ae5c62SRichard Henderson TCGReg reg; 384240ae5c62SRichard Henderson 384340ae5c62SRichard Henderson switch (ts->val_type) { 384440ae5c62SRichard Henderson case TEMP_VAL_REG: 384540ae5c62SRichard Henderson return; 384640ae5c62SRichard Henderson case TEMP_VAL_CONST: 3847b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3848b722452aSRichard Henderson preferred_regs, ts->indirect_base); 38490a6a8bc8SRichard Henderson if (ts->type <= TCG_TYPE_I64) { 385040ae5c62SRichard Henderson tcg_out_movi(s, ts->type, reg, ts->val); 38510a6a8bc8SRichard Henderson } else { 38524e186175SRichard Henderson uint64_t val = ts->val; 38534e186175SRichard Henderson MemOp vece = MO_64; 38544e186175SRichard Henderson 38554e186175SRichard Henderson /* 38564e186175SRichard Henderson * Find the minimal vector element that matches the constant. 38574e186175SRichard Henderson * The targets will, in general, have to do this search anyway, 38584e186175SRichard Henderson * do this generically. 38594e186175SRichard Henderson */ 38604e186175SRichard Henderson if (val == dup_const(MO_8, val)) { 38614e186175SRichard Henderson vece = MO_8; 38624e186175SRichard Henderson } else if (val == dup_const(MO_16, val)) { 38634e186175SRichard Henderson vece = MO_16; 38640b4286ddSRichard Henderson } else if (val == dup_const(MO_32, val)) { 38654e186175SRichard Henderson vece = MO_32; 38664e186175SRichard Henderson } 38674e186175SRichard Henderson 38684e186175SRichard Henderson tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 38690a6a8bc8SRichard Henderson } 387040ae5c62SRichard Henderson ts->mem_coherent = 0; 387140ae5c62SRichard Henderson break; 387240ae5c62SRichard Henderson case TEMP_VAL_MEM: 3873b016486eSRichard Henderson reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3874b722452aSRichard Henderson preferred_regs, ts->indirect_base); 387540ae5c62SRichard Henderson tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 387640ae5c62SRichard Henderson ts->mem_coherent = 1; 387740ae5c62SRichard Henderson break; 387840ae5c62SRichard Henderson case TEMP_VAL_DEAD: 387940ae5c62SRichard Henderson default: 388040ae5c62SRichard Henderson tcg_abort(); 388140ae5c62SRichard Henderson } 3882098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 388340ae5c62SRichard Henderson } 388440ae5c62SRichard Henderson 388559d7c14eSRichard Henderson /* Save a temporary to memory. 'allocated_regs' is used in case a 3886e8996ee0Sbellard temporary registers needs to be allocated to store a constant. */ 388759d7c14eSRichard Henderson static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 38881ad80729SAurelien Jarno { 38892c0366f0SAurelien Jarno /* The liveness analysis already ensures that globals are back 3890eabb7b91SAurelien Jarno in memory. Keep an tcg_debug_assert for safety. */ 3891e01fa97dSRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 38921ad80729SAurelien Jarno } 38931ad80729SAurelien Jarno 38949814dd27SDong Xu Wang /* save globals to their canonical location and assume they can be 3895641d5fbeSbellard modified be the following code. 'allocated_regs' is used in case a 3896641d5fbeSbellard temporary registers needs to be allocated to store a constant. */ 3897641d5fbeSbellard static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3898641d5fbeSbellard { 3899ac3b8891SRichard Henderson int i, n; 3900641d5fbeSbellard 3901ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 3902b13eb728SRichard Henderson temp_save(s, &s->temps[i], allocated_regs); 3903641d5fbeSbellard } 3904e5097dc8Sbellard } 3905e5097dc8Sbellard 39063d5c5f87SAurelien Jarno /* sync globals to their canonical location and assume they can be 39073d5c5f87SAurelien Jarno read by the following code. 'allocated_regs' is used in case a 39083d5c5f87SAurelien Jarno temporary registers needs to be allocated to store a constant. */ 39093d5c5f87SAurelien Jarno static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 39103d5c5f87SAurelien Jarno { 3911ac3b8891SRichard Henderson int i, n; 39123d5c5f87SAurelien Jarno 3913ac3b8891SRichard Henderson for (i = 0, n = s->nb_globals; i < n; i++) { 391412b9b11aSRichard Henderson TCGTemp *ts = &s->temps[i]; 391512b9b11aSRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3916ee17db83SRichard Henderson || ts->kind == TEMP_FIXED 391712b9b11aSRichard Henderson || ts->mem_coherent); 39183d5c5f87SAurelien Jarno } 39193d5c5f87SAurelien Jarno } 39203d5c5f87SAurelien Jarno 3921e5097dc8Sbellard /* at the end of a basic block, we assume all temporaries are dead and 3922e8996ee0Sbellard all globals are stored at their canonical location. */ 3923e8996ee0Sbellard static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3924e5097dc8Sbellard { 3925e5097dc8Sbellard int i; 3926e5097dc8Sbellard 3927c896fe29Sbellard for (i = s->nb_globals; i < s->nb_temps; i++) { 3928b13eb728SRichard Henderson TCGTemp *ts = &s->temps[i]; 3929c0522136SRichard Henderson 3930c0522136SRichard Henderson switch (ts->kind) { 3931f57c6915SRichard Henderson case TEMP_TB: 3932b13eb728SRichard Henderson temp_save(s, ts, allocated_regs); 3933c0522136SRichard Henderson break; 3934c7482438SRichard Henderson case TEMP_EBB: 39352c0366f0SAurelien Jarno /* The liveness analysis already ensures that temps are dead. 3936eabb7b91SAurelien Jarno Keep an tcg_debug_assert for safety. */ 3937eabb7b91SAurelien Jarno tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3938c0522136SRichard Henderson break; 3939c0522136SRichard Henderson case TEMP_CONST: 3940c0522136SRichard Henderson /* Similarly, we should have freed any allocated register. */ 3941c0522136SRichard Henderson tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3942c0522136SRichard Henderson break; 3943c0522136SRichard Henderson default: 3944c0522136SRichard Henderson g_assert_not_reached(); 3945c896fe29Sbellard } 3946641d5fbeSbellard } 3947e8996ee0Sbellard 3948e8996ee0Sbellard save_globals(s, allocated_regs); 3949c896fe29Sbellard } 3950c896fe29Sbellard 3951bab1671fSRichard Henderson /* 3952c7482438SRichard Henderson * At a conditional branch, we assume all temporaries are dead unless 3953c7482438SRichard Henderson * explicitly live-across-conditional-branch; all globals and local 3954c7482438SRichard Henderson * temps are synced to their location. 3955b4cb76e6SRichard Henderson */ 3956b4cb76e6SRichard Henderson static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3957b4cb76e6SRichard Henderson { 3958b4cb76e6SRichard Henderson sync_globals(s, allocated_regs); 3959b4cb76e6SRichard Henderson 3960b4cb76e6SRichard Henderson for (int i = s->nb_globals; i < s->nb_temps; i++) { 3961b4cb76e6SRichard Henderson TCGTemp *ts = &s->temps[i]; 3962b4cb76e6SRichard Henderson /* 3963b4cb76e6SRichard Henderson * The liveness analysis already ensures that temps are dead. 3964b4cb76e6SRichard Henderson * Keep tcg_debug_asserts for safety. 3965b4cb76e6SRichard Henderson */ 3966c0522136SRichard Henderson switch (ts->kind) { 3967f57c6915SRichard Henderson case TEMP_TB: 3968b4cb76e6SRichard Henderson tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3969c0522136SRichard Henderson break; 3970c7482438SRichard Henderson case TEMP_EBB: 3971c0522136SRichard Henderson case TEMP_CONST: 3972c0522136SRichard Henderson break; 3973c0522136SRichard Henderson default: 3974c0522136SRichard Henderson g_assert_not_reached(); 3975b4cb76e6SRichard Henderson } 3976b4cb76e6SRichard Henderson } 3977b4cb76e6SRichard Henderson } 3978b4cb76e6SRichard Henderson 3979b4cb76e6SRichard Henderson /* 3980c58f4c97SRichard Henderson * Specialized code generation for INDEX_op_mov_* with a constant. 3981bab1671fSRichard Henderson */ 39820fe4fca4SPaolo Bonzini static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3983ba87719cSRichard Henderson tcg_target_ulong val, TCGLifeData arg_life, 3984ba87719cSRichard Henderson TCGRegSet preferred_regs) 3985e8996ee0Sbellard { 3986d63e3b6eSRichard Henderson /* ENV should not be modified. */ 3987e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 398859d7c14eSRichard Henderson 398959d7c14eSRichard Henderson /* The movi is not explicitly generated here. */ 3990098859f1SRichard Henderson set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 3991e8996ee0Sbellard ots->val = val; 399259d7c14eSRichard Henderson ots->mem_coherent = 0; 3993ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 3994ba87719cSRichard Henderson temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 399559d7c14eSRichard Henderson } else if (IS_DEAD_ARG(0)) { 3996f8bf00f1SRichard Henderson temp_dead(s, ots); 39974c4e1ab2SAurelien Jarno } 3998e8996ee0Sbellard } 3999e8996ee0Sbellard 4000bab1671fSRichard Henderson /* 4001bab1671fSRichard Henderson * Specialized code generation for INDEX_op_mov_*. 4002bab1671fSRichard Henderson */ 4003dd186292SRichard Henderson static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 4004c896fe29Sbellard { 4005dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 400669e3706dSRichard Henderson TCGRegSet allocated_regs, preferred_regs; 4007c896fe29Sbellard TCGTemp *ts, *ots; 4008450445d5SRichard Henderson TCGType otype, itype; 4009098859f1SRichard Henderson TCGReg oreg, ireg; 4010c896fe29Sbellard 4011d21369f5SRichard Henderson allocated_regs = s->reserved_regs; 401231fd884bSRichard Henderson preferred_regs = output_pref(op, 0); 401343439139SRichard Henderson ots = arg_temp(op->args[0]); 401443439139SRichard Henderson ts = arg_temp(op->args[1]); 4015450445d5SRichard Henderson 4016d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4017e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4018d63e3b6eSRichard Henderson 4019450445d5SRichard Henderson /* Note that otype != itype for no-op truncation. */ 4020450445d5SRichard Henderson otype = ots->type; 4021450445d5SRichard Henderson itype = ts->type; 4022c896fe29Sbellard 40230fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_CONST) { 40240fe4fca4SPaolo Bonzini /* propagate constant or generate sti */ 40250fe4fca4SPaolo Bonzini tcg_target_ulong val = ts->val; 40260fe4fca4SPaolo Bonzini if (IS_DEAD_ARG(1)) { 40270fe4fca4SPaolo Bonzini temp_dead(s, ts); 40280fe4fca4SPaolo Bonzini } 402969e3706dSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 40300fe4fca4SPaolo Bonzini return; 40310fe4fca4SPaolo Bonzini } 40320fe4fca4SPaolo Bonzini 40330fe4fca4SPaolo Bonzini /* If the source value is in memory we're going to be forced 40340fe4fca4SPaolo Bonzini to have it in a register in order to perform the copy. Copy 40350fe4fca4SPaolo Bonzini the SOURCE value into its own register first, that way we 40360fe4fca4SPaolo Bonzini don't have to reload SOURCE the next time it is used. */ 40370fe4fca4SPaolo Bonzini if (ts->val_type == TEMP_VAL_MEM) { 403869e3706dSRichard Henderson temp_load(s, ts, tcg_target_available_regs[itype], 403969e3706dSRichard Henderson allocated_regs, preferred_regs); 4040c29c1d7eSAurelien Jarno } 40410fe4fca4SPaolo Bonzini tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 4042098859f1SRichard Henderson ireg = ts->reg; 4043098859f1SRichard Henderson 4044d63e3b6eSRichard Henderson if (IS_DEAD_ARG(0)) { 4045c29c1d7eSAurelien Jarno /* mov to a non-saved dead register makes no sense (even with 4046c29c1d7eSAurelien Jarno liveness analysis disabled). */ 4047eabb7b91SAurelien Jarno tcg_debug_assert(NEED_SYNC_ARG(0)); 4048c29c1d7eSAurelien Jarno if (!ots->mem_allocated) { 40492272e4a7SRichard Henderson temp_allocate_frame(s, ots); 4050c29c1d7eSAurelien Jarno } 4051098859f1SRichard Henderson tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 4052c29c1d7eSAurelien Jarno if (IS_DEAD_ARG(1)) { 4053f8bf00f1SRichard Henderson temp_dead(s, ts); 4054c29c1d7eSAurelien Jarno } 4055f8bf00f1SRichard Henderson temp_dead(s, ots); 4056098859f1SRichard Henderson return; 4057098859f1SRichard Henderson } 4058098859f1SRichard Henderson 4059ee17db83SRichard Henderson if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 4060098859f1SRichard Henderson /* 4061098859f1SRichard Henderson * The mov can be suppressed. Kill input first, so that it 4062098859f1SRichard Henderson * is unlinked from reg_to_temp, then set the output to the 4063098859f1SRichard Henderson * reg that we saved from the input. 4064098859f1SRichard Henderson */ 4065f8bf00f1SRichard Henderson temp_dead(s, ts); 4066098859f1SRichard Henderson oreg = ireg; 4067c29c1d7eSAurelien Jarno } else { 4068098859f1SRichard Henderson if (ots->val_type == TEMP_VAL_REG) { 4069098859f1SRichard Henderson oreg = ots->reg; 4070098859f1SRichard Henderson } else { 4071098859f1SRichard Henderson /* Make sure to not spill the input register during allocation. */ 4072098859f1SRichard Henderson oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 4073098859f1SRichard Henderson allocated_regs | ((TCGRegSet)1 << ireg), 4074098859f1SRichard Henderson preferred_regs, ots->indirect_base); 4075c29c1d7eSAurelien Jarno } 4076098859f1SRichard Henderson if (!tcg_out_mov(s, otype, oreg, ireg)) { 4077240c08d0SRichard Henderson /* 4078240c08d0SRichard Henderson * Cross register class move not supported. 4079240c08d0SRichard Henderson * Store the source register into the destination slot 4080240c08d0SRichard Henderson * and leave the destination temp as TEMP_VAL_MEM. 4081240c08d0SRichard Henderson */ 4082e01fa97dSRichard Henderson assert(!temp_readonly(ots)); 4083240c08d0SRichard Henderson if (!ts->mem_allocated) { 4084240c08d0SRichard Henderson temp_allocate_frame(s, ots); 4085240c08d0SRichard Henderson } 4086098859f1SRichard Henderson tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 4087098859f1SRichard Henderson set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 4088240c08d0SRichard Henderson ots->mem_coherent = 1; 4089240c08d0SRichard Henderson return; 409078113e83SRichard Henderson } 4091c29c1d7eSAurelien Jarno } 4092098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4093c896fe29Sbellard ots->mem_coherent = 0; 4094098859f1SRichard Henderson 4095ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(0)) { 409698b4e186SRichard Henderson temp_sync(s, ots, allocated_regs, 0, 0); 4097c29c1d7eSAurelien Jarno } 4098ec7a869dSAurelien Jarno } 4099c896fe29Sbellard 4100bab1671fSRichard Henderson /* 4101bab1671fSRichard Henderson * Specialized code generation for INDEX_op_dup_vec. 4102bab1671fSRichard Henderson */ 4103bab1671fSRichard Henderson static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 4104bab1671fSRichard Henderson { 4105bab1671fSRichard Henderson const TCGLifeData arg_life = op->life; 4106bab1671fSRichard Henderson TCGRegSet dup_out_regs, dup_in_regs; 4107bab1671fSRichard Henderson TCGTemp *its, *ots; 4108bab1671fSRichard Henderson TCGType itype, vtype; 4109bab1671fSRichard Henderson unsigned vece; 411031c96417SRichard Henderson int lowpart_ofs; 4111bab1671fSRichard Henderson bool ok; 4112bab1671fSRichard Henderson 4113bab1671fSRichard Henderson ots = arg_temp(op->args[0]); 4114bab1671fSRichard Henderson its = arg_temp(op->args[1]); 4115bab1671fSRichard Henderson 4116bab1671fSRichard Henderson /* ENV should not be modified. */ 4117e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4118bab1671fSRichard Henderson 4119bab1671fSRichard Henderson itype = its->type; 4120bab1671fSRichard Henderson vece = TCGOP_VECE(op); 4121bab1671fSRichard Henderson vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4122bab1671fSRichard Henderson 4123bab1671fSRichard Henderson if (its->val_type == TEMP_VAL_CONST) { 4124bab1671fSRichard Henderson /* Propagate constant via movi -> dupi. */ 4125bab1671fSRichard Henderson tcg_target_ulong val = its->val; 4126bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 4127bab1671fSRichard Henderson temp_dead(s, its); 4128bab1671fSRichard Henderson } 412931fd884bSRichard Henderson tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); 4130bab1671fSRichard Henderson return; 4131bab1671fSRichard Henderson } 4132bab1671fSRichard Henderson 41339be0d080SRichard Henderson dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 41349be0d080SRichard Henderson dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 4135bab1671fSRichard Henderson 4136bab1671fSRichard Henderson /* Allocate the output register now. */ 4137bab1671fSRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 4138bab1671fSRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 4139098859f1SRichard Henderson TCGReg oreg; 4140bab1671fSRichard Henderson 4141bab1671fSRichard Henderson if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 4142bab1671fSRichard Henderson /* Make sure to not spill the input register. */ 4143bab1671fSRichard Henderson tcg_regset_set_reg(allocated_regs, its->reg); 4144bab1671fSRichard Henderson } 4145098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 414631fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base); 4147098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4148bab1671fSRichard Henderson } 4149bab1671fSRichard Henderson 4150bab1671fSRichard Henderson switch (its->val_type) { 4151bab1671fSRichard Henderson case TEMP_VAL_REG: 4152bab1671fSRichard Henderson /* 4153bab1671fSRichard Henderson * The dup constriaints must be broad, covering all possible VECE. 4154bab1671fSRichard Henderson * However, tcg_op_dup_vec() gets to see the VECE and we allow it 4155bab1671fSRichard Henderson * to fail, indicating that extra moves are required for that case. 4156bab1671fSRichard Henderson */ 4157bab1671fSRichard Henderson if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 4158bab1671fSRichard Henderson if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 4159bab1671fSRichard Henderson goto done; 4160bab1671fSRichard Henderson } 4161bab1671fSRichard Henderson /* Try again from memory or a vector input register. */ 4162bab1671fSRichard Henderson } 4163bab1671fSRichard Henderson if (!its->mem_coherent) { 4164bab1671fSRichard Henderson /* 4165bab1671fSRichard Henderson * The input register is not synced, and so an extra store 4166bab1671fSRichard Henderson * would be required to use memory. Attempt an integer-vector 4167bab1671fSRichard Henderson * register move first. We do not have a TCGRegSet for this. 4168bab1671fSRichard Henderson */ 4169bab1671fSRichard Henderson if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 4170bab1671fSRichard Henderson break; 4171bab1671fSRichard Henderson } 4172bab1671fSRichard Henderson /* Sync the temp back to its slot and load from there. */ 4173bab1671fSRichard Henderson temp_sync(s, its, s->reserved_regs, 0, 0); 4174bab1671fSRichard Henderson } 4175bab1671fSRichard Henderson /* fall through */ 4176bab1671fSRichard Henderson 4177bab1671fSRichard Henderson case TEMP_VAL_MEM: 417831c96417SRichard Henderson lowpart_ofs = 0; 417931c96417SRichard Henderson if (HOST_BIG_ENDIAN) { 418031c96417SRichard Henderson lowpart_ofs = tcg_type_size(itype) - (1 << vece); 418131c96417SRichard Henderson } 4182d6ecb4a9SRichard Henderson if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 418331c96417SRichard Henderson its->mem_offset + lowpart_ofs)) { 4184d6ecb4a9SRichard Henderson goto done; 4185d6ecb4a9SRichard Henderson } 4186098859f1SRichard Henderson /* Load the input into the destination vector register. */ 4187bab1671fSRichard Henderson tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 4188bab1671fSRichard Henderson break; 4189bab1671fSRichard Henderson 4190bab1671fSRichard Henderson default: 4191bab1671fSRichard Henderson g_assert_not_reached(); 4192bab1671fSRichard Henderson } 4193bab1671fSRichard Henderson 4194bab1671fSRichard Henderson /* We now have a vector input register, so dup must succeed. */ 4195bab1671fSRichard Henderson ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 4196bab1671fSRichard Henderson tcg_debug_assert(ok); 4197bab1671fSRichard Henderson 4198bab1671fSRichard Henderson done: 419936f5539cSRichard Henderson ots->mem_coherent = 0; 4200bab1671fSRichard Henderson if (IS_DEAD_ARG(1)) { 4201bab1671fSRichard Henderson temp_dead(s, its); 4202bab1671fSRichard Henderson } 4203bab1671fSRichard Henderson if (NEED_SYNC_ARG(0)) { 4204bab1671fSRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, 0); 4205bab1671fSRichard Henderson } 4206bab1671fSRichard Henderson if (IS_DEAD_ARG(0)) { 4207bab1671fSRichard Henderson temp_dead(s, ots); 4208bab1671fSRichard Henderson } 4209bab1671fSRichard Henderson } 4210bab1671fSRichard Henderson 4211dd186292SRichard Henderson static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 4212c896fe29Sbellard { 4213dd186292SRichard Henderson const TCGLifeData arg_life = op->life; 4214dd186292SRichard Henderson const TCGOpDef * const def = &tcg_op_defs[op->opc]; 421582790a87SRichard Henderson TCGRegSet i_allocated_regs; 421682790a87SRichard Henderson TCGRegSet o_allocated_regs; 4217b6638662SRichard Henderson int i, k, nb_iargs, nb_oargs; 4218b6638662SRichard Henderson TCGReg reg; 4219c896fe29Sbellard TCGArg arg; 4220c896fe29Sbellard const TCGArgConstraint *arg_ct; 4221c896fe29Sbellard TCGTemp *ts; 4222c896fe29Sbellard TCGArg new_args[TCG_MAX_OP_ARGS]; 4223c896fe29Sbellard int const_args[TCG_MAX_OP_ARGS]; 4224c896fe29Sbellard 4225c896fe29Sbellard nb_oargs = def->nb_oargs; 4226c896fe29Sbellard nb_iargs = def->nb_iargs; 4227c896fe29Sbellard 4228c896fe29Sbellard /* copy constants */ 4229c896fe29Sbellard memcpy(new_args + nb_oargs + nb_iargs, 4230dd186292SRichard Henderson op->args + nb_oargs + nb_iargs, 4231c896fe29Sbellard sizeof(TCGArg) * def->nb_cargs); 4232c896fe29Sbellard 4233d21369f5SRichard Henderson i_allocated_regs = s->reserved_regs; 4234d21369f5SRichard Henderson o_allocated_regs = s->reserved_regs; 423582790a87SRichard Henderson 4236c896fe29Sbellard /* satisfy input constraints */ 4237c896fe29Sbellard for (k = 0; k < nb_iargs; k++) { 423829f5e925SRichard Henderson TCGRegSet i_preferred_regs, i_required_regs; 423929f5e925SRichard Henderson bool allocate_new_reg, copyto_new_reg; 424029f5e925SRichard Henderson TCGTemp *ts2; 424129f5e925SRichard Henderson int i1, i2; 4242d62816f2SRichard Henderson 424366792f90SRichard Henderson i = def->args_ct[nb_oargs + k].sort_index; 4244dd186292SRichard Henderson arg = op->args[i]; 4245c896fe29Sbellard arg_ct = &def->args_ct[i]; 424643439139SRichard Henderson ts = arg_temp(arg); 424740ae5c62SRichard Henderson 424840ae5c62SRichard Henderson if (ts->val_type == TEMP_VAL_CONST 4249a4fbbd77SRichard Henderson && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 4250c896fe29Sbellard /* constant is OK for instruction */ 4251c896fe29Sbellard const_args[i] = 1; 4252c896fe29Sbellard new_args[i] = ts->val; 4253d62816f2SRichard Henderson continue; 4254c896fe29Sbellard } 425540ae5c62SRichard Henderson 42561c1824dcSRichard Henderson reg = ts->reg; 42571c1824dcSRichard Henderson i_preferred_regs = 0; 425829f5e925SRichard Henderson i_required_regs = arg_ct->regs; 42591c1824dcSRichard Henderson allocate_new_reg = false; 426029f5e925SRichard Henderson copyto_new_reg = false; 42611c1824dcSRichard Henderson 426229f5e925SRichard Henderson switch (arg_ct->pair) { 426329f5e925SRichard Henderson case 0: /* not paired */ 4264bc2b17e6SRichard Henderson if (arg_ct->ialias) { 426531fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 4266c0522136SRichard Henderson 4267c0522136SRichard Henderson /* 4268c0522136SRichard Henderson * If the input is readonly, then it cannot also be an 4269c0522136SRichard Henderson * output and aliased to itself. If the input is not 4270c0522136SRichard Henderson * dead after the instruction, we must allocate a new 4271c0522136SRichard Henderson * register and move it. 4272c0522136SRichard Henderson */ 4273c0522136SRichard Henderson if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 42741c1824dcSRichard Henderson allocate_new_reg = true; 42751c1824dcSRichard Henderson } else if (ts->val_type == TEMP_VAL_REG) { 4276c0522136SRichard Henderson /* 42771c1824dcSRichard Henderson * Check if the current register has already been 42781c1824dcSRichard Henderson * allocated for another input. 4279c0522136SRichard Henderson */ 428029f5e925SRichard Henderson allocate_new_reg = 428129f5e925SRichard Henderson tcg_regset_test_reg(i_allocated_regs, reg); 42827e1df267SAurelien Jarno } 42837e1df267SAurelien Jarno } 42841c1824dcSRichard Henderson if (!allocate_new_reg) { 428529f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs, 428629f5e925SRichard Henderson i_preferred_regs); 4287c896fe29Sbellard reg = ts->reg; 428829f5e925SRichard Henderson allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 42891c1824dcSRichard Henderson } 42901c1824dcSRichard Henderson if (allocate_new_reg) { 4291c0522136SRichard Henderson /* 4292c0522136SRichard Henderson * Allocate a new register matching the constraint 4293c0522136SRichard Henderson * and move the temporary register into it. 4294c0522136SRichard Henderson */ 4295d62816f2SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 4296d62816f2SRichard Henderson i_allocated_regs, 0); 429729f5e925SRichard Henderson reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 42981c1824dcSRichard Henderson i_preferred_regs, ts->indirect_base); 429929f5e925SRichard Henderson copyto_new_reg = true; 430029f5e925SRichard Henderson } 430129f5e925SRichard Henderson break; 430229f5e925SRichard Henderson 430329f5e925SRichard Henderson case 1: 430429f5e925SRichard Henderson /* First of an input pair; if i1 == i2, the second is an output. */ 430529f5e925SRichard Henderson i1 = i; 430629f5e925SRichard Henderson i2 = arg_ct->pair_index; 430729f5e925SRichard Henderson ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 430829f5e925SRichard Henderson 430929f5e925SRichard Henderson /* 431029f5e925SRichard Henderson * It is easier to default to allocating a new pair 431129f5e925SRichard Henderson * and to identify a few cases where it's not required. 431229f5e925SRichard Henderson */ 431329f5e925SRichard Henderson if (arg_ct->ialias) { 431431fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 431529f5e925SRichard Henderson if (IS_DEAD_ARG(i1) && 431629f5e925SRichard Henderson IS_DEAD_ARG(i2) && 431729f5e925SRichard Henderson !temp_readonly(ts) && 431829f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG && 431929f5e925SRichard Henderson ts->reg < TCG_TARGET_NB_REGS - 1 && 432029f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) && 432129f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) && 432229f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 432329f5e925SRichard Henderson (ts2 432429f5e925SRichard Henderson ? ts2->val_type == TEMP_VAL_REG && 432529f5e925SRichard Henderson ts2->reg == reg + 1 && 432629f5e925SRichard Henderson !temp_readonly(ts2) 432729f5e925SRichard Henderson : s->reg_to_temp[reg + 1] == NULL)) { 432829f5e925SRichard Henderson break; 432929f5e925SRichard Henderson } 433029f5e925SRichard Henderson } else { 433129f5e925SRichard Henderson /* Without aliasing, the pair must also be an input. */ 433229f5e925SRichard Henderson tcg_debug_assert(ts2); 433329f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG && 433429f5e925SRichard Henderson ts2->val_type == TEMP_VAL_REG && 433529f5e925SRichard Henderson ts2->reg == reg + 1 && 433629f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg)) { 433729f5e925SRichard Henderson break; 433829f5e925SRichard Henderson } 433929f5e925SRichard Henderson } 434029f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 434129f5e925SRichard Henderson 0, ts->indirect_base); 434229f5e925SRichard Henderson goto do_pair; 434329f5e925SRichard Henderson 434429f5e925SRichard Henderson case 2: /* pair second */ 434529f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1; 434629f5e925SRichard Henderson goto do_pair; 434729f5e925SRichard Henderson 434829f5e925SRichard Henderson case 3: /* ialias with second output, no first input */ 434929f5e925SRichard Henderson tcg_debug_assert(arg_ct->ialias); 435031fd884bSRichard Henderson i_preferred_regs = output_pref(op, arg_ct->alias_index); 435129f5e925SRichard Henderson 435229f5e925SRichard Henderson if (IS_DEAD_ARG(i) && 435329f5e925SRichard Henderson !temp_readonly(ts) && 435429f5e925SRichard Henderson ts->val_type == TEMP_VAL_REG && 435529f5e925SRichard Henderson reg > 0 && 435629f5e925SRichard Henderson s->reg_to_temp[reg - 1] == NULL && 435729f5e925SRichard Henderson tcg_regset_test_reg(i_required_regs, reg) && 435829f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg) && 435929f5e925SRichard Henderson !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 436029f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg - 1); 436129f5e925SRichard Henderson break; 436229f5e925SRichard Henderson } 436329f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 436429f5e925SRichard Henderson i_allocated_regs, 0, 436529f5e925SRichard Henderson ts->indirect_base); 436629f5e925SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg); 436729f5e925SRichard Henderson reg += 1; 436829f5e925SRichard Henderson goto do_pair; 436929f5e925SRichard Henderson 437029f5e925SRichard Henderson do_pair: 437129f5e925SRichard Henderson /* 437229f5e925SRichard Henderson * If an aliased input is not dead after the instruction, 437329f5e925SRichard Henderson * we must allocate a new register and move it. 437429f5e925SRichard Henderson */ 437529f5e925SRichard Henderson if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 437629f5e925SRichard Henderson TCGRegSet t_allocated_regs = i_allocated_regs; 437729f5e925SRichard Henderson 437829f5e925SRichard Henderson /* 437929f5e925SRichard Henderson * Because of the alias, and the continued life, make sure 438029f5e925SRichard Henderson * that the temp is somewhere *other* than the reg pair, 438129f5e925SRichard Henderson * and we get a copy in reg. 438229f5e925SRichard Henderson */ 438329f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg); 438429f5e925SRichard Henderson tcg_regset_set_reg(t_allocated_regs, reg + 1); 438529f5e925SRichard Henderson if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 438629f5e925SRichard Henderson /* If ts was already in reg, copy it somewhere else. */ 438729f5e925SRichard Henderson TCGReg nr; 438829f5e925SRichard Henderson bool ok; 438929f5e925SRichard Henderson 439029f5e925SRichard Henderson tcg_debug_assert(ts->kind != TEMP_FIXED); 439129f5e925SRichard Henderson nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 439229f5e925SRichard Henderson t_allocated_regs, 0, ts->indirect_base); 439329f5e925SRichard Henderson ok = tcg_out_mov(s, ts->type, nr, reg); 439429f5e925SRichard Henderson tcg_debug_assert(ok); 439529f5e925SRichard Henderson 439629f5e925SRichard Henderson set_temp_val_reg(s, ts, nr); 439729f5e925SRichard Henderson } else { 439829f5e925SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], 439929f5e925SRichard Henderson t_allocated_regs, 0); 440029f5e925SRichard Henderson copyto_new_reg = true; 440129f5e925SRichard Henderson } 440229f5e925SRichard Henderson } else { 440329f5e925SRichard Henderson /* Preferably allocate to reg, otherwise copy. */ 440429f5e925SRichard Henderson i_required_regs = (TCGRegSet)1 << reg; 440529f5e925SRichard Henderson temp_load(s, ts, i_required_regs, i_allocated_regs, 440629f5e925SRichard Henderson i_preferred_regs); 440729f5e925SRichard Henderson copyto_new_reg = ts->reg != reg; 440829f5e925SRichard Henderson } 440929f5e925SRichard Henderson break; 441029f5e925SRichard Henderson 441129f5e925SRichard Henderson default: 441229f5e925SRichard Henderson g_assert_not_reached(); 441329f5e925SRichard Henderson } 441429f5e925SRichard Henderson 441529f5e925SRichard Henderson if (copyto_new_reg) { 441678113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4417240c08d0SRichard Henderson /* 4418240c08d0SRichard Henderson * Cross register class move not supported. Sync the 4419240c08d0SRichard Henderson * temp back to its slot and load from there. 4420240c08d0SRichard Henderson */ 4421240c08d0SRichard Henderson temp_sync(s, ts, i_allocated_regs, 0, 0); 4422240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 4423240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 442478113e83SRichard Henderson } 4425c896fe29Sbellard } 4426c896fe29Sbellard new_args[i] = reg; 4427c896fe29Sbellard const_args[i] = 0; 442882790a87SRichard Henderson tcg_regset_set_reg(i_allocated_regs, reg); 4429c896fe29Sbellard } 4430c896fe29Sbellard 4431c896fe29Sbellard /* mark dead temporaries and free the associated registers */ 4432866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4433866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 443443439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 4435c896fe29Sbellard } 4436c896fe29Sbellard } 4437c896fe29Sbellard 4438b4cb76e6SRichard Henderson if (def->flags & TCG_OPF_COND_BRANCH) { 4439b4cb76e6SRichard Henderson tcg_reg_alloc_cbranch(s, i_allocated_regs); 4440b4cb76e6SRichard Henderson } else if (def->flags & TCG_OPF_BB_END) { 444182790a87SRichard Henderson tcg_reg_alloc_bb_end(s, i_allocated_regs); 4442a52ad07eSAurelien Jarno } else { 4443c896fe29Sbellard if (def->flags & TCG_OPF_CALL_CLOBBER) { 4444b03cce8eSbellard /* XXX: permit generic clobber register list ? */ 4445c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4446c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 444782790a87SRichard Henderson tcg_reg_free(s, i, i_allocated_regs); 4448c896fe29Sbellard } 4449c896fe29Sbellard } 44503d5c5f87SAurelien Jarno } 44513d5c5f87SAurelien Jarno if (def->flags & TCG_OPF_SIDE_EFFECTS) { 44523d5c5f87SAurelien Jarno /* sync globals if the op has side effects and might trigger 44533d5c5f87SAurelien Jarno an exception. */ 445482790a87SRichard Henderson sync_globals(s, i_allocated_regs); 4455c896fe29Sbellard } 4456c896fe29Sbellard 4457c896fe29Sbellard /* satisfy the output constraints */ 4458c896fe29Sbellard for(k = 0; k < nb_oargs; k++) { 445966792f90SRichard Henderson i = def->args_ct[k].sort_index; 4460dd186292SRichard Henderson arg = op->args[i]; 4461c896fe29Sbellard arg_ct = &def->args_ct[i]; 446243439139SRichard Henderson ts = arg_temp(arg); 4463d63e3b6eSRichard Henderson 4464d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4465e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4466d63e3b6eSRichard Henderson 446729f5e925SRichard Henderson switch (arg_ct->pair) { 446829f5e925SRichard Henderson case 0: /* not paired */ 4469bc2b17e6SRichard Henderson if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 44705ff9d6a4Sbellard reg = new_args[arg_ct->alias_index]; 4471bc2b17e6SRichard Henderson } else if (arg_ct->newreg) { 44729be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, 447382790a87SRichard Henderson i_allocated_regs | o_allocated_regs, 447431fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 4475c896fe29Sbellard } else { 44769be0d080SRichard Henderson reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 447731fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 4478c896fe29Sbellard } 447929f5e925SRichard Henderson break; 448029f5e925SRichard Henderson 448129f5e925SRichard Henderson case 1: /* first of pair */ 448229f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 448329f5e925SRichard Henderson if (arg_ct->oalias) { 448429f5e925SRichard Henderson reg = new_args[arg_ct->alias_index]; 448529f5e925SRichard Henderson break; 448629f5e925SRichard Henderson } 448729f5e925SRichard Henderson reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 448831fd884bSRichard Henderson output_pref(op, k), ts->indirect_base); 448929f5e925SRichard Henderson break; 449029f5e925SRichard Henderson 449129f5e925SRichard Henderson case 2: /* second of pair */ 449229f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 449329f5e925SRichard Henderson if (arg_ct->oalias) { 449429f5e925SRichard Henderson reg = new_args[arg_ct->alias_index]; 449529f5e925SRichard Henderson } else { 449629f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] + 1; 449729f5e925SRichard Henderson } 449829f5e925SRichard Henderson break; 449929f5e925SRichard Henderson 450029f5e925SRichard Henderson case 3: /* first of pair, aliasing with a second input */ 450129f5e925SRichard Henderson tcg_debug_assert(!arg_ct->newreg); 450229f5e925SRichard Henderson reg = new_args[arg_ct->pair_index] - 1; 450329f5e925SRichard Henderson break; 450429f5e925SRichard Henderson 450529f5e925SRichard Henderson default: 450629f5e925SRichard Henderson g_assert_not_reached(); 450729f5e925SRichard Henderson } 450882790a87SRichard Henderson tcg_regset_set_reg(o_allocated_regs, reg); 4509098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 4510c896fe29Sbellard ts->mem_coherent = 0; 4511c896fe29Sbellard new_args[i] = reg; 4512c896fe29Sbellard } 4513e8996ee0Sbellard } 4514c896fe29Sbellard 4515c896fe29Sbellard /* emit instruction */ 4516d2fd745fSRichard Henderson if (def->flags & TCG_OPF_VECTOR) { 4517d2fd745fSRichard Henderson tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 4518d2fd745fSRichard Henderson new_args, const_args); 4519d2fd745fSRichard Henderson } else { 4520dd186292SRichard Henderson tcg_out_op(s, op->opc, new_args, const_args); 4521d2fd745fSRichard Henderson } 4522c896fe29Sbellard 4523c896fe29Sbellard /* move the outputs in the correct register if needed */ 4524c896fe29Sbellard for(i = 0; i < nb_oargs; i++) { 452543439139SRichard Henderson ts = arg_temp(op->args[i]); 4526d63e3b6eSRichard Henderson 4527d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4528e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4529d63e3b6eSRichard Henderson 4530ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 453198b4e186SRichard Henderson temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 453259d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 4533f8bf00f1SRichard Henderson temp_dead(s, ts); 4534ec7a869dSAurelien Jarno } 4535c896fe29Sbellard } 4536c896fe29Sbellard } 4537c896fe29Sbellard 4538efe86b21SRichard Henderson static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 4539efe86b21SRichard Henderson { 4540efe86b21SRichard Henderson const TCGLifeData arg_life = op->life; 4541efe86b21SRichard Henderson TCGTemp *ots, *itsl, *itsh; 4542efe86b21SRichard Henderson TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4543efe86b21SRichard Henderson 4544efe86b21SRichard Henderson /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 4545efe86b21SRichard Henderson tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 4546efe86b21SRichard Henderson tcg_debug_assert(TCGOP_VECE(op) == MO_64); 4547efe86b21SRichard Henderson 4548efe86b21SRichard Henderson ots = arg_temp(op->args[0]); 4549efe86b21SRichard Henderson itsl = arg_temp(op->args[1]); 4550efe86b21SRichard Henderson itsh = arg_temp(op->args[2]); 4551efe86b21SRichard Henderson 4552efe86b21SRichard Henderson /* ENV should not be modified. */ 4553efe86b21SRichard Henderson tcg_debug_assert(!temp_readonly(ots)); 4554efe86b21SRichard Henderson 4555efe86b21SRichard Henderson /* Allocate the output register now. */ 4556efe86b21SRichard Henderson if (ots->val_type != TEMP_VAL_REG) { 4557efe86b21SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 4558efe86b21SRichard Henderson TCGRegSet dup_out_regs = 4559efe86b21SRichard Henderson tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 4560098859f1SRichard Henderson TCGReg oreg; 4561efe86b21SRichard Henderson 4562efe86b21SRichard Henderson /* Make sure to not spill the input registers. */ 4563efe86b21SRichard Henderson if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 4564efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsl->reg); 4565efe86b21SRichard Henderson } 4566efe86b21SRichard Henderson if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 4567efe86b21SRichard Henderson tcg_regset_set_reg(allocated_regs, itsh->reg); 4568efe86b21SRichard Henderson } 4569efe86b21SRichard Henderson 4570098859f1SRichard Henderson oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 457131fd884bSRichard Henderson output_pref(op, 0), ots->indirect_base); 4572098859f1SRichard Henderson set_temp_val_reg(s, ots, oreg); 4573efe86b21SRichard Henderson } 4574efe86b21SRichard Henderson 4575efe86b21SRichard Henderson /* Promote dup2 of immediates to dupi_vec. */ 4576efe86b21SRichard Henderson if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 4577efe86b21SRichard Henderson uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 4578efe86b21SRichard Henderson MemOp vece = MO_64; 4579efe86b21SRichard Henderson 4580efe86b21SRichard Henderson if (val == dup_const(MO_8, val)) { 4581efe86b21SRichard Henderson vece = MO_8; 4582efe86b21SRichard Henderson } else if (val == dup_const(MO_16, val)) { 4583efe86b21SRichard Henderson vece = MO_16; 4584efe86b21SRichard Henderson } else if (val == dup_const(MO_32, val)) { 4585efe86b21SRichard Henderson vece = MO_32; 4586efe86b21SRichard Henderson } 4587efe86b21SRichard Henderson 4588efe86b21SRichard Henderson tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 4589efe86b21SRichard Henderson goto done; 4590efe86b21SRichard Henderson } 4591efe86b21SRichard Henderson 4592efe86b21SRichard Henderson /* If the two inputs form one 64-bit value, try dupm_vec. */ 4593aef85402SRichard Henderson if (itsl->temp_subindex == HOST_BIG_ENDIAN && 4594aef85402SRichard Henderson itsh->temp_subindex == !HOST_BIG_ENDIAN && 4595aef85402SRichard Henderson itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { 4596aef85402SRichard Henderson TCGTemp *its = itsl - HOST_BIG_ENDIAN; 4597aef85402SRichard Henderson 4598aef85402SRichard Henderson temp_sync(s, its + 0, s->reserved_regs, 0, 0); 4599aef85402SRichard Henderson temp_sync(s, its + 1, s->reserved_regs, 0, 0); 4600aef85402SRichard Henderson 4601efe86b21SRichard Henderson if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 4602efe86b21SRichard Henderson its->mem_base->reg, its->mem_offset)) { 4603efe86b21SRichard Henderson goto done; 4604efe86b21SRichard Henderson } 4605efe86b21SRichard Henderson } 4606efe86b21SRichard Henderson 4607efe86b21SRichard Henderson /* Fall back to generic expansion. */ 4608efe86b21SRichard Henderson return false; 4609efe86b21SRichard Henderson 4610efe86b21SRichard Henderson done: 461136f5539cSRichard Henderson ots->mem_coherent = 0; 4612efe86b21SRichard Henderson if (IS_DEAD_ARG(1)) { 4613efe86b21SRichard Henderson temp_dead(s, itsl); 4614efe86b21SRichard Henderson } 4615efe86b21SRichard Henderson if (IS_DEAD_ARG(2)) { 4616efe86b21SRichard Henderson temp_dead(s, itsh); 4617efe86b21SRichard Henderson } 4618efe86b21SRichard Henderson if (NEED_SYNC_ARG(0)) { 4619efe86b21SRichard Henderson temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 4620efe86b21SRichard Henderson } else if (IS_DEAD_ARG(0)) { 4621efe86b21SRichard Henderson temp_dead(s, ots); 4622efe86b21SRichard Henderson } 4623efe86b21SRichard Henderson return true; 4624efe86b21SRichard Henderson } 4625efe86b21SRichard Henderson 462639004a71SRichard Henderson static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, 462739004a71SRichard Henderson TCGRegSet allocated_regs) 4628c896fe29Sbellard { 4629c896fe29Sbellard if (ts->val_type == TEMP_VAL_REG) { 4630c896fe29Sbellard if (ts->reg != reg) { 46314250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 463278113e83SRichard Henderson if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4633240c08d0SRichard Henderson /* 4634240c08d0SRichard Henderson * Cross register class move not supported. Sync the 4635240c08d0SRichard Henderson * temp back to its slot and load from there. 4636240c08d0SRichard Henderson */ 4637240c08d0SRichard Henderson temp_sync(s, ts, allocated_regs, 0, 0); 4638240c08d0SRichard Henderson tcg_out_ld(s, ts->type, reg, 4639240c08d0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 464078113e83SRichard Henderson } 4641c896fe29Sbellard } 4642c896fe29Sbellard } else { 4643ccb1bb66SRichard Henderson TCGRegSet arg_set = 0; 464440ae5c62SRichard Henderson 46454250da10SRichard Henderson tcg_reg_free(s, reg, allocated_regs); 464640ae5c62SRichard Henderson tcg_regset_set_reg(arg_set, reg); 4647b722452aSRichard Henderson temp_load(s, ts, arg_set, allocated_regs, 0); 4648c896fe29Sbellard } 464939004a71SRichard Henderson } 465040ae5c62SRichard Henderson 465139004a71SRichard Henderson static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, 465239004a71SRichard Henderson TCGRegSet allocated_regs) 465339004a71SRichard Henderson { 465439004a71SRichard Henderson /* 465539004a71SRichard Henderson * When the destination is on the stack, load up the temp and store. 465639004a71SRichard Henderson * If there are many call-saved registers, the temp might live to 465739004a71SRichard Henderson * see another use; otherwise it'll be discarded. 465839004a71SRichard Henderson */ 465939004a71SRichard Henderson temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); 466039004a71SRichard Henderson tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, 466139004a71SRichard Henderson TCG_TARGET_CALL_STACK_OFFSET + 466239004a71SRichard Henderson stk_slot * sizeof(tcg_target_long)); 466339004a71SRichard Henderson } 466439004a71SRichard Henderson 466539004a71SRichard Henderson static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, 466639004a71SRichard Henderson TCGTemp *ts, TCGRegSet *allocated_regs) 466739004a71SRichard Henderson { 466839004a71SRichard Henderson if (REG_P(l)) { 466939004a71SRichard Henderson TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; 467039004a71SRichard Henderson load_arg_reg(s, reg, ts, *allocated_regs); 467139004a71SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg); 467239004a71SRichard Henderson } else { 467339004a71SRichard Henderson load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs), 467439004a71SRichard Henderson ts, *allocated_regs); 4675c896fe29Sbellard } 467639cf05d3Sbellard } 4677c896fe29Sbellard 4678313bdea8SRichard Henderson static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base, 4679313bdea8SRichard Henderson intptr_t ref_off, TCGRegSet *allocated_regs) 4680313bdea8SRichard Henderson { 4681313bdea8SRichard Henderson TCGReg reg; 4682313bdea8SRichard Henderson int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); 4683313bdea8SRichard Henderson 4684313bdea8SRichard Henderson if (stk_slot < 0) { 4685313bdea8SRichard Henderson reg = tcg_target_call_iarg_regs[arg_slot]; 4686313bdea8SRichard Henderson tcg_reg_free(s, reg, *allocated_regs); 4687313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4688313bdea8SRichard Henderson tcg_regset_set_reg(*allocated_regs, reg); 4689313bdea8SRichard Henderson } else { 4690313bdea8SRichard Henderson reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR], 4691313bdea8SRichard Henderson *allocated_regs, 0, false); 4692313bdea8SRichard Henderson tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4693313bdea8SRichard Henderson tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, 4694313bdea8SRichard Henderson TCG_TARGET_CALL_STACK_OFFSET 4695313bdea8SRichard Henderson + stk_slot * sizeof(tcg_target_long)); 4696313bdea8SRichard Henderson } 4697313bdea8SRichard Henderson } 4698313bdea8SRichard Henderson 469939004a71SRichard Henderson static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 470039004a71SRichard Henderson { 470139004a71SRichard Henderson const int nb_oargs = TCGOP_CALLO(op); 470239004a71SRichard Henderson const int nb_iargs = TCGOP_CALLI(op); 470339004a71SRichard Henderson const TCGLifeData arg_life = op->life; 470439004a71SRichard Henderson const TCGHelperInfo *info = tcg_call_info(op); 470539004a71SRichard Henderson TCGRegSet allocated_regs = s->reserved_regs; 470639004a71SRichard Henderson int i; 470739004a71SRichard Henderson 470839004a71SRichard Henderson /* 470939004a71SRichard Henderson * Move inputs into place in reverse order, 471039004a71SRichard Henderson * so that we place stacked arguments first. 471139004a71SRichard Henderson */ 471239004a71SRichard Henderson for (i = nb_iargs - 1; i >= 0; --i) { 471339004a71SRichard Henderson const TCGCallArgumentLoc *loc = &info->in[i]; 471439004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); 471539004a71SRichard Henderson 471639004a71SRichard Henderson switch (loc->kind) { 471739004a71SRichard Henderson case TCG_CALL_ARG_NORMAL: 471839004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_U: 471939004a71SRichard Henderson case TCG_CALL_ARG_EXTEND_S: 472039004a71SRichard Henderson load_arg_normal(s, loc, ts, &allocated_regs); 472139004a71SRichard Henderson break; 4722313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF: 4723313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4724313bdea8SRichard Henderson load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, 4725313bdea8SRichard Henderson TCG_TARGET_CALL_STACK_OFFSET 4726313bdea8SRichard Henderson + loc->ref_slot * sizeof(tcg_target_long), 4727313bdea8SRichard Henderson &allocated_regs); 4728313bdea8SRichard Henderson break; 4729313bdea8SRichard Henderson case TCG_CALL_ARG_BY_REF_N: 4730313bdea8SRichard Henderson load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4731313bdea8SRichard Henderson break; 473239004a71SRichard Henderson default: 473339004a71SRichard Henderson g_assert_not_reached(); 473439004a71SRichard Henderson } 473539004a71SRichard Henderson } 473639004a71SRichard Henderson 473739004a71SRichard Henderson /* Mark dead temporaries and free the associated registers. */ 4738866cb6cbSAurelien Jarno for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4739866cb6cbSAurelien Jarno if (IS_DEAD_ARG(i)) { 474043439139SRichard Henderson temp_dead(s, arg_temp(op->args[i])); 4741c896fe29Sbellard } 4742c896fe29Sbellard } 4743c896fe29Sbellard 474439004a71SRichard Henderson /* Clobber call registers. */ 4745c8074023SRichard Henderson for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4746c8074023SRichard Henderson if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4747b3915dbbSRichard Henderson tcg_reg_free(s, i, allocated_regs); 4748c896fe29Sbellard } 4749c896fe29Sbellard } 4750c896fe29Sbellard 475139004a71SRichard Henderson /* 475239004a71SRichard Henderson * Save globals if they might be written by the helper, 475339004a71SRichard Henderson * sync them if they might be read. 475439004a71SRichard Henderson */ 475539004a71SRichard Henderson if (info->flags & TCG_CALL_NO_READ_GLOBALS) { 475678505279SAurelien Jarno /* Nothing to do */ 475739004a71SRichard Henderson } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { 475878505279SAurelien Jarno sync_globals(s, allocated_regs); 475978505279SAurelien Jarno } else { 4760e8996ee0Sbellard save_globals(s, allocated_regs); 4761b9c18f56Saurel32 } 4762c896fe29Sbellard 4763313bdea8SRichard Henderson /* 4764313bdea8SRichard Henderson * If the ABI passes a pointer to the returned struct as the first 4765313bdea8SRichard Henderson * argument, load that now. Pass a pointer to the output home slot. 4766313bdea8SRichard Henderson */ 4767313bdea8SRichard Henderson if (info->out_kind == TCG_CALL_RET_BY_REF) { 4768313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]); 4769313bdea8SRichard Henderson 4770313bdea8SRichard Henderson if (!ts->mem_allocated) { 4771313bdea8SRichard Henderson temp_allocate_frame(s, ts); 4772313bdea8SRichard Henderson } 4773313bdea8SRichard Henderson load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); 4774313bdea8SRichard Henderson } 4775313bdea8SRichard Henderson 4776cee44b03SRichard Henderson tcg_out_call(s, tcg_call_func(op), info); 4777c896fe29Sbellard 477839004a71SRichard Henderson /* Assign output registers and emit moves if needed. */ 477939004a71SRichard Henderson switch (info->out_kind) { 478039004a71SRichard Henderson case TCG_CALL_RET_NORMAL: 4781c896fe29Sbellard for (i = 0; i < nb_oargs; i++) { 478239004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 47835e3d0c19SRichard Henderson TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i); 4784d63e3b6eSRichard Henderson 4785d63e3b6eSRichard Henderson /* ENV should not be modified. */ 4786e01fa97dSRichard Henderson tcg_debug_assert(!temp_readonly(ts)); 4787d63e3b6eSRichard Henderson 4788098859f1SRichard Henderson set_temp_val_reg(s, ts, reg); 4789c896fe29Sbellard ts->mem_coherent = 0; 479039004a71SRichard Henderson } 479139004a71SRichard Henderson break; 4792313bdea8SRichard Henderson 4793c6556aa0SRichard Henderson case TCG_CALL_RET_BY_VEC: 4794c6556aa0SRichard Henderson { 4795c6556aa0SRichard Henderson TCGTemp *ts = arg_temp(op->args[0]); 4796c6556aa0SRichard Henderson 4797c6556aa0SRichard Henderson tcg_debug_assert(ts->base_type == TCG_TYPE_I128); 4798c6556aa0SRichard Henderson tcg_debug_assert(ts->temp_subindex == 0); 4799c6556aa0SRichard Henderson if (!ts->mem_allocated) { 4800c6556aa0SRichard Henderson temp_allocate_frame(s, ts); 4801c6556aa0SRichard Henderson } 4802c6556aa0SRichard Henderson tcg_out_st(s, TCG_TYPE_V128, 4803c6556aa0SRichard Henderson tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 4804c6556aa0SRichard Henderson ts->mem_base->reg, ts->mem_offset); 4805c6556aa0SRichard Henderson } 4806c6556aa0SRichard Henderson /* fall through to mark all parts in memory */ 4807c6556aa0SRichard Henderson 4808313bdea8SRichard Henderson case TCG_CALL_RET_BY_REF: 4809313bdea8SRichard Henderson /* The callee has performed a write through the reference. */ 4810313bdea8SRichard Henderson for (i = 0; i < nb_oargs; i++) { 4811313bdea8SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 4812313bdea8SRichard Henderson ts->val_type = TEMP_VAL_MEM; 4813313bdea8SRichard Henderson } 4814313bdea8SRichard Henderson break; 4815313bdea8SRichard Henderson 481639004a71SRichard Henderson default: 481739004a71SRichard Henderson g_assert_not_reached(); 481839004a71SRichard Henderson } 481939004a71SRichard Henderson 482039004a71SRichard Henderson /* Flush or discard output registers as needed. */ 482139004a71SRichard Henderson for (i = 0; i < nb_oargs; i++) { 482239004a71SRichard Henderson TCGTemp *ts = arg_temp(op->args[i]); 4823ec7a869dSAurelien Jarno if (NEED_SYNC_ARG(i)) { 482439004a71SRichard Henderson temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); 482559d7c14eSRichard Henderson } else if (IS_DEAD_ARG(i)) { 4826f8bf00f1SRichard Henderson temp_dead(s, ts); 4827c896fe29Sbellard } 4828c896fe29Sbellard } 48298c11ad25SAurelien Jarno } 4830c896fe29Sbellard 4831c896fe29Sbellard #ifdef CONFIG_PROFILER 4832c896fe29Sbellard 4833c3fac113SEmilio G. Cota /* avoid copy/paste errors */ 4834c3fac113SEmilio G. Cota #define PROF_ADD(to, from, field) \ 4835c3fac113SEmilio G. Cota do { \ 4836d73415a3SStefan Hajnoczi (to)->field += qatomic_read(&((from)->field)); \ 4837c3fac113SEmilio G. Cota } while (0) 4838c896fe29Sbellard 4839c3fac113SEmilio G. Cota #define PROF_MAX(to, from, field) \ 4840c3fac113SEmilio G. Cota do { \ 4841d73415a3SStefan Hajnoczi typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4842c3fac113SEmilio G. Cota if (val__ > (to)->field) { \ 4843c3fac113SEmilio G. Cota (to)->field = val__; \ 4844c3fac113SEmilio G. Cota } \ 4845c3fac113SEmilio G. Cota } while (0) 4846c3fac113SEmilio G. Cota 4847c3fac113SEmilio G. Cota /* Pass in a zero'ed @prof */ 4848c3fac113SEmilio G. Cota static inline 4849c3fac113SEmilio G. Cota void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4850c896fe29Sbellard { 48510e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4852c3fac113SEmilio G. Cota unsigned int i; 4853c3fac113SEmilio G. Cota 48543468b59eSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 4855d73415a3SStefan Hajnoczi TCGContext *s = qatomic_read(&tcg_ctxs[i]); 48563468b59eSEmilio G. Cota const TCGProfile *orig = &s->prof; 4857c3fac113SEmilio G. Cota 4858c3fac113SEmilio G. Cota if (counters) { 485972fd2efbSEmilio G. Cota PROF_ADD(prof, orig, cpu_exec_time); 4860c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count1); 4861c3fac113SEmilio G. Cota PROF_ADD(prof, orig, tb_count); 4862c3fac113SEmilio G. Cota PROF_ADD(prof, orig, op_count); 4863c3fac113SEmilio G. Cota PROF_MAX(prof, orig, op_count_max); 4864c3fac113SEmilio G. Cota PROF_ADD(prof, orig, temp_count); 4865c3fac113SEmilio G. Cota PROF_MAX(prof, orig, temp_count_max); 4866c3fac113SEmilio G. Cota PROF_ADD(prof, orig, del_op_count); 4867c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_in_len); 4868c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_out_len); 4869c3fac113SEmilio G. Cota PROF_ADD(prof, orig, search_out_len); 4870c3fac113SEmilio G. Cota PROF_ADD(prof, orig, interm_time); 4871c3fac113SEmilio G. Cota PROF_ADD(prof, orig, code_time); 4872c3fac113SEmilio G. Cota PROF_ADD(prof, orig, la_time); 4873c3fac113SEmilio G. Cota PROF_ADD(prof, orig, opt_time); 4874c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_count); 4875c3fac113SEmilio G. Cota PROF_ADD(prof, orig, restore_time); 4876c3fac113SEmilio G. Cota } 4877c3fac113SEmilio G. Cota if (table) { 4878c896fe29Sbellard int i; 4879d70724ceSzhanghailiang 488015fc7daaSRichard Henderson for (i = 0; i < NB_OPS; i++) { 4881c3fac113SEmilio G. Cota PROF_ADD(prof, orig, table_op_count[i]); 4882c3fac113SEmilio G. Cota } 4883c3fac113SEmilio G. Cota } 4884c3fac113SEmilio G. Cota } 4885c3fac113SEmilio G. Cota } 4886c3fac113SEmilio G. Cota 4887c3fac113SEmilio G. Cota #undef PROF_ADD 4888c3fac113SEmilio G. Cota #undef PROF_MAX 4889c3fac113SEmilio G. Cota 4890c3fac113SEmilio G. Cota static void tcg_profile_snapshot_counters(TCGProfile *prof) 4891c3fac113SEmilio G. Cota { 4892c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, true, false); 4893c3fac113SEmilio G. Cota } 4894c3fac113SEmilio G. Cota 4895c3fac113SEmilio G. Cota static void tcg_profile_snapshot_table(TCGProfile *prof) 4896c3fac113SEmilio G. Cota { 4897c3fac113SEmilio G. Cota tcg_profile_snapshot(prof, false, true); 4898c3fac113SEmilio G. Cota } 4899c3fac113SEmilio G. Cota 4900b6a7f3e0SDaniel P. Berrangé void tcg_dump_op_count(GString *buf) 4901c3fac113SEmilio G. Cota { 4902c3fac113SEmilio G. Cota TCGProfile prof = {}; 4903c3fac113SEmilio G. Cota int i; 4904c3fac113SEmilio G. Cota 4905c3fac113SEmilio G. Cota tcg_profile_snapshot_table(&prof); 4906c3fac113SEmilio G. Cota for (i = 0; i < NB_OPS; i++) { 4907b6a7f3e0SDaniel P. Berrangé g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name, 4908c3fac113SEmilio G. Cota prof.table_op_count[i]); 4909c896fe29Sbellard } 4910c896fe29Sbellard } 491172fd2efbSEmilio G. Cota 491272fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 491372fd2efbSEmilio G. Cota { 49140e2d61cfSRichard Henderson unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 491572fd2efbSEmilio G. Cota unsigned int i; 491672fd2efbSEmilio G. Cota int64_t ret = 0; 491772fd2efbSEmilio G. Cota 491872fd2efbSEmilio G. Cota for (i = 0; i < n_ctxs; i++) { 4919d73415a3SStefan Hajnoczi const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 492072fd2efbSEmilio G. Cota const TCGProfile *prof = &s->prof; 492172fd2efbSEmilio G. Cota 4922d73415a3SStefan Hajnoczi ret += qatomic_read(&prof->cpu_exec_time); 492372fd2efbSEmilio G. Cota } 492472fd2efbSEmilio G. Cota return ret; 492572fd2efbSEmilio G. Cota } 4926246ae24dSMax Filippov #else 4927b6a7f3e0SDaniel P. Berrangé void tcg_dump_op_count(GString *buf) 4928246ae24dSMax Filippov { 4929b6a7f3e0SDaniel P. Berrangé g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 4930246ae24dSMax Filippov } 493172fd2efbSEmilio G. Cota 493272fd2efbSEmilio G. Cota int64_t tcg_cpu_exec_time(void) 493372fd2efbSEmilio G. Cota { 493472fd2efbSEmilio G. Cota error_report("%s: TCG profiler not compiled", __func__); 493572fd2efbSEmilio G. Cota exit(EXIT_FAILURE); 493672fd2efbSEmilio G. Cota } 4937c896fe29Sbellard #endif 4938c896fe29Sbellard 4939c896fe29Sbellard 4940fbf59aadSRichard Henderson int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) 4941c896fe29Sbellard { 4942c3fac113SEmilio G. Cota #ifdef CONFIG_PROFILER 4943c3fac113SEmilio G. Cota TCGProfile *prof = &s->prof; 4944c3fac113SEmilio G. Cota #endif 494515fa08f8SRichard Henderson int i, num_insns; 494615fa08f8SRichard Henderson TCGOp *op; 4947c896fe29Sbellard 494804fe6400SRichard Henderson #ifdef CONFIG_PROFILER 494904fe6400SRichard Henderson { 4950c1f543b7SEmilio G. Cota int n = 0; 495104fe6400SRichard Henderson 495215fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 495315fa08f8SRichard Henderson n++; 495415fa08f8SRichard Henderson } 4955d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count, prof->op_count + n); 4956c3fac113SEmilio G. Cota if (n > prof->op_count_max) { 4957d73415a3SStefan Hajnoczi qatomic_set(&prof->op_count_max, n); 495804fe6400SRichard Henderson } 495904fe6400SRichard Henderson 496004fe6400SRichard Henderson n = s->nb_temps; 4961d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count, prof->temp_count + n); 4962c3fac113SEmilio G. Cota if (n > prof->temp_count_max) { 4963d73415a3SStefan Hajnoczi qatomic_set(&prof->temp_count_max, n); 496404fe6400SRichard Henderson } 496504fe6400SRichard Henderson } 496604fe6400SRichard Henderson #endif 496704fe6400SRichard Henderson 4968c896fe29Sbellard #ifdef DEBUG_DISAS 4969d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4970fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 4971c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 497278b54858SRichard Henderson if (logfile) { 497378b54858SRichard Henderson fprintf(logfile, "OP:\n"); 4974b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false); 497578b54858SRichard Henderson fprintf(logfile, "\n"); 4976fc59d2d8SRobert Foley qemu_log_unlock(logfile); 4977c896fe29Sbellard } 497878b54858SRichard Henderson } 4979c896fe29Sbellard #endif 4980c896fe29Sbellard 4981bef16ab4SRichard Henderson #ifdef CONFIG_DEBUG_TCG 4982bef16ab4SRichard Henderson /* Ensure all labels referenced have been emitted. */ 4983bef16ab4SRichard Henderson { 4984bef16ab4SRichard Henderson TCGLabel *l; 4985bef16ab4SRichard Henderson bool error = false; 4986bef16ab4SRichard Henderson 4987bef16ab4SRichard Henderson QSIMPLEQ_FOREACH(l, &s->labels, next) { 4988f85b1fc4SRichard Henderson if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { 4989bef16ab4SRichard Henderson qemu_log_mask(CPU_LOG_TB_OP, 4990bef16ab4SRichard Henderson "$L%d referenced but not present.\n", l->id); 4991bef16ab4SRichard Henderson error = true; 4992bef16ab4SRichard Henderson } 4993bef16ab4SRichard Henderson } 4994bef16ab4SRichard Henderson assert(!error); 4995bef16ab4SRichard Henderson } 4996bef16ab4SRichard Henderson #endif 4997bef16ab4SRichard Henderson 4998c5cc28ffSAurelien Jarno #ifdef CONFIG_PROFILER 4999d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 5000c5cc28ffSAurelien Jarno #endif 5001c5cc28ffSAurelien Jarno 50028f2e8c07SKirill Batuzov #ifdef USE_TCG_OPTIMIZATIONS 5003c45cb8bbSRichard Henderson tcg_optimize(s); 50048f2e8c07SKirill Batuzov #endif 50058f2e8c07SKirill Batuzov 5006a23a9ec6Sbellard #ifdef CONFIG_PROFILER 5007d73415a3SStefan Hajnoczi qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 5008d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 5009a23a9ec6Sbellard #endif 5010c5cc28ffSAurelien Jarno 5011b4fc67c7SRichard Henderson reachable_code_pass(s); 5012874b8574SRichard Henderson liveness_pass_0(s); 5013b83eabeaSRichard Henderson liveness_pass_1(s); 50145a18407fSRichard Henderson 50155a18407fSRichard Henderson if (s->nb_indirects > 0) { 50165a18407fSRichard Henderson #ifdef DEBUG_DISAS 50175a18407fSRichard Henderson if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 5018fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 5019c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 502078b54858SRichard Henderson if (logfile) { 502178b54858SRichard Henderson fprintf(logfile, "OP before indirect lowering:\n"); 5022b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, false); 502378b54858SRichard Henderson fprintf(logfile, "\n"); 5024fc59d2d8SRobert Foley qemu_log_unlock(logfile); 50255a18407fSRichard Henderson } 502678b54858SRichard Henderson } 50275a18407fSRichard Henderson #endif 50285a18407fSRichard Henderson /* Replace indirect temps with direct temps. */ 5029b83eabeaSRichard Henderson if (liveness_pass_2(s)) { 50305a18407fSRichard Henderson /* If changes were made, re-run liveness. */ 5031b83eabeaSRichard Henderson liveness_pass_1(s); 50325a18407fSRichard Henderson } 50335a18407fSRichard Henderson } 5034c5cc28ffSAurelien Jarno 5035a23a9ec6Sbellard #ifdef CONFIG_PROFILER 5036d73415a3SStefan Hajnoczi qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 5037a23a9ec6Sbellard #endif 5038c896fe29Sbellard 5039c896fe29Sbellard #ifdef DEBUG_DISAS 5040d977e1c2SAlex Bennée if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 5041fbf59aadSRichard Henderson && qemu_log_in_addr_range(pc_start))) { 5042c60f599bSRichard Henderson FILE *logfile = qemu_log_trylock(); 504378b54858SRichard Henderson if (logfile) { 504478b54858SRichard Henderson fprintf(logfile, "OP after optimization and liveness analysis:\n"); 5045b7a83ff8SRichard Henderson tcg_dump_ops(s, logfile, true); 504678b54858SRichard Henderson fprintf(logfile, "\n"); 5047fc59d2d8SRobert Foley qemu_log_unlock(logfile); 5048c896fe29Sbellard } 504978b54858SRichard Henderson } 5050c896fe29Sbellard #endif 5051c896fe29Sbellard 505235abb009SRichard Henderson /* Initialize goto_tb jump offsets. */ 50533a50f424SRichard Henderson tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; 50543a50f424SRichard Henderson tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; 50559da6079bSRichard Henderson tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; 50569da6079bSRichard Henderson tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; 505735abb009SRichard Henderson 5058c896fe29Sbellard tcg_reg_alloc_start(s); 5059c896fe29Sbellard 5060db0c51a3SRichard Henderson /* 5061db0c51a3SRichard Henderson * Reset the buffer pointers when restarting after overflow. 5062db0c51a3SRichard Henderson * TODO: Move this into translate-all.c with the rest of the 5063db0c51a3SRichard Henderson * buffer management. Having only this done here is confusing. 5064db0c51a3SRichard Henderson */ 5065db0c51a3SRichard Henderson s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 5066db0c51a3SRichard Henderson s->code_ptr = s->code_buf; 5067c896fe29Sbellard 5068659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 50696001f772SLaurent Vivier QSIMPLEQ_INIT(&s->ldst_labels); 5070659ef5cbSRichard Henderson #endif 507157a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 507257a26946SRichard Henderson s->pool_labels = NULL; 507357a26946SRichard Henderson #endif 50749ecefc84SRichard Henderson 5075fca8a500SRichard Henderson num_insns = -1; 507615fa08f8SRichard Henderson QTAILQ_FOREACH(op, &s->ops, link) { 5077c45cb8bbSRichard Henderson TCGOpcode opc = op->opc; 5078b3db8758Sblueswir1 5079c896fe29Sbellard #ifdef CONFIG_PROFILER 5080d73415a3SStefan Hajnoczi qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 5081c896fe29Sbellard #endif 5082c45cb8bbSRichard Henderson 5083c896fe29Sbellard switch (opc) { 5084c896fe29Sbellard case INDEX_op_mov_i32: 5085c896fe29Sbellard case INDEX_op_mov_i64: 5086d2fd745fSRichard Henderson case INDEX_op_mov_vec: 5087dd186292SRichard Henderson tcg_reg_alloc_mov(s, op); 5088c896fe29Sbellard break; 5089bab1671fSRichard Henderson case INDEX_op_dup_vec: 5090bab1671fSRichard Henderson tcg_reg_alloc_dup(s, op); 5091bab1671fSRichard Henderson break; 5092765b842aSRichard Henderson case INDEX_op_insn_start: 5093fca8a500SRichard Henderson if (num_insns >= 0) { 50949f754620SRichard Henderson size_t off = tcg_current_code_size(s); 50959f754620SRichard Henderson s->gen_insn_end_off[num_insns] = off; 50969f754620SRichard Henderson /* Assert that we do not overflow our stored offset. */ 50979f754620SRichard Henderson assert(s->gen_insn_end_off[num_insns] == off); 5098fca8a500SRichard Henderson } 5099fca8a500SRichard Henderson num_insns++; 5100bad729e2SRichard Henderson for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 5101bad729e2SRichard Henderson target_ulong a; 5102bad729e2SRichard Henderson #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 5103efee3746SRichard Henderson a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 5104bad729e2SRichard Henderson #else 5105efee3746SRichard Henderson a = op->args[i]; 5106bad729e2SRichard Henderson #endif 5107fca8a500SRichard Henderson s->gen_insn_data[num_insns][i] = a; 5108bad729e2SRichard Henderson } 5109c896fe29Sbellard break; 51105ff9d6a4Sbellard case INDEX_op_discard: 511143439139SRichard Henderson temp_dead(s, arg_temp(op->args[0])); 51125ff9d6a4Sbellard break; 5113c896fe29Sbellard case INDEX_op_set_label: 5114e8996ee0Sbellard tcg_reg_alloc_bb_end(s, s->reserved_regs); 511592ab8e7dSRichard Henderson tcg_out_label(s, arg_label(op->args[0])); 5116c896fe29Sbellard break; 5117c896fe29Sbellard case INDEX_op_call: 5118dd186292SRichard Henderson tcg_reg_alloc_call(s, op); 5119c45cb8bbSRichard Henderson break; 5120b55a8d9dSRichard Henderson case INDEX_op_exit_tb: 5121b55a8d9dSRichard Henderson tcg_out_exit_tb(s, op->args[0]); 5122b55a8d9dSRichard Henderson break; 5123cf7d6b8eSRichard Henderson case INDEX_op_goto_tb: 5124cf7d6b8eSRichard Henderson tcg_out_goto_tb(s, op->args[0]); 5125cf7d6b8eSRichard Henderson break; 5126efe86b21SRichard Henderson case INDEX_op_dup2_vec: 5127efe86b21SRichard Henderson if (tcg_reg_alloc_dup2(s, op)) { 5128efe86b21SRichard Henderson break; 5129efe86b21SRichard Henderson } 5130efe86b21SRichard Henderson /* fall through */ 5131c896fe29Sbellard default: 513225c4d9ccSRichard Henderson /* Sanity check that we've not introduced any unhandled opcodes. */ 5133be0f34b5SRichard Henderson tcg_debug_assert(tcg_op_supported(opc)); 5134c896fe29Sbellard /* Note: in order to speed up the code, it would be much 5135c896fe29Sbellard faster to have specialized register allocator functions for 5136c896fe29Sbellard some common argument patterns */ 5137dd186292SRichard Henderson tcg_reg_alloc_op(s, op); 5138c896fe29Sbellard break; 5139c896fe29Sbellard } 5140b125f9dcSRichard Henderson /* Test for (pending) buffer overflow. The assumption is that any 5141b125f9dcSRichard Henderson one operation beginning below the high water mark cannot overrun 5142b125f9dcSRichard Henderson the buffer completely. Thus we can test for overflow after 5143b125f9dcSRichard Henderson generating code without having to check during generation. */ 5144644da9b3SJohn Clarke if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 5145b125f9dcSRichard Henderson return -1; 5146b125f9dcSRichard Henderson } 51476e6c4efeSRichard Henderson /* Test for TB overflow, as seen by gen_insn_end_off. */ 51486e6c4efeSRichard Henderson if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 51496e6c4efeSRichard Henderson return -2; 51506e6c4efeSRichard Henderson } 5151c896fe29Sbellard } 5152fca8a500SRichard Henderson tcg_debug_assert(num_insns >= 0); 5153fca8a500SRichard Henderson s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 5154c45cb8bbSRichard Henderson 5155b76f0d8cSYeongkyoon Lee /* Generate TB finalization at the end of block */ 5156659ef5cbSRichard Henderson #ifdef TCG_TARGET_NEED_LDST_LABELS 5157aeee05f5SRichard Henderson i = tcg_out_ldst_finalize(s); 5158aeee05f5SRichard Henderson if (i < 0) { 5159aeee05f5SRichard Henderson return i; 516023dceda6SRichard Henderson } 5161659ef5cbSRichard Henderson #endif 516257a26946SRichard Henderson #ifdef TCG_TARGET_NEED_POOL_LABELS 51631768987bSRichard Henderson i = tcg_out_pool_finalize(s); 51641768987bSRichard Henderson if (i < 0) { 51651768987bSRichard Henderson return i; 516657a26946SRichard Henderson } 516757a26946SRichard Henderson #endif 51687ecd02a0SRichard Henderson if (!tcg_resolve_relocs(s)) { 51697ecd02a0SRichard Henderson return -2; 51707ecd02a0SRichard Henderson } 5171c896fe29Sbellard 5172df5d2b16SRichard Henderson #ifndef CONFIG_TCG_INTERPRETER 5173c896fe29Sbellard /* flush instruction cache */ 5174db0c51a3SRichard Henderson flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 5175db0c51a3SRichard Henderson (uintptr_t)s->code_buf, 51761da8de39SRichard Henderson tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 5177df5d2b16SRichard Henderson #endif 51782aeabc08SStefan Weil 51791813e175SRichard Henderson return tcg_current_code_size(s); 5180c896fe29Sbellard } 5181c896fe29Sbellard 5182a23a9ec6Sbellard #ifdef CONFIG_PROFILER 51833a841ab5SDaniel P. Berrangé void tcg_dump_info(GString *buf) 5184a23a9ec6Sbellard { 5185c3fac113SEmilio G. Cota TCGProfile prof = {}; 5186c3fac113SEmilio G. Cota const TCGProfile *s; 5187c3fac113SEmilio G. Cota int64_t tb_count; 5188c3fac113SEmilio G. Cota int64_t tb_div_count; 5189c3fac113SEmilio G. Cota int64_t tot; 5190c3fac113SEmilio G. Cota 5191c3fac113SEmilio G. Cota tcg_profile_snapshot_counters(&prof); 5192c3fac113SEmilio G. Cota s = &prof; 5193c3fac113SEmilio G. Cota tb_count = s->tb_count; 5194c3fac113SEmilio G. Cota tb_div_count = tb_count ? tb_count : 1; 5195c3fac113SEmilio G. Cota tot = s->interm_time + s->code_time; 5196a23a9ec6Sbellard 51973a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "JIT cycles %" PRId64 51983a841ab5SDaniel P. Berrangé " (%0.3f s at 2.4 GHz)\n", 5199a23a9ec6Sbellard tot, tot / 2.4e9); 52003a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "translated TBs %" PRId64 52013a841ab5SDaniel P. Berrangé " (aborted=%" PRId64 " %0.1f%%)\n", 5202fca8a500SRichard Henderson tb_count, s->tb_count1 - tb_count, 5203fca8a500SRichard Henderson (double)(s->tb_count1 - s->tb_count) 5204fca8a500SRichard Henderson / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 52053a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n", 5206fca8a500SRichard Henderson (double)s->op_count / tb_div_count, s->op_count_max); 52073a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "deleted ops/TB %0.2f\n", 5208fca8a500SRichard Henderson (double)s->del_op_count / tb_div_count); 52093a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n", 52103a841ab5SDaniel P. Berrangé (double)s->temp_count / tb_div_count, 52113a841ab5SDaniel P. Berrangé s->temp_count_max); 52123a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg host code/TB %0.1f\n", 5213fca8a500SRichard Henderson (double)s->code_out_len / tb_div_count); 52143a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "avg search data/TB %0.1f\n", 5215fca8a500SRichard Henderson (double)s->search_out_len / tb_div_count); 5216a23a9ec6Sbellard 52173a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/op %0.1f\n", 5218a23a9ec6Sbellard s->op_count ? (double)tot / s->op_count : 0); 52193a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/in byte %0.1f\n", 5220a23a9ec6Sbellard s->code_in_len ? (double)tot / s->code_in_len : 0); 52213a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/out byte %0.1f\n", 5222a23a9ec6Sbellard s->code_out_len ? (double)tot / s->code_out_len : 0); 52233a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cycles/search byte %0.1f\n", 52243a841ab5SDaniel P. Berrangé s->search_out_len ? 52253a841ab5SDaniel P. Berrangé (double)tot / s->search_out_len : 0); 5226fca8a500SRichard Henderson if (tot == 0) { 5227a23a9ec6Sbellard tot = 1; 5228fca8a500SRichard Henderson } 52293a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " gen_interm time %0.1f%%\n", 5230a23a9ec6Sbellard (double)s->interm_time / tot * 100.0); 52313a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " gen_code time %0.1f%%\n", 5232a23a9ec6Sbellard (double)s->code_time / tot * 100.0); 52333a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "optim./code time %0.1f%%\n", 52343a841ab5SDaniel P. Berrangé (double)s->opt_time / (s->code_time ? 52353a841ab5SDaniel P. Berrangé s->code_time : 1) 5236c5cc28ffSAurelien Jarno * 100.0); 52373a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "liveness/code time %0.1f%%\n", 52383a841ab5SDaniel P. Berrangé (double)s->la_time / (s->code_time ? 52393a841ab5SDaniel P. Berrangé s->code_time : 1) * 100.0); 52403a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n", 5241a23a9ec6Sbellard s->restore_count); 52423a841ab5SDaniel P. Berrangé g_string_append_printf(buf, " avg cycles %0.1f\n", 52433a841ab5SDaniel P. Berrangé s->restore_count ? 52443a841ab5SDaniel P. Berrangé (double)s->restore_time / s->restore_count : 0); 5245a23a9ec6Sbellard } 5246a23a9ec6Sbellard #else 52473a841ab5SDaniel P. Berrangé void tcg_dump_info(GString *buf) 5248a23a9ec6Sbellard { 52493a841ab5SDaniel P. Berrangé g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 5250a23a9ec6Sbellard } 5251a23a9ec6Sbellard #endif 5252813da627SRichard Henderson 5253813da627SRichard Henderson #ifdef ELF_HOST_MACHINE 52545872bbf2SRichard Henderson /* In order to use this feature, the backend needs to do three things: 52555872bbf2SRichard Henderson 52565872bbf2SRichard Henderson (1) Define ELF_HOST_MACHINE to indicate both what value to 52575872bbf2SRichard Henderson put into the ELF image and to indicate support for the feature. 52585872bbf2SRichard Henderson 52595872bbf2SRichard Henderson (2) Define tcg_register_jit. This should create a buffer containing 52605872bbf2SRichard Henderson the contents of a .debug_frame section that describes the post- 52615872bbf2SRichard Henderson prologue unwind info for the tcg machine. 52625872bbf2SRichard Henderson 52635872bbf2SRichard Henderson (3) Call tcg_register_jit_int, with the constructed .debug_frame. 52645872bbf2SRichard Henderson */ 5265813da627SRichard Henderson 5266813da627SRichard Henderson /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 5267813da627SRichard Henderson typedef enum { 5268813da627SRichard Henderson JIT_NOACTION = 0, 5269813da627SRichard Henderson JIT_REGISTER_FN, 5270813da627SRichard Henderson JIT_UNREGISTER_FN 5271813da627SRichard Henderson } jit_actions_t; 5272813da627SRichard Henderson 5273813da627SRichard Henderson struct jit_code_entry { 5274813da627SRichard Henderson struct jit_code_entry *next_entry; 5275813da627SRichard Henderson struct jit_code_entry *prev_entry; 5276813da627SRichard Henderson const void *symfile_addr; 5277813da627SRichard Henderson uint64_t symfile_size; 5278813da627SRichard Henderson }; 5279813da627SRichard Henderson 5280813da627SRichard Henderson struct jit_descriptor { 5281813da627SRichard Henderson uint32_t version; 5282813da627SRichard Henderson uint32_t action_flag; 5283813da627SRichard Henderson struct jit_code_entry *relevant_entry; 5284813da627SRichard Henderson struct jit_code_entry *first_entry; 5285813da627SRichard Henderson }; 5286813da627SRichard Henderson 5287813da627SRichard Henderson void __jit_debug_register_code(void) __attribute__((noinline)); 5288813da627SRichard Henderson void __jit_debug_register_code(void) 5289813da627SRichard Henderson { 5290813da627SRichard Henderson asm(""); 5291813da627SRichard Henderson } 5292813da627SRichard Henderson 5293813da627SRichard Henderson /* Must statically initialize the version, because GDB may check 5294813da627SRichard Henderson the version before we can set it. */ 5295813da627SRichard Henderson struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 5296813da627SRichard Henderson 5297813da627SRichard Henderson /* End GDB interface. */ 5298813da627SRichard Henderson 5299813da627SRichard Henderson static int find_string(const char *strtab, const char *str) 5300813da627SRichard Henderson { 5301813da627SRichard Henderson const char *p = strtab + 1; 5302813da627SRichard Henderson 5303813da627SRichard Henderson while (1) { 5304813da627SRichard Henderson if (strcmp(p, str) == 0) { 5305813da627SRichard Henderson return p - strtab; 5306813da627SRichard Henderson } 5307813da627SRichard Henderson p += strlen(p) + 1; 5308813da627SRichard Henderson } 5309813da627SRichard Henderson } 5310813da627SRichard Henderson 5311755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 53122c90784aSRichard Henderson const void *debug_frame, 53132c90784aSRichard Henderson size_t debug_frame_size) 5314813da627SRichard Henderson { 53155872bbf2SRichard Henderson struct __attribute__((packed)) DebugInfo { 53165872bbf2SRichard Henderson uint32_t len; 53175872bbf2SRichard Henderson uint16_t version; 53185872bbf2SRichard Henderson uint32_t abbrev; 53195872bbf2SRichard Henderson uint8_t ptr_size; 53205872bbf2SRichard Henderson uint8_t cu_die; 53215872bbf2SRichard Henderson uint16_t cu_lang; 53225872bbf2SRichard Henderson uintptr_t cu_low_pc; 53235872bbf2SRichard Henderson uintptr_t cu_high_pc; 53245872bbf2SRichard Henderson uint8_t fn_die; 53255872bbf2SRichard Henderson char fn_name[16]; 53265872bbf2SRichard Henderson uintptr_t fn_low_pc; 53275872bbf2SRichard Henderson uintptr_t fn_high_pc; 53285872bbf2SRichard Henderson uint8_t cu_eoc; 53295872bbf2SRichard Henderson }; 5330813da627SRichard Henderson 5331813da627SRichard Henderson struct ElfImage { 5332813da627SRichard Henderson ElfW(Ehdr) ehdr; 5333813da627SRichard Henderson ElfW(Phdr) phdr; 53345872bbf2SRichard Henderson ElfW(Shdr) shdr[7]; 53355872bbf2SRichard Henderson ElfW(Sym) sym[2]; 53365872bbf2SRichard Henderson struct DebugInfo di; 53375872bbf2SRichard Henderson uint8_t da[24]; 53385872bbf2SRichard Henderson char str[80]; 53395872bbf2SRichard Henderson }; 53405872bbf2SRichard Henderson 53415872bbf2SRichard Henderson struct ElfImage *img; 53425872bbf2SRichard Henderson 53435872bbf2SRichard Henderson static const struct ElfImage img_template = { 53445872bbf2SRichard Henderson .ehdr = { 53455872bbf2SRichard Henderson .e_ident[EI_MAG0] = ELFMAG0, 53465872bbf2SRichard Henderson .e_ident[EI_MAG1] = ELFMAG1, 53475872bbf2SRichard Henderson .e_ident[EI_MAG2] = ELFMAG2, 53485872bbf2SRichard Henderson .e_ident[EI_MAG3] = ELFMAG3, 53495872bbf2SRichard Henderson .e_ident[EI_CLASS] = ELF_CLASS, 53505872bbf2SRichard Henderson .e_ident[EI_DATA] = ELF_DATA, 53515872bbf2SRichard Henderson .e_ident[EI_VERSION] = EV_CURRENT, 53525872bbf2SRichard Henderson .e_type = ET_EXEC, 53535872bbf2SRichard Henderson .e_machine = ELF_HOST_MACHINE, 53545872bbf2SRichard Henderson .e_version = EV_CURRENT, 53555872bbf2SRichard Henderson .e_phoff = offsetof(struct ElfImage, phdr), 53565872bbf2SRichard Henderson .e_shoff = offsetof(struct ElfImage, shdr), 53575872bbf2SRichard Henderson .e_ehsize = sizeof(ElfW(Shdr)), 53585872bbf2SRichard Henderson .e_phentsize = sizeof(ElfW(Phdr)), 53595872bbf2SRichard Henderson .e_phnum = 1, 53605872bbf2SRichard Henderson .e_shentsize = sizeof(ElfW(Shdr)), 53615872bbf2SRichard Henderson .e_shnum = ARRAY_SIZE(img->shdr), 53625872bbf2SRichard Henderson .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 5363abbb3eaeSRichard Henderson #ifdef ELF_HOST_FLAGS 5364abbb3eaeSRichard Henderson .e_flags = ELF_HOST_FLAGS, 5365abbb3eaeSRichard Henderson #endif 5366abbb3eaeSRichard Henderson #ifdef ELF_OSABI 5367abbb3eaeSRichard Henderson .e_ident[EI_OSABI] = ELF_OSABI, 5368abbb3eaeSRichard Henderson #endif 53695872bbf2SRichard Henderson }, 53705872bbf2SRichard Henderson .phdr = { 53715872bbf2SRichard Henderson .p_type = PT_LOAD, 53725872bbf2SRichard Henderson .p_flags = PF_X, 53735872bbf2SRichard Henderson }, 53745872bbf2SRichard Henderson .shdr = { 53755872bbf2SRichard Henderson [0] = { .sh_type = SHT_NULL }, 53765872bbf2SRichard Henderson /* Trick: The contents of code_gen_buffer are not present in 53775872bbf2SRichard Henderson this fake ELF file; that got allocated elsewhere. Therefore 53785872bbf2SRichard Henderson we mark .text as SHT_NOBITS (similar to .bss) so that readers 53795872bbf2SRichard Henderson will not look for contents. We can record any address. */ 53805872bbf2SRichard Henderson [1] = { /* .text */ 53815872bbf2SRichard Henderson .sh_type = SHT_NOBITS, 53825872bbf2SRichard Henderson .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 53835872bbf2SRichard Henderson }, 53845872bbf2SRichard Henderson [2] = { /* .debug_info */ 53855872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 53865872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, di), 53875872bbf2SRichard Henderson .sh_size = sizeof(struct DebugInfo), 53885872bbf2SRichard Henderson }, 53895872bbf2SRichard Henderson [3] = { /* .debug_abbrev */ 53905872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 53915872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, da), 53925872bbf2SRichard Henderson .sh_size = sizeof(img->da), 53935872bbf2SRichard Henderson }, 53945872bbf2SRichard Henderson [4] = { /* .debug_frame */ 53955872bbf2SRichard Henderson .sh_type = SHT_PROGBITS, 53965872bbf2SRichard Henderson .sh_offset = sizeof(struct ElfImage), 53975872bbf2SRichard Henderson }, 53985872bbf2SRichard Henderson [5] = { /* .symtab */ 53995872bbf2SRichard Henderson .sh_type = SHT_SYMTAB, 54005872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, sym), 54015872bbf2SRichard Henderson .sh_size = sizeof(img->sym), 54025872bbf2SRichard Henderson .sh_info = 1, 54035872bbf2SRichard Henderson .sh_link = ARRAY_SIZE(img->shdr) - 1, 54045872bbf2SRichard Henderson .sh_entsize = sizeof(ElfW(Sym)), 54055872bbf2SRichard Henderson }, 54065872bbf2SRichard Henderson [6] = { /* .strtab */ 54075872bbf2SRichard Henderson .sh_type = SHT_STRTAB, 54085872bbf2SRichard Henderson .sh_offset = offsetof(struct ElfImage, str), 54095872bbf2SRichard Henderson .sh_size = sizeof(img->str), 54105872bbf2SRichard Henderson } 54115872bbf2SRichard Henderson }, 54125872bbf2SRichard Henderson .sym = { 54135872bbf2SRichard Henderson [1] = { /* code_gen_buffer */ 54145872bbf2SRichard Henderson .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 54155872bbf2SRichard Henderson .st_shndx = 1, 54165872bbf2SRichard Henderson } 54175872bbf2SRichard Henderson }, 54185872bbf2SRichard Henderson .di = { 54195872bbf2SRichard Henderson .len = sizeof(struct DebugInfo) - 4, 54205872bbf2SRichard Henderson .version = 2, 54215872bbf2SRichard Henderson .ptr_size = sizeof(void *), 54225872bbf2SRichard Henderson .cu_die = 1, 54235872bbf2SRichard Henderson .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 54245872bbf2SRichard Henderson .fn_die = 2, 54255872bbf2SRichard Henderson .fn_name = "code_gen_buffer" 54265872bbf2SRichard Henderson }, 54275872bbf2SRichard Henderson .da = { 54285872bbf2SRichard Henderson 1, /* abbrev number (the cu) */ 54295872bbf2SRichard Henderson 0x11, 1, /* DW_TAG_compile_unit, has children */ 54305872bbf2SRichard Henderson 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 54315872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 54325872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 54335872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 54345872bbf2SRichard Henderson 2, /* abbrev number (the fn) */ 54355872bbf2SRichard Henderson 0x2e, 0, /* DW_TAG_subprogram, no children */ 54365872bbf2SRichard Henderson 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 54375872bbf2SRichard Henderson 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 54385872bbf2SRichard Henderson 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 54395872bbf2SRichard Henderson 0, 0, /* end of abbrev */ 54405872bbf2SRichard Henderson 0 /* no more abbrev */ 54415872bbf2SRichard Henderson }, 54425872bbf2SRichard Henderson .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 54435872bbf2SRichard Henderson ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 5444813da627SRichard Henderson }; 5445813da627SRichard Henderson 5446813da627SRichard Henderson /* We only need a single jit entry; statically allocate it. */ 5447813da627SRichard Henderson static struct jit_code_entry one_entry; 5448813da627SRichard Henderson 54495872bbf2SRichard Henderson uintptr_t buf = (uintptr_t)buf_ptr; 5450813da627SRichard Henderson size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 54512c90784aSRichard Henderson DebugFrameHeader *dfh; 5452813da627SRichard Henderson 54535872bbf2SRichard Henderson img = g_malloc(img_size); 54545872bbf2SRichard Henderson *img = img_template; 5455813da627SRichard Henderson 54565872bbf2SRichard Henderson img->phdr.p_vaddr = buf; 54575872bbf2SRichard Henderson img->phdr.p_paddr = buf; 54585872bbf2SRichard Henderson img->phdr.p_memsz = buf_size; 5459813da627SRichard Henderson 54605872bbf2SRichard Henderson img->shdr[1].sh_name = find_string(img->str, ".text"); 54615872bbf2SRichard Henderson img->shdr[1].sh_addr = buf; 54625872bbf2SRichard Henderson img->shdr[1].sh_size = buf_size; 5463813da627SRichard Henderson 54645872bbf2SRichard Henderson img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 54655872bbf2SRichard Henderson img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 54665872bbf2SRichard Henderson 54675872bbf2SRichard Henderson img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 54685872bbf2SRichard Henderson img->shdr[4].sh_size = debug_frame_size; 54695872bbf2SRichard Henderson 54705872bbf2SRichard Henderson img->shdr[5].sh_name = find_string(img->str, ".symtab"); 54715872bbf2SRichard Henderson img->shdr[6].sh_name = find_string(img->str, ".strtab"); 54725872bbf2SRichard Henderson 54735872bbf2SRichard Henderson img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 54745872bbf2SRichard Henderson img->sym[1].st_value = buf; 54755872bbf2SRichard Henderson img->sym[1].st_size = buf_size; 54765872bbf2SRichard Henderson 54775872bbf2SRichard Henderson img->di.cu_low_pc = buf; 547845aba097SRichard Henderson img->di.cu_high_pc = buf + buf_size; 54795872bbf2SRichard Henderson img->di.fn_low_pc = buf; 548045aba097SRichard Henderson img->di.fn_high_pc = buf + buf_size; 5481813da627SRichard Henderson 54822c90784aSRichard Henderson dfh = (DebugFrameHeader *)(img + 1); 54832c90784aSRichard Henderson memcpy(dfh, debug_frame, debug_frame_size); 54842c90784aSRichard Henderson dfh->fde.func_start = buf; 54852c90784aSRichard Henderson dfh->fde.func_len = buf_size; 54862c90784aSRichard Henderson 5487813da627SRichard Henderson #ifdef DEBUG_JIT 5488813da627SRichard Henderson /* Enable this block to be able to debug the ELF image file creation. 5489813da627SRichard Henderson One can use readelf, objdump, or other inspection utilities. */ 5490813da627SRichard Henderson { 5491eb6b2edfSBin Meng g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 5492eb6b2edfSBin Meng FILE *f = fopen(jit, "w+b"); 5493813da627SRichard Henderson if (f) { 54945872bbf2SRichard Henderson if (fwrite(img, img_size, 1, f) != img_size) { 5495813da627SRichard Henderson /* Avoid stupid unused return value warning for fwrite. */ 5496813da627SRichard Henderson } 5497813da627SRichard Henderson fclose(f); 5498813da627SRichard Henderson } 5499813da627SRichard Henderson } 5500813da627SRichard Henderson #endif 5501813da627SRichard Henderson 5502813da627SRichard Henderson one_entry.symfile_addr = img; 5503813da627SRichard Henderson one_entry.symfile_size = img_size; 5504813da627SRichard Henderson 5505813da627SRichard Henderson __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 5506813da627SRichard Henderson __jit_debug_descriptor.relevant_entry = &one_entry; 5507813da627SRichard Henderson __jit_debug_descriptor.first_entry = &one_entry; 5508813da627SRichard Henderson __jit_debug_register_code(); 5509813da627SRichard Henderson } 5510813da627SRichard Henderson #else 55115872bbf2SRichard Henderson /* No support for the feature. Provide the entry point expected by exec.c, 55125872bbf2SRichard Henderson and implement the internal function we declared earlier. */ 5513813da627SRichard Henderson 5514755bf9e5SRichard Henderson static void tcg_register_jit_int(const void *buf, size_t size, 55152c90784aSRichard Henderson const void *debug_frame, 55162c90784aSRichard Henderson size_t debug_frame_size) 5517813da627SRichard Henderson { 5518813da627SRichard Henderson } 5519813da627SRichard Henderson 5520755bf9e5SRichard Henderson void tcg_register_jit(const void *buf, size_t buf_size) 5521813da627SRichard Henderson { 5522813da627SRichard Henderson } 5523813da627SRichard Henderson #endif /* ELF_HOST_MACHINE */ 5524db432672SRichard Henderson 5525db432672SRichard Henderson #if !TCG_TARGET_MAYBE_vec 5526db432672SRichard Henderson void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 5527db432672SRichard Henderson { 5528db432672SRichard Henderson g_assert_not_reached(); 5529db432672SRichard Henderson } 5530db432672SRichard Henderson #endif 5531