xref: /openbmc/qemu/tcg/tcg.c (revision e8d1e0cd)
1 /*
2  * Tiny Code Generator for QEMU
3  *
4  * Copyright (c) 2008 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
27 
28 #include "qemu/osdep.h"
29 
30 /* Define to jump the ELF file used to communicate with GDB.  */
31 #undef DEBUG_JIT
32 
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/cacheflush.h"
38 #include "qemu/cacheinfo.h"
39 #include "qemu/timer.h"
40 
41 /* Note: the long term plan is to reduce the dependencies on the QEMU
42    CPU definitions. Currently they are used for qemu_ld/st
43    instructions */
44 #define NO_CPU_IO_DEFS
45 
46 #include "exec/exec-all.h"
47 #include "tcg/tcg-op.h"
48 
49 #if UINTPTR_MAX == UINT32_MAX
50 # define ELF_CLASS  ELFCLASS32
51 #else
52 # define ELF_CLASS  ELFCLASS64
53 #endif
54 #if HOST_BIG_ENDIAN
55 # define ELF_DATA   ELFDATA2MSB
56 #else
57 # define ELF_DATA   ELFDATA2LSB
58 #endif
59 
60 #include "elf.h"
61 #include "exec/log.h"
62 #include "tcg/tcg-ldst.h"
63 #include "tcg/tcg-temp-internal.h"
64 #include "tcg-internal.h"
65 #include "accel/tcg/perf.h"
66 
67 /* Forward declarations for functions declared in tcg-target.c.inc and
68    used here. */
69 static void tcg_target_init(TCGContext *s);
70 static void tcg_target_qemu_prologue(TCGContext *s);
71 static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
72                         intptr_t value, intptr_t addend);
73 
74 /* The CIE and FDE header definitions will be common to all hosts.  */
75 typedef struct {
76     uint32_t len __attribute__((aligned((sizeof(void *)))));
77     uint32_t id;
78     uint8_t version;
79     char augmentation[1];
80     uint8_t code_align;
81     uint8_t data_align;
82     uint8_t return_column;
83 } DebugFrameCIE;
84 
85 typedef struct QEMU_PACKED {
86     uint32_t len __attribute__((aligned((sizeof(void *)))));
87     uint32_t cie_offset;
88     uintptr_t func_start;
89     uintptr_t func_len;
90 } DebugFrameFDEHeader;
91 
92 typedef struct QEMU_PACKED {
93     DebugFrameCIE cie;
94     DebugFrameFDEHeader fde;
95 } DebugFrameHeader;
96 
97 static void tcg_register_jit_int(const void *buf, size_t size,
98                                  const void *debug_frame,
99                                  size_t debug_frame_size)
100     __attribute__((unused));
101 
102 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
103 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
104                        intptr_t arg2);
105 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
106 static void tcg_out_movi(TCGContext *s, TCGType type,
107                          TCGReg ret, tcg_target_long arg);
108 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
109 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
110 static void tcg_out_goto_tb(TCGContext *s, int which);
111 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
112                        const TCGArg args[TCG_MAX_OP_ARGS],
113                        const int const_args[TCG_MAX_OP_ARGS]);
114 #if TCG_TARGET_MAYBE_vec
115 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
116                             TCGReg dst, TCGReg src);
117 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
118                              TCGReg dst, TCGReg base, intptr_t offset);
119 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
120                              TCGReg dst, int64_t arg);
121 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
122                            unsigned vecl, unsigned vece,
123                            const TCGArg args[TCG_MAX_OP_ARGS],
124                            const int const_args[TCG_MAX_OP_ARGS]);
125 #else
126 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
127                                    TCGReg dst, TCGReg src)
128 {
129     g_assert_not_reached();
130 }
131 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
132                                     TCGReg dst, TCGReg base, intptr_t offset)
133 {
134     g_assert_not_reached();
135 }
136 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
137                                     TCGReg dst, int64_t arg)
138 {
139     g_assert_not_reached();
140 }
141 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
142                                   unsigned vecl, unsigned vece,
143                                   const TCGArg args[TCG_MAX_OP_ARGS],
144                                   const int const_args[TCG_MAX_OP_ARGS])
145 {
146     g_assert_not_reached();
147 }
148 #endif
149 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
150                        intptr_t arg2);
151 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
152                         TCGReg base, intptr_t ofs);
153 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
154                          const TCGHelperInfo *info);
155 static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
156 static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
157 #ifdef TCG_TARGET_NEED_LDST_LABELS
158 static int tcg_out_ldst_finalize(TCGContext *s);
159 #endif
160 
161 TCGContext tcg_init_ctx;
162 __thread TCGContext *tcg_ctx;
163 
164 TCGContext **tcg_ctxs;
165 unsigned int tcg_cur_ctxs;
166 unsigned int tcg_max_ctxs;
167 TCGv_env cpu_env = 0;
168 const void *tcg_code_gen_epilogue;
169 uintptr_t tcg_splitwx_diff;
170 
171 #ifndef CONFIG_TCG_INTERPRETER
172 tcg_prologue_fn *tcg_qemu_tb_exec;
173 #endif
174 
175 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
176 static TCGRegSet tcg_target_call_clobber_regs;
177 
178 #if TCG_TARGET_INSN_UNIT_SIZE == 1
179 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
180 {
181     *s->code_ptr++ = v;
182 }
183 
184 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
185                                                       uint8_t v)
186 {
187     *p = v;
188 }
189 #endif
190 
191 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
192 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
193 {
194     if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
195         *s->code_ptr++ = v;
196     } else {
197         tcg_insn_unit *p = s->code_ptr;
198         memcpy(p, &v, sizeof(v));
199         s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
200     }
201 }
202 
203 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
204                                                        uint16_t v)
205 {
206     if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
207         *p = v;
208     } else {
209         memcpy(p, &v, sizeof(v));
210     }
211 }
212 #endif
213 
214 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
215 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
216 {
217     if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
218         *s->code_ptr++ = v;
219     } else {
220         tcg_insn_unit *p = s->code_ptr;
221         memcpy(p, &v, sizeof(v));
222         s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
223     }
224 }
225 
226 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
227                                                        uint32_t v)
228 {
229     if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
230         *p = v;
231     } else {
232         memcpy(p, &v, sizeof(v));
233     }
234 }
235 #endif
236 
237 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
238 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
239 {
240     if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
241         *s->code_ptr++ = v;
242     } else {
243         tcg_insn_unit *p = s->code_ptr;
244         memcpy(p, &v, sizeof(v));
245         s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
246     }
247 }
248 
249 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
250                                                        uint64_t v)
251 {
252     if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
253         *p = v;
254     } else {
255         memcpy(p, &v, sizeof(v));
256     }
257 }
258 #endif
259 
260 /* label relocation processing */
261 
262 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
263                           TCGLabel *l, intptr_t addend)
264 {
265     TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
266 
267     r->type = type;
268     r->ptr = code_ptr;
269     r->addend = addend;
270     QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
271 }
272 
273 static void tcg_out_label(TCGContext *s, TCGLabel *l)
274 {
275     tcg_debug_assert(!l->has_value);
276     l->has_value = 1;
277     l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
278 }
279 
280 TCGLabel *gen_new_label(void)
281 {
282     TCGContext *s = tcg_ctx;
283     TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
284 
285     memset(l, 0, sizeof(TCGLabel));
286     l->id = s->nb_labels++;
287     QSIMPLEQ_INIT(&l->branches);
288     QSIMPLEQ_INIT(&l->relocs);
289 
290     QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
291 
292     return l;
293 }
294 
295 static bool tcg_resolve_relocs(TCGContext *s)
296 {
297     TCGLabel *l;
298 
299     QSIMPLEQ_FOREACH(l, &s->labels, next) {
300         TCGRelocation *r;
301         uintptr_t value = l->u.value;
302 
303         QSIMPLEQ_FOREACH(r, &l->relocs, next) {
304             if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
305                 return false;
306             }
307         }
308     }
309     return true;
310 }
311 
312 static void set_jmp_reset_offset(TCGContext *s, int which)
313 {
314     /*
315      * We will check for overflow at the end of the opcode loop in
316      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
317      */
318     s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
319 }
320 
321 static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
322 {
323     /*
324      * We will check for overflow at the end of the opcode loop in
325      * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
326      */
327     s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
328 }
329 
330 static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
331 {
332     /*
333      * Return the read-execute version of the pointer, for the benefit
334      * of any pc-relative addressing mode.
335      */
336     return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
337 }
338 
339 /* Signal overflow, starting over with fewer guest insns. */
340 static G_NORETURN
341 void tcg_raise_tb_overflow(TCGContext *s)
342 {
343     siglongjmp(s->jmp_trans, -2);
344 }
345 
346 #define C_PFX1(P, A)                    P##A
347 #define C_PFX2(P, A, B)                 P##A##_##B
348 #define C_PFX3(P, A, B, C)              P##A##_##B##_##C
349 #define C_PFX4(P, A, B, C, D)           P##A##_##B##_##C##_##D
350 #define C_PFX5(P, A, B, C, D, E)        P##A##_##B##_##C##_##D##_##E
351 #define C_PFX6(P, A, B, C, D, E, F)     P##A##_##B##_##C##_##D##_##E##_##F
352 
353 /* Define an enumeration for the various combinations. */
354 
355 #define C_O0_I1(I1)                     C_PFX1(c_o0_i1_, I1),
356 #define C_O0_I2(I1, I2)                 C_PFX2(c_o0_i2_, I1, I2),
357 #define C_O0_I3(I1, I2, I3)             C_PFX3(c_o0_i3_, I1, I2, I3),
358 #define C_O0_I4(I1, I2, I3, I4)         C_PFX4(c_o0_i4_, I1, I2, I3, I4),
359 
360 #define C_O1_I1(O1, I1)                 C_PFX2(c_o1_i1_, O1, I1),
361 #define C_O1_I2(O1, I1, I2)             C_PFX3(c_o1_i2_, O1, I1, I2),
362 #define C_O1_I3(O1, I1, I2, I3)         C_PFX4(c_o1_i3_, O1, I1, I2, I3),
363 #define C_O1_I4(O1, I1, I2, I3, I4)     C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
364 
365 #define C_N1_I2(O1, I1, I2)             C_PFX3(c_n1_i2_, O1, I1, I2),
366 
367 #define C_O2_I1(O1, O2, I1)             C_PFX3(c_o2_i1_, O1, O2, I1),
368 #define C_O2_I2(O1, O2, I1, I2)         C_PFX4(c_o2_i2_, O1, O2, I1, I2),
369 #define C_O2_I3(O1, O2, I1, I2, I3)     C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
370 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
371 
372 typedef enum {
373 #include "tcg-target-con-set.h"
374 } TCGConstraintSetIndex;
375 
376 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
377 
378 #undef C_O0_I1
379 #undef C_O0_I2
380 #undef C_O0_I3
381 #undef C_O0_I4
382 #undef C_O1_I1
383 #undef C_O1_I2
384 #undef C_O1_I3
385 #undef C_O1_I4
386 #undef C_N1_I2
387 #undef C_O2_I1
388 #undef C_O2_I2
389 #undef C_O2_I3
390 #undef C_O2_I4
391 
392 /* Put all of the constraint sets into an array, indexed by the enum. */
393 
394 #define C_O0_I1(I1)                     { .args_ct_str = { #I1 } },
395 #define C_O0_I2(I1, I2)                 { .args_ct_str = { #I1, #I2 } },
396 #define C_O0_I3(I1, I2, I3)             { .args_ct_str = { #I1, #I2, #I3 } },
397 #define C_O0_I4(I1, I2, I3, I4)         { .args_ct_str = { #I1, #I2, #I3, #I4 } },
398 
399 #define C_O1_I1(O1, I1)                 { .args_ct_str = { #O1, #I1 } },
400 #define C_O1_I2(O1, I1, I2)             { .args_ct_str = { #O1, #I1, #I2 } },
401 #define C_O1_I3(O1, I1, I2, I3)         { .args_ct_str = { #O1, #I1, #I2, #I3 } },
402 #define C_O1_I4(O1, I1, I2, I3, I4)     { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
403 
404 #define C_N1_I2(O1, I1, I2)             { .args_ct_str = { "&" #O1, #I1, #I2 } },
405 
406 #define C_O2_I1(O1, O2, I1)             { .args_ct_str = { #O1, #O2, #I1 } },
407 #define C_O2_I2(O1, O2, I1, I2)         { .args_ct_str = { #O1, #O2, #I1, #I2 } },
408 #define C_O2_I3(O1, O2, I1, I2, I3)     { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
409 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
410 
411 static const TCGTargetOpDef constraint_sets[] = {
412 #include "tcg-target-con-set.h"
413 };
414 
415 
416 #undef C_O0_I1
417 #undef C_O0_I2
418 #undef C_O0_I3
419 #undef C_O0_I4
420 #undef C_O1_I1
421 #undef C_O1_I2
422 #undef C_O1_I3
423 #undef C_O1_I4
424 #undef C_N1_I2
425 #undef C_O2_I1
426 #undef C_O2_I2
427 #undef C_O2_I3
428 #undef C_O2_I4
429 
430 /* Expand the enumerator to be returned from tcg_target_op_def(). */
431 
432 #define C_O0_I1(I1)                     C_PFX1(c_o0_i1_, I1)
433 #define C_O0_I2(I1, I2)                 C_PFX2(c_o0_i2_, I1, I2)
434 #define C_O0_I3(I1, I2, I3)             C_PFX3(c_o0_i3_, I1, I2, I3)
435 #define C_O0_I4(I1, I2, I3, I4)         C_PFX4(c_o0_i4_, I1, I2, I3, I4)
436 
437 #define C_O1_I1(O1, I1)                 C_PFX2(c_o1_i1_, O1, I1)
438 #define C_O1_I2(O1, I1, I2)             C_PFX3(c_o1_i2_, O1, I1, I2)
439 #define C_O1_I3(O1, I1, I2, I3)         C_PFX4(c_o1_i3_, O1, I1, I2, I3)
440 #define C_O1_I4(O1, I1, I2, I3, I4)     C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
441 
442 #define C_N1_I2(O1, I1, I2)             C_PFX3(c_n1_i2_, O1, I1, I2)
443 
444 #define C_O2_I1(O1, O2, I1)             C_PFX3(c_o2_i1_, O1, O2, I1)
445 #define C_O2_I2(O1, O2, I1, I2)         C_PFX4(c_o2_i2_, O1, O2, I1, I2)
446 #define C_O2_I3(O1, O2, I1, I2, I3)     C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
447 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
448 
449 #include "tcg-target.c.inc"
450 
451 static void alloc_tcg_plugin_context(TCGContext *s)
452 {
453 #ifdef CONFIG_PLUGIN
454     s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
455     s->plugin_tb->insns =
456         g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
457 #endif
458 }
459 
460 /*
461  * All TCG threads except the parent (i.e. the one that called tcg_context_init
462  * and registered the target's TCG globals) must register with this function
463  * before initiating translation.
464  *
465  * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
466  * of tcg_region_init() for the reasoning behind this.
467  *
468  * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
469  * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
470  * is not used anymore for translation once this function is called.
471  *
472  * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
473  * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
474  */
475 #ifdef CONFIG_USER_ONLY
476 void tcg_register_thread(void)
477 {
478     tcg_ctx = &tcg_init_ctx;
479 }
480 #else
481 void tcg_register_thread(void)
482 {
483     TCGContext *s = g_malloc(sizeof(*s));
484     unsigned int i, n;
485 
486     *s = tcg_init_ctx;
487 
488     /* Relink mem_base.  */
489     for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
490         if (tcg_init_ctx.temps[i].mem_base) {
491             ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
492             tcg_debug_assert(b >= 0 && b < n);
493             s->temps[i].mem_base = &s->temps[b];
494         }
495     }
496 
497     /* Claim an entry in tcg_ctxs */
498     n = qatomic_fetch_inc(&tcg_cur_ctxs);
499     g_assert(n < tcg_max_ctxs);
500     qatomic_set(&tcg_ctxs[n], s);
501 
502     if (n > 0) {
503         alloc_tcg_plugin_context(s);
504         tcg_region_initial_alloc(s);
505     }
506 
507     tcg_ctx = s;
508 }
509 #endif /* !CONFIG_USER_ONLY */
510 
511 /* pool based memory allocation */
512 void *tcg_malloc_internal(TCGContext *s, int size)
513 {
514     TCGPool *p;
515     int pool_size;
516 
517     if (size > TCG_POOL_CHUNK_SIZE) {
518         /* big malloc: insert a new pool (XXX: could optimize) */
519         p = g_malloc(sizeof(TCGPool) + size);
520         p->size = size;
521         p->next = s->pool_first_large;
522         s->pool_first_large = p;
523         return p->data;
524     } else {
525         p = s->pool_current;
526         if (!p) {
527             p = s->pool_first;
528             if (!p)
529                 goto new_pool;
530         } else {
531             if (!p->next) {
532             new_pool:
533                 pool_size = TCG_POOL_CHUNK_SIZE;
534                 p = g_malloc(sizeof(TCGPool) + pool_size);
535                 p->size = pool_size;
536                 p->next = NULL;
537                 if (s->pool_current) {
538                     s->pool_current->next = p;
539                 } else {
540                     s->pool_first = p;
541                 }
542             } else {
543                 p = p->next;
544             }
545         }
546     }
547     s->pool_current = p;
548     s->pool_cur = p->data + size;
549     s->pool_end = p->data + p->size;
550     return p->data;
551 }
552 
553 void tcg_pool_reset(TCGContext *s)
554 {
555     TCGPool *p, *t;
556     for (p = s->pool_first_large; p; p = t) {
557         t = p->next;
558         g_free(p);
559     }
560     s->pool_first_large = NULL;
561     s->pool_cur = s->pool_end = NULL;
562     s->pool_current = NULL;
563 }
564 
565 #include "exec/helper-proto.h"
566 
567 static TCGHelperInfo all_helpers[] = {
568 #include "exec/helper-tcg.h"
569 };
570 static GHashTable *helper_table;
571 
572 #ifdef CONFIG_TCG_INTERPRETER
573 static ffi_type *typecode_to_ffi(int argmask)
574 {
575     /*
576      * libffi does not support __int128_t, so we have forced Int128
577      * to use the structure definition instead of the builtin type.
578      */
579     static ffi_type *ffi_type_i128_elements[3] = {
580         &ffi_type_uint64,
581         &ffi_type_uint64,
582         NULL
583     };
584     static ffi_type ffi_type_i128 = {
585         .size = 16,
586         .alignment = __alignof__(Int128),
587         .type = FFI_TYPE_STRUCT,
588         .elements = ffi_type_i128_elements,
589     };
590 
591     switch (argmask) {
592     case dh_typecode_void:
593         return &ffi_type_void;
594     case dh_typecode_i32:
595         return &ffi_type_uint32;
596     case dh_typecode_s32:
597         return &ffi_type_sint32;
598     case dh_typecode_i64:
599         return &ffi_type_uint64;
600     case dh_typecode_s64:
601         return &ffi_type_sint64;
602     case dh_typecode_ptr:
603         return &ffi_type_pointer;
604     case dh_typecode_i128:
605         return &ffi_type_i128;
606     }
607     g_assert_not_reached();
608 }
609 
610 static void init_ffi_layouts(void)
611 {
612     /* g_direct_hash/equal for direct comparisons on uint32_t.  */
613     GHashTable *ffi_table = g_hash_table_new(NULL, NULL);
614 
615     for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
616         TCGHelperInfo *info = &all_helpers[i];
617         unsigned typemask = info->typemask;
618         gpointer hash = (gpointer)(uintptr_t)typemask;
619         struct {
620             ffi_cif cif;
621             ffi_type *args[];
622         } *ca;
623         ffi_status status;
624         int nargs;
625         ffi_cif *cif;
626 
627         cif = g_hash_table_lookup(ffi_table, hash);
628         if (cif) {
629             info->cif = cif;
630             continue;
631         }
632 
633         /* Ignoring the return type, find the last non-zero field. */
634         nargs = 32 - clz32(typemask >> 3);
635         nargs = DIV_ROUND_UP(nargs, 3);
636         assert(nargs <= MAX_CALL_IARGS);
637 
638         ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
639         ca->cif.rtype = typecode_to_ffi(typemask & 7);
640         ca->cif.nargs = nargs;
641 
642         if (nargs != 0) {
643             ca->cif.arg_types = ca->args;
644             for (int j = 0; j < nargs; ++j) {
645                 int typecode = extract32(typemask, (j + 1) * 3, 3);
646                 ca->args[j] = typecode_to_ffi(typecode);
647             }
648         }
649 
650         status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
651                               ca->cif.rtype, ca->cif.arg_types);
652         assert(status == FFI_OK);
653 
654         cif = &ca->cif;
655         info->cif = cif;
656         g_hash_table_insert(ffi_table, hash, (gpointer)cif);
657     }
658 
659     g_hash_table_destroy(ffi_table);
660 }
661 #endif /* CONFIG_TCG_INTERPRETER */
662 
663 typedef struct TCGCumulativeArgs {
664     int arg_idx;                /* tcg_gen_callN args[] */
665     int info_in_idx;            /* TCGHelperInfo in[] */
666     int arg_slot;               /* regs+stack slot */
667     int ref_slot;               /* stack slots for references */
668 } TCGCumulativeArgs;
669 
670 static void layout_arg_even(TCGCumulativeArgs *cum)
671 {
672     cum->arg_slot += cum->arg_slot & 1;
673 }
674 
675 static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
676                          TCGCallArgumentKind kind)
677 {
678     TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
679 
680     *loc = (TCGCallArgumentLoc){
681         .kind = kind,
682         .arg_idx = cum->arg_idx,
683         .arg_slot = cum->arg_slot,
684     };
685     cum->info_in_idx++;
686     cum->arg_slot++;
687 }
688 
689 static void layout_arg_normal_n(TCGCumulativeArgs *cum,
690                                 TCGHelperInfo *info, int n)
691 {
692     TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
693 
694     for (int i = 0; i < n; ++i) {
695         /* Layout all using the same arg_idx, adjusting the subindex. */
696         loc[i] = (TCGCallArgumentLoc){
697             .kind = TCG_CALL_ARG_NORMAL,
698             .arg_idx = cum->arg_idx,
699             .tmp_subindex = i,
700             .arg_slot = cum->arg_slot + i,
701         };
702     }
703     cum->info_in_idx += n;
704     cum->arg_slot += n;
705 }
706 
707 static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
708 {
709     TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
710     int n = 128 / TCG_TARGET_REG_BITS;
711 
712     /* The first subindex carries the pointer. */
713     layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
714 
715     /*
716      * The callee is allowed to clobber memory associated with
717      * structure pass by-reference.  Therefore we must make copies.
718      * Allocate space from "ref_slot", which will be adjusted to
719      * follow the parameters on the stack.
720      */
721     loc[0].ref_slot = cum->ref_slot;
722 
723     /*
724      * Subsequent words also go into the reference slot, but
725      * do not accumulate into the regular arguments.
726      */
727     for (int i = 1; i < n; ++i) {
728         loc[i] = (TCGCallArgumentLoc){
729             .kind = TCG_CALL_ARG_BY_REF_N,
730             .arg_idx = cum->arg_idx,
731             .tmp_subindex = i,
732             .ref_slot = cum->ref_slot + i,
733         };
734     }
735     cum->info_in_idx += n;
736     cum->ref_slot += n;
737 }
738 
739 static void init_call_layout(TCGHelperInfo *info)
740 {
741     int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
742     int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
743     unsigned typemask = info->typemask;
744     unsigned typecode;
745     TCGCumulativeArgs cum = { };
746 
747     /*
748      * Parse and place any function return value.
749      */
750     typecode = typemask & 7;
751     switch (typecode) {
752     case dh_typecode_void:
753         info->nr_out = 0;
754         break;
755     case dh_typecode_i32:
756     case dh_typecode_s32:
757     case dh_typecode_ptr:
758         info->nr_out = 1;
759         info->out_kind = TCG_CALL_RET_NORMAL;
760         break;
761     case dh_typecode_i64:
762     case dh_typecode_s64:
763         info->nr_out = 64 / TCG_TARGET_REG_BITS;
764         info->out_kind = TCG_CALL_RET_NORMAL;
765         /* Query the last register now to trigger any assert early. */
766         tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
767         break;
768     case dh_typecode_i128:
769         info->nr_out = 128 / TCG_TARGET_REG_BITS;
770         info->out_kind = TCG_TARGET_CALL_RET_I128;
771         switch (TCG_TARGET_CALL_RET_I128) {
772         case TCG_CALL_RET_NORMAL:
773             /* Query the last register now to trigger any assert early. */
774             tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
775             break;
776         case TCG_CALL_RET_BY_VEC:
777             /* Query the single register now to trigger any assert early. */
778             tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
779             break;
780         case TCG_CALL_RET_BY_REF:
781             /*
782              * Allocate the first argument to the output.
783              * We don't need to store this anywhere, just make it
784              * unavailable for use in the input loop below.
785              */
786             cum.arg_slot = 1;
787             break;
788         default:
789             qemu_build_not_reached();
790         }
791         break;
792     default:
793         g_assert_not_reached();
794     }
795 
796     /*
797      * Parse and place function arguments.
798      */
799     for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
800         TCGCallArgumentKind kind;
801         TCGType type;
802 
803         typecode = typemask & 7;
804         switch (typecode) {
805         case dh_typecode_i32:
806         case dh_typecode_s32:
807             type = TCG_TYPE_I32;
808             break;
809         case dh_typecode_i64:
810         case dh_typecode_s64:
811             type = TCG_TYPE_I64;
812             break;
813         case dh_typecode_ptr:
814             type = TCG_TYPE_PTR;
815             break;
816         case dh_typecode_i128:
817             type = TCG_TYPE_I128;
818             break;
819         default:
820             g_assert_not_reached();
821         }
822 
823         switch (type) {
824         case TCG_TYPE_I32:
825             switch (TCG_TARGET_CALL_ARG_I32) {
826             case TCG_CALL_ARG_EVEN:
827                 layout_arg_even(&cum);
828                 /* fall through */
829             case TCG_CALL_ARG_NORMAL:
830                 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
831                 break;
832             case TCG_CALL_ARG_EXTEND:
833                 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
834                 layout_arg_1(&cum, info, kind);
835                 break;
836             default:
837                 qemu_build_not_reached();
838             }
839             break;
840 
841         case TCG_TYPE_I64:
842             switch (TCG_TARGET_CALL_ARG_I64) {
843             case TCG_CALL_ARG_EVEN:
844                 layout_arg_even(&cum);
845                 /* fall through */
846             case TCG_CALL_ARG_NORMAL:
847                 if (TCG_TARGET_REG_BITS == 32) {
848                     layout_arg_normal_n(&cum, info, 2);
849                 } else {
850                     layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
851                 }
852                 break;
853             default:
854                 qemu_build_not_reached();
855             }
856             break;
857 
858         case TCG_TYPE_I128:
859             switch (TCG_TARGET_CALL_ARG_I128) {
860             case TCG_CALL_ARG_EVEN:
861                 layout_arg_even(&cum);
862                 /* fall through */
863             case TCG_CALL_ARG_NORMAL:
864                 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
865                 break;
866             case TCG_CALL_ARG_BY_REF:
867                 layout_arg_by_ref(&cum, info);
868                 break;
869             default:
870                 qemu_build_not_reached();
871             }
872             break;
873 
874         default:
875             g_assert_not_reached();
876         }
877     }
878     info->nr_in = cum.info_in_idx;
879 
880     /* Validate that we didn't overrun the input array. */
881     assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
882     /* Validate the backend has enough argument space. */
883     assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
884 
885     /*
886      * Relocate the "ref_slot" area to the end of the parameters.
887      * Minimizing this stack offset helps code size for x86,
888      * which has a signed 8-bit offset encoding.
889      */
890     if (cum.ref_slot != 0) {
891         int ref_base = 0;
892 
893         if (cum.arg_slot > max_reg_slots) {
894             int align = __alignof(Int128) / sizeof(tcg_target_long);
895 
896             ref_base = cum.arg_slot - max_reg_slots;
897             if (align > 1) {
898                 ref_base = ROUND_UP(ref_base, align);
899             }
900         }
901         assert(ref_base + cum.ref_slot <= max_stk_slots);
902 
903         if (ref_base != 0) {
904             for (int i = cum.info_in_idx - 1; i >= 0; --i) {
905                 TCGCallArgumentLoc *loc = &info->in[i];
906                 switch (loc->kind) {
907                 case TCG_CALL_ARG_BY_REF:
908                 case TCG_CALL_ARG_BY_REF_N:
909                     loc->ref_slot += ref_base;
910                     break;
911                 default:
912                     break;
913                 }
914             }
915         }
916     }
917 }
918 
919 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
920 static void process_op_defs(TCGContext *s);
921 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
922                                             TCGReg reg, const char *name);
923 
924 static void tcg_context_init(unsigned max_cpus)
925 {
926     TCGContext *s = &tcg_init_ctx;
927     int op, total_args, n, i;
928     TCGOpDef *def;
929     TCGArgConstraint *args_ct;
930     TCGTemp *ts;
931 
932     memset(s, 0, sizeof(*s));
933     s->nb_globals = 0;
934 
935     /* Count total number of arguments and allocate the corresponding
936        space */
937     total_args = 0;
938     for(op = 0; op < NB_OPS; op++) {
939         def = &tcg_op_defs[op];
940         n = def->nb_iargs + def->nb_oargs;
941         total_args += n;
942     }
943 
944     args_ct = g_new0(TCGArgConstraint, total_args);
945 
946     for(op = 0; op < NB_OPS; op++) {
947         def = &tcg_op_defs[op];
948         def->args_ct = args_ct;
949         n = def->nb_iargs + def->nb_oargs;
950         args_ct += n;
951     }
952 
953     /* Register helpers.  */
954     /* Use g_direct_hash/equal for direct pointer comparisons on func.  */
955     helper_table = g_hash_table_new(NULL, NULL);
956 
957     for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
958         init_call_layout(&all_helpers[i]);
959         g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
960                             (gpointer)&all_helpers[i]);
961     }
962 
963 #ifdef CONFIG_TCG_INTERPRETER
964     init_ffi_layouts();
965 #endif
966 
967     tcg_target_init(s);
968     process_op_defs(s);
969 
970     /* Reverse the order of the saved registers, assuming they're all at
971        the start of tcg_target_reg_alloc_order.  */
972     for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
973         int r = tcg_target_reg_alloc_order[n];
974         if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
975             break;
976         }
977     }
978     for (i = 0; i < n; ++i) {
979         indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
980     }
981     for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
982         indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
983     }
984 
985     alloc_tcg_plugin_context(s);
986 
987     tcg_ctx = s;
988     /*
989      * In user-mode we simply share the init context among threads, since we
990      * use a single region. See the documentation tcg_region_init() for the
991      * reasoning behind this.
992      * In softmmu we will have at most max_cpus TCG threads.
993      */
994 #ifdef CONFIG_USER_ONLY
995     tcg_ctxs = &tcg_ctx;
996     tcg_cur_ctxs = 1;
997     tcg_max_ctxs = 1;
998 #else
999     tcg_max_ctxs = max_cpus;
1000     tcg_ctxs = g_new0(TCGContext *, max_cpus);
1001 #endif
1002 
1003     tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1004     ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1005     cpu_env = temp_tcgv_ptr(ts);
1006 }
1007 
1008 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
1009 {
1010     tcg_context_init(max_cpus);
1011     tcg_region_init(tb_size, splitwx, max_cpus);
1012 }
1013 
1014 /*
1015  * Allocate TBs right before their corresponding translated code, making
1016  * sure that TBs and code are on different cache lines.
1017  */
1018 TranslationBlock *tcg_tb_alloc(TCGContext *s)
1019 {
1020     uintptr_t align = qemu_icache_linesize;
1021     TranslationBlock *tb;
1022     void *next;
1023 
1024  retry:
1025     tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1026     next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1027 
1028     if (unlikely(next > s->code_gen_highwater)) {
1029         if (tcg_region_alloc(s)) {
1030             return NULL;
1031         }
1032         goto retry;
1033     }
1034     qatomic_set(&s->code_gen_ptr, next);
1035     s->data_gen_ptr = NULL;
1036     return tb;
1037 }
1038 
1039 void tcg_prologue_init(TCGContext *s)
1040 {
1041     size_t prologue_size;
1042 
1043     s->code_ptr = s->code_gen_ptr;
1044     s->code_buf = s->code_gen_ptr;
1045     s->data_gen_ptr = NULL;
1046 
1047 #ifndef CONFIG_TCG_INTERPRETER
1048     tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
1049 #endif
1050 
1051 #ifdef TCG_TARGET_NEED_POOL_LABELS
1052     s->pool_labels = NULL;
1053 #endif
1054 
1055     qemu_thread_jit_write();
1056     /* Generate the prologue.  */
1057     tcg_target_qemu_prologue(s);
1058 
1059 #ifdef TCG_TARGET_NEED_POOL_LABELS
1060     /* Allow the prologue to put e.g. guest_base into a pool entry.  */
1061     {
1062         int result = tcg_out_pool_finalize(s);
1063         tcg_debug_assert(result == 0);
1064     }
1065 #endif
1066 
1067     prologue_size = tcg_current_code_size(s);
1068     perf_report_prologue(s->code_gen_ptr, prologue_size);
1069 
1070 #ifndef CONFIG_TCG_INTERPRETER
1071     flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1072                         (uintptr_t)s->code_buf, prologue_size);
1073 #endif
1074 
1075 #ifdef DEBUG_DISAS
1076     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
1077         FILE *logfile = qemu_log_trylock();
1078         if (logfile) {
1079             fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1080             if (s->data_gen_ptr) {
1081                 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1082                 size_t data_size = prologue_size - code_size;
1083                 size_t i;
1084 
1085                 disas(logfile, s->code_gen_ptr, code_size);
1086 
1087                 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1088                     if (sizeof(tcg_target_ulong) == 8) {
1089                         fprintf(logfile,
1090                                 "0x%08" PRIxPTR ":  .quad  0x%016" PRIx64 "\n",
1091                                 (uintptr_t)s->data_gen_ptr + i,
1092                                 *(uint64_t *)(s->data_gen_ptr + i));
1093                     } else {
1094                         fprintf(logfile,
1095                                 "0x%08" PRIxPTR ":  .long  0x%08x\n",
1096                                 (uintptr_t)s->data_gen_ptr + i,
1097                                 *(uint32_t *)(s->data_gen_ptr + i));
1098                     }
1099                 }
1100             } else {
1101                 disas(logfile, s->code_gen_ptr, prologue_size);
1102             }
1103             fprintf(logfile, "\n");
1104             qemu_log_unlock(logfile);
1105         }
1106     }
1107 #endif
1108 
1109 #ifndef CONFIG_TCG_INTERPRETER
1110     /*
1111      * Assert that goto_ptr is implemented completely, setting an epilogue.
1112      * For tci, we use NULL as the signal to return from the interpreter,
1113      * so skip this check.
1114      */
1115     tcg_debug_assert(tcg_code_gen_epilogue != NULL);
1116 #endif
1117 
1118     tcg_region_prologue_set(s);
1119 }
1120 
1121 void tcg_func_start(TCGContext *s)
1122 {
1123     tcg_pool_reset(s);
1124     s->nb_temps = s->nb_globals;
1125 
1126     /* No temps have been previously allocated for size or locality.  */
1127     memset(s->free_temps, 0, sizeof(s->free_temps));
1128 
1129     /* No constant temps have been previously allocated. */
1130     for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1131         if (s->const_table[i]) {
1132             g_hash_table_remove_all(s->const_table[i]);
1133         }
1134     }
1135 
1136     s->nb_ops = 0;
1137     s->nb_labels = 0;
1138     s->current_frame_offset = s->frame_start;
1139 
1140 #ifdef CONFIG_DEBUG_TCG
1141     s->goto_tb_issue_mask = 0;
1142 #endif
1143 
1144     QTAILQ_INIT(&s->ops);
1145     QTAILQ_INIT(&s->free_ops);
1146     QSIMPLEQ_INIT(&s->labels);
1147 }
1148 
1149 static TCGTemp *tcg_temp_alloc(TCGContext *s)
1150 {
1151     int n = s->nb_temps++;
1152 
1153     if (n >= TCG_MAX_TEMPS) {
1154         tcg_raise_tb_overflow(s);
1155     }
1156     return memset(&s->temps[n], 0, sizeof(TCGTemp));
1157 }
1158 
1159 static TCGTemp *tcg_global_alloc(TCGContext *s)
1160 {
1161     TCGTemp *ts;
1162 
1163     tcg_debug_assert(s->nb_globals == s->nb_temps);
1164     tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
1165     s->nb_globals++;
1166     ts = tcg_temp_alloc(s);
1167     ts->kind = TEMP_GLOBAL;
1168 
1169     return ts;
1170 }
1171 
1172 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1173                                             TCGReg reg, const char *name)
1174 {
1175     TCGTemp *ts;
1176 
1177     if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
1178         tcg_abort();
1179     }
1180 
1181     ts = tcg_global_alloc(s);
1182     ts->base_type = type;
1183     ts->type = type;
1184     ts->kind = TEMP_FIXED;
1185     ts->reg = reg;
1186     ts->name = name;
1187     tcg_regset_set_reg(s->reserved_regs, reg);
1188 
1189     return ts;
1190 }
1191 
1192 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
1193 {
1194     s->frame_start = start;
1195     s->frame_end = start + size;
1196     s->frame_temp
1197         = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
1198 }
1199 
1200 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1201                                      intptr_t offset, const char *name)
1202 {
1203     TCGContext *s = tcg_ctx;
1204     TCGTemp *base_ts = tcgv_ptr_temp(base);
1205     TCGTemp *ts = tcg_global_alloc(s);
1206     int indirect_reg = 0;
1207 
1208     switch (base_ts->kind) {
1209     case TEMP_FIXED:
1210         break;
1211     case TEMP_GLOBAL:
1212         /* We do not support double-indirect registers.  */
1213         tcg_debug_assert(!base_ts->indirect_reg);
1214         base_ts->indirect_base = 1;
1215         s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1216                             ? 2 : 1);
1217         indirect_reg = 1;
1218         break;
1219     default:
1220         g_assert_not_reached();
1221     }
1222 
1223     if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1224         TCGTemp *ts2 = tcg_global_alloc(s);
1225         char buf[64];
1226 
1227         ts->base_type = TCG_TYPE_I64;
1228         ts->type = TCG_TYPE_I32;
1229         ts->indirect_reg = indirect_reg;
1230         ts->mem_allocated = 1;
1231         ts->mem_base = base_ts;
1232         ts->mem_offset = offset;
1233         pstrcpy(buf, sizeof(buf), name);
1234         pstrcat(buf, sizeof(buf), "_0");
1235         ts->name = strdup(buf);
1236 
1237         tcg_debug_assert(ts2 == ts + 1);
1238         ts2->base_type = TCG_TYPE_I64;
1239         ts2->type = TCG_TYPE_I32;
1240         ts2->indirect_reg = indirect_reg;
1241         ts2->mem_allocated = 1;
1242         ts2->mem_base = base_ts;
1243         ts2->mem_offset = offset + 4;
1244         ts2->temp_subindex = 1;
1245         pstrcpy(buf, sizeof(buf), name);
1246         pstrcat(buf, sizeof(buf), "_1");
1247         ts2->name = strdup(buf);
1248     } else {
1249         ts->base_type = type;
1250         ts->type = type;
1251         ts->indirect_reg = indirect_reg;
1252         ts->mem_allocated = 1;
1253         ts->mem_base = base_ts;
1254         ts->mem_offset = offset;
1255         ts->name = name;
1256     }
1257     return ts;
1258 }
1259 
1260 TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
1261 {
1262     TCGContext *s = tcg_ctx;
1263     TCGTemp *ts;
1264     int n;
1265 
1266     if (kind == TEMP_EBB) {
1267         int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
1268 
1269         if (idx < TCG_MAX_TEMPS) {
1270             /* There is already an available temp with the right type.  */
1271             clear_bit(idx, s->free_temps[type].l);
1272 
1273             ts = &s->temps[idx];
1274             ts->temp_allocated = 1;
1275             tcg_debug_assert(ts->base_type == type);
1276             tcg_debug_assert(ts->kind == kind);
1277             return ts;
1278         }
1279     } else {
1280         tcg_debug_assert(kind == TEMP_TB);
1281     }
1282 
1283     switch (type) {
1284     case TCG_TYPE_I32:
1285     case TCG_TYPE_V64:
1286     case TCG_TYPE_V128:
1287     case TCG_TYPE_V256:
1288         n = 1;
1289         break;
1290     case TCG_TYPE_I64:
1291         n = 64 / TCG_TARGET_REG_BITS;
1292         break;
1293     case TCG_TYPE_I128:
1294         n = 128 / TCG_TARGET_REG_BITS;
1295         break;
1296     default:
1297         g_assert_not_reached();
1298     }
1299 
1300     ts = tcg_temp_alloc(s);
1301     ts->base_type = type;
1302     ts->temp_allocated = 1;
1303     ts->kind = kind;
1304 
1305     if (n == 1) {
1306         ts->type = type;
1307     } else {
1308         ts->type = TCG_TYPE_REG;
1309 
1310         for (int i = 1; i < n; ++i) {
1311             TCGTemp *ts2 = tcg_temp_alloc(s);
1312 
1313             tcg_debug_assert(ts2 == ts + i);
1314             ts2->base_type = type;
1315             ts2->type = TCG_TYPE_REG;
1316             ts2->temp_allocated = 1;
1317             ts2->temp_subindex = i;
1318             ts2->kind = kind;
1319         }
1320     }
1321     return ts;
1322 }
1323 
1324 TCGv_vec tcg_temp_new_vec(TCGType type)
1325 {
1326     TCGTemp *t;
1327 
1328 #ifdef CONFIG_DEBUG_TCG
1329     switch (type) {
1330     case TCG_TYPE_V64:
1331         assert(TCG_TARGET_HAS_v64);
1332         break;
1333     case TCG_TYPE_V128:
1334         assert(TCG_TARGET_HAS_v128);
1335         break;
1336     case TCG_TYPE_V256:
1337         assert(TCG_TARGET_HAS_v256);
1338         break;
1339     default:
1340         g_assert_not_reached();
1341     }
1342 #endif
1343 
1344     t = tcg_temp_new_internal(type, TEMP_EBB);
1345     return temp_tcgv_vec(t);
1346 }
1347 
1348 /* Create a new temp of the same type as an existing temp.  */
1349 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1350 {
1351     TCGTemp *t = tcgv_vec_temp(match);
1352 
1353     tcg_debug_assert(t->temp_allocated != 0);
1354 
1355     t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
1356     return temp_tcgv_vec(t);
1357 }
1358 
1359 void tcg_temp_free_internal(TCGTemp *ts)
1360 {
1361     TCGContext *s = tcg_ctx;
1362 
1363     switch (ts->kind) {
1364     case TEMP_CONST:
1365     case TEMP_TB:
1366         /* Silently ignore free. */
1367         break;
1368     case TEMP_EBB:
1369         tcg_debug_assert(ts->temp_allocated != 0);
1370         ts->temp_allocated = 0;
1371         set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
1372         break;
1373     default:
1374         /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
1375         g_assert_not_reached();
1376     }
1377 }
1378 
1379 TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1380 {
1381     TCGContext *s = tcg_ctx;
1382     GHashTable *h = s->const_table[type];
1383     TCGTemp *ts;
1384 
1385     if (h == NULL) {
1386         h = g_hash_table_new(g_int64_hash, g_int64_equal);
1387         s->const_table[type] = h;
1388     }
1389 
1390     ts = g_hash_table_lookup(h, &val);
1391     if (ts == NULL) {
1392         int64_t *val_ptr;
1393 
1394         ts = tcg_temp_alloc(s);
1395 
1396         if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1397             TCGTemp *ts2 = tcg_temp_alloc(s);
1398 
1399             tcg_debug_assert(ts2 == ts + 1);
1400 
1401             ts->base_type = TCG_TYPE_I64;
1402             ts->type = TCG_TYPE_I32;
1403             ts->kind = TEMP_CONST;
1404             ts->temp_allocated = 1;
1405 
1406             ts2->base_type = TCG_TYPE_I64;
1407             ts2->type = TCG_TYPE_I32;
1408             ts2->kind = TEMP_CONST;
1409             ts2->temp_allocated = 1;
1410             ts2->temp_subindex = 1;
1411 
1412             /*
1413              * Retain the full value of the 64-bit constant in the low
1414              * part, so that the hash table works.  Actual uses will
1415              * truncate the value to the low part.
1416              */
1417             ts[HOST_BIG_ENDIAN].val = val;
1418             ts[!HOST_BIG_ENDIAN].val = val >> 32;
1419             val_ptr = &ts[HOST_BIG_ENDIAN].val;
1420         } else {
1421             ts->base_type = type;
1422             ts->type = type;
1423             ts->kind = TEMP_CONST;
1424             ts->temp_allocated = 1;
1425             ts->val = val;
1426             val_ptr = &ts->val;
1427         }
1428         g_hash_table_insert(h, val_ptr, ts);
1429     }
1430 
1431     return ts;
1432 }
1433 
1434 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1435 {
1436     val = dup_const(vece, val);
1437     return temp_tcgv_vec(tcg_constant_internal(type, val));
1438 }
1439 
1440 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1441 {
1442     TCGTemp *t = tcgv_vec_temp(match);
1443 
1444     tcg_debug_assert(t->temp_allocated != 0);
1445     return tcg_constant_vec(t->base_type, vece, val);
1446 }
1447 
1448 /* Return true if OP may appear in the opcode stream.
1449    Test the runtime variable that controls each opcode.  */
1450 bool tcg_op_supported(TCGOpcode op)
1451 {
1452     const bool have_vec
1453         = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1454 
1455     switch (op) {
1456     case INDEX_op_discard:
1457     case INDEX_op_set_label:
1458     case INDEX_op_call:
1459     case INDEX_op_br:
1460     case INDEX_op_mb:
1461     case INDEX_op_insn_start:
1462     case INDEX_op_exit_tb:
1463     case INDEX_op_goto_tb:
1464     case INDEX_op_goto_ptr:
1465     case INDEX_op_qemu_ld_i32:
1466     case INDEX_op_qemu_st_i32:
1467     case INDEX_op_qemu_ld_i64:
1468     case INDEX_op_qemu_st_i64:
1469         return true;
1470 
1471     case INDEX_op_qemu_st8_i32:
1472         return TCG_TARGET_HAS_qemu_st8_i32;
1473 
1474     case INDEX_op_mov_i32:
1475     case INDEX_op_setcond_i32:
1476     case INDEX_op_brcond_i32:
1477     case INDEX_op_ld8u_i32:
1478     case INDEX_op_ld8s_i32:
1479     case INDEX_op_ld16u_i32:
1480     case INDEX_op_ld16s_i32:
1481     case INDEX_op_ld_i32:
1482     case INDEX_op_st8_i32:
1483     case INDEX_op_st16_i32:
1484     case INDEX_op_st_i32:
1485     case INDEX_op_add_i32:
1486     case INDEX_op_sub_i32:
1487     case INDEX_op_mul_i32:
1488     case INDEX_op_and_i32:
1489     case INDEX_op_or_i32:
1490     case INDEX_op_xor_i32:
1491     case INDEX_op_shl_i32:
1492     case INDEX_op_shr_i32:
1493     case INDEX_op_sar_i32:
1494         return true;
1495 
1496     case INDEX_op_movcond_i32:
1497         return TCG_TARGET_HAS_movcond_i32;
1498     case INDEX_op_div_i32:
1499     case INDEX_op_divu_i32:
1500         return TCG_TARGET_HAS_div_i32;
1501     case INDEX_op_rem_i32:
1502     case INDEX_op_remu_i32:
1503         return TCG_TARGET_HAS_rem_i32;
1504     case INDEX_op_div2_i32:
1505     case INDEX_op_divu2_i32:
1506         return TCG_TARGET_HAS_div2_i32;
1507     case INDEX_op_rotl_i32:
1508     case INDEX_op_rotr_i32:
1509         return TCG_TARGET_HAS_rot_i32;
1510     case INDEX_op_deposit_i32:
1511         return TCG_TARGET_HAS_deposit_i32;
1512     case INDEX_op_extract_i32:
1513         return TCG_TARGET_HAS_extract_i32;
1514     case INDEX_op_sextract_i32:
1515         return TCG_TARGET_HAS_sextract_i32;
1516     case INDEX_op_extract2_i32:
1517         return TCG_TARGET_HAS_extract2_i32;
1518     case INDEX_op_add2_i32:
1519         return TCG_TARGET_HAS_add2_i32;
1520     case INDEX_op_sub2_i32:
1521         return TCG_TARGET_HAS_sub2_i32;
1522     case INDEX_op_mulu2_i32:
1523         return TCG_TARGET_HAS_mulu2_i32;
1524     case INDEX_op_muls2_i32:
1525         return TCG_TARGET_HAS_muls2_i32;
1526     case INDEX_op_muluh_i32:
1527         return TCG_TARGET_HAS_muluh_i32;
1528     case INDEX_op_mulsh_i32:
1529         return TCG_TARGET_HAS_mulsh_i32;
1530     case INDEX_op_ext8s_i32:
1531         return TCG_TARGET_HAS_ext8s_i32;
1532     case INDEX_op_ext16s_i32:
1533         return TCG_TARGET_HAS_ext16s_i32;
1534     case INDEX_op_ext8u_i32:
1535         return TCG_TARGET_HAS_ext8u_i32;
1536     case INDEX_op_ext16u_i32:
1537         return TCG_TARGET_HAS_ext16u_i32;
1538     case INDEX_op_bswap16_i32:
1539         return TCG_TARGET_HAS_bswap16_i32;
1540     case INDEX_op_bswap32_i32:
1541         return TCG_TARGET_HAS_bswap32_i32;
1542     case INDEX_op_not_i32:
1543         return TCG_TARGET_HAS_not_i32;
1544     case INDEX_op_neg_i32:
1545         return TCG_TARGET_HAS_neg_i32;
1546     case INDEX_op_andc_i32:
1547         return TCG_TARGET_HAS_andc_i32;
1548     case INDEX_op_orc_i32:
1549         return TCG_TARGET_HAS_orc_i32;
1550     case INDEX_op_eqv_i32:
1551         return TCG_TARGET_HAS_eqv_i32;
1552     case INDEX_op_nand_i32:
1553         return TCG_TARGET_HAS_nand_i32;
1554     case INDEX_op_nor_i32:
1555         return TCG_TARGET_HAS_nor_i32;
1556     case INDEX_op_clz_i32:
1557         return TCG_TARGET_HAS_clz_i32;
1558     case INDEX_op_ctz_i32:
1559         return TCG_TARGET_HAS_ctz_i32;
1560     case INDEX_op_ctpop_i32:
1561         return TCG_TARGET_HAS_ctpop_i32;
1562 
1563     case INDEX_op_brcond2_i32:
1564     case INDEX_op_setcond2_i32:
1565         return TCG_TARGET_REG_BITS == 32;
1566 
1567     case INDEX_op_mov_i64:
1568     case INDEX_op_setcond_i64:
1569     case INDEX_op_brcond_i64:
1570     case INDEX_op_ld8u_i64:
1571     case INDEX_op_ld8s_i64:
1572     case INDEX_op_ld16u_i64:
1573     case INDEX_op_ld16s_i64:
1574     case INDEX_op_ld32u_i64:
1575     case INDEX_op_ld32s_i64:
1576     case INDEX_op_ld_i64:
1577     case INDEX_op_st8_i64:
1578     case INDEX_op_st16_i64:
1579     case INDEX_op_st32_i64:
1580     case INDEX_op_st_i64:
1581     case INDEX_op_add_i64:
1582     case INDEX_op_sub_i64:
1583     case INDEX_op_mul_i64:
1584     case INDEX_op_and_i64:
1585     case INDEX_op_or_i64:
1586     case INDEX_op_xor_i64:
1587     case INDEX_op_shl_i64:
1588     case INDEX_op_shr_i64:
1589     case INDEX_op_sar_i64:
1590     case INDEX_op_ext_i32_i64:
1591     case INDEX_op_extu_i32_i64:
1592         return TCG_TARGET_REG_BITS == 64;
1593 
1594     case INDEX_op_movcond_i64:
1595         return TCG_TARGET_HAS_movcond_i64;
1596     case INDEX_op_div_i64:
1597     case INDEX_op_divu_i64:
1598         return TCG_TARGET_HAS_div_i64;
1599     case INDEX_op_rem_i64:
1600     case INDEX_op_remu_i64:
1601         return TCG_TARGET_HAS_rem_i64;
1602     case INDEX_op_div2_i64:
1603     case INDEX_op_divu2_i64:
1604         return TCG_TARGET_HAS_div2_i64;
1605     case INDEX_op_rotl_i64:
1606     case INDEX_op_rotr_i64:
1607         return TCG_TARGET_HAS_rot_i64;
1608     case INDEX_op_deposit_i64:
1609         return TCG_TARGET_HAS_deposit_i64;
1610     case INDEX_op_extract_i64:
1611         return TCG_TARGET_HAS_extract_i64;
1612     case INDEX_op_sextract_i64:
1613         return TCG_TARGET_HAS_sextract_i64;
1614     case INDEX_op_extract2_i64:
1615         return TCG_TARGET_HAS_extract2_i64;
1616     case INDEX_op_extrl_i64_i32:
1617         return TCG_TARGET_HAS_extrl_i64_i32;
1618     case INDEX_op_extrh_i64_i32:
1619         return TCG_TARGET_HAS_extrh_i64_i32;
1620     case INDEX_op_ext8s_i64:
1621         return TCG_TARGET_HAS_ext8s_i64;
1622     case INDEX_op_ext16s_i64:
1623         return TCG_TARGET_HAS_ext16s_i64;
1624     case INDEX_op_ext32s_i64:
1625         return TCG_TARGET_HAS_ext32s_i64;
1626     case INDEX_op_ext8u_i64:
1627         return TCG_TARGET_HAS_ext8u_i64;
1628     case INDEX_op_ext16u_i64:
1629         return TCG_TARGET_HAS_ext16u_i64;
1630     case INDEX_op_ext32u_i64:
1631         return TCG_TARGET_HAS_ext32u_i64;
1632     case INDEX_op_bswap16_i64:
1633         return TCG_TARGET_HAS_bswap16_i64;
1634     case INDEX_op_bswap32_i64:
1635         return TCG_TARGET_HAS_bswap32_i64;
1636     case INDEX_op_bswap64_i64:
1637         return TCG_TARGET_HAS_bswap64_i64;
1638     case INDEX_op_not_i64:
1639         return TCG_TARGET_HAS_not_i64;
1640     case INDEX_op_neg_i64:
1641         return TCG_TARGET_HAS_neg_i64;
1642     case INDEX_op_andc_i64:
1643         return TCG_TARGET_HAS_andc_i64;
1644     case INDEX_op_orc_i64:
1645         return TCG_TARGET_HAS_orc_i64;
1646     case INDEX_op_eqv_i64:
1647         return TCG_TARGET_HAS_eqv_i64;
1648     case INDEX_op_nand_i64:
1649         return TCG_TARGET_HAS_nand_i64;
1650     case INDEX_op_nor_i64:
1651         return TCG_TARGET_HAS_nor_i64;
1652     case INDEX_op_clz_i64:
1653         return TCG_TARGET_HAS_clz_i64;
1654     case INDEX_op_ctz_i64:
1655         return TCG_TARGET_HAS_ctz_i64;
1656     case INDEX_op_ctpop_i64:
1657         return TCG_TARGET_HAS_ctpop_i64;
1658     case INDEX_op_add2_i64:
1659         return TCG_TARGET_HAS_add2_i64;
1660     case INDEX_op_sub2_i64:
1661         return TCG_TARGET_HAS_sub2_i64;
1662     case INDEX_op_mulu2_i64:
1663         return TCG_TARGET_HAS_mulu2_i64;
1664     case INDEX_op_muls2_i64:
1665         return TCG_TARGET_HAS_muls2_i64;
1666     case INDEX_op_muluh_i64:
1667         return TCG_TARGET_HAS_muluh_i64;
1668     case INDEX_op_mulsh_i64:
1669         return TCG_TARGET_HAS_mulsh_i64;
1670 
1671     case INDEX_op_mov_vec:
1672     case INDEX_op_dup_vec:
1673     case INDEX_op_dupm_vec:
1674     case INDEX_op_ld_vec:
1675     case INDEX_op_st_vec:
1676     case INDEX_op_add_vec:
1677     case INDEX_op_sub_vec:
1678     case INDEX_op_and_vec:
1679     case INDEX_op_or_vec:
1680     case INDEX_op_xor_vec:
1681     case INDEX_op_cmp_vec:
1682         return have_vec;
1683     case INDEX_op_dup2_vec:
1684         return have_vec && TCG_TARGET_REG_BITS == 32;
1685     case INDEX_op_not_vec:
1686         return have_vec && TCG_TARGET_HAS_not_vec;
1687     case INDEX_op_neg_vec:
1688         return have_vec && TCG_TARGET_HAS_neg_vec;
1689     case INDEX_op_abs_vec:
1690         return have_vec && TCG_TARGET_HAS_abs_vec;
1691     case INDEX_op_andc_vec:
1692         return have_vec && TCG_TARGET_HAS_andc_vec;
1693     case INDEX_op_orc_vec:
1694         return have_vec && TCG_TARGET_HAS_orc_vec;
1695     case INDEX_op_nand_vec:
1696         return have_vec && TCG_TARGET_HAS_nand_vec;
1697     case INDEX_op_nor_vec:
1698         return have_vec && TCG_TARGET_HAS_nor_vec;
1699     case INDEX_op_eqv_vec:
1700         return have_vec && TCG_TARGET_HAS_eqv_vec;
1701     case INDEX_op_mul_vec:
1702         return have_vec && TCG_TARGET_HAS_mul_vec;
1703     case INDEX_op_shli_vec:
1704     case INDEX_op_shri_vec:
1705     case INDEX_op_sari_vec:
1706         return have_vec && TCG_TARGET_HAS_shi_vec;
1707     case INDEX_op_shls_vec:
1708     case INDEX_op_shrs_vec:
1709     case INDEX_op_sars_vec:
1710         return have_vec && TCG_TARGET_HAS_shs_vec;
1711     case INDEX_op_shlv_vec:
1712     case INDEX_op_shrv_vec:
1713     case INDEX_op_sarv_vec:
1714         return have_vec && TCG_TARGET_HAS_shv_vec;
1715     case INDEX_op_rotli_vec:
1716         return have_vec && TCG_TARGET_HAS_roti_vec;
1717     case INDEX_op_rotls_vec:
1718         return have_vec && TCG_TARGET_HAS_rots_vec;
1719     case INDEX_op_rotlv_vec:
1720     case INDEX_op_rotrv_vec:
1721         return have_vec && TCG_TARGET_HAS_rotv_vec;
1722     case INDEX_op_ssadd_vec:
1723     case INDEX_op_usadd_vec:
1724     case INDEX_op_sssub_vec:
1725     case INDEX_op_ussub_vec:
1726         return have_vec && TCG_TARGET_HAS_sat_vec;
1727     case INDEX_op_smin_vec:
1728     case INDEX_op_umin_vec:
1729     case INDEX_op_smax_vec:
1730     case INDEX_op_umax_vec:
1731         return have_vec && TCG_TARGET_HAS_minmax_vec;
1732     case INDEX_op_bitsel_vec:
1733         return have_vec && TCG_TARGET_HAS_bitsel_vec;
1734     case INDEX_op_cmpsel_vec:
1735         return have_vec && TCG_TARGET_HAS_cmpsel_vec;
1736 
1737     default:
1738         tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1739         return true;
1740     }
1741 }
1742 
1743 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
1744 
1745 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
1746 {
1747     const TCGHelperInfo *info;
1748     TCGv_i64 extend_free[MAX_CALL_IARGS];
1749     int n_extend = 0;
1750     TCGOp *op;
1751     int i, n, pi = 0, total_args;
1752 
1753     info = g_hash_table_lookup(helper_table, (gpointer)func);
1754     total_args = info->nr_out + info->nr_in + 2;
1755     op = tcg_op_alloc(INDEX_op_call, total_args);
1756 
1757 #ifdef CONFIG_PLUGIN
1758     /* Flag helpers that may affect guest state */
1759     if (tcg_ctx->plugin_insn &&
1760         !(info->flags & TCG_CALL_PLUGIN) &&
1761         !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
1762         tcg_ctx->plugin_insn->calls_helpers = true;
1763     }
1764 #endif
1765 
1766     TCGOP_CALLO(op) = n = info->nr_out;
1767     switch (n) {
1768     case 0:
1769         tcg_debug_assert(ret == NULL);
1770         break;
1771     case 1:
1772         tcg_debug_assert(ret != NULL);
1773         op->args[pi++] = temp_arg(ret);
1774         break;
1775     case 2:
1776     case 4:
1777         tcg_debug_assert(ret != NULL);
1778         tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
1779         tcg_debug_assert(ret->temp_subindex == 0);
1780         for (i = 0; i < n; ++i) {
1781             op->args[pi++] = temp_arg(ret + i);
1782         }
1783         break;
1784     default:
1785         g_assert_not_reached();
1786     }
1787 
1788     TCGOP_CALLI(op) = n = info->nr_in;
1789     for (i = 0; i < n; i++) {
1790         const TCGCallArgumentLoc *loc = &info->in[i];
1791         TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
1792 
1793         switch (loc->kind) {
1794         case TCG_CALL_ARG_NORMAL:
1795         case TCG_CALL_ARG_BY_REF:
1796         case TCG_CALL_ARG_BY_REF_N:
1797             op->args[pi++] = temp_arg(ts);
1798             break;
1799 
1800         case TCG_CALL_ARG_EXTEND_U:
1801         case TCG_CALL_ARG_EXTEND_S:
1802             {
1803                 TCGv_i64 temp = tcg_temp_ebb_new_i64();
1804                 TCGv_i32 orig = temp_tcgv_i32(ts);
1805 
1806                 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
1807                     tcg_gen_ext_i32_i64(temp, orig);
1808                 } else {
1809                     tcg_gen_extu_i32_i64(temp, orig);
1810                 }
1811                 op->args[pi++] = tcgv_i64_arg(temp);
1812                 extend_free[n_extend++] = temp;
1813             }
1814             break;
1815 
1816         default:
1817             g_assert_not_reached();
1818         }
1819     }
1820     op->args[pi++] = (uintptr_t)func;
1821     op->args[pi++] = (uintptr_t)info;
1822     tcg_debug_assert(pi == total_args);
1823 
1824     QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
1825 
1826     tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
1827     for (i = 0; i < n_extend; ++i) {
1828         tcg_temp_free_i64(extend_free[i]);
1829     }
1830 }
1831 
1832 static void tcg_reg_alloc_start(TCGContext *s)
1833 {
1834     int i, n;
1835 
1836     for (i = 0, n = s->nb_temps; i < n; i++) {
1837         TCGTemp *ts = &s->temps[i];
1838         TCGTempVal val = TEMP_VAL_MEM;
1839 
1840         switch (ts->kind) {
1841         case TEMP_CONST:
1842             val = TEMP_VAL_CONST;
1843             break;
1844         case TEMP_FIXED:
1845             val = TEMP_VAL_REG;
1846             break;
1847         case TEMP_GLOBAL:
1848             break;
1849         case TEMP_EBB:
1850             val = TEMP_VAL_DEAD;
1851             /* fall through */
1852         case TEMP_TB:
1853             ts->mem_allocated = 0;
1854             break;
1855         default:
1856             g_assert_not_reached();
1857         }
1858         ts->val_type = val;
1859     }
1860 
1861     memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
1862 }
1863 
1864 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1865                                  TCGTemp *ts)
1866 {
1867     int idx = temp_idx(ts);
1868 
1869     switch (ts->kind) {
1870     case TEMP_FIXED:
1871     case TEMP_GLOBAL:
1872         pstrcpy(buf, buf_size, ts->name);
1873         break;
1874     case TEMP_TB:
1875         snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
1876         break;
1877     case TEMP_EBB:
1878         snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
1879         break;
1880     case TEMP_CONST:
1881         switch (ts->type) {
1882         case TCG_TYPE_I32:
1883             snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1884             break;
1885 #if TCG_TARGET_REG_BITS > 32
1886         case TCG_TYPE_I64:
1887             snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1888             break;
1889 #endif
1890         case TCG_TYPE_V64:
1891         case TCG_TYPE_V128:
1892         case TCG_TYPE_V256:
1893             snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1894                      64 << (ts->type - TCG_TYPE_V64), ts->val);
1895             break;
1896         default:
1897             g_assert_not_reached();
1898         }
1899         break;
1900     }
1901     return buf;
1902 }
1903 
1904 static char *tcg_get_arg_str(TCGContext *s, char *buf,
1905                              int buf_size, TCGArg arg)
1906 {
1907     return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
1908 }
1909 
1910 static const char * const cond_name[] =
1911 {
1912     [TCG_COND_NEVER] = "never",
1913     [TCG_COND_ALWAYS] = "always",
1914     [TCG_COND_EQ] = "eq",
1915     [TCG_COND_NE] = "ne",
1916     [TCG_COND_LT] = "lt",
1917     [TCG_COND_GE] = "ge",
1918     [TCG_COND_LE] = "le",
1919     [TCG_COND_GT] = "gt",
1920     [TCG_COND_LTU] = "ltu",
1921     [TCG_COND_GEU] = "geu",
1922     [TCG_COND_LEU] = "leu",
1923     [TCG_COND_GTU] = "gtu"
1924 };
1925 
1926 static const char * const ldst_name[] =
1927 {
1928     [MO_UB]   = "ub",
1929     [MO_SB]   = "sb",
1930     [MO_LEUW] = "leuw",
1931     [MO_LESW] = "lesw",
1932     [MO_LEUL] = "leul",
1933     [MO_LESL] = "lesl",
1934     [MO_LEUQ] = "leq",
1935     [MO_BEUW] = "beuw",
1936     [MO_BESW] = "besw",
1937     [MO_BEUL] = "beul",
1938     [MO_BESL] = "besl",
1939     [MO_BEUQ] = "beq",
1940 };
1941 
1942 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
1943 #ifdef TARGET_ALIGNED_ONLY
1944     [MO_UNALN >> MO_ASHIFT]    = "un+",
1945     [MO_ALIGN >> MO_ASHIFT]    = "",
1946 #else
1947     [MO_UNALN >> MO_ASHIFT]    = "",
1948     [MO_ALIGN >> MO_ASHIFT]    = "al+",
1949 #endif
1950     [MO_ALIGN_2 >> MO_ASHIFT]  = "al2+",
1951     [MO_ALIGN_4 >> MO_ASHIFT]  = "al4+",
1952     [MO_ALIGN_8 >> MO_ASHIFT]  = "al8+",
1953     [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1954     [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1955     [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1956 };
1957 
1958 static const char bswap_flag_name[][6] = {
1959     [TCG_BSWAP_IZ] = "iz",
1960     [TCG_BSWAP_OZ] = "oz",
1961     [TCG_BSWAP_OS] = "os",
1962     [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1963     [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1964 };
1965 
1966 static inline bool tcg_regset_single(TCGRegSet d)
1967 {
1968     return (d & (d - 1)) == 0;
1969 }
1970 
1971 static inline TCGReg tcg_regset_first(TCGRegSet d)
1972 {
1973     if (TCG_TARGET_NB_REGS <= 32) {
1974         return ctz32(d);
1975     } else {
1976         return ctz64(d);
1977     }
1978 }
1979 
1980 /* Return only the number of characters output -- no error return. */
1981 #define ne_fprintf(...) \
1982     ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1983 
1984 static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
1985 {
1986     char buf[128];
1987     TCGOp *op;
1988 
1989     QTAILQ_FOREACH(op, &s->ops, link) {
1990         int i, k, nb_oargs, nb_iargs, nb_cargs;
1991         const TCGOpDef *def;
1992         TCGOpcode c;
1993         int col = 0;
1994 
1995         c = op->opc;
1996         def = &tcg_op_defs[c];
1997 
1998         if (c == INDEX_op_insn_start) {
1999             nb_oargs = 0;
2000             col += ne_fprintf(f, "\n ----");
2001 
2002             for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
2003                 target_ulong a;
2004 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2005                 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
2006 #else
2007                 a = op->args[i];
2008 #endif
2009                 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
2010             }
2011         } else if (c == INDEX_op_call) {
2012             const TCGHelperInfo *info = tcg_call_info(op);
2013             void *func = tcg_call_func(op);
2014 
2015             /* variable number of arguments */
2016             nb_oargs = TCGOP_CALLO(op);
2017             nb_iargs = TCGOP_CALLI(op);
2018             nb_cargs = def->nb_cargs;
2019 
2020             col += ne_fprintf(f, " %s ", def->name);
2021 
2022             /*
2023              * Print the function name from TCGHelperInfo, if available.
2024              * Note that plugins have a template function for the info,
2025              * but the actual function pointer comes from the plugin.
2026              */
2027             if (func == info->func) {
2028                 col += ne_fprintf(f, "%s", info->name);
2029             } else {
2030                 col += ne_fprintf(f, "plugin(%p)", func);
2031             }
2032 
2033             col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
2034             for (i = 0; i < nb_oargs; i++) {
2035                 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2036                                                             op->args[i]));
2037             }
2038             for (i = 0; i < nb_iargs; i++) {
2039                 TCGArg arg = op->args[nb_oargs + i];
2040                 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
2041                 col += ne_fprintf(f, ",%s", t);
2042             }
2043         } else {
2044             col += ne_fprintf(f, " %s ", def->name);
2045 
2046             nb_oargs = def->nb_oargs;
2047             nb_iargs = def->nb_iargs;
2048             nb_cargs = def->nb_cargs;
2049 
2050             if (def->flags & TCG_OPF_VECTOR) {
2051                 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
2052                                   8 << TCGOP_VECE(op));
2053             }
2054 
2055             k = 0;
2056             for (i = 0; i < nb_oargs; i++) {
2057                 const char *sep =  k ? "," : "";
2058                 col += ne_fprintf(f, "%s%s", sep,
2059                                   tcg_get_arg_str(s, buf, sizeof(buf),
2060                                                   op->args[k++]));
2061             }
2062             for (i = 0; i < nb_iargs; i++) {
2063                 const char *sep =  k ? "," : "";
2064                 col += ne_fprintf(f, "%s%s", sep,
2065                                   tcg_get_arg_str(s, buf, sizeof(buf),
2066                                                   op->args[k++]));
2067             }
2068             switch (c) {
2069             case INDEX_op_brcond_i32:
2070             case INDEX_op_setcond_i32:
2071             case INDEX_op_movcond_i32:
2072             case INDEX_op_brcond2_i32:
2073             case INDEX_op_setcond2_i32:
2074             case INDEX_op_brcond_i64:
2075             case INDEX_op_setcond_i64:
2076             case INDEX_op_movcond_i64:
2077             case INDEX_op_cmp_vec:
2078             case INDEX_op_cmpsel_vec:
2079                 if (op->args[k] < ARRAY_SIZE(cond_name)
2080                     && cond_name[op->args[k]]) {
2081                     col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
2082                 } else {
2083                     col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
2084                 }
2085                 i = 1;
2086                 break;
2087             case INDEX_op_qemu_ld_i32:
2088             case INDEX_op_qemu_st_i32:
2089             case INDEX_op_qemu_st8_i32:
2090             case INDEX_op_qemu_ld_i64:
2091             case INDEX_op_qemu_st_i64:
2092                 {
2093                     MemOpIdx oi = op->args[k++];
2094                     MemOp op = get_memop(oi);
2095                     unsigned ix = get_mmuidx(oi);
2096 
2097                     if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
2098                         col += ne_fprintf(f, ",$0x%x,%u", op, ix);
2099                     } else {
2100                         const char *s_al, *s_op;
2101                         s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
2102                         s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
2103                         col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
2104                     }
2105                     i = 1;
2106                 }
2107                 break;
2108             case INDEX_op_bswap16_i32:
2109             case INDEX_op_bswap16_i64:
2110             case INDEX_op_bswap32_i32:
2111             case INDEX_op_bswap32_i64:
2112             case INDEX_op_bswap64_i64:
2113                 {
2114                     TCGArg flags = op->args[k];
2115                     const char *name = NULL;
2116 
2117                     if (flags < ARRAY_SIZE(bswap_flag_name)) {
2118                         name = bswap_flag_name[flags];
2119                     }
2120                     if (name) {
2121                         col += ne_fprintf(f, ",%s", name);
2122                     } else {
2123                         col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
2124                     }
2125                     i = k = 1;
2126                 }
2127                 break;
2128             default:
2129                 i = 0;
2130                 break;
2131             }
2132             switch (c) {
2133             case INDEX_op_set_label:
2134             case INDEX_op_br:
2135             case INDEX_op_brcond_i32:
2136             case INDEX_op_brcond_i64:
2137             case INDEX_op_brcond2_i32:
2138                 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
2139                                   arg_label(op->args[k])->id);
2140                 i++, k++;
2141                 break;
2142             case INDEX_op_mb:
2143                 {
2144                     TCGBar membar = op->args[k];
2145                     const char *b_op, *m_op;
2146 
2147                     switch (membar & TCG_BAR_SC) {
2148                     case 0:
2149                         b_op = "none";
2150                         break;
2151                     case TCG_BAR_LDAQ:
2152                         b_op = "acq";
2153                         break;
2154                     case TCG_BAR_STRL:
2155                         b_op = "rel";
2156                         break;
2157                     case TCG_BAR_SC:
2158                         b_op = "seq";
2159                         break;
2160                     default:
2161                         g_assert_not_reached();
2162                     }
2163 
2164                     switch (membar & TCG_MO_ALL) {
2165                     case 0:
2166                         m_op = "none";
2167                         break;
2168                     case TCG_MO_LD_LD:
2169                         m_op = "rr";
2170                         break;
2171                     case TCG_MO_LD_ST:
2172                         m_op = "rw";
2173                         break;
2174                     case TCG_MO_ST_LD:
2175                         m_op = "wr";
2176                         break;
2177                     case TCG_MO_ST_ST:
2178                         m_op = "ww";
2179                         break;
2180                     case TCG_MO_LD_LD | TCG_MO_LD_ST:
2181                         m_op = "rr+rw";
2182                         break;
2183                     case TCG_MO_LD_LD | TCG_MO_ST_LD:
2184                         m_op = "rr+wr";
2185                         break;
2186                     case TCG_MO_LD_LD | TCG_MO_ST_ST:
2187                         m_op = "rr+ww";
2188                         break;
2189                     case TCG_MO_LD_ST | TCG_MO_ST_LD:
2190                         m_op = "rw+wr";
2191                         break;
2192                     case TCG_MO_LD_ST | TCG_MO_ST_ST:
2193                         m_op = "rw+ww";
2194                         break;
2195                     case TCG_MO_ST_LD | TCG_MO_ST_ST:
2196                         m_op = "wr+ww";
2197                         break;
2198                     case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
2199                         m_op = "rr+rw+wr";
2200                         break;
2201                     case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
2202                         m_op = "rr+rw+ww";
2203                         break;
2204                     case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
2205                         m_op = "rr+wr+ww";
2206                         break;
2207                     case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
2208                         m_op = "rw+wr+ww";
2209                         break;
2210                     case TCG_MO_ALL:
2211                         m_op = "all";
2212                         break;
2213                     default:
2214                         g_assert_not_reached();
2215                     }
2216 
2217                     col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
2218                     i++, k++;
2219                 }
2220                 break;
2221             default:
2222                 break;
2223             }
2224             for (; i < nb_cargs; i++, k++) {
2225                 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
2226                                   op->args[k]);
2227             }
2228         }
2229 
2230         if (have_prefs || op->life) {
2231             for (; col < 40; ++col) {
2232                 putc(' ', f);
2233             }
2234         }
2235 
2236         if (op->life) {
2237             unsigned life = op->life;
2238 
2239             if (life & (SYNC_ARG * 3)) {
2240                 ne_fprintf(f, "  sync:");
2241                 for (i = 0; i < 2; ++i) {
2242                     if (life & (SYNC_ARG << i)) {
2243                         ne_fprintf(f, " %d", i);
2244                     }
2245                 }
2246             }
2247             life /= DEAD_ARG;
2248             if (life) {
2249                 ne_fprintf(f, "  dead:");
2250                 for (i = 0; life; ++i, life >>= 1) {
2251                     if (life & 1) {
2252                         ne_fprintf(f, " %d", i);
2253                     }
2254                 }
2255             }
2256         }
2257 
2258         if (have_prefs) {
2259             for (i = 0; i < nb_oargs; ++i) {
2260                 TCGRegSet set = output_pref(op, i);
2261 
2262                 if (i == 0) {
2263                     ne_fprintf(f, "  pref=");
2264                 } else {
2265                     ne_fprintf(f, ",");
2266                 }
2267                 if (set == 0) {
2268                     ne_fprintf(f, "none");
2269                 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2270                     ne_fprintf(f, "all");
2271 #ifdef CONFIG_DEBUG_TCG
2272                 } else if (tcg_regset_single(set)) {
2273                     TCGReg reg = tcg_regset_first(set);
2274                     ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
2275 #endif
2276                 } else if (TCG_TARGET_NB_REGS <= 32) {
2277                     ne_fprintf(f, "0x%x", (uint32_t)set);
2278                 } else {
2279                     ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
2280                 }
2281             }
2282         }
2283 
2284         putc('\n', f);
2285     }
2286 }
2287 
2288 /* we give more priority to constraints with less registers */
2289 static int get_constraint_priority(const TCGOpDef *def, int k)
2290 {
2291     const TCGArgConstraint *arg_ct = &def->args_ct[k];
2292     int n = ctpop64(arg_ct->regs);
2293 
2294     /*
2295      * Sort constraints of a single register first, which includes output
2296      * aliases (which must exactly match the input already allocated).
2297      */
2298     if (n == 1 || arg_ct->oalias) {
2299         return INT_MAX;
2300     }
2301 
2302     /*
2303      * Sort register pairs next, first then second immediately after.
2304      * Arbitrarily sort multiple pairs by the index of the first reg;
2305      * there shouldn't be many pairs.
2306      */
2307     switch (arg_ct->pair) {
2308     case 1:
2309     case 3:
2310         return (k + 1) * 2;
2311     case 2:
2312         return (arg_ct->pair_index + 1) * 2 - 1;
2313     }
2314 
2315     /* Finally, sort by decreasing register count. */
2316     assert(n > 1);
2317     return -n;
2318 }
2319 
2320 /* sort from highest priority to lowest */
2321 static void sort_constraints(TCGOpDef *def, int start, int n)
2322 {
2323     int i, j;
2324     TCGArgConstraint *a = def->args_ct;
2325 
2326     for (i = 0; i < n; i++) {
2327         a[start + i].sort_index = start + i;
2328     }
2329     if (n <= 1) {
2330         return;
2331     }
2332     for (i = 0; i < n - 1; i++) {
2333         for (j = i + 1; j < n; j++) {
2334             int p1 = get_constraint_priority(def, a[start + i].sort_index);
2335             int p2 = get_constraint_priority(def, a[start + j].sort_index);
2336             if (p1 < p2) {
2337                 int tmp = a[start + i].sort_index;
2338                 a[start + i].sort_index = a[start + j].sort_index;
2339                 a[start + j].sort_index = tmp;
2340             }
2341         }
2342     }
2343 }
2344 
2345 static void process_op_defs(TCGContext *s)
2346 {
2347     TCGOpcode op;
2348 
2349     for (op = 0; op < NB_OPS; op++) {
2350         TCGOpDef *def = &tcg_op_defs[op];
2351         const TCGTargetOpDef *tdefs;
2352         bool saw_alias_pair = false;
2353         int i, o, i2, o2, nb_args;
2354 
2355         if (def->flags & TCG_OPF_NOT_PRESENT) {
2356             continue;
2357         }
2358 
2359         nb_args = def->nb_iargs + def->nb_oargs;
2360         if (nb_args == 0) {
2361             continue;
2362         }
2363 
2364         /*
2365          * Macro magic should make it impossible, but double-check that
2366          * the array index is in range.  Since the signness of an enum
2367          * is implementation defined, force the result to unsigned.
2368          */
2369         unsigned con_set = tcg_target_op_def(op);
2370         tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2371         tdefs = &constraint_sets[con_set];
2372 
2373         for (i = 0; i < nb_args; i++) {
2374             const char *ct_str = tdefs->args_ct_str[i];
2375             bool input_p = i >= def->nb_oargs;
2376 
2377             /* Incomplete TCGTargetOpDef entry. */
2378             tcg_debug_assert(ct_str != NULL);
2379 
2380             switch (*ct_str) {
2381             case '0' ... '9':
2382                 o = *ct_str - '0';
2383                 tcg_debug_assert(input_p);
2384                 tcg_debug_assert(o < def->nb_oargs);
2385                 tcg_debug_assert(def->args_ct[o].regs != 0);
2386                 tcg_debug_assert(!def->args_ct[o].oalias);
2387                 def->args_ct[i] = def->args_ct[o];
2388                 /* The output sets oalias.  */
2389                 def->args_ct[o].oalias = 1;
2390                 def->args_ct[o].alias_index = i;
2391                 /* The input sets ialias. */
2392                 def->args_ct[i].ialias = 1;
2393                 def->args_ct[i].alias_index = o;
2394                 if (def->args_ct[i].pair) {
2395                     saw_alias_pair = true;
2396                 }
2397                 tcg_debug_assert(ct_str[1] == '\0');
2398                 continue;
2399 
2400             case '&':
2401                 tcg_debug_assert(!input_p);
2402                 def->args_ct[i].newreg = true;
2403                 ct_str++;
2404                 break;
2405 
2406             case 'p': /* plus */
2407                 /* Allocate to the register after the previous. */
2408                 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2409                 o = i - 1;
2410                 tcg_debug_assert(!def->args_ct[o].pair);
2411                 tcg_debug_assert(!def->args_ct[o].ct);
2412                 def->args_ct[i] = (TCGArgConstraint){
2413                     .pair = 2,
2414                     .pair_index = o,
2415                     .regs = def->args_ct[o].regs << 1,
2416                 };
2417                 def->args_ct[o].pair = 1;
2418                 def->args_ct[o].pair_index = i;
2419                 tcg_debug_assert(ct_str[1] == '\0');
2420                 continue;
2421 
2422             case 'm': /* minus */
2423                 /* Allocate to the register before the previous. */
2424                 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0));
2425                 o = i - 1;
2426                 tcg_debug_assert(!def->args_ct[o].pair);
2427                 tcg_debug_assert(!def->args_ct[o].ct);
2428                 def->args_ct[i] = (TCGArgConstraint){
2429                     .pair = 1,
2430                     .pair_index = o,
2431                     .regs = def->args_ct[o].regs >> 1,
2432                 };
2433                 def->args_ct[o].pair = 2;
2434                 def->args_ct[o].pair_index = i;
2435                 tcg_debug_assert(ct_str[1] == '\0');
2436                 continue;
2437             }
2438 
2439             do {
2440                 switch (*ct_str) {
2441                 case 'i':
2442                     def->args_ct[i].ct |= TCG_CT_CONST;
2443                     break;
2444 
2445                 /* Include all of the target-specific constraints. */
2446 
2447 #undef CONST
2448 #define CONST(CASE, MASK) \
2449     case CASE: def->args_ct[i].ct |= MASK; break;
2450 #define REGS(CASE, MASK) \
2451     case CASE: def->args_ct[i].regs |= MASK; break;
2452 
2453 #include "tcg-target-con-str.h"
2454 
2455 #undef REGS
2456 #undef CONST
2457                 default:
2458                 case '0' ... '9':
2459                 case '&':
2460                 case 'p':
2461                 case 'm':
2462                     /* Typo in TCGTargetOpDef constraint. */
2463                     g_assert_not_reached();
2464                 }
2465             } while (*++ct_str != '\0');
2466         }
2467 
2468         /* TCGTargetOpDef entry with too much information? */
2469         tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
2470 
2471         /*
2472          * Fix up output pairs that are aliased with inputs.
2473          * When we created the alias, we copied pair from the output.
2474          * There are three cases:
2475          *    (1a) Pairs of inputs alias pairs of outputs.
2476          *    (1b) One input aliases the first of a pair of outputs.
2477          *    (2)  One input aliases the second of a pair of outputs.
2478          *
2479          * Case 1a is handled by making sure that the pair_index'es are
2480          * properly updated so that they appear the same as a pair of inputs.
2481          *
2482          * Case 1b is handled by setting the pair_index of the input to
2483          * itself, simply so it doesn't point to an unrelated argument.
2484          * Since we don't encounter the "second" during the input allocation
2485          * phase, nothing happens with the second half of the input pair.
2486          *
2487          * Case 2 is handled by setting the second input to pair=3, the
2488          * first output to pair=3, and the pair_index'es to match.
2489          */
2490         if (saw_alias_pair) {
2491             for (i = def->nb_oargs; i < nb_args; i++) {
2492                 /*
2493                  * Since [0-9pm] must be alone in the constraint string,
2494                  * the only way they can both be set is if the pair comes
2495                  * from the output alias.
2496                  */
2497                 if (!def->args_ct[i].ialias) {
2498                     continue;
2499                 }
2500                 switch (def->args_ct[i].pair) {
2501                 case 0:
2502                     break;
2503                 case 1:
2504                     o = def->args_ct[i].alias_index;
2505                     o2 = def->args_ct[o].pair_index;
2506                     tcg_debug_assert(def->args_ct[o].pair == 1);
2507                     tcg_debug_assert(def->args_ct[o2].pair == 2);
2508                     if (def->args_ct[o2].oalias) {
2509                         /* Case 1a */
2510                         i2 = def->args_ct[o2].alias_index;
2511                         tcg_debug_assert(def->args_ct[i2].pair == 2);
2512                         def->args_ct[i2].pair_index = i;
2513                         def->args_ct[i].pair_index = i2;
2514                     } else {
2515                         /* Case 1b */
2516                         def->args_ct[i].pair_index = i;
2517                     }
2518                     break;
2519                 case 2:
2520                     o = def->args_ct[i].alias_index;
2521                     o2 = def->args_ct[o].pair_index;
2522                     tcg_debug_assert(def->args_ct[o].pair == 2);
2523                     tcg_debug_assert(def->args_ct[o2].pair == 1);
2524                     if (def->args_ct[o2].oalias) {
2525                         /* Case 1a */
2526                         i2 = def->args_ct[o2].alias_index;
2527                         tcg_debug_assert(def->args_ct[i2].pair == 1);
2528                         def->args_ct[i2].pair_index = i;
2529                         def->args_ct[i].pair_index = i2;
2530                     } else {
2531                         /* Case 2 */
2532                         def->args_ct[i].pair = 3;
2533                         def->args_ct[o2].pair = 3;
2534                         def->args_ct[i].pair_index = o2;
2535                         def->args_ct[o2].pair_index = i;
2536                     }
2537                     break;
2538                 default:
2539                     g_assert_not_reached();
2540                 }
2541             }
2542         }
2543 
2544         /* sort the constraints (XXX: this is just an heuristic) */
2545         sort_constraints(def, 0, def->nb_oargs);
2546         sort_constraints(def, def->nb_oargs, def->nb_iargs);
2547     }
2548 }
2549 
2550 static void remove_label_use(TCGOp *op, int idx)
2551 {
2552     TCGLabel *label = arg_label(op->args[idx]);
2553     TCGLabelUse *use;
2554 
2555     QSIMPLEQ_FOREACH(use, &label->branches, next) {
2556         if (use->op == op) {
2557             QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
2558             return;
2559         }
2560     }
2561     g_assert_not_reached();
2562 }
2563 
2564 void tcg_op_remove(TCGContext *s, TCGOp *op)
2565 {
2566     switch (op->opc) {
2567     case INDEX_op_br:
2568         remove_label_use(op, 0);
2569         break;
2570     case INDEX_op_brcond_i32:
2571     case INDEX_op_brcond_i64:
2572         remove_label_use(op, 3);
2573         break;
2574     case INDEX_op_brcond2_i32:
2575         remove_label_use(op, 5);
2576         break;
2577     default:
2578         break;
2579     }
2580 
2581     QTAILQ_REMOVE(&s->ops, op, link);
2582     QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
2583     s->nb_ops--;
2584 
2585 #ifdef CONFIG_PROFILER
2586     qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
2587 #endif
2588 }
2589 
2590 void tcg_remove_ops_after(TCGOp *op)
2591 {
2592     TCGContext *s = tcg_ctx;
2593 
2594     while (true) {
2595         TCGOp *last = tcg_last_op();
2596         if (last == op) {
2597             return;
2598         }
2599         tcg_op_remove(s, last);
2600     }
2601 }
2602 
2603 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
2604 {
2605     TCGContext *s = tcg_ctx;
2606     TCGOp *op = NULL;
2607 
2608     if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
2609         QTAILQ_FOREACH(op, &s->free_ops, link) {
2610             if (nargs <= op->nargs) {
2611                 QTAILQ_REMOVE(&s->free_ops, op, link);
2612                 nargs = op->nargs;
2613                 goto found;
2614             }
2615         }
2616     }
2617 
2618     /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
2619     nargs = MAX(4, nargs);
2620     op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
2621 
2622  found:
2623     memset(op, 0, offsetof(TCGOp, link));
2624     op->opc = opc;
2625     op->nargs = nargs;
2626 
2627     /* Check for bitfield overflow. */
2628     tcg_debug_assert(op->nargs == nargs);
2629 
2630     s->nb_ops++;
2631     return op;
2632 }
2633 
2634 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
2635 {
2636     TCGOp *op = tcg_op_alloc(opc, nargs);
2637     QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2638     return op;
2639 }
2640 
2641 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
2642                             TCGOpcode opc, unsigned nargs)
2643 {
2644     TCGOp *new_op = tcg_op_alloc(opc, nargs);
2645     QTAILQ_INSERT_BEFORE(old_op, new_op, link);
2646     return new_op;
2647 }
2648 
2649 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
2650                            TCGOpcode opc, unsigned nargs)
2651 {
2652     TCGOp *new_op = tcg_op_alloc(opc, nargs);
2653     QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
2654     return new_op;
2655 }
2656 
2657 static void move_label_uses(TCGLabel *to, TCGLabel *from)
2658 {
2659     TCGLabelUse *u;
2660 
2661     QSIMPLEQ_FOREACH(u, &from->branches, next) {
2662         TCGOp *op = u->op;
2663         switch (op->opc) {
2664         case INDEX_op_br:
2665             op->args[0] = label_arg(to);
2666             break;
2667         case INDEX_op_brcond_i32:
2668         case INDEX_op_brcond_i64:
2669             op->args[3] = label_arg(to);
2670             break;
2671         case INDEX_op_brcond2_i32:
2672             op->args[5] = label_arg(to);
2673             break;
2674         default:
2675             g_assert_not_reached();
2676         }
2677     }
2678 
2679     QSIMPLEQ_CONCAT(&to->branches, &from->branches);
2680 }
2681 
2682 /* Reachable analysis : remove unreachable code.  */
2683 static void __attribute__((noinline))
2684 reachable_code_pass(TCGContext *s)
2685 {
2686     TCGOp *op, *op_next, *op_prev;
2687     bool dead = false;
2688 
2689     QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2690         bool remove = dead;
2691         TCGLabel *label;
2692 
2693         switch (op->opc) {
2694         case INDEX_op_set_label:
2695             label = arg_label(op->args[0]);
2696 
2697             /*
2698              * Note that the first op in the TB is always a load,
2699              * so there is always something before a label.
2700              */
2701             op_prev = QTAILQ_PREV(op, link);
2702 
2703             /*
2704              * If we find two sequential labels, move all branches to
2705              * reference the second label and remove the first label.
2706              * Do this before branch to next optimization, so that the
2707              * middle label is out of the way.
2708              */
2709             if (op_prev->opc == INDEX_op_set_label) {
2710                 move_label_uses(label, arg_label(op_prev->args[0]));
2711                 tcg_op_remove(s, op_prev);
2712                 op_prev = QTAILQ_PREV(op, link);
2713             }
2714 
2715             /*
2716              * Optimization can fold conditional branches to unconditional.
2717              * If we find a label which is preceded by an unconditional
2718              * branch to next, remove the branch.  We couldn't do this when
2719              * processing the branch because any dead code between the branch
2720              * and label had not yet been removed.
2721              */
2722             if (op_prev->opc == INDEX_op_br &&
2723                 label == arg_label(op_prev->args[0])) {
2724                 tcg_op_remove(s, op_prev);
2725                 /* Fall through means insns become live again.  */
2726                 dead = false;
2727             }
2728 
2729             if (QSIMPLEQ_EMPTY(&label->branches)) {
2730                 /*
2731                  * While there is an occasional backward branch, virtually
2732                  * all branches generated by the translators are forward.
2733                  * Which means that generally we will have already removed
2734                  * all references to the label that will be, and there is
2735                  * little to be gained by iterating.
2736                  */
2737                 remove = true;
2738             } else {
2739                 /* Once we see a label, insns become live again.  */
2740                 dead = false;
2741                 remove = false;
2742             }
2743             break;
2744 
2745         case INDEX_op_br:
2746         case INDEX_op_exit_tb:
2747         case INDEX_op_goto_ptr:
2748             /* Unconditional branches; everything following is dead.  */
2749             dead = true;
2750             break;
2751 
2752         case INDEX_op_call:
2753             /* Notice noreturn helper calls, raising exceptions.  */
2754             if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
2755                 dead = true;
2756             }
2757             break;
2758 
2759         case INDEX_op_insn_start:
2760             /* Never remove -- we need to keep these for unwind.  */
2761             remove = false;
2762             break;
2763 
2764         default:
2765             break;
2766         }
2767 
2768         if (remove) {
2769             tcg_op_remove(s, op);
2770         }
2771     }
2772 }
2773 
2774 #define TS_DEAD  1
2775 #define TS_MEM   2
2776 
2777 #define IS_DEAD_ARG(n)   (arg_life & (DEAD_ARG << (n)))
2778 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2779 
2780 /* For liveness_pass_1, the register preferences for a given temp.  */
2781 static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2782 {
2783     return ts->state_ptr;
2784 }
2785 
2786 /* For liveness_pass_1, reset the preferences for a given temp to the
2787  * maximal regset for its type.
2788  */
2789 static inline void la_reset_pref(TCGTemp *ts)
2790 {
2791     *la_temp_pref(ts)
2792         = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2793 }
2794 
2795 /* liveness analysis: end of function: all temps are dead, and globals
2796    should be in memory. */
2797 static void la_func_end(TCGContext *s, int ng, int nt)
2798 {
2799     int i;
2800 
2801     for (i = 0; i < ng; ++i) {
2802         s->temps[i].state = TS_DEAD | TS_MEM;
2803         la_reset_pref(&s->temps[i]);
2804     }
2805     for (i = ng; i < nt; ++i) {
2806         s->temps[i].state = TS_DEAD;
2807         la_reset_pref(&s->temps[i]);
2808     }
2809 }
2810 
2811 /* liveness analysis: end of basic block: all temps are dead, globals
2812    and local temps should be in memory. */
2813 static void la_bb_end(TCGContext *s, int ng, int nt)
2814 {
2815     int i;
2816 
2817     for (i = 0; i < nt; ++i) {
2818         TCGTemp *ts = &s->temps[i];
2819         int state;
2820 
2821         switch (ts->kind) {
2822         case TEMP_FIXED:
2823         case TEMP_GLOBAL:
2824         case TEMP_TB:
2825             state = TS_DEAD | TS_MEM;
2826             break;
2827         case TEMP_EBB:
2828         case TEMP_CONST:
2829             state = TS_DEAD;
2830             break;
2831         default:
2832             g_assert_not_reached();
2833         }
2834         ts->state = state;
2835         la_reset_pref(ts);
2836     }
2837 }
2838 
2839 /* liveness analysis: sync globals back to memory.  */
2840 static void la_global_sync(TCGContext *s, int ng)
2841 {
2842     int i;
2843 
2844     for (i = 0; i < ng; ++i) {
2845         int state = s->temps[i].state;
2846         s->temps[i].state = state | TS_MEM;
2847         if (state == TS_DEAD) {
2848             /* If the global was previously dead, reset prefs.  */
2849             la_reset_pref(&s->temps[i]);
2850         }
2851     }
2852 }
2853 
2854 /*
2855  * liveness analysis: conditional branch: all temps are dead unless
2856  * explicitly live-across-conditional-branch, globals and local temps
2857  * should be synced.
2858  */
2859 static void la_bb_sync(TCGContext *s, int ng, int nt)
2860 {
2861     la_global_sync(s, ng);
2862 
2863     for (int i = ng; i < nt; ++i) {
2864         TCGTemp *ts = &s->temps[i];
2865         int state;
2866 
2867         switch (ts->kind) {
2868         case TEMP_TB:
2869             state = ts->state;
2870             ts->state = state | TS_MEM;
2871             if (state != TS_DEAD) {
2872                 continue;
2873             }
2874             break;
2875         case TEMP_EBB:
2876         case TEMP_CONST:
2877             continue;
2878         default:
2879             g_assert_not_reached();
2880         }
2881         la_reset_pref(&s->temps[i]);
2882     }
2883 }
2884 
2885 /* liveness analysis: sync globals back to memory and kill.  */
2886 static void la_global_kill(TCGContext *s, int ng)
2887 {
2888     int i;
2889 
2890     for (i = 0; i < ng; i++) {
2891         s->temps[i].state = TS_DEAD | TS_MEM;
2892         la_reset_pref(&s->temps[i]);
2893     }
2894 }
2895 
2896 /* liveness analysis: note live globals crossing calls.  */
2897 static void la_cross_call(TCGContext *s, int nt)
2898 {
2899     TCGRegSet mask = ~tcg_target_call_clobber_regs;
2900     int i;
2901 
2902     for (i = 0; i < nt; i++) {
2903         TCGTemp *ts = &s->temps[i];
2904         if (!(ts->state & TS_DEAD)) {
2905             TCGRegSet *pset = la_temp_pref(ts);
2906             TCGRegSet set = *pset;
2907 
2908             set &= mask;
2909             /* If the combination is not possible, restart.  */
2910             if (set == 0) {
2911                 set = tcg_target_available_regs[ts->type] & mask;
2912             }
2913             *pset = set;
2914         }
2915     }
2916 }
2917 
2918 /*
2919  * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
2920  * to TEMP_EBB, if possible.
2921  */
2922 static void __attribute__((noinline))
2923 liveness_pass_0(TCGContext *s)
2924 {
2925     void * const multiple_ebb = (void *)(uintptr_t)-1;
2926     int nb_temps = s->nb_temps;
2927     TCGOp *op, *ebb;
2928 
2929     for (int i = s->nb_globals; i < nb_temps; ++i) {
2930         s->temps[i].state_ptr = NULL;
2931     }
2932 
2933     /*
2934      * Represent each EBB by the op at which it begins.  In the case of
2935      * the first EBB, this is the first op, otherwise it is a label.
2936      * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
2937      * within a single EBB, else MULTIPLE_EBB.
2938      */
2939     ebb = QTAILQ_FIRST(&s->ops);
2940     QTAILQ_FOREACH(op, &s->ops, link) {
2941         const TCGOpDef *def;
2942         int nb_oargs, nb_iargs;
2943 
2944         switch (op->opc) {
2945         case INDEX_op_set_label:
2946             ebb = op;
2947             continue;
2948         case INDEX_op_discard:
2949             continue;
2950         case INDEX_op_call:
2951             nb_oargs = TCGOP_CALLO(op);
2952             nb_iargs = TCGOP_CALLI(op);
2953             break;
2954         default:
2955             def = &tcg_op_defs[op->opc];
2956             nb_oargs = def->nb_oargs;
2957             nb_iargs = def->nb_iargs;
2958             break;
2959         }
2960 
2961         for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
2962             TCGTemp *ts = arg_temp(op->args[i]);
2963 
2964             if (ts->kind != TEMP_TB) {
2965                 continue;
2966             }
2967             if (ts->state_ptr == NULL) {
2968                 ts->state_ptr = ebb;
2969             } else if (ts->state_ptr != ebb) {
2970                 ts->state_ptr = multiple_ebb;
2971             }
2972         }
2973     }
2974 
2975     /*
2976      * For TEMP_TB that turned out not to be used beyond one EBB,
2977      * reduce the liveness to TEMP_EBB.
2978      */
2979     for (int i = s->nb_globals; i < nb_temps; ++i) {
2980         TCGTemp *ts = &s->temps[i];
2981         if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
2982             ts->kind = TEMP_EBB;
2983         }
2984     }
2985 }
2986 
2987 /* Liveness analysis : update the opc_arg_life array to tell if a
2988    given input arguments is dead. Instructions updating dead
2989    temporaries are removed. */
2990 static void __attribute__((noinline))
2991 liveness_pass_1(TCGContext *s)
2992 {
2993     int nb_globals = s->nb_globals;
2994     int nb_temps = s->nb_temps;
2995     TCGOp *op, *op_prev;
2996     TCGRegSet *prefs;
2997     int i;
2998 
2999     prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
3000     for (i = 0; i < nb_temps; ++i) {
3001         s->temps[i].state_ptr = prefs + i;
3002     }
3003 
3004     /* ??? Should be redundant with the exit_tb that ends the TB.  */
3005     la_func_end(s, nb_globals, nb_temps);
3006 
3007     QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
3008         int nb_iargs, nb_oargs;
3009         TCGOpcode opc_new, opc_new2;
3010         bool have_opc_new2;
3011         TCGLifeData arg_life = 0;
3012         TCGTemp *ts;
3013         TCGOpcode opc = op->opc;
3014         const TCGOpDef *def = &tcg_op_defs[opc];
3015 
3016         switch (opc) {
3017         case INDEX_op_call:
3018             {
3019                 const TCGHelperInfo *info = tcg_call_info(op);
3020                 int call_flags = tcg_call_flags(op);
3021 
3022                 nb_oargs = TCGOP_CALLO(op);
3023                 nb_iargs = TCGOP_CALLI(op);
3024 
3025                 /* pure functions can be removed if their result is unused */
3026                 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
3027                     for (i = 0; i < nb_oargs; i++) {
3028                         ts = arg_temp(op->args[i]);
3029                         if (ts->state != TS_DEAD) {
3030                             goto do_not_remove_call;
3031                         }
3032                     }
3033                     goto do_remove;
3034                 }
3035             do_not_remove_call:
3036 
3037                 /* Output args are dead.  */
3038                 for (i = 0; i < nb_oargs; i++) {
3039                     ts = arg_temp(op->args[i]);
3040                     if (ts->state & TS_DEAD) {
3041                         arg_life |= DEAD_ARG << i;
3042                     }
3043                     if (ts->state & TS_MEM) {
3044                         arg_life |= SYNC_ARG << i;
3045                     }
3046                     ts->state = TS_DEAD;
3047                     la_reset_pref(ts);
3048                 }
3049 
3050                 /* Not used -- it will be tcg_target_call_oarg_reg().  */
3051                 memset(op->output_pref, 0, sizeof(op->output_pref));
3052 
3053                 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
3054                                     TCG_CALL_NO_READ_GLOBALS))) {
3055                     la_global_kill(s, nb_globals);
3056                 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
3057                     la_global_sync(s, nb_globals);
3058                 }
3059 
3060                 /* Record arguments that die in this helper.  */
3061                 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3062                     ts = arg_temp(op->args[i]);
3063                     if (ts->state & TS_DEAD) {
3064                         arg_life |= DEAD_ARG << i;
3065                     }
3066                 }
3067 
3068                 /* For all live registers, remove call-clobbered prefs.  */
3069                 la_cross_call(s, nb_temps);
3070 
3071                 /*
3072                  * Input arguments are live for preceding opcodes.
3073                  *
3074                  * For those arguments that die, and will be allocated in
3075                  * registers, clear the register set for that arg, to be
3076                  * filled in below.  For args that will be on the stack,
3077                  * reset to any available reg.  Process arguments in reverse
3078                  * order so that if a temp is used more than once, the stack
3079                  * reset to max happens before the register reset to 0.
3080                  */
3081                 for (i = nb_iargs - 1; i >= 0; i--) {
3082                     const TCGCallArgumentLoc *loc = &info->in[i];
3083                     ts = arg_temp(op->args[nb_oargs + i]);
3084 
3085                     if (ts->state & TS_DEAD) {
3086                         switch (loc->kind) {
3087                         case TCG_CALL_ARG_NORMAL:
3088                         case TCG_CALL_ARG_EXTEND_U:
3089                         case TCG_CALL_ARG_EXTEND_S:
3090                             if (REG_P(loc)) {
3091                                 *la_temp_pref(ts) = 0;
3092                                 break;
3093                             }
3094                             /* fall through */
3095                         default:
3096                             *la_temp_pref(ts) =
3097                                 tcg_target_available_regs[ts->type];
3098                             break;
3099                         }
3100                         ts->state &= ~TS_DEAD;
3101                     }
3102                 }
3103 
3104                 /*
3105                  * For each input argument, add its input register to prefs.
3106                  * If a temp is used once, this produces a single set bit;
3107                  * if a temp is used multiple times, this produces a set.
3108                  */
3109                 for (i = 0; i < nb_iargs; i++) {
3110                     const TCGCallArgumentLoc *loc = &info->in[i];
3111                     ts = arg_temp(op->args[nb_oargs + i]);
3112 
3113                     switch (loc->kind) {
3114                     case TCG_CALL_ARG_NORMAL:
3115                     case TCG_CALL_ARG_EXTEND_U:
3116                     case TCG_CALL_ARG_EXTEND_S:
3117                         if (REG_P(loc)) {
3118                             tcg_regset_set_reg(*la_temp_pref(ts),
3119                                 tcg_target_call_iarg_regs[loc->arg_slot]);
3120                         }
3121                         break;
3122                     default:
3123                         break;
3124                     }
3125                 }
3126             }
3127             break;
3128         case INDEX_op_insn_start:
3129             break;
3130         case INDEX_op_discard:
3131             /* mark the temporary as dead */
3132             ts = arg_temp(op->args[0]);
3133             ts->state = TS_DEAD;
3134             la_reset_pref(ts);
3135             break;
3136 
3137         case INDEX_op_add2_i32:
3138             opc_new = INDEX_op_add_i32;
3139             goto do_addsub2;
3140         case INDEX_op_sub2_i32:
3141             opc_new = INDEX_op_sub_i32;
3142             goto do_addsub2;
3143         case INDEX_op_add2_i64:
3144             opc_new = INDEX_op_add_i64;
3145             goto do_addsub2;
3146         case INDEX_op_sub2_i64:
3147             opc_new = INDEX_op_sub_i64;
3148         do_addsub2:
3149             nb_iargs = 4;
3150             nb_oargs = 2;
3151             /* Test if the high part of the operation is dead, but not
3152                the low part.  The result can be optimized to a simple
3153                add or sub.  This happens often for x86_64 guest when the
3154                cpu mode is set to 32 bit.  */
3155             if (arg_temp(op->args[1])->state == TS_DEAD) {
3156                 if (arg_temp(op->args[0])->state == TS_DEAD) {
3157                     goto do_remove;
3158                 }
3159                 /* Replace the opcode and adjust the args in place,
3160                    leaving 3 unused args at the end.  */
3161                 op->opc = opc = opc_new;
3162                 op->args[1] = op->args[2];
3163                 op->args[2] = op->args[4];
3164                 /* Fall through and mark the single-word operation live.  */
3165                 nb_iargs = 2;
3166                 nb_oargs = 1;
3167             }
3168             goto do_not_remove;
3169 
3170         case INDEX_op_mulu2_i32:
3171             opc_new = INDEX_op_mul_i32;
3172             opc_new2 = INDEX_op_muluh_i32;
3173             have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
3174             goto do_mul2;
3175         case INDEX_op_muls2_i32:
3176             opc_new = INDEX_op_mul_i32;
3177             opc_new2 = INDEX_op_mulsh_i32;
3178             have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
3179             goto do_mul2;
3180         case INDEX_op_mulu2_i64:
3181             opc_new = INDEX_op_mul_i64;
3182             opc_new2 = INDEX_op_muluh_i64;
3183             have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
3184             goto do_mul2;
3185         case INDEX_op_muls2_i64:
3186             opc_new = INDEX_op_mul_i64;
3187             opc_new2 = INDEX_op_mulsh_i64;
3188             have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
3189             goto do_mul2;
3190         do_mul2:
3191             nb_iargs = 2;
3192             nb_oargs = 2;
3193             if (arg_temp(op->args[1])->state == TS_DEAD) {
3194                 if (arg_temp(op->args[0])->state == TS_DEAD) {
3195                     /* Both parts of the operation are dead.  */
3196                     goto do_remove;
3197                 }
3198                 /* The high part of the operation is dead; generate the low. */
3199                 op->opc = opc = opc_new;
3200                 op->args[1] = op->args[2];
3201                 op->args[2] = op->args[3];
3202             } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
3203                 /* The low part of the operation is dead; generate the high. */
3204                 op->opc = opc = opc_new2;
3205                 op->args[0] = op->args[1];
3206                 op->args[1] = op->args[2];
3207                 op->args[2] = op->args[3];
3208             } else {
3209                 goto do_not_remove;
3210             }
3211             /* Mark the single-word operation live.  */
3212             nb_oargs = 1;
3213             goto do_not_remove;
3214 
3215         default:
3216             /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
3217             nb_iargs = def->nb_iargs;
3218             nb_oargs = def->nb_oargs;
3219 
3220             /* Test if the operation can be removed because all
3221                its outputs are dead. We assume that nb_oargs == 0
3222                implies side effects */
3223             if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
3224                 for (i = 0; i < nb_oargs; i++) {
3225                     if (arg_temp(op->args[i])->state != TS_DEAD) {
3226                         goto do_not_remove;
3227                     }
3228                 }
3229                 goto do_remove;
3230             }
3231             goto do_not_remove;
3232 
3233         do_remove:
3234             tcg_op_remove(s, op);
3235             break;
3236 
3237         do_not_remove:
3238             for (i = 0; i < nb_oargs; i++) {
3239                 ts = arg_temp(op->args[i]);
3240 
3241                 /* Remember the preference of the uses that followed.  */
3242                 if (i < ARRAY_SIZE(op->output_pref)) {
3243                     op->output_pref[i] = *la_temp_pref(ts);
3244                 }
3245 
3246                 /* Output args are dead.  */
3247                 if (ts->state & TS_DEAD) {
3248                     arg_life |= DEAD_ARG << i;
3249                 }
3250                 if (ts->state & TS_MEM) {
3251                     arg_life |= SYNC_ARG << i;
3252                 }
3253                 ts->state = TS_DEAD;
3254                 la_reset_pref(ts);
3255             }
3256 
3257             /* If end of basic block, update.  */
3258             if (def->flags & TCG_OPF_BB_EXIT) {
3259                 la_func_end(s, nb_globals, nb_temps);
3260             } else if (def->flags & TCG_OPF_COND_BRANCH) {
3261                 la_bb_sync(s, nb_globals, nb_temps);
3262             } else if (def->flags & TCG_OPF_BB_END) {
3263                 la_bb_end(s, nb_globals, nb_temps);
3264             } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3265                 la_global_sync(s, nb_globals);
3266                 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3267                     la_cross_call(s, nb_temps);
3268                 }
3269             }
3270 
3271             /* Record arguments that die in this opcode.  */
3272             for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3273                 ts = arg_temp(op->args[i]);
3274                 if (ts->state & TS_DEAD) {
3275                     arg_life |= DEAD_ARG << i;
3276                 }
3277             }
3278 
3279             /* Input arguments are live for preceding opcodes.  */
3280             for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3281                 ts = arg_temp(op->args[i]);
3282                 if (ts->state & TS_DEAD) {
3283                     /* For operands that were dead, initially allow
3284                        all regs for the type.  */
3285                     *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
3286                     ts->state &= ~TS_DEAD;
3287                 }
3288             }
3289 
3290             /* Incorporate constraints for this operand.  */
3291             switch (opc) {
3292             case INDEX_op_mov_i32:
3293             case INDEX_op_mov_i64:
3294                 /* Note that these are TCG_OPF_NOT_PRESENT and do not
3295                    have proper constraints.  That said, special case
3296                    moves to propagate preferences backward.  */
3297                 if (IS_DEAD_ARG(1)) {
3298                     *la_temp_pref(arg_temp(op->args[0]))
3299                         = *la_temp_pref(arg_temp(op->args[1]));
3300                 }
3301                 break;
3302 
3303             default:
3304                 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3305                     const TCGArgConstraint *ct = &def->args_ct[i];
3306                     TCGRegSet set, *pset;
3307 
3308                     ts = arg_temp(op->args[i]);
3309                     pset = la_temp_pref(ts);
3310                     set = *pset;
3311 
3312                     set &= ct->regs;
3313                     if (ct->ialias) {
3314                         set &= output_pref(op, ct->alias_index);
3315                     }
3316                     /* If the combination is not possible, restart.  */
3317                     if (set == 0) {
3318                         set = ct->regs;
3319                     }
3320                     *pset = set;
3321                 }
3322                 break;
3323             }
3324             break;
3325         }
3326         op->life = arg_life;
3327     }
3328 }
3329 
3330 /* Liveness analysis: Convert indirect regs to direct temporaries.  */
3331 static bool __attribute__((noinline))
3332 liveness_pass_2(TCGContext *s)
3333 {
3334     int nb_globals = s->nb_globals;
3335     int nb_temps, i;
3336     bool changes = false;
3337     TCGOp *op, *op_next;
3338 
3339     /* Create a temporary for each indirect global.  */
3340     for (i = 0; i < nb_globals; ++i) {
3341         TCGTemp *its = &s->temps[i];
3342         if (its->indirect_reg) {
3343             TCGTemp *dts = tcg_temp_alloc(s);
3344             dts->type = its->type;
3345             dts->base_type = its->base_type;
3346             dts->temp_subindex = its->temp_subindex;
3347             dts->kind = TEMP_EBB;
3348             its->state_ptr = dts;
3349         } else {
3350             its->state_ptr = NULL;
3351         }
3352         /* All globals begin dead.  */
3353         its->state = TS_DEAD;
3354     }
3355     for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
3356         TCGTemp *its = &s->temps[i];
3357         its->state_ptr = NULL;
3358         its->state = TS_DEAD;
3359     }
3360 
3361     QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3362         TCGOpcode opc = op->opc;
3363         const TCGOpDef *def = &tcg_op_defs[opc];
3364         TCGLifeData arg_life = op->life;
3365         int nb_iargs, nb_oargs, call_flags;
3366         TCGTemp *arg_ts, *dir_ts;
3367 
3368         if (opc == INDEX_op_call) {
3369             nb_oargs = TCGOP_CALLO(op);
3370             nb_iargs = TCGOP_CALLI(op);
3371             call_flags = tcg_call_flags(op);
3372         } else {
3373             nb_iargs = def->nb_iargs;
3374             nb_oargs = def->nb_oargs;
3375 
3376             /* Set flags similar to how calls require.  */
3377             if (def->flags & TCG_OPF_COND_BRANCH) {
3378                 /* Like reading globals: sync_globals */
3379                 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3380             } else if (def->flags & TCG_OPF_BB_END) {
3381                 /* Like writing globals: save_globals */
3382                 call_flags = 0;
3383             } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3384                 /* Like reading globals: sync_globals */
3385                 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
3386             } else {
3387                 /* No effect on globals.  */
3388                 call_flags = (TCG_CALL_NO_READ_GLOBALS |
3389                               TCG_CALL_NO_WRITE_GLOBALS);
3390             }
3391         }
3392 
3393         /* Make sure that input arguments are available.  */
3394         for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3395             arg_ts = arg_temp(op->args[i]);
3396             dir_ts = arg_ts->state_ptr;
3397             if (dir_ts && arg_ts->state == TS_DEAD) {
3398                 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
3399                                   ? INDEX_op_ld_i32
3400                                   : INDEX_op_ld_i64);
3401                 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
3402 
3403                 lop->args[0] = temp_arg(dir_ts);
3404                 lop->args[1] = temp_arg(arg_ts->mem_base);
3405                 lop->args[2] = arg_ts->mem_offset;
3406 
3407                 /* Loaded, but synced with memory.  */
3408                 arg_ts->state = TS_MEM;
3409             }
3410         }
3411 
3412         /* Perform input replacement, and mark inputs that became dead.
3413            No action is required except keeping temp_state up to date
3414            so that we reload when needed.  */
3415         for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3416             arg_ts = arg_temp(op->args[i]);
3417             dir_ts = arg_ts->state_ptr;
3418             if (dir_ts) {
3419                 op->args[i] = temp_arg(dir_ts);
3420                 changes = true;
3421                 if (IS_DEAD_ARG(i)) {
3422                     arg_ts->state = TS_DEAD;
3423                 }
3424             }
3425         }
3426 
3427         /* Liveness analysis should ensure that the following are
3428            all correct, for call sites and basic block end points.  */
3429         if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
3430             /* Nothing to do */
3431         } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
3432             for (i = 0; i < nb_globals; ++i) {
3433                 /* Liveness should see that globals are synced back,
3434                    that is, either TS_DEAD or TS_MEM.  */
3435                 arg_ts = &s->temps[i];
3436                 tcg_debug_assert(arg_ts->state_ptr == 0
3437                                  || arg_ts->state != 0);
3438             }
3439         } else {
3440             for (i = 0; i < nb_globals; ++i) {
3441                 /* Liveness should see that globals are saved back,
3442                    that is, TS_DEAD, waiting to be reloaded.  */
3443                 arg_ts = &s->temps[i];
3444                 tcg_debug_assert(arg_ts->state_ptr == 0
3445                                  || arg_ts->state == TS_DEAD);
3446             }
3447         }
3448 
3449         /* Outputs become available.  */
3450         if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
3451             arg_ts = arg_temp(op->args[0]);
3452             dir_ts = arg_ts->state_ptr;
3453             if (dir_ts) {
3454                 op->args[0] = temp_arg(dir_ts);
3455                 changes = true;
3456 
3457                 /* The output is now live and modified.  */
3458                 arg_ts->state = 0;
3459 
3460                 if (NEED_SYNC_ARG(0)) {
3461                     TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3462                                       ? INDEX_op_st_i32
3463                                       : INDEX_op_st_i64);
3464                     TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
3465                     TCGTemp *out_ts = dir_ts;
3466 
3467                     if (IS_DEAD_ARG(0)) {
3468                         out_ts = arg_temp(op->args[1]);
3469                         arg_ts->state = TS_DEAD;
3470                         tcg_op_remove(s, op);
3471                     } else {
3472                         arg_ts->state = TS_MEM;
3473                     }
3474 
3475                     sop->args[0] = temp_arg(out_ts);
3476                     sop->args[1] = temp_arg(arg_ts->mem_base);
3477                     sop->args[2] = arg_ts->mem_offset;
3478                 } else {
3479                     tcg_debug_assert(!IS_DEAD_ARG(0));
3480                 }
3481             }
3482         } else {
3483             for (i = 0; i < nb_oargs; i++) {
3484                 arg_ts = arg_temp(op->args[i]);
3485                 dir_ts = arg_ts->state_ptr;
3486                 if (!dir_ts) {
3487                     continue;
3488                 }
3489                 op->args[i] = temp_arg(dir_ts);
3490                 changes = true;
3491 
3492                 /* The output is now live and modified.  */
3493                 arg_ts->state = 0;
3494 
3495                 /* Sync outputs upon their last write.  */
3496                 if (NEED_SYNC_ARG(i)) {
3497                     TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3498                                       ? INDEX_op_st_i32
3499                                       : INDEX_op_st_i64);
3500                     TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
3501 
3502                     sop->args[0] = temp_arg(dir_ts);
3503                     sop->args[1] = temp_arg(arg_ts->mem_base);
3504                     sop->args[2] = arg_ts->mem_offset;
3505 
3506                     arg_ts->state = TS_MEM;
3507                 }
3508                 /* Drop outputs that are dead.  */
3509                 if (IS_DEAD_ARG(i)) {
3510                     arg_ts->state = TS_DEAD;
3511                 }
3512             }
3513         }
3514     }
3515 
3516     return changes;
3517 }
3518 
3519 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
3520 {
3521     intptr_t off;
3522     int size, align;
3523 
3524     /* When allocating an object, look at the full type. */
3525     size = tcg_type_size(ts->base_type);
3526     switch (ts->base_type) {
3527     case TCG_TYPE_I32:
3528         align = 4;
3529         break;
3530     case TCG_TYPE_I64:
3531     case TCG_TYPE_V64:
3532         align = 8;
3533         break;
3534     case TCG_TYPE_I128:
3535     case TCG_TYPE_V128:
3536     case TCG_TYPE_V256:
3537         /*
3538          * Note that we do not require aligned storage for V256,
3539          * and that we provide alignment for I128 to match V128,
3540          * even if that's above what the host ABI requires.
3541          */
3542         align = 16;
3543         break;
3544     default:
3545         g_assert_not_reached();
3546     }
3547 
3548     /*
3549      * Assume the stack is sufficiently aligned.
3550      * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3551      * and do not require 16 byte vector alignment.  This seems slightly
3552      * easier than fully parameterizing the above switch statement.
3553      */
3554     align = MIN(TCG_TARGET_STACK_ALIGN, align);
3555     off = ROUND_UP(s->current_frame_offset, align);
3556 
3557     /* If we've exhausted the stack frame, restart with a smaller TB. */
3558     if (off + size > s->frame_end) {
3559         tcg_raise_tb_overflow(s);
3560     }
3561     s->current_frame_offset = off + size;
3562 #if defined(__sparc__)
3563     off += TCG_TARGET_STACK_BIAS;
3564 #endif
3565 
3566     /* If the object was subdivided, assign memory to all the parts. */
3567     if (ts->base_type != ts->type) {
3568         int part_size = tcg_type_size(ts->type);
3569         int part_count = size / part_size;
3570 
3571         /*
3572          * Each part is allocated sequentially in tcg_temp_new_internal.
3573          * Jump back to the first part by subtracting the current index.
3574          */
3575         ts -= ts->temp_subindex;
3576         for (int i = 0; i < part_count; ++i) {
3577             ts[i].mem_offset = off + i * part_size;
3578             ts[i].mem_base = s->frame_temp;
3579             ts[i].mem_allocated = 1;
3580         }
3581     } else {
3582         ts->mem_offset = off;
3583         ts->mem_base = s->frame_temp;
3584         ts->mem_allocated = 1;
3585     }
3586 }
3587 
3588 /* Assign @reg to @ts, and update reg_to_temp[]. */
3589 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
3590 {
3591     if (ts->val_type == TEMP_VAL_REG) {
3592         TCGReg old = ts->reg;
3593         tcg_debug_assert(s->reg_to_temp[old] == ts);
3594         if (old == reg) {
3595             return;
3596         }
3597         s->reg_to_temp[old] = NULL;
3598     }
3599     tcg_debug_assert(s->reg_to_temp[reg] == NULL);
3600     s->reg_to_temp[reg] = ts;
3601     ts->val_type = TEMP_VAL_REG;
3602     ts->reg = reg;
3603 }
3604 
3605 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
3606 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
3607 {
3608     tcg_debug_assert(type != TEMP_VAL_REG);
3609     if (ts->val_type == TEMP_VAL_REG) {
3610         TCGReg reg = ts->reg;
3611         tcg_debug_assert(s->reg_to_temp[reg] == ts);
3612         s->reg_to_temp[reg] = NULL;
3613     }
3614     ts->val_type = type;
3615 }
3616 
3617 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
3618 
3619 /* Mark a temporary as free or dead.  If 'free_or_dead' is negative,
3620    mark it free; otherwise mark it dead.  */
3621 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
3622 {
3623     TCGTempVal new_type;
3624 
3625     switch (ts->kind) {
3626     case TEMP_FIXED:
3627         return;
3628     case TEMP_GLOBAL:
3629     case TEMP_TB:
3630         new_type = TEMP_VAL_MEM;
3631         break;
3632     case TEMP_EBB:
3633         new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3634         break;
3635     case TEMP_CONST:
3636         new_type = TEMP_VAL_CONST;
3637         break;
3638     default:
3639         g_assert_not_reached();
3640     }
3641     set_temp_val_nonreg(s, ts, new_type);
3642 }
3643 
3644 /* Mark a temporary as dead.  */
3645 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3646 {
3647     temp_free_or_dead(s, ts, 1);
3648 }
3649 
3650 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3651    registers needs to be allocated to store a constant.  If 'free_or_dead'
3652    is non-zero, subsequently release the temporary; if it is positive, the
3653    temp is dead; if it is negative, the temp is free.  */
3654 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3655                       TCGRegSet preferred_regs, int free_or_dead)
3656 {
3657     if (!temp_readonly(ts) && !ts->mem_coherent) {
3658         if (!ts->mem_allocated) {
3659             temp_allocate_frame(s, ts);
3660         }
3661         switch (ts->val_type) {
3662         case TEMP_VAL_CONST:
3663             /* If we're going to free the temp immediately, then we won't
3664                require it later in a register, so attempt to store the
3665                constant to memory directly.  */
3666             if (free_or_dead
3667                 && tcg_out_sti(s, ts->type, ts->val,
3668                                ts->mem_base->reg, ts->mem_offset)) {
3669                 break;
3670             }
3671             temp_load(s, ts, tcg_target_available_regs[ts->type],
3672                       allocated_regs, preferred_regs);
3673             /* fallthrough */
3674 
3675         case TEMP_VAL_REG:
3676             tcg_out_st(s, ts->type, ts->reg,
3677                        ts->mem_base->reg, ts->mem_offset);
3678             break;
3679 
3680         case TEMP_VAL_MEM:
3681             break;
3682 
3683         case TEMP_VAL_DEAD:
3684         default:
3685             tcg_abort();
3686         }
3687         ts->mem_coherent = 1;
3688     }
3689     if (free_or_dead) {
3690         temp_free_or_dead(s, ts, free_or_dead);
3691     }
3692 }
3693 
3694 /* free register 'reg' by spilling the corresponding temporary if necessary */
3695 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
3696 {
3697     TCGTemp *ts = s->reg_to_temp[reg];
3698     if (ts != NULL) {
3699         temp_sync(s, ts, allocated_regs, 0, -1);
3700     }
3701 }
3702 
3703 /**
3704  * tcg_reg_alloc:
3705  * @required_regs: Set of registers in which we must allocate.
3706  * @allocated_regs: Set of registers which must be avoided.
3707  * @preferred_regs: Set of registers we should prefer.
3708  * @rev: True if we search the registers in "indirect" order.
3709  *
3710  * The allocated register must be in @required_regs & ~@allocated_regs,
3711  * but if we can put it in @preferred_regs we may save a move later.
3712  */
3713 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3714                             TCGRegSet allocated_regs,
3715                             TCGRegSet preferred_regs, bool rev)
3716 {
3717     int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3718     TCGRegSet reg_ct[2];
3719     const int *order;
3720 
3721     reg_ct[1] = required_regs & ~allocated_regs;
3722     tcg_debug_assert(reg_ct[1] != 0);
3723     reg_ct[0] = reg_ct[1] & preferred_regs;
3724 
3725     /* Skip the preferred_regs option if it cannot be satisfied,
3726        or if the preference made no difference.  */
3727     f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3728 
3729     order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3730 
3731     /* Try free registers, preferences first.  */
3732     for (j = f; j < 2; j++) {
3733         TCGRegSet set = reg_ct[j];
3734 
3735         if (tcg_regset_single(set)) {
3736             /* One register in the set.  */
3737             TCGReg reg = tcg_regset_first(set);
3738             if (s->reg_to_temp[reg] == NULL) {
3739                 return reg;
3740             }
3741         } else {
3742             for (i = 0; i < n; i++) {
3743                 TCGReg reg = order[i];
3744                 if (s->reg_to_temp[reg] == NULL &&
3745                     tcg_regset_test_reg(set, reg)) {
3746                     return reg;
3747                 }
3748             }
3749         }
3750     }
3751 
3752     /* We must spill something.  */
3753     for (j = f; j < 2; j++) {
3754         TCGRegSet set = reg_ct[j];
3755 
3756         if (tcg_regset_single(set)) {
3757             /* One register in the set.  */
3758             TCGReg reg = tcg_regset_first(set);
3759             tcg_reg_free(s, reg, allocated_regs);
3760             return reg;
3761         } else {
3762             for (i = 0; i < n; i++) {
3763                 TCGReg reg = order[i];
3764                 if (tcg_regset_test_reg(set, reg)) {
3765                     tcg_reg_free(s, reg, allocated_regs);
3766                     return reg;
3767                 }
3768             }
3769         }
3770     }
3771 
3772     tcg_abort();
3773 }
3774 
3775 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
3776                                  TCGRegSet allocated_regs,
3777                                  TCGRegSet preferred_regs, bool rev)
3778 {
3779     int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3780     TCGRegSet reg_ct[2];
3781     const int *order;
3782 
3783     /* Ensure that if I is not in allocated_regs, I+1 is not either. */
3784     reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
3785     tcg_debug_assert(reg_ct[1] != 0);
3786     reg_ct[0] = reg_ct[1] & preferred_regs;
3787 
3788     order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3789 
3790     /*
3791      * Skip the preferred_regs option if it cannot be satisfied,
3792      * or if the preference made no difference.
3793      */
3794     k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3795 
3796     /*
3797      * Minimize the number of flushes by looking for 2 free registers first,
3798      * then a single flush, then two flushes.
3799      */
3800     for (fmin = 2; fmin >= 0; fmin--) {
3801         for (j = k; j < 2; j++) {
3802             TCGRegSet set = reg_ct[j];
3803 
3804             for (i = 0; i < n; i++) {
3805                 TCGReg reg = order[i];
3806 
3807                 if (tcg_regset_test_reg(set, reg)) {
3808                     int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
3809                     if (f >= fmin) {
3810                         tcg_reg_free(s, reg, allocated_regs);
3811                         tcg_reg_free(s, reg + 1, allocated_regs);
3812                         return reg;
3813                     }
3814                 }
3815             }
3816         }
3817     }
3818     tcg_abort();
3819 }
3820 
3821 /* Make sure the temporary is in a register.  If needed, allocate the register
3822    from DESIRED while avoiding ALLOCATED.  */
3823 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
3824                       TCGRegSet allocated_regs, TCGRegSet preferred_regs)
3825 {
3826     TCGReg reg;
3827 
3828     switch (ts->val_type) {
3829     case TEMP_VAL_REG:
3830         return;
3831     case TEMP_VAL_CONST:
3832         reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3833                             preferred_regs, ts->indirect_base);
3834         if (ts->type <= TCG_TYPE_I64) {
3835             tcg_out_movi(s, ts->type, reg, ts->val);
3836         } else {
3837             uint64_t val = ts->val;
3838             MemOp vece = MO_64;
3839 
3840             /*
3841              * Find the minimal vector element that matches the constant.
3842              * The targets will, in general, have to do this search anyway,
3843              * do this generically.
3844              */
3845             if (val == dup_const(MO_8, val)) {
3846                 vece = MO_8;
3847             } else if (val == dup_const(MO_16, val)) {
3848                 vece = MO_16;
3849             } else if (val == dup_const(MO_32, val)) {
3850                 vece = MO_32;
3851             }
3852 
3853             tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
3854         }
3855         ts->mem_coherent = 0;
3856         break;
3857     case TEMP_VAL_MEM:
3858         reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3859                             preferred_regs, ts->indirect_base);
3860         tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3861         ts->mem_coherent = 1;
3862         break;
3863     case TEMP_VAL_DEAD:
3864     default:
3865         tcg_abort();
3866     }
3867     set_temp_val_reg(s, ts, reg);
3868 }
3869 
3870 /* Save a temporary to memory. 'allocated_regs' is used in case a
3871    temporary registers needs to be allocated to store a constant.  */
3872 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
3873 {
3874     /* The liveness analysis already ensures that globals are back
3875        in memory. Keep an tcg_debug_assert for safety. */
3876     tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
3877 }
3878 
3879 /* save globals to their canonical location and assume they can be
3880    modified be the following code. 'allocated_regs' is used in case a
3881    temporary registers needs to be allocated to store a constant. */
3882 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
3883 {
3884     int i, n;
3885 
3886     for (i = 0, n = s->nb_globals; i < n; i++) {
3887         temp_save(s, &s->temps[i], allocated_regs);
3888     }
3889 }
3890 
3891 /* sync globals to their canonical location and assume they can be
3892    read by the following code. 'allocated_regs' is used in case a
3893    temporary registers needs to be allocated to store a constant. */
3894 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3895 {
3896     int i, n;
3897 
3898     for (i = 0, n = s->nb_globals; i < n; i++) {
3899         TCGTemp *ts = &s->temps[i];
3900         tcg_debug_assert(ts->val_type != TEMP_VAL_REG
3901                          || ts->kind == TEMP_FIXED
3902                          || ts->mem_coherent);
3903     }
3904 }
3905 
3906 /* at the end of a basic block, we assume all temporaries are dead and
3907    all globals are stored at their canonical location. */
3908 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
3909 {
3910     int i;
3911 
3912     for (i = s->nb_globals; i < s->nb_temps; i++) {
3913         TCGTemp *ts = &s->temps[i];
3914 
3915         switch (ts->kind) {
3916         case TEMP_TB:
3917             temp_save(s, ts, allocated_regs);
3918             break;
3919         case TEMP_EBB:
3920             /* The liveness analysis already ensures that temps are dead.
3921                Keep an tcg_debug_assert for safety. */
3922             tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3923             break;
3924         case TEMP_CONST:
3925             /* Similarly, we should have freed any allocated register. */
3926             tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3927             break;
3928         default:
3929             g_assert_not_reached();
3930         }
3931     }
3932 
3933     save_globals(s, allocated_regs);
3934 }
3935 
3936 /*
3937  * At a conditional branch, we assume all temporaries are dead unless
3938  * explicitly live-across-conditional-branch; all globals and local
3939  * temps are synced to their location.
3940  */
3941 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3942 {
3943     sync_globals(s, allocated_regs);
3944 
3945     for (int i = s->nb_globals; i < s->nb_temps; i++) {
3946         TCGTemp *ts = &s->temps[i];
3947         /*
3948          * The liveness analysis already ensures that temps are dead.
3949          * Keep tcg_debug_asserts for safety.
3950          */
3951         switch (ts->kind) {
3952         case TEMP_TB:
3953             tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
3954             break;
3955         case TEMP_EBB:
3956         case TEMP_CONST:
3957             break;
3958         default:
3959             g_assert_not_reached();
3960         }
3961     }
3962 }
3963 
3964 /*
3965  * Specialized code generation for INDEX_op_mov_* with a constant.
3966  */
3967 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
3968                                   tcg_target_ulong val, TCGLifeData arg_life,
3969                                   TCGRegSet preferred_regs)
3970 {
3971     /* ENV should not be modified.  */
3972     tcg_debug_assert(!temp_readonly(ots));
3973 
3974     /* The movi is not explicitly generated here.  */
3975     set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
3976     ots->val = val;
3977     ots->mem_coherent = 0;
3978     if (NEED_SYNC_ARG(0)) {
3979         temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
3980     } else if (IS_DEAD_ARG(0)) {
3981         temp_dead(s, ots);
3982     }
3983 }
3984 
3985 /*
3986  * Specialized code generation for INDEX_op_mov_*.
3987  */
3988 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
3989 {
3990     const TCGLifeData arg_life = op->life;
3991     TCGRegSet allocated_regs, preferred_regs;
3992     TCGTemp *ts, *ots;
3993     TCGType otype, itype;
3994     TCGReg oreg, ireg;
3995 
3996     allocated_regs = s->reserved_regs;
3997     preferred_regs = output_pref(op, 0);
3998     ots = arg_temp(op->args[0]);
3999     ts = arg_temp(op->args[1]);
4000 
4001     /* ENV should not be modified.  */
4002     tcg_debug_assert(!temp_readonly(ots));
4003 
4004     /* Note that otype != itype for no-op truncation.  */
4005     otype = ots->type;
4006     itype = ts->type;
4007 
4008     if (ts->val_type == TEMP_VAL_CONST) {
4009         /* propagate constant or generate sti */
4010         tcg_target_ulong val = ts->val;
4011         if (IS_DEAD_ARG(1)) {
4012             temp_dead(s, ts);
4013         }
4014         tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
4015         return;
4016     }
4017 
4018     /* If the source value is in memory we're going to be forced
4019        to have it in a register in order to perform the copy.  Copy
4020        the SOURCE value into its own register first, that way we
4021        don't have to reload SOURCE the next time it is used. */
4022     if (ts->val_type == TEMP_VAL_MEM) {
4023         temp_load(s, ts, tcg_target_available_regs[itype],
4024                   allocated_regs, preferred_regs);
4025     }
4026     tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
4027     ireg = ts->reg;
4028 
4029     if (IS_DEAD_ARG(0)) {
4030         /* mov to a non-saved dead register makes no sense (even with
4031            liveness analysis disabled). */
4032         tcg_debug_assert(NEED_SYNC_ARG(0));
4033         if (!ots->mem_allocated) {
4034             temp_allocate_frame(s, ots);
4035         }
4036         tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
4037         if (IS_DEAD_ARG(1)) {
4038             temp_dead(s, ts);
4039         }
4040         temp_dead(s, ots);
4041         return;
4042     }
4043 
4044     if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
4045         /*
4046          * The mov can be suppressed.  Kill input first, so that it
4047          * is unlinked from reg_to_temp, then set the output to the
4048          * reg that we saved from the input.
4049          */
4050         temp_dead(s, ts);
4051         oreg = ireg;
4052     } else {
4053         if (ots->val_type == TEMP_VAL_REG) {
4054             oreg = ots->reg;
4055         } else {
4056             /* Make sure to not spill the input register during allocation. */
4057             oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
4058                                  allocated_regs | ((TCGRegSet)1 << ireg),
4059                                  preferred_regs, ots->indirect_base);
4060         }
4061         if (!tcg_out_mov(s, otype, oreg, ireg)) {
4062             /*
4063              * Cross register class move not supported.
4064              * Store the source register into the destination slot
4065              * and leave the destination temp as TEMP_VAL_MEM.
4066              */
4067             assert(!temp_readonly(ots));
4068             if (!ts->mem_allocated) {
4069                 temp_allocate_frame(s, ots);
4070             }
4071             tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
4072             set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
4073             ots->mem_coherent = 1;
4074             return;
4075         }
4076     }
4077     set_temp_val_reg(s, ots, oreg);
4078     ots->mem_coherent = 0;
4079 
4080     if (NEED_SYNC_ARG(0)) {
4081         temp_sync(s, ots, allocated_regs, 0, 0);
4082     }
4083 }
4084 
4085 /*
4086  * Specialized code generation for INDEX_op_dup_vec.
4087  */
4088 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
4089 {
4090     const TCGLifeData arg_life = op->life;
4091     TCGRegSet dup_out_regs, dup_in_regs;
4092     TCGTemp *its, *ots;
4093     TCGType itype, vtype;
4094     unsigned vece;
4095     int lowpart_ofs;
4096     bool ok;
4097 
4098     ots = arg_temp(op->args[0]);
4099     its = arg_temp(op->args[1]);
4100 
4101     /* ENV should not be modified.  */
4102     tcg_debug_assert(!temp_readonly(ots));
4103 
4104     itype = its->type;
4105     vece = TCGOP_VECE(op);
4106     vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4107 
4108     if (its->val_type == TEMP_VAL_CONST) {
4109         /* Propagate constant via movi -> dupi.  */
4110         tcg_target_ulong val = its->val;
4111         if (IS_DEAD_ARG(1)) {
4112             temp_dead(s, its);
4113         }
4114         tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
4115         return;
4116     }
4117 
4118     dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4119     dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
4120 
4121     /* Allocate the output register now.  */
4122     if (ots->val_type != TEMP_VAL_REG) {
4123         TCGRegSet allocated_regs = s->reserved_regs;
4124         TCGReg oreg;
4125 
4126         if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
4127             /* Make sure to not spill the input register. */
4128             tcg_regset_set_reg(allocated_regs, its->reg);
4129         }
4130         oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
4131                              output_pref(op, 0), ots->indirect_base);
4132         set_temp_val_reg(s, ots, oreg);
4133     }
4134 
4135     switch (its->val_type) {
4136     case TEMP_VAL_REG:
4137         /*
4138          * The dup constriaints must be broad, covering all possible VECE.
4139          * However, tcg_op_dup_vec() gets to see the VECE and we allow it
4140          * to fail, indicating that extra moves are required for that case.
4141          */
4142         if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
4143             if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
4144                 goto done;
4145             }
4146             /* Try again from memory or a vector input register.  */
4147         }
4148         if (!its->mem_coherent) {
4149             /*
4150              * The input register is not synced, and so an extra store
4151              * would be required to use memory.  Attempt an integer-vector
4152              * register move first.  We do not have a TCGRegSet for this.
4153              */
4154             if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
4155                 break;
4156             }
4157             /* Sync the temp back to its slot and load from there.  */
4158             temp_sync(s, its, s->reserved_regs, 0, 0);
4159         }
4160         /* fall through */
4161 
4162     case TEMP_VAL_MEM:
4163         lowpart_ofs = 0;
4164         if (HOST_BIG_ENDIAN) {
4165             lowpart_ofs = tcg_type_size(itype) - (1 << vece);
4166         }
4167         if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
4168                              its->mem_offset + lowpart_ofs)) {
4169             goto done;
4170         }
4171         /* Load the input into the destination vector register. */
4172         tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
4173         break;
4174 
4175     default:
4176         g_assert_not_reached();
4177     }
4178 
4179     /* We now have a vector input register, so dup must succeed. */
4180     ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
4181     tcg_debug_assert(ok);
4182 
4183  done:
4184     ots->mem_coherent = 0;
4185     if (IS_DEAD_ARG(1)) {
4186         temp_dead(s, its);
4187     }
4188     if (NEED_SYNC_ARG(0)) {
4189         temp_sync(s, ots, s->reserved_regs, 0, 0);
4190     }
4191     if (IS_DEAD_ARG(0)) {
4192         temp_dead(s, ots);
4193     }
4194 }
4195 
4196 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
4197 {
4198     const TCGLifeData arg_life = op->life;
4199     const TCGOpDef * const def = &tcg_op_defs[op->opc];
4200     TCGRegSet i_allocated_regs;
4201     TCGRegSet o_allocated_regs;
4202     int i, k, nb_iargs, nb_oargs;
4203     TCGReg reg;
4204     TCGArg arg;
4205     const TCGArgConstraint *arg_ct;
4206     TCGTemp *ts;
4207     TCGArg new_args[TCG_MAX_OP_ARGS];
4208     int const_args[TCG_MAX_OP_ARGS];
4209 
4210     nb_oargs = def->nb_oargs;
4211     nb_iargs = def->nb_iargs;
4212 
4213     /* copy constants */
4214     memcpy(new_args + nb_oargs + nb_iargs,
4215            op->args + nb_oargs + nb_iargs,
4216            sizeof(TCGArg) * def->nb_cargs);
4217 
4218     i_allocated_regs = s->reserved_regs;
4219     o_allocated_regs = s->reserved_regs;
4220 
4221     /* satisfy input constraints */
4222     for (k = 0; k < nb_iargs; k++) {
4223         TCGRegSet i_preferred_regs, i_required_regs;
4224         bool allocate_new_reg, copyto_new_reg;
4225         TCGTemp *ts2;
4226         int i1, i2;
4227 
4228         i = def->args_ct[nb_oargs + k].sort_index;
4229         arg = op->args[i];
4230         arg_ct = &def->args_ct[i];
4231         ts = arg_temp(arg);
4232 
4233         if (ts->val_type == TEMP_VAL_CONST
4234             && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
4235             /* constant is OK for instruction */
4236             const_args[i] = 1;
4237             new_args[i] = ts->val;
4238             continue;
4239         }
4240 
4241         reg = ts->reg;
4242         i_preferred_regs = 0;
4243         i_required_regs = arg_ct->regs;
4244         allocate_new_reg = false;
4245         copyto_new_reg = false;
4246 
4247         switch (arg_ct->pair) {
4248         case 0: /* not paired */
4249             if (arg_ct->ialias) {
4250                 i_preferred_regs = output_pref(op, arg_ct->alias_index);
4251 
4252                 /*
4253                  * If the input is readonly, then it cannot also be an
4254                  * output and aliased to itself.  If the input is not
4255                  * dead after the instruction, we must allocate a new
4256                  * register and move it.
4257                  */
4258                 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
4259                     allocate_new_reg = true;
4260                 } else if (ts->val_type == TEMP_VAL_REG) {
4261                     /*
4262                      * Check if the current register has already been
4263                      * allocated for another input.
4264                      */
4265                     allocate_new_reg =
4266                         tcg_regset_test_reg(i_allocated_regs, reg);
4267                 }
4268             }
4269             if (!allocate_new_reg) {
4270                 temp_load(s, ts, i_required_regs, i_allocated_regs,
4271                           i_preferred_regs);
4272                 reg = ts->reg;
4273                 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
4274             }
4275             if (allocate_new_reg) {
4276                 /*
4277                  * Allocate a new register matching the constraint
4278                  * and move the temporary register into it.
4279                  */
4280                 temp_load(s, ts, tcg_target_available_regs[ts->type],
4281                           i_allocated_regs, 0);
4282                 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
4283                                     i_preferred_regs, ts->indirect_base);
4284                 copyto_new_reg = true;
4285             }
4286             break;
4287 
4288         case 1:
4289             /* First of an input pair; if i1 == i2, the second is an output. */
4290             i1 = i;
4291             i2 = arg_ct->pair_index;
4292             ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
4293 
4294             /*
4295              * It is easier to default to allocating a new pair
4296              * and to identify a few cases where it's not required.
4297              */
4298             if (arg_ct->ialias) {
4299                 i_preferred_regs = output_pref(op, arg_ct->alias_index);
4300                 if (IS_DEAD_ARG(i1) &&
4301                     IS_DEAD_ARG(i2) &&
4302                     !temp_readonly(ts) &&
4303                     ts->val_type == TEMP_VAL_REG &&
4304                     ts->reg < TCG_TARGET_NB_REGS - 1 &&
4305                     tcg_regset_test_reg(i_required_regs, reg) &&
4306                     !tcg_regset_test_reg(i_allocated_regs, reg) &&
4307                     !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
4308                     (ts2
4309                      ? ts2->val_type == TEMP_VAL_REG &&
4310                        ts2->reg == reg + 1 &&
4311                        !temp_readonly(ts2)
4312                      : s->reg_to_temp[reg + 1] == NULL)) {
4313                     break;
4314                 }
4315             } else {
4316                 /* Without aliasing, the pair must also be an input. */
4317                 tcg_debug_assert(ts2);
4318                 if (ts->val_type == TEMP_VAL_REG &&
4319                     ts2->val_type == TEMP_VAL_REG &&
4320                     ts2->reg == reg + 1 &&
4321                     tcg_regset_test_reg(i_required_regs, reg)) {
4322                     break;
4323                 }
4324             }
4325             reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
4326                                      0, ts->indirect_base);
4327             goto do_pair;
4328 
4329         case 2: /* pair second */
4330             reg = new_args[arg_ct->pair_index] + 1;
4331             goto do_pair;
4332 
4333         case 3: /* ialias with second output, no first input */
4334             tcg_debug_assert(arg_ct->ialias);
4335             i_preferred_regs = output_pref(op, arg_ct->alias_index);
4336 
4337             if (IS_DEAD_ARG(i) &&
4338                 !temp_readonly(ts) &&
4339                 ts->val_type == TEMP_VAL_REG &&
4340                 reg > 0 &&
4341                 s->reg_to_temp[reg - 1] == NULL &&
4342                 tcg_regset_test_reg(i_required_regs, reg) &&
4343                 !tcg_regset_test_reg(i_allocated_regs, reg) &&
4344                 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
4345                 tcg_regset_set_reg(i_allocated_regs, reg - 1);
4346                 break;
4347             }
4348             reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
4349                                      i_allocated_regs, 0,
4350                                      ts->indirect_base);
4351             tcg_regset_set_reg(i_allocated_regs, reg);
4352             reg += 1;
4353             goto do_pair;
4354 
4355         do_pair:
4356             /*
4357              * If an aliased input is not dead after the instruction,
4358              * we must allocate a new register and move it.
4359              */
4360             if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
4361                 TCGRegSet t_allocated_regs = i_allocated_regs;
4362 
4363                 /*
4364                  * Because of the alias, and the continued life, make sure
4365                  * that the temp is somewhere *other* than the reg pair,
4366                  * and we get a copy in reg.
4367                  */
4368                 tcg_regset_set_reg(t_allocated_regs, reg);
4369                 tcg_regset_set_reg(t_allocated_regs, reg + 1);
4370                 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
4371                     /* If ts was already in reg, copy it somewhere else. */
4372                     TCGReg nr;
4373                     bool ok;
4374 
4375                     tcg_debug_assert(ts->kind != TEMP_FIXED);
4376                     nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
4377                                        t_allocated_regs, 0, ts->indirect_base);
4378                     ok = tcg_out_mov(s, ts->type, nr, reg);
4379                     tcg_debug_assert(ok);
4380 
4381                     set_temp_val_reg(s, ts, nr);
4382                 } else {
4383                     temp_load(s, ts, tcg_target_available_regs[ts->type],
4384                               t_allocated_regs, 0);
4385                     copyto_new_reg = true;
4386                 }
4387             } else {
4388                 /* Preferably allocate to reg, otherwise copy. */
4389                 i_required_regs = (TCGRegSet)1 << reg;
4390                 temp_load(s, ts, i_required_regs, i_allocated_regs,
4391                           i_preferred_regs);
4392                 copyto_new_reg = ts->reg != reg;
4393             }
4394             break;
4395 
4396         default:
4397             g_assert_not_reached();
4398         }
4399 
4400         if (copyto_new_reg) {
4401             if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4402                 /*
4403                  * Cross register class move not supported.  Sync the
4404                  * temp back to its slot and load from there.
4405                  */
4406                 temp_sync(s, ts, i_allocated_regs, 0, 0);
4407                 tcg_out_ld(s, ts->type, reg,
4408                            ts->mem_base->reg, ts->mem_offset);
4409             }
4410         }
4411         new_args[i] = reg;
4412         const_args[i] = 0;
4413         tcg_regset_set_reg(i_allocated_regs, reg);
4414     }
4415 
4416     /* mark dead temporaries and free the associated registers */
4417     for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4418         if (IS_DEAD_ARG(i)) {
4419             temp_dead(s, arg_temp(op->args[i]));
4420         }
4421     }
4422 
4423     if (def->flags & TCG_OPF_COND_BRANCH) {
4424         tcg_reg_alloc_cbranch(s, i_allocated_regs);
4425     } else if (def->flags & TCG_OPF_BB_END) {
4426         tcg_reg_alloc_bb_end(s, i_allocated_regs);
4427     } else {
4428         if (def->flags & TCG_OPF_CALL_CLOBBER) {
4429             /* XXX: permit generic clobber register list ? */
4430             for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4431                 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
4432                     tcg_reg_free(s, i, i_allocated_regs);
4433                 }
4434             }
4435         }
4436         if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4437             /* sync globals if the op has side effects and might trigger
4438                an exception. */
4439             sync_globals(s, i_allocated_regs);
4440         }
4441 
4442         /* satisfy the output constraints */
4443         for(k = 0; k < nb_oargs; k++) {
4444             i = def->args_ct[k].sort_index;
4445             arg = op->args[i];
4446             arg_ct = &def->args_ct[i];
4447             ts = arg_temp(arg);
4448 
4449             /* ENV should not be modified.  */
4450             tcg_debug_assert(!temp_readonly(ts));
4451 
4452             switch (arg_ct->pair) {
4453             case 0: /* not paired */
4454                 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
4455                     reg = new_args[arg_ct->alias_index];
4456                 } else if (arg_ct->newreg) {
4457                     reg = tcg_reg_alloc(s, arg_ct->regs,
4458                                         i_allocated_regs | o_allocated_regs,
4459                                         output_pref(op, k), ts->indirect_base);
4460                 } else {
4461                     reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
4462                                         output_pref(op, k), ts->indirect_base);
4463                 }
4464                 break;
4465 
4466             case 1: /* first of pair */
4467                 tcg_debug_assert(!arg_ct->newreg);
4468                 if (arg_ct->oalias) {
4469                     reg = new_args[arg_ct->alias_index];
4470                     break;
4471                 }
4472                 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
4473                                          output_pref(op, k), ts->indirect_base);
4474                 break;
4475 
4476             case 2: /* second of pair */
4477                 tcg_debug_assert(!arg_ct->newreg);
4478                 if (arg_ct->oalias) {
4479                     reg = new_args[arg_ct->alias_index];
4480                 } else {
4481                     reg = new_args[arg_ct->pair_index] + 1;
4482                 }
4483                 break;
4484 
4485             case 3: /* first of pair, aliasing with a second input */
4486                 tcg_debug_assert(!arg_ct->newreg);
4487                 reg = new_args[arg_ct->pair_index] - 1;
4488                 break;
4489 
4490             default:
4491                 g_assert_not_reached();
4492             }
4493             tcg_regset_set_reg(o_allocated_regs, reg);
4494             set_temp_val_reg(s, ts, reg);
4495             ts->mem_coherent = 0;
4496             new_args[i] = reg;
4497         }
4498     }
4499 
4500     /* emit instruction */
4501     if (def->flags & TCG_OPF_VECTOR) {
4502         tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
4503                        new_args, const_args);
4504     } else {
4505         tcg_out_op(s, op->opc, new_args, const_args);
4506     }
4507 
4508     /* move the outputs in the correct register if needed */
4509     for(i = 0; i < nb_oargs; i++) {
4510         ts = arg_temp(op->args[i]);
4511 
4512         /* ENV should not be modified.  */
4513         tcg_debug_assert(!temp_readonly(ts));
4514 
4515         if (NEED_SYNC_ARG(i)) {
4516             temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
4517         } else if (IS_DEAD_ARG(i)) {
4518             temp_dead(s, ts);
4519         }
4520     }
4521 }
4522 
4523 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
4524 {
4525     const TCGLifeData arg_life = op->life;
4526     TCGTemp *ots, *itsl, *itsh;
4527     TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
4528 
4529     /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
4530     tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
4531     tcg_debug_assert(TCGOP_VECE(op) == MO_64);
4532 
4533     ots = arg_temp(op->args[0]);
4534     itsl = arg_temp(op->args[1]);
4535     itsh = arg_temp(op->args[2]);
4536 
4537     /* ENV should not be modified.  */
4538     tcg_debug_assert(!temp_readonly(ots));
4539 
4540     /* Allocate the output register now.  */
4541     if (ots->val_type != TEMP_VAL_REG) {
4542         TCGRegSet allocated_regs = s->reserved_regs;
4543         TCGRegSet dup_out_regs =
4544             tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
4545         TCGReg oreg;
4546 
4547         /* Make sure to not spill the input registers. */
4548         if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
4549             tcg_regset_set_reg(allocated_regs, itsl->reg);
4550         }
4551         if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
4552             tcg_regset_set_reg(allocated_regs, itsh->reg);
4553         }
4554 
4555         oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
4556                              output_pref(op, 0), ots->indirect_base);
4557         set_temp_val_reg(s, ots, oreg);
4558     }
4559 
4560     /* Promote dup2 of immediates to dupi_vec. */
4561     if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
4562         uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
4563         MemOp vece = MO_64;
4564 
4565         if (val == dup_const(MO_8, val)) {
4566             vece = MO_8;
4567         } else if (val == dup_const(MO_16, val)) {
4568             vece = MO_16;
4569         } else if (val == dup_const(MO_32, val)) {
4570             vece = MO_32;
4571         }
4572 
4573         tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
4574         goto done;
4575     }
4576 
4577     /* If the two inputs form one 64-bit value, try dupm_vec. */
4578     if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
4579         itsh->temp_subindex == !HOST_BIG_ENDIAN &&
4580         itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
4581         TCGTemp *its = itsl - HOST_BIG_ENDIAN;
4582 
4583         temp_sync(s, its + 0, s->reserved_regs, 0, 0);
4584         temp_sync(s, its + 1, s->reserved_regs, 0, 0);
4585 
4586         if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
4587                              its->mem_base->reg, its->mem_offset)) {
4588             goto done;
4589         }
4590     }
4591 
4592     /* Fall back to generic expansion. */
4593     return false;
4594 
4595  done:
4596     ots->mem_coherent = 0;
4597     if (IS_DEAD_ARG(1)) {
4598         temp_dead(s, itsl);
4599     }
4600     if (IS_DEAD_ARG(2)) {
4601         temp_dead(s, itsh);
4602     }
4603     if (NEED_SYNC_ARG(0)) {
4604         temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
4605     } else if (IS_DEAD_ARG(0)) {
4606         temp_dead(s, ots);
4607     }
4608     return true;
4609 }
4610 
4611 static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
4612                          TCGRegSet allocated_regs)
4613 {
4614     if (ts->val_type == TEMP_VAL_REG) {
4615         if (ts->reg != reg) {
4616             tcg_reg_free(s, reg, allocated_regs);
4617             if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
4618                 /*
4619                  * Cross register class move not supported.  Sync the
4620                  * temp back to its slot and load from there.
4621                  */
4622                 temp_sync(s, ts, allocated_regs, 0, 0);
4623                 tcg_out_ld(s, ts->type, reg,
4624                            ts->mem_base->reg, ts->mem_offset);
4625             }
4626         }
4627     } else {
4628         TCGRegSet arg_set = 0;
4629 
4630         tcg_reg_free(s, reg, allocated_regs);
4631         tcg_regset_set_reg(arg_set, reg);
4632         temp_load(s, ts, arg_set, allocated_regs, 0);
4633     }
4634 }
4635 
4636 static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts,
4637                          TCGRegSet allocated_regs)
4638 {
4639     /*
4640      * When the destination is on the stack, load up the temp and store.
4641      * If there are many call-saved registers, the temp might live to
4642      * see another use; otherwise it'll be discarded.
4643      */
4644     temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
4645     tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
4646                TCG_TARGET_CALL_STACK_OFFSET +
4647                stk_slot * sizeof(tcg_target_long));
4648 }
4649 
4650 static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
4651                             TCGTemp *ts, TCGRegSet *allocated_regs)
4652 {
4653     if (REG_P(l)) {
4654         TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
4655         load_arg_reg(s, reg, ts, *allocated_regs);
4656         tcg_regset_set_reg(*allocated_regs, reg);
4657     } else {
4658         load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs),
4659                      ts, *allocated_regs);
4660     }
4661 }
4662 
4663 static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base,
4664                          intptr_t ref_off, TCGRegSet *allocated_regs)
4665 {
4666     TCGReg reg;
4667     int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
4668 
4669     if (stk_slot < 0) {
4670         reg = tcg_target_call_iarg_regs[arg_slot];
4671         tcg_reg_free(s, reg, *allocated_regs);
4672         tcg_out_addi_ptr(s, reg, ref_base, ref_off);
4673         tcg_regset_set_reg(*allocated_regs, reg);
4674     } else {
4675         reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
4676                             *allocated_regs, 0, false);
4677         tcg_out_addi_ptr(s, reg, ref_base, ref_off);
4678         tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
4679                    TCG_TARGET_CALL_STACK_OFFSET
4680                    + stk_slot * sizeof(tcg_target_long));
4681     }
4682 }
4683 
4684 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
4685 {
4686     const int nb_oargs = TCGOP_CALLO(op);
4687     const int nb_iargs = TCGOP_CALLI(op);
4688     const TCGLifeData arg_life = op->life;
4689     const TCGHelperInfo *info = tcg_call_info(op);
4690     TCGRegSet allocated_regs = s->reserved_regs;
4691     int i;
4692 
4693     /*
4694      * Move inputs into place in reverse order,
4695      * so that we place stacked arguments first.
4696      */
4697     for (i = nb_iargs - 1; i >= 0; --i) {
4698         const TCGCallArgumentLoc *loc = &info->in[i];
4699         TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
4700 
4701         switch (loc->kind) {
4702         case TCG_CALL_ARG_NORMAL:
4703         case TCG_CALL_ARG_EXTEND_U:
4704         case TCG_CALL_ARG_EXTEND_S:
4705             load_arg_normal(s, loc, ts, &allocated_regs);
4706             break;
4707         case TCG_CALL_ARG_BY_REF:
4708             load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
4709             load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
4710                          TCG_TARGET_CALL_STACK_OFFSET
4711                          + loc->ref_slot * sizeof(tcg_target_long),
4712                          &allocated_regs);
4713             break;
4714         case TCG_CALL_ARG_BY_REF_N:
4715             load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
4716             break;
4717         default:
4718             g_assert_not_reached();
4719         }
4720     }
4721 
4722     /* Mark dead temporaries and free the associated registers.  */
4723     for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4724         if (IS_DEAD_ARG(i)) {
4725             temp_dead(s, arg_temp(op->args[i]));
4726         }
4727     }
4728 
4729     /* Clobber call registers.  */
4730     for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4731         if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
4732             tcg_reg_free(s, i, allocated_regs);
4733         }
4734     }
4735 
4736     /*
4737      * Save globals if they might be written by the helper,
4738      * sync them if they might be read.
4739      */
4740     if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
4741         /* Nothing to do */
4742     } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
4743         sync_globals(s, allocated_regs);
4744     } else {
4745         save_globals(s, allocated_regs);
4746     }
4747 
4748     /*
4749      * If the ABI passes a pointer to the returned struct as the first
4750      * argument, load that now.  Pass a pointer to the output home slot.
4751      */
4752     if (info->out_kind == TCG_CALL_RET_BY_REF) {
4753         TCGTemp *ts = arg_temp(op->args[0]);
4754 
4755         if (!ts->mem_allocated) {
4756             temp_allocate_frame(s, ts);
4757         }
4758         load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
4759     }
4760 
4761     tcg_out_call(s, tcg_call_func(op), info);
4762 
4763     /* Assign output registers and emit moves if needed.  */
4764     switch (info->out_kind) {
4765     case TCG_CALL_RET_NORMAL:
4766         for (i = 0; i < nb_oargs; i++) {
4767             TCGTemp *ts = arg_temp(op->args[i]);
4768             TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
4769 
4770             /* ENV should not be modified.  */
4771             tcg_debug_assert(!temp_readonly(ts));
4772 
4773             set_temp_val_reg(s, ts, reg);
4774             ts->mem_coherent = 0;
4775         }
4776         break;
4777 
4778     case TCG_CALL_RET_BY_VEC:
4779         {
4780             TCGTemp *ts = arg_temp(op->args[0]);
4781 
4782             tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
4783             tcg_debug_assert(ts->temp_subindex == 0);
4784             if (!ts->mem_allocated) {
4785                 temp_allocate_frame(s, ts);
4786             }
4787             tcg_out_st(s, TCG_TYPE_V128,
4788                        tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
4789                        ts->mem_base->reg, ts->mem_offset);
4790         }
4791         /* fall through to mark all parts in memory */
4792 
4793     case TCG_CALL_RET_BY_REF:
4794         /* The callee has performed a write through the reference. */
4795         for (i = 0; i < nb_oargs; i++) {
4796             TCGTemp *ts = arg_temp(op->args[i]);
4797             ts->val_type = TEMP_VAL_MEM;
4798         }
4799         break;
4800 
4801     default:
4802         g_assert_not_reached();
4803     }
4804 
4805     /* Flush or discard output registers as needed. */
4806     for (i = 0; i < nb_oargs; i++) {
4807         TCGTemp *ts = arg_temp(op->args[i]);
4808         if (NEED_SYNC_ARG(i)) {
4809             temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
4810         } else if (IS_DEAD_ARG(i)) {
4811             temp_dead(s, ts);
4812         }
4813     }
4814 }
4815 
4816 #ifdef CONFIG_PROFILER
4817 
4818 /* avoid copy/paste errors */
4819 #define PROF_ADD(to, from, field)                       \
4820     do {                                                \
4821         (to)->field += qatomic_read(&((from)->field));  \
4822     } while (0)
4823 
4824 #define PROF_MAX(to, from, field)                                       \
4825     do {                                                                \
4826         typeof((from)->field) val__ = qatomic_read(&((from)->field));   \
4827         if (val__ > (to)->field) {                                      \
4828             (to)->field = val__;                                        \
4829         }                                                               \
4830     } while (0)
4831 
4832 /* Pass in a zero'ed @prof */
4833 static inline
4834 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4835 {
4836     unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
4837     unsigned int i;
4838 
4839     for (i = 0; i < n_ctxs; i++) {
4840         TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4841         const TCGProfile *orig = &s->prof;
4842 
4843         if (counters) {
4844             PROF_ADD(prof, orig, cpu_exec_time);
4845             PROF_ADD(prof, orig, tb_count1);
4846             PROF_ADD(prof, orig, tb_count);
4847             PROF_ADD(prof, orig, op_count);
4848             PROF_MAX(prof, orig, op_count_max);
4849             PROF_ADD(prof, orig, temp_count);
4850             PROF_MAX(prof, orig, temp_count_max);
4851             PROF_ADD(prof, orig, del_op_count);
4852             PROF_ADD(prof, orig, code_in_len);
4853             PROF_ADD(prof, orig, code_out_len);
4854             PROF_ADD(prof, orig, search_out_len);
4855             PROF_ADD(prof, orig, interm_time);
4856             PROF_ADD(prof, orig, code_time);
4857             PROF_ADD(prof, orig, la_time);
4858             PROF_ADD(prof, orig, opt_time);
4859             PROF_ADD(prof, orig, restore_count);
4860             PROF_ADD(prof, orig, restore_time);
4861         }
4862         if (table) {
4863             int i;
4864 
4865             for (i = 0; i < NB_OPS; i++) {
4866                 PROF_ADD(prof, orig, table_op_count[i]);
4867             }
4868         }
4869     }
4870 }
4871 
4872 #undef PROF_ADD
4873 #undef PROF_MAX
4874 
4875 static void tcg_profile_snapshot_counters(TCGProfile *prof)
4876 {
4877     tcg_profile_snapshot(prof, true, false);
4878 }
4879 
4880 static void tcg_profile_snapshot_table(TCGProfile *prof)
4881 {
4882     tcg_profile_snapshot(prof, false, true);
4883 }
4884 
4885 void tcg_dump_op_count(GString *buf)
4886 {
4887     TCGProfile prof = {};
4888     int i;
4889 
4890     tcg_profile_snapshot_table(&prof);
4891     for (i = 0; i < NB_OPS; i++) {
4892         g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4893                                prof.table_op_count[i]);
4894     }
4895 }
4896 
4897 int64_t tcg_cpu_exec_time(void)
4898 {
4899     unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
4900     unsigned int i;
4901     int64_t ret = 0;
4902 
4903     for (i = 0; i < n_ctxs; i++) {
4904         const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
4905         const TCGProfile *prof = &s->prof;
4906 
4907         ret += qatomic_read(&prof->cpu_exec_time);
4908     }
4909     return ret;
4910 }
4911 #else
4912 void tcg_dump_op_count(GString *buf)
4913 {
4914     g_string_append_printf(buf, "[TCG profiler not compiled]\n");
4915 }
4916 
4917 int64_t tcg_cpu_exec_time(void)
4918 {
4919     error_report("%s: TCG profiler not compiled", __func__);
4920     exit(EXIT_FAILURE);
4921 }
4922 #endif
4923 
4924 
4925 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
4926 {
4927 #ifdef CONFIG_PROFILER
4928     TCGProfile *prof = &s->prof;
4929 #endif
4930     int i, num_insns;
4931     TCGOp *op;
4932 
4933 #ifdef CONFIG_PROFILER
4934     {
4935         int n = 0;
4936 
4937         QTAILQ_FOREACH(op, &s->ops, link) {
4938             n++;
4939         }
4940         qatomic_set(&prof->op_count, prof->op_count + n);
4941         if (n > prof->op_count_max) {
4942             qatomic_set(&prof->op_count_max, n);
4943         }
4944 
4945         n = s->nb_temps;
4946         qatomic_set(&prof->temp_count, prof->temp_count + n);
4947         if (n > prof->temp_count_max) {
4948             qatomic_set(&prof->temp_count_max, n);
4949         }
4950     }
4951 #endif
4952 
4953 #ifdef DEBUG_DISAS
4954     if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4955                  && qemu_log_in_addr_range(pc_start))) {
4956         FILE *logfile = qemu_log_trylock();
4957         if (logfile) {
4958             fprintf(logfile, "OP:\n");
4959             tcg_dump_ops(s, logfile, false);
4960             fprintf(logfile, "\n");
4961             qemu_log_unlock(logfile);
4962         }
4963     }
4964 #endif
4965 
4966 #ifdef CONFIG_DEBUG_TCG
4967     /* Ensure all labels referenced have been emitted.  */
4968     {
4969         TCGLabel *l;
4970         bool error = false;
4971 
4972         QSIMPLEQ_FOREACH(l, &s->labels, next) {
4973             if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
4974                 qemu_log_mask(CPU_LOG_TB_OP,
4975                               "$L%d referenced but not present.\n", l->id);
4976                 error = true;
4977             }
4978         }
4979         assert(!error);
4980     }
4981 #endif
4982 
4983 #ifdef CONFIG_PROFILER
4984     qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
4985 #endif
4986 
4987 #ifdef USE_TCG_OPTIMIZATIONS
4988     tcg_optimize(s);
4989 #endif
4990 
4991 #ifdef CONFIG_PROFILER
4992     qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4993     qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
4994 #endif
4995 
4996     reachable_code_pass(s);
4997     liveness_pass_0(s);
4998     liveness_pass_1(s);
4999 
5000     if (s->nb_indirects > 0) {
5001 #ifdef DEBUG_DISAS
5002         if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
5003                      && qemu_log_in_addr_range(pc_start))) {
5004             FILE *logfile = qemu_log_trylock();
5005             if (logfile) {
5006                 fprintf(logfile, "OP before indirect lowering:\n");
5007                 tcg_dump_ops(s, logfile, false);
5008                 fprintf(logfile, "\n");
5009                 qemu_log_unlock(logfile);
5010             }
5011         }
5012 #endif
5013         /* Replace indirect temps with direct temps.  */
5014         if (liveness_pass_2(s)) {
5015             /* If changes were made, re-run liveness.  */
5016             liveness_pass_1(s);
5017         }
5018     }
5019 
5020 #ifdef CONFIG_PROFILER
5021     qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
5022 #endif
5023 
5024 #ifdef DEBUG_DISAS
5025     if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
5026                  && qemu_log_in_addr_range(pc_start))) {
5027         FILE *logfile = qemu_log_trylock();
5028         if (logfile) {
5029             fprintf(logfile, "OP after optimization and liveness analysis:\n");
5030             tcg_dump_ops(s, logfile, true);
5031             fprintf(logfile, "\n");
5032             qemu_log_unlock(logfile);
5033         }
5034     }
5035 #endif
5036 
5037     /* Initialize goto_tb jump offsets. */
5038     tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
5039     tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
5040     tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
5041     tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
5042 
5043     tcg_reg_alloc_start(s);
5044 
5045     /*
5046      * Reset the buffer pointers when restarting after overflow.
5047      * TODO: Move this into translate-all.c with the rest of the
5048      * buffer management.  Having only this done here is confusing.
5049      */
5050     s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
5051     s->code_ptr = s->code_buf;
5052 
5053 #ifdef TCG_TARGET_NEED_LDST_LABELS
5054     QSIMPLEQ_INIT(&s->ldst_labels);
5055 #endif
5056 #ifdef TCG_TARGET_NEED_POOL_LABELS
5057     s->pool_labels = NULL;
5058 #endif
5059 
5060     num_insns = -1;
5061     QTAILQ_FOREACH(op, &s->ops, link) {
5062         TCGOpcode opc = op->opc;
5063 
5064 #ifdef CONFIG_PROFILER
5065         qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
5066 #endif
5067 
5068         switch (opc) {
5069         case INDEX_op_mov_i32:
5070         case INDEX_op_mov_i64:
5071         case INDEX_op_mov_vec:
5072             tcg_reg_alloc_mov(s, op);
5073             break;
5074         case INDEX_op_dup_vec:
5075             tcg_reg_alloc_dup(s, op);
5076             break;
5077         case INDEX_op_insn_start:
5078             if (num_insns >= 0) {
5079                 size_t off = tcg_current_code_size(s);
5080                 s->gen_insn_end_off[num_insns] = off;
5081                 /* Assert that we do not overflow our stored offset.  */
5082                 assert(s->gen_insn_end_off[num_insns] == off);
5083             }
5084             num_insns++;
5085             for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
5086                 target_ulong a;
5087 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
5088                 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
5089 #else
5090                 a = op->args[i];
5091 #endif
5092                 s->gen_insn_data[num_insns][i] = a;
5093             }
5094             break;
5095         case INDEX_op_discard:
5096             temp_dead(s, arg_temp(op->args[0]));
5097             break;
5098         case INDEX_op_set_label:
5099             tcg_reg_alloc_bb_end(s, s->reserved_regs);
5100             tcg_out_label(s, arg_label(op->args[0]));
5101             break;
5102         case INDEX_op_call:
5103             tcg_reg_alloc_call(s, op);
5104             break;
5105         case INDEX_op_exit_tb:
5106             tcg_out_exit_tb(s, op->args[0]);
5107             break;
5108         case INDEX_op_goto_tb:
5109             tcg_out_goto_tb(s, op->args[0]);
5110             break;
5111         case INDEX_op_dup2_vec:
5112             if (tcg_reg_alloc_dup2(s, op)) {
5113                 break;
5114             }
5115             /* fall through */
5116         default:
5117             /* Sanity check that we've not introduced any unhandled opcodes. */
5118             tcg_debug_assert(tcg_op_supported(opc));
5119             /* Note: in order to speed up the code, it would be much
5120                faster to have specialized register allocator functions for
5121                some common argument patterns */
5122             tcg_reg_alloc_op(s, op);
5123             break;
5124         }
5125         /* Test for (pending) buffer overflow.  The assumption is that any
5126            one operation beginning below the high water mark cannot overrun
5127            the buffer completely.  Thus we can test for overflow after
5128            generating code without having to check during generation.  */
5129         if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
5130             return -1;
5131         }
5132         /* Test for TB overflow, as seen by gen_insn_end_off.  */
5133         if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
5134             return -2;
5135         }
5136     }
5137     tcg_debug_assert(num_insns >= 0);
5138     s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
5139 
5140     /* Generate TB finalization at the end of block */
5141 #ifdef TCG_TARGET_NEED_LDST_LABELS
5142     i = tcg_out_ldst_finalize(s);
5143     if (i < 0) {
5144         return i;
5145     }
5146 #endif
5147 #ifdef TCG_TARGET_NEED_POOL_LABELS
5148     i = tcg_out_pool_finalize(s);
5149     if (i < 0) {
5150         return i;
5151     }
5152 #endif
5153     if (!tcg_resolve_relocs(s)) {
5154         return -2;
5155     }
5156 
5157 #ifndef CONFIG_TCG_INTERPRETER
5158     /* flush instruction cache */
5159     flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
5160                         (uintptr_t)s->code_buf,
5161                         tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
5162 #endif
5163 
5164     return tcg_current_code_size(s);
5165 }
5166 
5167 #ifdef CONFIG_PROFILER
5168 void tcg_dump_info(GString *buf)
5169 {
5170     TCGProfile prof = {};
5171     const TCGProfile *s;
5172     int64_t tb_count;
5173     int64_t tb_div_count;
5174     int64_t tot;
5175 
5176     tcg_profile_snapshot_counters(&prof);
5177     s = &prof;
5178     tb_count = s->tb_count;
5179     tb_div_count = tb_count ? tb_count : 1;
5180     tot = s->interm_time + s->code_time;
5181 
5182     g_string_append_printf(buf, "JIT cycles          %" PRId64
5183                            " (%0.3f s at 2.4 GHz)\n",
5184                            tot, tot / 2.4e9);
5185     g_string_append_printf(buf, "translated TBs      %" PRId64
5186                            " (aborted=%" PRId64 " %0.1f%%)\n",
5187                            tb_count, s->tb_count1 - tb_count,
5188                            (double)(s->tb_count1 - s->tb_count)
5189                            / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
5190     g_string_append_printf(buf, "avg ops/TB          %0.1f max=%d\n",
5191                            (double)s->op_count / tb_div_count, s->op_count_max);
5192     g_string_append_printf(buf, "deleted ops/TB      %0.2f\n",
5193                            (double)s->del_op_count / tb_div_count);
5194     g_string_append_printf(buf, "avg temps/TB        %0.2f max=%d\n",
5195                            (double)s->temp_count / tb_div_count,
5196                            s->temp_count_max);
5197     g_string_append_printf(buf, "avg host code/TB    %0.1f\n",
5198                            (double)s->code_out_len / tb_div_count);
5199     g_string_append_printf(buf, "avg search data/TB  %0.1f\n",
5200                            (double)s->search_out_len / tb_div_count);
5201 
5202     g_string_append_printf(buf, "cycles/op           %0.1f\n",
5203                            s->op_count ? (double)tot / s->op_count : 0);
5204     g_string_append_printf(buf, "cycles/in byte      %0.1f\n",
5205                            s->code_in_len ? (double)tot / s->code_in_len : 0);
5206     g_string_append_printf(buf, "cycles/out byte     %0.1f\n",
5207                            s->code_out_len ? (double)tot / s->code_out_len : 0);
5208     g_string_append_printf(buf, "cycles/search byte     %0.1f\n",
5209                            s->search_out_len ?
5210                            (double)tot / s->search_out_len : 0);
5211     if (tot == 0) {
5212         tot = 1;
5213     }
5214     g_string_append_printf(buf, "  gen_interm time   %0.1f%%\n",
5215                            (double)s->interm_time / tot * 100.0);
5216     g_string_append_printf(buf, "  gen_code time     %0.1f%%\n",
5217                            (double)s->code_time / tot * 100.0);
5218     g_string_append_printf(buf, "optim./code time    %0.1f%%\n",
5219                            (double)s->opt_time / (s->code_time ?
5220                                                   s->code_time : 1)
5221                            * 100.0);
5222     g_string_append_printf(buf, "liveness/code time  %0.1f%%\n",
5223                            (double)s->la_time / (s->code_time ?
5224                                                  s->code_time : 1) * 100.0);
5225     g_string_append_printf(buf, "cpu_restore count   %" PRId64 "\n",
5226                            s->restore_count);
5227     g_string_append_printf(buf, "  avg cycles        %0.1f\n",
5228                            s->restore_count ?
5229                            (double)s->restore_time / s->restore_count : 0);
5230 }
5231 #else
5232 void tcg_dump_info(GString *buf)
5233 {
5234     g_string_append_printf(buf, "[TCG profiler not compiled]\n");
5235 }
5236 #endif
5237 
5238 #ifdef ELF_HOST_MACHINE
5239 /* In order to use this feature, the backend needs to do three things:
5240 
5241    (1) Define ELF_HOST_MACHINE to indicate both what value to
5242        put into the ELF image and to indicate support for the feature.
5243 
5244    (2) Define tcg_register_jit.  This should create a buffer containing
5245        the contents of a .debug_frame section that describes the post-
5246        prologue unwind info for the tcg machine.
5247 
5248    (3) Call tcg_register_jit_int, with the constructed .debug_frame.
5249 */
5250 
5251 /* Begin GDB interface.  THE FOLLOWING MUST MATCH GDB DOCS.  */
5252 typedef enum {
5253     JIT_NOACTION = 0,
5254     JIT_REGISTER_FN,
5255     JIT_UNREGISTER_FN
5256 } jit_actions_t;
5257 
5258 struct jit_code_entry {
5259     struct jit_code_entry *next_entry;
5260     struct jit_code_entry *prev_entry;
5261     const void *symfile_addr;
5262     uint64_t symfile_size;
5263 };
5264 
5265 struct jit_descriptor {
5266     uint32_t version;
5267     uint32_t action_flag;
5268     struct jit_code_entry *relevant_entry;
5269     struct jit_code_entry *first_entry;
5270 };
5271 
5272 void __jit_debug_register_code(void) __attribute__((noinline));
5273 void __jit_debug_register_code(void)
5274 {
5275     asm("");
5276 }
5277 
5278 /* Must statically initialize the version, because GDB may check
5279    the version before we can set it.  */
5280 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
5281 
5282 /* End GDB interface.  */
5283 
5284 static int find_string(const char *strtab, const char *str)
5285 {
5286     const char *p = strtab + 1;
5287 
5288     while (1) {
5289         if (strcmp(p, str) == 0) {
5290             return p - strtab;
5291         }
5292         p += strlen(p) + 1;
5293     }
5294 }
5295 
5296 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
5297                                  const void *debug_frame,
5298                                  size_t debug_frame_size)
5299 {
5300     struct __attribute__((packed)) DebugInfo {
5301         uint32_t  len;
5302         uint16_t  version;
5303         uint32_t  abbrev;
5304         uint8_t   ptr_size;
5305         uint8_t   cu_die;
5306         uint16_t  cu_lang;
5307         uintptr_t cu_low_pc;
5308         uintptr_t cu_high_pc;
5309         uint8_t   fn_die;
5310         char      fn_name[16];
5311         uintptr_t fn_low_pc;
5312         uintptr_t fn_high_pc;
5313         uint8_t   cu_eoc;
5314     };
5315 
5316     struct ElfImage {
5317         ElfW(Ehdr) ehdr;
5318         ElfW(Phdr) phdr;
5319         ElfW(Shdr) shdr[7];
5320         ElfW(Sym)  sym[2];
5321         struct DebugInfo di;
5322         uint8_t    da[24];
5323         char       str[80];
5324     };
5325 
5326     struct ElfImage *img;
5327 
5328     static const struct ElfImage img_template = {
5329         .ehdr = {
5330             .e_ident[EI_MAG0] = ELFMAG0,
5331             .e_ident[EI_MAG1] = ELFMAG1,
5332             .e_ident[EI_MAG2] = ELFMAG2,
5333             .e_ident[EI_MAG3] = ELFMAG3,
5334             .e_ident[EI_CLASS] = ELF_CLASS,
5335             .e_ident[EI_DATA] = ELF_DATA,
5336             .e_ident[EI_VERSION] = EV_CURRENT,
5337             .e_type = ET_EXEC,
5338             .e_machine = ELF_HOST_MACHINE,
5339             .e_version = EV_CURRENT,
5340             .e_phoff = offsetof(struct ElfImage, phdr),
5341             .e_shoff = offsetof(struct ElfImage, shdr),
5342             .e_ehsize = sizeof(ElfW(Shdr)),
5343             .e_phentsize = sizeof(ElfW(Phdr)),
5344             .e_phnum = 1,
5345             .e_shentsize = sizeof(ElfW(Shdr)),
5346             .e_shnum = ARRAY_SIZE(img->shdr),
5347             .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
5348 #ifdef ELF_HOST_FLAGS
5349             .e_flags = ELF_HOST_FLAGS,
5350 #endif
5351 #ifdef ELF_OSABI
5352             .e_ident[EI_OSABI] = ELF_OSABI,
5353 #endif
5354         },
5355         .phdr = {
5356             .p_type = PT_LOAD,
5357             .p_flags = PF_X,
5358         },
5359         .shdr = {
5360             [0] = { .sh_type = SHT_NULL },
5361             /* Trick: The contents of code_gen_buffer are not present in
5362                this fake ELF file; that got allocated elsewhere.  Therefore
5363                we mark .text as SHT_NOBITS (similar to .bss) so that readers
5364                will not look for contents.  We can record any address.  */
5365             [1] = { /* .text */
5366                 .sh_type = SHT_NOBITS,
5367                 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
5368             },
5369             [2] = { /* .debug_info */
5370                 .sh_type = SHT_PROGBITS,
5371                 .sh_offset = offsetof(struct ElfImage, di),
5372                 .sh_size = sizeof(struct DebugInfo),
5373             },
5374             [3] = { /* .debug_abbrev */
5375                 .sh_type = SHT_PROGBITS,
5376                 .sh_offset = offsetof(struct ElfImage, da),
5377                 .sh_size = sizeof(img->da),
5378             },
5379             [4] = { /* .debug_frame */
5380                 .sh_type = SHT_PROGBITS,
5381                 .sh_offset = sizeof(struct ElfImage),
5382             },
5383             [5] = { /* .symtab */
5384                 .sh_type = SHT_SYMTAB,
5385                 .sh_offset = offsetof(struct ElfImage, sym),
5386                 .sh_size = sizeof(img->sym),
5387                 .sh_info = 1,
5388                 .sh_link = ARRAY_SIZE(img->shdr) - 1,
5389                 .sh_entsize = sizeof(ElfW(Sym)),
5390             },
5391             [6] = { /* .strtab */
5392                 .sh_type = SHT_STRTAB,
5393                 .sh_offset = offsetof(struct ElfImage, str),
5394                 .sh_size = sizeof(img->str),
5395             }
5396         },
5397         .sym = {
5398             [1] = { /* code_gen_buffer */
5399                 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
5400                 .st_shndx = 1,
5401             }
5402         },
5403         .di = {
5404             .len = sizeof(struct DebugInfo) - 4,
5405             .version = 2,
5406             .ptr_size = sizeof(void *),
5407             .cu_die = 1,
5408             .cu_lang = 0x8001,  /* DW_LANG_Mips_Assembler */
5409             .fn_die = 2,
5410             .fn_name = "code_gen_buffer"
5411         },
5412         .da = {
5413             1,          /* abbrev number (the cu) */
5414             0x11, 1,    /* DW_TAG_compile_unit, has children */
5415             0x13, 0x5,  /* DW_AT_language, DW_FORM_data2 */
5416             0x11, 0x1,  /* DW_AT_low_pc, DW_FORM_addr */
5417             0x12, 0x1,  /* DW_AT_high_pc, DW_FORM_addr */
5418             0, 0,       /* end of abbrev */
5419             2,          /* abbrev number (the fn) */
5420             0x2e, 0,    /* DW_TAG_subprogram, no children */
5421             0x3, 0x8,   /* DW_AT_name, DW_FORM_string */
5422             0x11, 0x1,  /* DW_AT_low_pc, DW_FORM_addr */
5423             0x12, 0x1,  /* DW_AT_high_pc, DW_FORM_addr */
5424             0, 0,       /* end of abbrev */
5425             0           /* no more abbrev */
5426         },
5427         .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
5428                ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
5429     };
5430 
5431     /* We only need a single jit entry; statically allocate it.  */
5432     static struct jit_code_entry one_entry;
5433 
5434     uintptr_t buf = (uintptr_t)buf_ptr;
5435     size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
5436     DebugFrameHeader *dfh;
5437 
5438     img = g_malloc(img_size);
5439     *img = img_template;
5440 
5441     img->phdr.p_vaddr = buf;
5442     img->phdr.p_paddr = buf;
5443     img->phdr.p_memsz = buf_size;
5444 
5445     img->shdr[1].sh_name = find_string(img->str, ".text");
5446     img->shdr[1].sh_addr = buf;
5447     img->shdr[1].sh_size = buf_size;
5448 
5449     img->shdr[2].sh_name = find_string(img->str, ".debug_info");
5450     img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
5451 
5452     img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
5453     img->shdr[4].sh_size = debug_frame_size;
5454 
5455     img->shdr[5].sh_name = find_string(img->str, ".symtab");
5456     img->shdr[6].sh_name = find_string(img->str, ".strtab");
5457 
5458     img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
5459     img->sym[1].st_value = buf;
5460     img->sym[1].st_size = buf_size;
5461 
5462     img->di.cu_low_pc = buf;
5463     img->di.cu_high_pc = buf + buf_size;
5464     img->di.fn_low_pc = buf;
5465     img->di.fn_high_pc = buf + buf_size;
5466 
5467     dfh = (DebugFrameHeader *)(img + 1);
5468     memcpy(dfh, debug_frame, debug_frame_size);
5469     dfh->fde.func_start = buf;
5470     dfh->fde.func_len = buf_size;
5471 
5472 #ifdef DEBUG_JIT
5473     /* Enable this block to be able to debug the ELF image file creation.
5474        One can use readelf, objdump, or other inspection utilities.  */
5475     {
5476         g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
5477         FILE *f = fopen(jit, "w+b");
5478         if (f) {
5479             if (fwrite(img, img_size, 1, f) != img_size) {
5480                 /* Avoid stupid unused return value warning for fwrite.  */
5481             }
5482             fclose(f);
5483         }
5484     }
5485 #endif
5486 
5487     one_entry.symfile_addr = img;
5488     one_entry.symfile_size = img_size;
5489 
5490     __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
5491     __jit_debug_descriptor.relevant_entry = &one_entry;
5492     __jit_debug_descriptor.first_entry = &one_entry;
5493     __jit_debug_register_code();
5494 }
5495 #else
5496 /* No support for the feature.  Provide the entry point expected by exec.c,
5497    and implement the internal function we declared earlier.  */
5498 
5499 static void tcg_register_jit_int(const void *buf, size_t size,
5500                                  const void *debug_frame,
5501                                  size_t debug_frame_size)
5502 {
5503 }
5504 
5505 void tcg_register_jit(const void *buf, size_t buf_size)
5506 {
5507 }
5508 #endif /* ELF_HOST_MACHINE */
5509 
5510 #if !TCG_TARGET_MAYBE_vec
5511 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
5512 {
5513     g_assert_not_reached();
5514 }
5515 #endif
5516