xref: /openbmc/qemu/include/tcg/tcg.h (revision 3161f9f4)
1 /*
2  * Tiny Code Generator for QEMU
3  *
4  * Copyright (c) 2008 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
25 #ifndef TCG_H
26 #define TCG_H
27 
28 #include "exec/memop.h"
29 #include "exec/memopidx.h"
30 #include "qemu/bitops.h"
31 #include "qemu/plugin.h"
32 #include "qemu/queue.h"
33 #include "tcg/tcg-mo.h"
34 #include "tcg-target-reg-bits.h"
35 #include "tcg-target.h"
36 #include "tcg/tcg-cond.h"
37 #include "tcg/debug-assert.h"
38 
39 /* XXX: make safe guess about sizes */
40 #define MAX_OP_PER_INSTR 266
41 
42 #define MAX_CALL_IARGS  7
43 
44 #define CPU_TEMP_BUF_NLONGS 128
45 #define TCG_STATIC_FRAME_SIZE  (CPU_TEMP_BUF_NLONGS * sizeof(long))
46 
47 #if TCG_TARGET_REG_BITS == 32
48 typedef int32_t tcg_target_long;
49 typedef uint32_t tcg_target_ulong;
50 #define TCG_PRIlx PRIx32
51 #define TCG_PRIld PRId32
52 #elif TCG_TARGET_REG_BITS == 64
53 typedef int64_t tcg_target_long;
54 typedef uint64_t tcg_target_ulong;
55 #define TCG_PRIlx PRIx64
56 #define TCG_PRIld PRId64
57 #else
58 #error unsupported
59 #endif
60 
61 #if TCG_TARGET_NB_REGS <= 32
62 typedef uint32_t TCGRegSet;
63 #elif TCG_TARGET_NB_REGS <= 64
64 typedef uint64_t TCGRegSet;
65 #else
66 #error unsupported
67 #endif
68 
69 #if TCG_TARGET_REG_BITS == 32
70 /* Turn some undef macros into false macros.  */
71 #define TCG_TARGET_HAS_extr_i64_i32     0
72 #define TCG_TARGET_HAS_div_i64          0
73 #define TCG_TARGET_HAS_rem_i64          0
74 #define TCG_TARGET_HAS_div2_i64         0
75 #define TCG_TARGET_HAS_rot_i64          0
76 #define TCG_TARGET_HAS_ext8s_i64        0
77 #define TCG_TARGET_HAS_ext16s_i64       0
78 #define TCG_TARGET_HAS_ext32s_i64       0
79 #define TCG_TARGET_HAS_ext8u_i64        0
80 #define TCG_TARGET_HAS_ext16u_i64       0
81 #define TCG_TARGET_HAS_ext32u_i64       0
82 #define TCG_TARGET_HAS_bswap16_i64      0
83 #define TCG_TARGET_HAS_bswap32_i64      0
84 #define TCG_TARGET_HAS_bswap64_i64      0
85 #define TCG_TARGET_HAS_not_i64          0
86 #define TCG_TARGET_HAS_andc_i64         0
87 #define TCG_TARGET_HAS_orc_i64          0
88 #define TCG_TARGET_HAS_eqv_i64          0
89 #define TCG_TARGET_HAS_nand_i64         0
90 #define TCG_TARGET_HAS_nor_i64          0
91 #define TCG_TARGET_HAS_clz_i64          0
92 #define TCG_TARGET_HAS_ctz_i64          0
93 #define TCG_TARGET_HAS_ctpop_i64        0
94 #define TCG_TARGET_HAS_deposit_i64      0
95 #define TCG_TARGET_HAS_extract_i64      0
96 #define TCG_TARGET_HAS_sextract_i64     0
97 #define TCG_TARGET_HAS_extract2_i64     0
98 #define TCG_TARGET_HAS_negsetcond_i64   0
99 #define TCG_TARGET_HAS_add2_i64         0
100 #define TCG_TARGET_HAS_sub2_i64         0
101 #define TCG_TARGET_HAS_mulu2_i64        0
102 #define TCG_TARGET_HAS_muls2_i64        0
103 #define TCG_TARGET_HAS_muluh_i64        0
104 #define TCG_TARGET_HAS_mulsh_i64        0
105 /* Turn some undef macros into true macros.  */
106 #define TCG_TARGET_HAS_add2_i32         1
107 #define TCG_TARGET_HAS_sub2_i32         1
108 #endif
109 
110 #ifndef TCG_TARGET_deposit_i32_valid
111 #define TCG_TARGET_deposit_i32_valid(ofs, len) 1
112 #endif
113 #ifndef TCG_TARGET_deposit_i64_valid
114 #define TCG_TARGET_deposit_i64_valid(ofs, len) 1
115 #endif
116 #ifndef TCG_TARGET_extract_i32_valid
117 #define TCG_TARGET_extract_i32_valid(ofs, len) 1
118 #endif
119 #ifndef TCG_TARGET_extract_i64_valid
120 #define TCG_TARGET_extract_i64_valid(ofs, len) 1
121 #endif
122 
123 /* Only one of DIV or DIV2 should be defined.  */
124 #if defined(TCG_TARGET_HAS_div_i32)
125 #define TCG_TARGET_HAS_div2_i32         0
126 #elif defined(TCG_TARGET_HAS_div2_i32)
127 #define TCG_TARGET_HAS_div_i32          0
128 #define TCG_TARGET_HAS_rem_i32          0
129 #endif
130 #if defined(TCG_TARGET_HAS_div_i64)
131 #define TCG_TARGET_HAS_div2_i64         0
132 #elif defined(TCG_TARGET_HAS_div2_i64)
133 #define TCG_TARGET_HAS_div_i64          0
134 #define TCG_TARGET_HAS_rem_i64          0
135 #endif
136 
137 #if !defined(TCG_TARGET_HAS_v64) \
138     && !defined(TCG_TARGET_HAS_v128) \
139     && !defined(TCG_TARGET_HAS_v256)
140 #define TCG_TARGET_MAYBE_vec            0
141 #define TCG_TARGET_HAS_abs_vec          0
142 #define TCG_TARGET_HAS_neg_vec          0
143 #define TCG_TARGET_HAS_not_vec          0
144 #define TCG_TARGET_HAS_andc_vec         0
145 #define TCG_TARGET_HAS_orc_vec          0
146 #define TCG_TARGET_HAS_nand_vec         0
147 #define TCG_TARGET_HAS_nor_vec          0
148 #define TCG_TARGET_HAS_eqv_vec          0
149 #define TCG_TARGET_HAS_roti_vec         0
150 #define TCG_TARGET_HAS_rots_vec         0
151 #define TCG_TARGET_HAS_rotv_vec         0
152 #define TCG_TARGET_HAS_shi_vec          0
153 #define TCG_TARGET_HAS_shs_vec          0
154 #define TCG_TARGET_HAS_shv_vec          0
155 #define TCG_TARGET_HAS_mul_vec          0
156 #define TCG_TARGET_HAS_sat_vec          0
157 #define TCG_TARGET_HAS_minmax_vec       0
158 #define TCG_TARGET_HAS_bitsel_vec       0
159 #define TCG_TARGET_HAS_cmpsel_vec       0
160 #else
161 #define TCG_TARGET_MAYBE_vec            1
162 #endif
163 #ifndef TCG_TARGET_HAS_v64
164 #define TCG_TARGET_HAS_v64              0
165 #endif
166 #ifndef TCG_TARGET_HAS_v128
167 #define TCG_TARGET_HAS_v128             0
168 #endif
169 #ifndef TCG_TARGET_HAS_v256
170 #define TCG_TARGET_HAS_v256             0
171 #endif
172 
173 typedef enum TCGOpcode {
174 #define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
175 #include "tcg/tcg-opc.h"
176 #undef DEF
177     NB_OPS,
178 } TCGOpcode;
179 
180 #define tcg_regset_set_reg(d, r)   ((d) |= (TCGRegSet)1 << (r))
181 #define tcg_regset_reset_reg(d, r) ((d) &= ~((TCGRegSet)1 << (r)))
182 #define tcg_regset_test_reg(d, r)  (((d) >> (r)) & 1)
183 
184 #ifndef TCG_TARGET_INSN_UNIT_SIZE
185 # error "Missing TCG_TARGET_INSN_UNIT_SIZE"
186 #elif TCG_TARGET_INSN_UNIT_SIZE == 1
187 typedef uint8_t tcg_insn_unit;
188 #elif TCG_TARGET_INSN_UNIT_SIZE == 2
189 typedef uint16_t tcg_insn_unit;
190 #elif TCG_TARGET_INSN_UNIT_SIZE == 4
191 typedef uint32_t tcg_insn_unit;
192 #elif TCG_TARGET_INSN_UNIT_SIZE == 8
193 typedef uint64_t tcg_insn_unit;
194 #else
195 /* The port better have done this.  */
196 #endif
197 
198 typedef struct TCGRelocation TCGRelocation;
199 struct TCGRelocation {
200     QSIMPLEQ_ENTRY(TCGRelocation) next;
201     tcg_insn_unit *ptr;
202     intptr_t addend;
203     int type;
204 };
205 
206 typedef struct TCGOp TCGOp;
207 typedef struct TCGLabelUse TCGLabelUse;
208 struct TCGLabelUse {
209     QSIMPLEQ_ENTRY(TCGLabelUse) next;
210     TCGOp *op;
211 };
212 
213 typedef struct TCGLabel TCGLabel;
214 struct TCGLabel {
215     bool present;
216     bool has_value;
217     uint16_t id;
218     union {
219         uintptr_t value;
220         const tcg_insn_unit *value_ptr;
221     } u;
222     QSIMPLEQ_HEAD(, TCGLabelUse) branches;
223     QSIMPLEQ_HEAD(, TCGRelocation) relocs;
224     QSIMPLEQ_ENTRY(TCGLabel) next;
225 };
226 
227 typedef struct TCGPool {
228     struct TCGPool *next;
229     int size;
230     uint8_t data[] __attribute__ ((aligned));
231 } TCGPool;
232 
233 #define TCG_POOL_CHUNK_SIZE 32768
234 
235 #define TCG_MAX_TEMPS 512
236 #define TCG_MAX_INSNS 512
237 
238 /* when the size of the arguments of a called function is smaller than
239    this value, they are statically allocated in the TB stack frame */
240 #define TCG_STATIC_CALL_ARGS_SIZE 128
241 
242 typedef enum TCGType {
243     TCG_TYPE_I32,
244     TCG_TYPE_I64,
245     TCG_TYPE_I128,
246 
247     TCG_TYPE_V64,
248     TCG_TYPE_V128,
249     TCG_TYPE_V256,
250 
251     /* Number of different types (integer not enum) */
252 #define TCG_TYPE_COUNT  (TCG_TYPE_V256 + 1)
253 
254     /* An alias for the size of the host register.  */
255 #if TCG_TARGET_REG_BITS == 32
256     TCG_TYPE_REG = TCG_TYPE_I32,
257 #else
258     TCG_TYPE_REG = TCG_TYPE_I64,
259 #endif
260 
261     /* An alias for the size of the native pointer.  */
262 #if UINTPTR_MAX == UINT32_MAX
263     TCG_TYPE_PTR = TCG_TYPE_I32,
264 #else
265     TCG_TYPE_PTR = TCG_TYPE_I64,
266 #endif
267 } TCGType;
268 
269 /**
270  * tcg_type_size
271  * @t: type
272  *
273  * Return the size of the type in bytes.
274  */
275 static inline int tcg_type_size(TCGType t)
276 {
277     unsigned i = t;
278     if (i >= TCG_TYPE_V64) {
279         tcg_debug_assert(i < TCG_TYPE_COUNT);
280         i -= TCG_TYPE_V64 - 1;
281     }
282     return 4 << i;
283 }
284 
285 /**
286  * get_alignment_bits
287  * @memop: MemOp value
288  *
289  * Extract the alignment size from the memop.
290  */
291 static inline unsigned get_alignment_bits(MemOp memop)
292 {
293     unsigned a = memop & MO_AMASK;
294 
295     if (a == MO_UNALN) {
296         /* No alignment required.  */
297         a = 0;
298     } else if (a == MO_ALIGN) {
299         /* A natural alignment requirement.  */
300         a = memop & MO_SIZE;
301     } else {
302         /* A specific alignment requirement.  */
303         a = a >> MO_ASHIFT;
304     }
305     return a;
306 }
307 
308 typedef tcg_target_ulong TCGArg;
309 
310 /* Define type and accessor macros for TCG variables.
311 
312    TCG variables are the inputs and outputs of TCG ops, as described
313    in tcg/README. Target CPU front-end code uses these types to deal
314    with TCG variables as it emits TCG code via the tcg_gen_* functions.
315    They come in several flavours:
316     * TCGv_i32  : 32 bit integer type
317     * TCGv_i64  : 64 bit integer type
318     * TCGv_i128 : 128 bit integer type
319     * TCGv_ptr  : a host pointer type
320     * TCGv_vec  : a host vector type; the exact size is not exposed
321                   to the CPU front-end code.
322     * TCGv      : an integer type the same size as target_ulong
323                   (an alias for either TCGv_i32 or TCGv_i64)
324    The compiler's type checking will complain if you mix them
325    up and pass the wrong sized TCGv to a function.
326 
327    Users of tcg_gen_* don't need to know about any of the internal
328    details of these, and should treat them as opaque types.
329    You won't be able to look inside them in a debugger either.
330 
331    Internal implementation details follow:
332 
333    Note that there is no definition of the structs TCGv_i32_d etc anywhere.
334    This is deliberate, because the values we store in variables of type
335    TCGv_i32 are not really pointers-to-structures. They're just small
336    integers, but keeping them in pointer types like this means that the
337    compiler will complain if you accidentally pass a TCGv_i32 to a
338    function which takes a TCGv_i64, and so on. Only the internals of
339    TCG need to care about the actual contents of the types.  */
340 
341 typedef struct TCGv_i32_d *TCGv_i32;
342 typedef struct TCGv_i64_d *TCGv_i64;
343 typedef struct TCGv_i128_d *TCGv_i128;
344 typedef struct TCGv_ptr_d *TCGv_ptr;
345 typedef struct TCGv_vec_d *TCGv_vec;
346 typedef TCGv_ptr TCGv_env;
347 
348 /* call flags */
349 /* Helper does not read globals (either directly or through an exception). It
350    implies TCG_CALL_NO_WRITE_GLOBALS. */
351 #define TCG_CALL_NO_READ_GLOBALS    0x0001
352 /* Helper does not write globals */
353 #define TCG_CALL_NO_WRITE_GLOBALS   0x0002
354 /* Helper can be safely suppressed if the return value is not used. */
355 #define TCG_CALL_NO_SIDE_EFFECTS    0x0004
356 /* Helper is G_NORETURN.  */
357 #define TCG_CALL_NO_RETURN          0x0008
358 /* Helper is part of Plugins.  */
359 #define TCG_CALL_PLUGIN             0x0010
360 
361 /* convenience version of most used call flags */
362 #define TCG_CALL_NO_RWG         TCG_CALL_NO_READ_GLOBALS
363 #define TCG_CALL_NO_WG          TCG_CALL_NO_WRITE_GLOBALS
364 #define TCG_CALL_NO_SE          TCG_CALL_NO_SIDE_EFFECTS
365 #define TCG_CALL_NO_RWG_SE      (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
366 #define TCG_CALL_NO_WG_SE       (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
367 
368 /*
369  * Flags for the bswap opcodes.
370  * If IZ, the input is zero-extended, otherwise unknown.
371  * If OZ or OS, the output is zero- or sign-extended respectively,
372  * otherwise the high bits are undefined.
373  */
374 enum {
375     TCG_BSWAP_IZ = 1,
376     TCG_BSWAP_OZ = 2,
377     TCG_BSWAP_OS = 4,
378 };
379 
380 typedef enum TCGTempVal {
381     TEMP_VAL_DEAD,
382     TEMP_VAL_REG,
383     TEMP_VAL_MEM,
384     TEMP_VAL_CONST,
385 } TCGTempVal;
386 
387 typedef enum TCGTempKind {
388     /*
389      * Temp is dead at the end of the extended basic block (EBB),
390      * the single-entry multiple-exit region that falls through
391      * conditional branches.
392      */
393     TEMP_EBB,
394     /* Temp is live across the entire translation block, but dead at end. */
395     TEMP_TB,
396     /* Temp is live across the entire translation block, and between them. */
397     TEMP_GLOBAL,
398     /* Temp is in a fixed register. */
399     TEMP_FIXED,
400     /* Temp is a fixed constant. */
401     TEMP_CONST,
402 } TCGTempKind;
403 
404 typedef struct TCGTemp {
405     TCGReg reg:8;
406     TCGTempVal val_type:8;
407     TCGType base_type:8;
408     TCGType type:8;
409     TCGTempKind kind:3;
410     unsigned int indirect_reg:1;
411     unsigned int indirect_base:1;
412     unsigned int mem_coherent:1;
413     unsigned int mem_allocated:1;
414     unsigned int temp_allocated:1;
415     unsigned int temp_subindex:1;
416 
417     int64_t val;
418     struct TCGTemp *mem_base;
419     intptr_t mem_offset;
420     const char *name;
421 
422     /* Pass-specific information that can be stored for a temporary.
423        One word worth of integer data, and one pointer to data
424        allocated separately.  */
425     uintptr_t state;
426     void *state_ptr;
427 } TCGTemp;
428 
429 typedef struct TCGContext TCGContext;
430 
431 typedef struct TCGTempSet {
432     unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
433 } TCGTempSet;
434 
435 /*
436  * With 1 128-bit output, a 32-bit host requires 4 output parameters,
437  * which leaves a maximum of 28 other slots.  Which is enough for 7
438  * 128-bit operands.
439  */
440 #define DEAD_ARG  (1 << 4)
441 #define SYNC_ARG  (1 << 0)
442 typedef uint32_t TCGLifeData;
443 
444 struct TCGOp {
445     TCGOpcode opc   : 8;
446     unsigned nargs  : 8;
447 
448     /* Parameters for this opcode.  See below.  */
449     unsigned param1 : 8;
450     unsigned param2 : 8;
451 
452     /* Lifetime data of the operands.  */
453     TCGLifeData life;
454 
455     /* Next and previous opcodes.  */
456     QTAILQ_ENTRY(TCGOp) link;
457 
458     /* Register preferences for the output(s).  */
459     TCGRegSet output_pref[2];
460 
461     /* Arguments for the opcode.  */
462     TCGArg args[];
463 };
464 
465 #define TCGOP_CALLI(X)    (X)->param1
466 #define TCGOP_CALLO(X)    (X)->param2
467 
468 #define TCGOP_VECL(X)     (X)->param1
469 #define TCGOP_VECE(X)     (X)->param2
470 
471 /* Make sure operands fit in the bitfields above.  */
472 QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
473 
474 static inline TCGRegSet output_pref(const TCGOp *op, unsigned i)
475 {
476     return i < ARRAY_SIZE(op->output_pref) ? op->output_pref[i] : 0;
477 }
478 
479 struct TCGContext {
480     uint8_t *pool_cur, *pool_end;
481     TCGPool *pool_first, *pool_current, *pool_first_large;
482     int nb_labels;
483     int nb_globals;
484     int nb_temps;
485     int nb_indirects;
486     int nb_ops;
487     TCGType addr_type;            /* TCG_TYPE_I32 or TCG_TYPE_I64 */
488 
489     int page_mask;
490     uint8_t page_bits;
491     uint8_t tlb_dyn_max_bits;
492     uint8_t insn_start_words;
493     TCGBar guest_mo;
494 
495     TCGRegSet reserved_regs;
496     intptr_t current_frame_offset;
497     intptr_t frame_start;
498     intptr_t frame_end;
499     TCGTemp *frame_temp;
500 
501     TranslationBlock *gen_tb;     /* tb for which code is being generated */
502     tcg_insn_unit *code_buf;      /* pointer for start of tb */
503     tcg_insn_unit *code_ptr;      /* pointer for running end of tb */
504 
505 #ifdef CONFIG_DEBUG_TCG
506     int goto_tb_issue_mask;
507     const TCGOpcode *vecop_list;
508 #endif
509 
510     /* Code generation.  Note that we specifically do not use tcg_insn_unit
511        here, because there's too much arithmetic throughout that relies
512        on addition and subtraction working on bytes.  Rely on the GCC
513        extension that allows arithmetic on void*.  */
514     void *code_gen_buffer;
515     size_t code_gen_buffer_size;
516     void *code_gen_ptr;
517     void *data_gen_ptr;
518 
519     /* Threshold to flush the translated code buffer.  */
520     void *code_gen_highwater;
521 
522     /* Track which vCPU triggers events */
523     CPUState *cpu;                      /* *_trans */
524 
525     /* These structures are private to tcg-target.c.inc.  */
526 #ifdef TCG_TARGET_NEED_LDST_LABELS
527     QSIMPLEQ_HEAD(, TCGLabelQemuLdst) ldst_labels;
528 #endif
529 #ifdef TCG_TARGET_NEED_POOL_LABELS
530     struct TCGLabelPoolData *pool_labels;
531 #endif
532 
533     TCGLabel *exitreq_label;
534 
535 #ifdef CONFIG_PLUGIN
536     /*
537      * We keep one plugin_tb struct per TCGContext. Note that on every TB
538      * translation we clear but do not free its contents; this way we
539      * avoid a lot of malloc/free churn, since after a few TB's it's
540      * unlikely that we'll need to allocate either more instructions or more
541      * space for instructions (for variable-instruction-length ISAs).
542      */
543     struct qemu_plugin_tb *plugin_tb;
544 
545     /* descriptor of the instruction being translated */
546     struct qemu_plugin_insn *plugin_insn;
547 #endif
548 
549     GHashTable *const_table[TCG_TYPE_COUNT];
550     TCGTempSet free_temps[TCG_TYPE_COUNT];
551     TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */
552 
553     QTAILQ_HEAD(, TCGOp) ops, free_ops;
554     QSIMPLEQ_HEAD(, TCGLabel) labels;
555 
556     /* Tells which temporary holds a given register.
557        It does not take into account fixed registers */
558     TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
559 
560     uint16_t gen_insn_end_off[TCG_MAX_INSNS];
561     uint64_t *gen_insn_data;
562 
563     /* Exit to translator on overflow. */
564     sigjmp_buf jmp_trans;
565 };
566 
567 static inline bool temp_readonly(TCGTemp *ts)
568 {
569     return ts->kind >= TEMP_FIXED;
570 }
571 
572 #ifdef CONFIG_USER_ONLY
573 extern bool tcg_use_softmmu;
574 #else
575 #define tcg_use_softmmu  true
576 #endif
577 
578 extern __thread TCGContext *tcg_ctx;
579 extern const void *tcg_code_gen_epilogue;
580 extern uintptr_t tcg_splitwx_diff;
581 extern TCGv_env tcg_env;
582 
583 bool in_code_gen_buffer(const void *p);
584 
585 #ifdef CONFIG_DEBUG_TCG
586 const void *tcg_splitwx_to_rx(void *rw);
587 void *tcg_splitwx_to_rw(const void *rx);
588 #else
589 static inline const void *tcg_splitwx_to_rx(void *rw)
590 {
591     return rw ? rw + tcg_splitwx_diff : NULL;
592 }
593 
594 static inline void *tcg_splitwx_to_rw(const void *rx)
595 {
596     return rx ? (void *)rx - tcg_splitwx_diff : NULL;
597 }
598 #endif
599 
600 static inline TCGArg temp_arg(TCGTemp *ts)
601 {
602     return (uintptr_t)ts;
603 }
604 
605 static inline TCGTemp *arg_temp(TCGArg a)
606 {
607     return (TCGTemp *)(uintptr_t)a;
608 }
609 
610 #ifdef CONFIG_DEBUG_TCG
611 size_t temp_idx(TCGTemp *ts);
612 TCGTemp *tcgv_i32_temp(TCGv_i32 v);
613 #else
614 static inline size_t temp_idx(TCGTemp *ts)
615 {
616     return ts - tcg_ctx->temps;
617 }
618 
619 /*
620  * Using the offset of a temporary, relative to TCGContext, rather than
621  * its index means that we don't use 0.  That leaves offset 0 free for
622  * a NULL representation without having to leave index 0 unused.
623  */
624 static inline TCGTemp *tcgv_i32_temp(TCGv_i32 v)
625 {
626     return (void *)tcg_ctx + (uintptr_t)v;
627 }
628 #endif
629 
630 static inline TCGTemp *tcgv_i64_temp(TCGv_i64 v)
631 {
632     return tcgv_i32_temp((TCGv_i32)v);
633 }
634 
635 static inline TCGTemp *tcgv_i128_temp(TCGv_i128 v)
636 {
637     return tcgv_i32_temp((TCGv_i32)v);
638 }
639 
640 static inline TCGTemp *tcgv_ptr_temp(TCGv_ptr v)
641 {
642     return tcgv_i32_temp((TCGv_i32)v);
643 }
644 
645 static inline TCGTemp *tcgv_vec_temp(TCGv_vec v)
646 {
647     return tcgv_i32_temp((TCGv_i32)v);
648 }
649 
650 static inline TCGArg tcgv_i32_arg(TCGv_i32 v)
651 {
652     return temp_arg(tcgv_i32_temp(v));
653 }
654 
655 static inline TCGArg tcgv_i64_arg(TCGv_i64 v)
656 {
657     return temp_arg(tcgv_i64_temp(v));
658 }
659 
660 static inline TCGArg tcgv_i128_arg(TCGv_i128 v)
661 {
662     return temp_arg(tcgv_i128_temp(v));
663 }
664 
665 static inline TCGArg tcgv_ptr_arg(TCGv_ptr v)
666 {
667     return temp_arg(tcgv_ptr_temp(v));
668 }
669 
670 static inline TCGArg tcgv_vec_arg(TCGv_vec v)
671 {
672     return temp_arg(tcgv_vec_temp(v));
673 }
674 
675 static inline TCGv_i32 temp_tcgv_i32(TCGTemp *t)
676 {
677     (void)temp_idx(t); /* trigger embedded assert */
678     return (TCGv_i32)((void *)t - (void *)tcg_ctx);
679 }
680 
681 static inline TCGv_i64 temp_tcgv_i64(TCGTemp *t)
682 {
683     return (TCGv_i64)temp_tcgv_i32(t);
684 }
685 
686 static inline TCGv_i128 temp_tcgv_i128(TCGTemp *t)
687 {
688     return (TCGv_i128)temp_tcgv_i32(t);
689 }
690 
691 static inline TCGv_ptr temp_tcgv_ptr(TCGTemp *t)
692 {
693     return (TCGv_ptr)temp_tcgv_i32(t);
694 }
695 
696 static inline TCGv_vec temp_tcgv_vec(TCGTemp *t)
697 {
698     return (TCGv_vec)temp_tcgv_i32(t);
699 }
700 
701 static inline TCGArg tcg_get_insn_param(TCGOp *op, int arg)
702 {
703     return op->args[arg];
704 }
705 
706 static inline void tcg_set_insn_param(TCGOp *op, int arg, TCGArg v)
707 {
708     op->args[arg] = v;
709 }
710 
711 static inline uint64_t tcg_get_insn_start_param(TCGOp *op, int arg)
712 {
713     if (TCG_TARGET_REG_BITS == 64) {
714         return tcg_get_insn_param(op, arg);
715     } else {
716         return deposit64(tcg_get_insn_param(op, arg * 2), 32, 32,
717                          tcg_get_insn_param(op, arg * 2 + 1));
718     }
719 }
720 
721 static inline void tcg_set_insn_start_param(TCGOp *op, int arg, uint64_t v)
722 {
723     if (TCG_TARGET_REG_BITS == 64) {
724         tcg_set_insn_param(op, arg, v);
725     } else {
726         tcg_set_insn_param(op, arg * 2, v);
727         tcg_set_insn_param(op, arg * 2 + 1, v >> 32);
728     }
729 }
730 
731 /* The last op that was emitted.  */
732 static inline TCGOp *tcg_last_op(void)
733 {
734     return QTAILQ_LAST(&tcg_ctx->ops);
735 }
736 
737 /* Test for whether to terminate the TB for using too many opcodes.  */
738 static inline bool tcg_op_buf_full(void)
739 {
740     /* This is not a hard limit, it merely stops translation when
741      * we have produced "enough" opcodes.  We want to limit TB size
742      * such that a RISC host can reasonably use a 16-bit signed
743      * branch within the TB.  We also need to be mindful of the
744      * 16-bit unsigned offsets, TranslationBlock.jmp_reset_offset[]
745      * and TCGContext.gen_insn_end_off[].
746      */
747     return tcg_ctx->nb_ops >= 4000;
748 }
749 
750 /* pool based memory allocation */
751 
752 /* user-mode: mmap_lock must be held for tcg_malloc_internal. */
753 void *tcg_malloc_internal(TCGContext *s, int size);
754 void tcg_pool_reset(TCGContext *s);
755 TranslationBlock *tcg_tb_alloc(TCGContext *s);
756 
757 void tcg_region_reset_all(void);
758 
759 size_t tcg_code_size(void);
760 size_t tcg_code_capacity(void);
761 
762 void tcg_tb_insert(TranslationBlock *tb);
763 void tcg_tb_remove(TranslationBlock *tb);
764 TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr);
765 void tcg_tb_foreach(GTraverseFunc func, gpointer user_data);
766 size_t tcg_nb_tbs(void);
767 
768 /* user-mode: Called with mmap_lock held.  */
769 static inline void *tcg_malloc(int size)
770 {
771     TCGContext *s = tcg_ctx;
772     uint8_t *ptr, *ptr_end;
773 
774     /* ??? This is a weak placeholder for minimum malloc alignment.  */
775     size = QEMU_ALIGN_UP(size, 8);
776 
777     ptr = s->pool_cur;
778     ptr_end = ptr + size;
779     if (unlikely(ptr_end > s->pool_end)) {
780         return tcg_malloc_internal(tcg_ctx, size);
781     } else {
782         s->pool_cur = ptr_end;
783         return ptr;
784     }
785 }
786 
787 void tcg_func_start(TCGContext *s);
788 
789 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start);
790 
791 void tb_target_set_jmp_target(const TranslationBlock *, int,
792                               uintptr_t, uintptr_t);
793 
794 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
795 
796 #define TCG_CT_CONST  1 /* any constant of register size */
797 
798 typedef struct TCGArgConstraint {
799     unsigned ct : 16;
800     unsigned alias_index : 4;
801     unsigned sort_index : 4;
802     unsigned pair_index : 4;
803     unsigned pair : 2;  /* 0: none, 1: first, 2: second, 3: second alias */
804     bool oalias : 1;
805     bool ialias : 1;
806     bool newreg : 1;
807     TCGRegSet regs;
808 } TCGArgConstraint;
809 
810 #define TCG_MAX_OP_ARGS 16
811 
812 /* Bits for TCGOpDef->flags, 8 bits available, all used.  */
813 enum {
814     /* Instruction exits the translation block.  */
815     TCG_OPF_BB_EXIT      = 0x01,
816     /* Instruction defines the end of a basic block.  */
817     TCG_OPF_BB_END       = 0x02,
818     /* Instruction clobbers call registers and potentially update globals.  */
819     TCG_OPF_CALL_CLOBBER = 0x04,
820     /* Instruction has side effects: it cannot be removed if its outputs
821        are not used, and might trigger exceptions.  */
822     TCG_OPF_SIDE_EFFECTS = 0x08,
823     /* Instruction operands are 64-bits (otherwise 32-bits).  */
824     TCG_OPF_64BIT        = 0x10,
825     /* Instruction is optional and not implemented by the host, or insn
826        is generic and should not be implemented by the host.  */
827     TCG_OPF_NOT_PRESENT  = 0x20,
828     /* Instruction operands are vectors.  */
829     TCG_OPF_VECTOR       = 0x40,
830     /* Instruction is a conditional branch. */
831     TCG_OPF_COND_BRANCH  = 0x80
832 };
833 
834 typedef struct TCGOpDef {
835     const char *name;
836     uint8_t nb_oargs, nb_iargs, nb_cargs, nb_args;
837     uint8_t flags;
838     TCGArgConstraint *args_ct;
839 } TCGOpDef;
840 
841 extern TCGOpDef tcg_op_defs[];
842 extern const size_t tcg_op_defs_max;
843 
844 typedef struct TCGTargetOpDef {
845     TCGOpcode op;
846     const char *args_ct_str[TCG_MAX_OP_ARGS];
847 } TCGTargetOpDef;
848 
849 bool tcg_op_supported(TCGOpcode op);
850 
851 void tcg_gen_call0(TCGHelperInfo *, TCGTemp *ret);
852 void tcg_gen_call1(TCGHelperInfo *, TCGTemp *ret, TCGTemp *);
853 void tcg_gen_call2(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *);
854 void tcg_gen_call3(TCGHelperInfo *, TCGTemp *ret, TCGTemp *,
855                    TCGTemp *, TCGTemp *);
856 void tcg_gen_call4(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
857                    TCGTemp *, TCGTemp *);
858 void tcg_gen_call5(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
859                    TCGTemp *, TCGTemp *, TCGTemp *);
860 void tcg_gen_call6(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
861                    TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *);
862 void tcg_gen_call7(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *,
863                    TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *);
864 
865 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs);
866 void tcg_op_remove(TCGContext *s, TCGOp *op);
867 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op,
868                             TCGOpcode opc, unsigned nargs);
869 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op,
870                            TCGOpcode opc, unsigned nargs);
871 
872 /**
873  * tcg_remove_ops_after:
874  * @op: target operation
875  *
876  * Discard any opcodes emitted since @op.  Expected usage is to save
877  * a starting point with tcg_last_op(), speculatively emit opcodes,
878  * then decide whether or not to keep those opcodes after the fact.
879  */
880 void tcg_remove_ops_after(TCGOp *op);
881 
882 void tcg_optimize(TCGContext *s);
883 
884 TCGLabel *gen_new_label(void);
885 
886 /**
887  * label_arg
888  * @l: label
889  *
890  * Encode a label for storage in the TCG opcode stream.
891  */
892 
893 static inline TCGArg label_arg(TCGLabel *l)
894 {
895     return (uintptr_t)l;
896 }
897 
898 /**
899  * arg_label
900  * @i: value
901  *
902  * The opposite of label_arg.  Retrieve a label from the
903  * encoding of the TCG opcode stream.
904  */
905 
906 static inline TCGLabel *arg_label(TCGArg i)
907 {
908     return (TCGLabel *)(uintptr_t)i;
909 }
910 
911 /**
912  * tcg_ptr_byte_diff
913  * @a, @b: addresses to be differenced
914  *
915  * There are many places within the TCG backends where we need a byte
916  * difference between two pointers.  While this can be accomplished
917  * with local casting, it's easy to get wrong -- especially if one is
918  * concerned with the signedness of the result.
919  *
920  * This version relies on GCC's void pointer arithmetic to get the
921  * correct result.
922  */
923 
924 static inline ptrdiff_t tcg_ptr_byte_diff(const void *a, const void *b)
925 {
926     return a - b;
927 }
928 
929 /**
930  * tcg_pcrel_diff
931  * @s: the tcg context
932  * @target: address of the target
933  *
934  * Produce a pc-relative difference, from the current code_ptr
935  * to the destination address.
936  */
937 
938 static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, const void *target)
939 {
940     return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_ptr));
941 }
942 
943 /**
944  * tcg_tbrel_diff
945  * @s: the tcg context
946  * @target: address of the target
947  *
948  * Produce a difference, from the beginning of the current TB code
949  * to the destination address.
950  */
951 static inline ptrdiff_t tcg_tbrel_diff(TCGContext *s, const void *target)
952 {
953     return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_buf));
954 }
955 
956 /**
957  * tcg_current_code_size
958  * @s: the tcg context
959  *
960  * Compute the current code size within the translation block.
961  * This is used to fill in qemu's data structures for goto_tb.
962  */
963 
964 static inline size_t tcg_current_code_size(TCGContext *s)
965 {
966     return tcg_ptr_byte_diff(s->code_ptr, s->code_buf);
967 }
968 
969 /**
970  * tcg_qemu_tb_exec:
971  * @env: pointer to CPUArchState for the CPU
972  * @tb_ptr: address of generated code for the TB to execute
973  *
974  * Start executing code from a given translation block.
975  * Where translation blocks have been linked, execution
976  * may proceed from the given TB into successive ones.
977  * Control eventually returns only when some action is needed
978  * from the top-level loop: either control must pass to a TB
979  * which has not yet been directly linked, or an asynchronous
980  * event such as an interrupt needs handling.
981  *
982  * Return: The return value is the value passed to the corresponding
983  * tcg_gen_exit_tb() at translation time of the last TB attempted to execute.
984  * The value is either zero or a 4-byte aligned pointer to that TB combined
985  * with additional information in its two least significant bits. The
986  * additional information is encoded as follows:
987  *  0, 1: the link between this TB and the next is via the specified
988  *        TB index (0 or 1). That is, we left the TB via (the equivalent
989  *        of) "goto_tb <index>". The main loop uses this to determine
990  *        how to link the TB just executed to the next.
991  *  2:    we are using instruction counting code generation, and we
992  *        did not start executing this TB because the instruction counter
993  *        would hit zero midway through it. In this case the pointer
994  *        returned is the TB we were about to execute, and the caller must
995  *        arrange to execute the remaining count of instructions.
996  *  3:    we stopped because the CPU's exit_request flag was set
997  *        (usually meaning that there is an interrupt that needs to be
998  *        handled). The pointer returned is the TB we were about to execute
999  *        when we noticed the pending exit request.
1000  *
1001  * If the bottom two bits indicate an exit-via-index then the CPU
1002  * state is correctly synchronised and ready for execution of the next
1003  * TB (and in particular the guest PC is the address to execute next).
1004  * Otherwise, we gave up on execution of this TB before it started, and
1005  * the caller must fix up the CPU state by calling the CPU's
1006  * synchronize_from_tb() method with the TB pointer we return (falling
1007  * back to calling the CPU's set_pc method with tb->pb if no
1008  * synchronize_from_tb() method exists).
1009  *
1010  * Note that TCG targets may use a different definition of tcg_qemu_tb_exec
1011  * to this default (which just calls the prologue.code emitted by
1012  * tcg_target_qemu_prologue()).
1013  */
1014 #define TB_EXIT_MASK      3
1015 #define TB_EXIT_IDX0      0
1016 #define TB_EXIT_IDX1      1
1017 #define TB_EXIT_IDXMAX    1
1018 #define TB_EXIT_REQUESTED 3
1019 
1020 #ifdef CONFIG_TCG_INTERPRETER
1021 uintptr_t tcg_qemu_tb_exec(CPUArchState *env, const void *tb_ptr);
1022 #else
1023 typedef uintptr_t tcg_prologue_fn(CPUArchState *env, const void *tb_ptr);
1024 extern tcg_prologue_fn *tcg_qemu_tb_exec;
1025 #endif
1026 
1027 void tcg_register_jit(const void *buf, size_t buf_size);
1028 
1029 #if TCG_TARGET_MAYBE_vec
1030 /* Return zero if the tuple (opc, type, vece) is unsupportable;
1031    return > 0 if it is directly supportable;
1032    return < 0 if we must call tcg_expand_vec_op.  */
1033 int tcg_can_emit_vec_op(TCGOpcode, TCGType, unsigned);
1034 #else
1035 static inline int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
1036 {
1037     return 0;
1038 }
1039 #endif
1040 
1041 /* Expand the tuple (opc, type, vece) on the given arguments.  */
1042 void tcg_expand_vec_op(TCGOpcode, TCGType, unsigned, TCGArg, ...);
1043 
1044 /* Replicate a constant C according to the log2 of the element size.  */
1045 uint64_t dup_const(unsigned vece, uint64_t c);
1046 
1047 #define dup_const(VECE, C)                                         \
1048     (__builtin_constant_p(VECE)                                    \
1049      ? (  (VECE) == MO_8  ? 0x0101010101010101ull * (uint8_t)(C)   \
1050         : (VECE) == MO_16 ? 0x0001000100010001ull * (uint16_t)(C)  \
1051         : (VECE) == MO_32 ? 0x0000000100000001ull * (uint32_t)(C)  \
1052         : (VECE) == MO_64 ? (uint64_t)(C)                          \
1053         : (qemu_build_not_reached_always(), 0))                    \
1054      : dup_const(VECE, C))
1055 
1056 static inline const TCGOpcode *tcg_swap_vecop_list(const TCGOpcode *n)
1057 {
1058 #ifdef CONFIG_DEBUG_TCG
1059     const TCGOpcode *o = tcg_ctx->vecop_list;
1060     tcg_ctx->vecop_list = n;
1061     return o;
1062 #else
1063     return NULL;
1064 #endif
1065 }
1066 
1067 bool tcg_can_emit_vecop_list(const TCGOpcode *, TCGType, unsigned);
1068 
1069 #endif /* TCG_H */
1070