xref: /openbmc/qemu/tcg/tcg.c (revision 63785678)
1 /*
2  * Tiny Code Generator for QEMU
3  *
4  * Copyright (c) 2008 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
25 /* define it to use liveness analysis (better code) */
26 #define USE_LIVENESS_ANALYSIS
27 #define USE_TCG_OPTIMIZATIONS
28 
29 #include "qemu/osdep.h"
30 
31 /* Define to jump the ELF file used to communicate with GDB.  */
32 #undef DEBUG_JIT
33 
34 #if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
35 /* define it to suppress various consistency checks (faster) */
36 #define NDEBUG
37 #endif
38 
39 #include "qemu/cutils.h"
40 #include "qemu/host-utils.h"
41 #include "qemu/timer.h"
42 
43 /* Note: the long term plan is to reduce the dependencies on the QEMU
44    CPU definitions. Currently they are used for qemu_ld/st
45    instructions */
46 #define NO_CPU_IO_DEFS
47 #include "cpu.h"
48 
49 #include "tcg-op.h"
50 
51 #if UINTPTR_MAX == UINT32_MAX
52 # define ELF_CLASS  ELFCLASS32
53 #else
54 # define ELF_CLASS  ELFCLASS64
55 #endif
56 #ifdef HOST_WORDS_BIGENDIAN
57 # define ELF_DATA   ELFDATA2MSB
58 #else
59 # define ELF_DATA   ELFDATA2LSB
60 #endif
61 
62 #include "elf.h"
63 #include "exec/log.h"
64 
65 /* Forward declarations for functions declared in tcg-target.inc.c and
66    used here. */
67 static void tcg_target_init(TCGContext *s);
68 static void tcg_target_qemu_prologue(TCGContext *s);
69 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
70                         intptr_t value, intptr_t addend);
71 
72 /* The CIE and FDE header definitions will be common to all hosts.  */
73 typedef struct {
74     uint32_t len __attribute__((aligned((sizeof(void *)))));
75     uint32_t id;
76     uint8_t version;
77     char augmentation[1];
78     uint8_t code_align;
79     uint8_t data_align;
80     uint8_t return_column;
81 } DebugFrameCIE;
82 
83 typedef struct QEMU_PACKED {
84     uint32_t len __attribute__((aligned((sizeof(void *)))));
85     uint32_t cie_offset;
86     uintptr_t func_start;
87     uintptr_t func_len;
88 } DebugFrameFDEHeader;
89 
90 typedef struct QEMU_PACKED {
91     DebugFrameCIE cie;
92     DebugFrameFDEHeader fde;
93 } DebugFrameHeader;
94 
95 static void tcg_register_jit_int(void *buf, size_t size,
96                                  const void *debug_frame,
97                                  size_t debug_frame_size)
98     __attribute__((unused));
99 
100 /* Forward declarations for functions declared and used in tcg-target.inc.c. */
101 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);
102 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
103                        intptr_t arg2);
104 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
105 static void tcg_out_movi(TCGContext *s, TCGType type,
106                          TCGReg ret, tcg_target_long arg);
107 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
108                        const int *const_args);
109 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
110                        intptr_t arg2);
111 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
112 static int tcg_target_const_match(tcg_target_long val, TCGType type,
113                                   const TCGArgConstraint *arg_ct);
114 static void tcg_out_tb_init(TCGContext *s);
115 static bool tcg_out_tb_finalize(TCGContext *s);
116 
117 
118 
119 static TCGRegSet tcg_target_available_regs[2];
120 static TCGRegSet tcg_target_call_clobber_regs;
121 
122 #if TCG_TARGET_INSN_UNIT_SIZE == 1
123 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
124 {
125     *s->code_ptr++ = v;
126 }
127 
128 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
129                                                       uint8_t v)
130 {
131     *p = v;
132 }
133 #endif
134 
135 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
136 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
137 {
138     if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
139         *s->code_ptr++ = v;
140     } else {
141         tcg_insn_unit *p = s->code_ptr;
142         memcpy(p, &v, sizeof(v));
143         s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
144     }
145 }
146 
147 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
148                                                        uint16_t v)
149 {
150     if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
151         *p = v;
152     } else {
153         memcpy(p, &v, sizeof(v));
154     }
155 }
156 #endif
157 
158 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
159 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
160 {
161     if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
162         *s->code_ptr++ = v;
163     } else {
164         tcg_insn_unit *p = s->code_ptr;
165         memcpy(p, &v, sizeof(v));
166         s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
167     }
168 }
169 
170 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
171                                                        uint32_t v)
172 {
173     if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
174         *p = v;
175     } else {
176         memcpy(p, &v, sizeof(v));
177     }
178 }
179 #endif
180 
181 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
182 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
183 {
184     if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
185         *s->code_ptr++ = v;
186     } else {
187         tcg_insn_unit *p = s->code_ptr;
188         memcpy(p, &v, sizeof(v));
189         s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
190     }
191 }
192 
193 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
194                                                        uint64_t v)
195 {
196     if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
197         *p = v;
198     } else {
199         memcpy(p, &v, sizeof(v));
200     }
201 }
202 #endif
203 
204 /* label relocation processing */
205 
206 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
207                           TCGLabel *l, intptr_t addend)
208 {
209     TCGRelocation *r;
210 
211     if (l->has_value) {
212         /* FIXME: This may break relocations on RISC targets that
213            modify instruction fields in place.  The caller may not have
214            written the initial value.  */
215         patch_reloc(code_ptr, type, l->u.value, addend);
216     } else {
217         /* add a new relocation entry */
218         r = tcg_malloc(sizeof(TCGRelocation));
219         r->type = type;
220         r->ptr = code_ptr;
221         r->addend = addend;
222         r->next = l->u.first_reloc;
223         l->u.first_reloc = r;
224     }
225 }
226 
227 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
228 {
229     intptr_t value = (intptr_t)ptr;
230     TCGRelocation *r;
231 
232     assert(!l->has_value);
233 
234     for (r = l->u.first_reloc; r != NULL; r = r->next) {
235         patch_reloc(r->ptr, r->type, value, r->addend);
236     }
237 
238     l->has_value = 1;
239     l->u.value_ptr = ptr;
240 }
241 
242 TCGLabel *gen_new_label(void)
243 {
244     TCGContext *s = &tcg_ctx;
245     TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
246 
247     *l = (TCGLabel){
248         .id = s->nb_labels++
249     };
250 
251     return l;
252 }
253 
254 #include "tcg-target.inc.c"
255 
256 /* pool based memory allocation */
257 void *tcg_malloc_internal(TCGContext *s, int size)
258 {
259     TCGPool *p;
260     int pool_size;
261 
262     if (size > TCG_POOL_CHUNK_SIZE) {
263         /* big malloc: insert a new pool (XXX: could optimize) */
264         p = g_malloc(sizeof(TCGPool) + size);
265         p->size = size;
266         p->next = s->pool_first_large;
267         s->pool_first_large = p;
268         return p->data;
269     } else {
270         p = s->pool_current;
271         if (!p) {
272             p = s->pool_first;
273             if (!p)
274                 goto new_pool;
275         } else {
276             if (!p->next) {
277             new_pool:
278                 pool_size = TCG_POOL_CHUNK_SIZE;
279                 p = g_malloc(sizeof(TCGPool) + pool_size);
280                 p->size = pool_size;
281                 p->next = NULL;
282                 if (s->pool_current)
283                     s->pool_current->next = p;
284                 else
285                     s->pool_first = p;
286             } else {
287                 p = p->next;
288             }
289         }
290     }
291     s->pool_current = p;
292     s->pool_cur = p->data + size;
293     s->pool_end = p->data + p->size;
294     return p->data;
295 }
296 
297 void tcg_pool_reset(TCGContext *s)
298 {
299     TCGPool *p, *t;
300     for (p = s->pool_first_large; p; p = t) {
301         t = p->next;
302         g_free(p);
303     }
304     s->pool_first_large = NULL;
305     s->pool_cur = s->pool_end = NULL;
306     s->pool_current = NULL;
307 }
308 
309 typedef struct TCGHelperInfo {
310     void *func;
311     const char *name;
312     unsigned flags;
313     unsigned sizemask;
314 } TCGHelperInfo;
315 
316 #include "exec/helper-proto.h"
317 
318 static const TCGHelperInfo all_helpers[] = {
319 #include "exec/helper-tcg.h"
320 };
321 
322 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
323 
324 void tcg_context_init(TCGContext *s)
325 {
326     int op, total_args, n, i;
327     TCGOpDef *def;
328     TCGArgConstraint *args_ct;
329     int *sorted_args;
330     GHashTable *helper_table;
331 
332     memset(s, 0, sizeof(*s));
333     s->nb_globals = 0;
334 
335     /* Count total number of arguments and allocate the corresponding
336        space */
337     total_args = 0;
338     for(op = 0; op < NB_OPS; op++) {
339         def = &tcg_op_defs[op];
340         n = def->nb_iargs + def->nb_oargs;
341         total_args += n;
342     }
343 
344     args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args);
345     sorted_args = g_malloc(sizeof(int) * total_args);
346 
347     for(op = 0; op < NB_OPS; op++) {
348         def = &tcg_op_defs[op];
349         def->args_ct = args_ct;
350         def->sorted_args = sorted_args;
351         n = def->nb_iargs + def->nb_oargs;
352         sorted_args += n;
353         args_ct += n;
354     }
355 
356     /* Register helpers.  */
357     /* Use g_direct_hash/equal for direct pointer comparisons on func.  */
358     s->helpers = helper_table = g_hash_table_new(NULL, NULL);
359 
360     for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
361         g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
362                             (gpointer)&all_helpers[i]);
363     }
364 
365     tcg_target_init(s);
366 
367     /* Reverse the order of the saved registers, assuming they're all at
368        the start of tcg_target_reg_alloc_order.  */
369     for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
370         int r = tcg_target_reg_alloc_order[n];
371         if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
372             break;
373         }
374     }
375     for (i = 0; i < n; ++i) {
376         indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
377     }
378     for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
379         indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
380     }
381 }
382 
383 void tcg_prologue_init(TCGContext *s)
384 {
385     size_t prologue_size, total_size;
386     void *buf0, *buf1;
387 
388     /* Put the prologue at the beginning of code_gen_buffer.  */
389     buf0 = s->code_gen_buffer;
390     s->code_ptr = buf0;
391     s->code_buf = buf0;
392     s->code_gen_prologue = buf0;
393 
394     /* Generate the prologue.  */
395     tcg_target_qemu_prologue(s);
396     buf1 = s->code_ptr;
397     flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
398 
399     /* Deduct the prologue from the buffer.  */
400     prologue_size = tcg_current_code_size(s);
401     s->code_gen_ptr = buf1;
402     s->code_gen_buffer = buf1;
403     s->code_buf = buf1;
404     total_size = s->code_gen_buffer_size - prologue_size;
405     s->code_gen_buffer_size = total_size;
406 
407     /* Compute a high-water mark, at which we voluntarily flush the buffer
408        and start over.  The size here is arbitrary, significantly larger
409        than we expect the code generation for any one opcode to require.  */
410     s->code_gen_highwater = s->code_gen_buffer + (total_size - 1024);
411 
412     tcg_register_jit(s->code_gen_buffer, total_size);
413 
414 #ifdef DEBUG_DISAS
415     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
416         qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
417         log_disas(buf0, prologue_size);
418         qemu_log("\n");
419         qemu_log_flush();
420     }
421 #endif
422 }
423 
424 void tcg_func_start(TCGContext *s)
425 {
426     tcg_pool_reset(s);
427     s->nb_temps = s->nb_globals;
428 
429     /* No temps have been previously allocated for size or locality.  */
430     memset(s->free_temps, 0, sizeof(s->free_temps));
431 
432     s->nb_labels = 0;
433     s->current_frame_offset = s->frame_start;
434 
435 #ifdef CONFIG_DEBUG_TCG
436     s->goto_tb_issue_mask = 0;
437 #endif
438 
439     s->gen_first_op_idx = 0;
440     s->gen_last_op_idx = -1;
441     s->gen_next_op_idx = 0;
442     s->gen_next_parm_idx = 0;
443 
444     s->be = tcg_malloc(sizeof(TCGBackendData));
445 }
446 
447 static inline int temp_idx(TCGContext *s, TCGTemp *ts)
448 {
449     ptrdiff_t n = ts - s->temps;
450     tcg_debug_assert(n >= 0 && n < s->nb_temps);
451     return n;
452 }
453 
454 static inline TCGTemp *tcg_temp_alloc(TCGContext *s)
455 {
456     int n = s->nb_temps++;
457     tcg_debug_assert(n < TCG_MAX_TEMPS);
458     return memset(&s->temps[n], 0, sizeof(TCGTemp));
459 }
460 
461 static inline TCGTemp *tcg_global_alloc(TCGContext *s)
462 {
463     tcg_debug_assert(s->nb_globals == s->nb_temps);
464     s->nb_globals++;
465     return tcg_temp_alloc(s);
466 }
467 
468 static int tcg_global_reg_new_internal(TCGContext *s, TCGType type,
469                                        TCGReg reg, const char *name)
470 {
471     TCGTemp *ts;
472 
473     if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
474         tcg_abort();
475     }
476 
477     ts = tcg_global_alloc(s);
478     ts->base_type = type;
479     ts->type = type;
480     ts->fixed_reg = 1;
481     ts->reg = reg;
482     ts->name = name;
483     tcg_regset_set_reg(s->reserved_regs, reg);
484 
485     return temp_idx(s, ts);
486 }
487 
488 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
489 {
490     int idx;
491     s->frame_start = start;
492     s->frame_end = start + size;
493     idx = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
494     s->frame_temp = &s->temps[idx];
495 }
496 
497 TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name)
498 {
499     TCGContext *s = &tcg_ctx;
500     int idx;
501 
502     if (tcg_regset_test_reg(s->reserved_regs, reg)) {
503         tcg_abort();
504     }
505     idx = tcg_global_reg_new_internal(s, TCG_TYPE_I32, reg, name);
506     return MAKE_TCGV_I32(idx);
507 }
508 
509 TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name)
510 {
511     TCGContext *s = &tcg_ctx;
512     int idx;
513 
514     if (tcg_regset_test_reg(s->reserved_regs, reg)) {
515         tcg_abort();
516     }
517     idx = tcg_global_reg_new_internal(s, TCG_TYPE_I64, reg, name);
518     return MAKE_TCGV_I64(idx);
519 }
520 
521 int tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
522                                 intptr_t offset, const char *name)
523 {
524     TCGContext *s = &tcg_ctx;
525     TCGTemp *base_ts = &s->temps[GET_TCGV_PTR(base)];
526     TCGTemp *ts = tcg_global_alloc(s);
527     int indirect_reg = 0, bigendian = 0;
528 #ifdef HOST_WORDS_BIGENDIAN
529     bigendian = 1;
530 #endif
531 
532     if (!base_ts->fixed_reg) {
533         indirect_reg = 1;
534         base_ts->indirect_base = 1;
535     }
536 
537     if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
538         TCGTemp *ts2 = tcg_global_alloc(s);
539         char buf[64];
540 
541         ts->base_type = TCG_TYPE_I64;
542         ts->type = TCG_TYPE_I32;
543         ts->indirect_reg = indirect_reg;
544         ts->mem_allocated = 1;
545         ts->mem_base = base_ts;
546         ts->mem_offset = offset + bigendian * 4;
547         pstrcpy(buf, sizeof(buf), name);
548         pstrcat(buf, sizeof(buf), "_0");
549         ts->name = strdup(buf);
550 
551         tcg_debug_assert(ts2 == ts + 1);
552         ts2->base_type = TCG_TYPE_I64;
553         ts2->type = TCG_TYPE_I32;
554         ts2->indirect_reg = indirect_reg;
555         ts2->mem_allocated = 1;
556         ts2->mem_base = base_ts;
557         ts2->mem_offset = offset + (1 - bigendian) * 4;
558         pstrcpy(buf, sizeof(buf), name);
559         pstrcat(buf, sizeof(buf), "_1");
560         ts->name = strdup(buf);
561     } else {
562         ts->base_type = type;
563         ts->type = type;
564         ts->indirect_reg = indirect_reg;
565         ts->mem_allocated = 1;
566         ts->mem_base = base_ts;
567         ts->mem_offset = offset;
568         ts->name = name;
569     }
570     return temp_idx(s, ts);
571 }
572 
573 static int tcg_temp_new_internal(TCGType type, int temp_local)
574 {
575     TCGContext *s = &tcg_ctx;
576     TCGTemp *ts;
577     int idx, k;
578 
579     k = type + (temp_local ? TCG_TYPE_COUNT : 0);
580     idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
581     if (idx < TCG_MAX_TEMPS) {
582         /* There is already an available temp with the right type.  */
583         clear_bit(idx, s->free_temps[k].l);
584 
585         ts = &s->temps[idx];
586         ts->temp_allocated = 1;
587         tcg_debug_assert(ts->base_type == type);
588         tcg_debug_assert(ts->temp_local == temp_local);
589     } else {
590         ts = tcg_temp_alloc(s);
591         if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
592             TCGTemp *ts2 = tcg_temp_alloc(s);
593 
594             ts->base_type = type;
595             ts->type = TCG_TYPE_I32;
596             ts->temp_allocated = 1;
597             ts->temp_local = temp_local;
598 
599             tcg_debug_assert(ts2 == ts + 1);
600             ts2->base_type = TCG_TYPE_I64;
601             ts2->type = TCG_TYPE_I32;
602             ts2->temp_allocated = 1;
603             ts2->temp_local = temp_local;
604         } else {
605             ts->base_type = type;
606             ts->type = type;
607             ts->temp_allocated = 1;
608             ts->temp_local = temp_local;
609         }
610         idx = temp_idx(s, ts);
611     }
612 
613 #if defined(CONFIG_DEBUG_TCG)
614     s->temps_in_use++;
615 #endif
616     return idx;
617 }
618 
619 TCGv_i32 tcg_temp_new_internal_i32(int temp_local)
620 {
621     int idx;
622 
623     idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local);
624     return MAKE_TCGV_I32(idx);
625 }
626 
627 TCGv_i64 tcg_temp_new_internal_i64(int temp_local)
628 {
629     int idx;
630 
631     idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local);
632     return MAKE_TCGV_I64(idx);
633 }
634 
635 static void tcg_temp_free_internal(int idx)
636 {
637     TCGContext *s = &tcg_ctx;
638     TCGTemp *ts;
639     int k;
640 
641 #if defined(CONFIG_DEBUG_TCG)
642     s->temps_in_use--;
643     if (s->temps_in_use < 0) {
644         fprintf(stderr, "More temporaries freed than allocated!\n");
645     }
646 #endif
647 
648     assert(idx >= s->nb_globals && idx < s->nb_temps);
649     ts = &s->temps[idx];
650     assert(ts->temp_allocated != 0);
651     ts->temp_allocated = 0;
652 
653     k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0);
654     set_bit(idx, s->free_temps[k].l);
655 }
656 
657 void tcg_temp_free_i32(TCGv_i32 arg)
658 {
659     tcg_temp_free_internal(GET_TCGV_I32(arg));
660 }
661 
662 void tcg_temp_free_i64(TCGv_i64 arg)
663 {
664     tcg_temp_free_internal(GET_TCGV_I64(arg));
665 }
666 
667 TCGv_i32 tcg_const_i32(int32_t val)
668 {
669     TCGv_i32 t0;
670     t0 = tcg_temp_new_i32();
671     tcg_gen_movi_i32(t0, val);
672     return t0;
673 }
674 
675 TCGv_i64 tcg_const_i64(int64_t val)
676 {
677     TCGv_i64 t0;
678     t0 = tcg_temp_new_i64();
679     tcg_gen_movi_i64(t0, val);
680     return t0;
681 }
682 
683 TCGv_i32 tcg_const_local_i32(int32_t val)
684 {
685     TCGv_i32 t0;
686     t0 = tcg_temp_local_new_i32();
687     tcg_gen_movi_i32(t0, val);
688     return t0;
689 }
690 
691 TCGv_i64 tcg_const_local_i64(int64_t val)
692 {
693     TCGv_i64 t0;
694     t0 = tcg_temp_local_new_i64();
695     tcg_gen_movi_i64(t0, val);
696     return t0;
697 }
698 
699 #if defined(CONFIG_DEBUG_TCG)
700 void tcg_clear_temp_count(void)
701 {
702     TCGContext *s = &tcg_ctx;
703     s->temps_in_use = 0;
704 }
705 
706 int tcg_check_temp_count(void)
707 {
708     TCGContext *s = &tcg_ctx;
709     if (s->temps_in_use) {
710         /* Clear the count so that we don't give another
711          * warning immediately next time around.
712          */
713         s->temps_in_use = 0;
714         return 1;
715     }
716     return 0;
717 }
718 #endif
719 
720 /* Note: we convert the 64 bit args to 32 bit and do some alignment
721    and endian swap. Maybe it would be better to do the alignment
722    and endian swap in tcg_reg_alloc_call(). */
723 void tcg_gen_callN(TCGContext *s, void *func, TCGArg ret,
724                    int nargs, TCGArg *args)
725 {
726     int i, real_args, nb_rets, pi, pi_first;
727     unsigned sizemask, flags;
728     TCGHelperInfo *info;
729 
730     info = g_hash_table_lookup(s->helpers, (gpointer)func);
731     flags = info->flags;
732     sizemask = info->sizemask;
733 
734 #if defined(__sparc__) && !defined(__arch64__) \
735     && !defined(CONFIG_TCG_INTERPRETER)
736     /* We have 64-bit values in one register, but need to pass as two
737        separate parameters.  Split them.  */
738     int orig_sizemask = sizemask;
739     int orig_nargs = nargs;
740     TCGv_i64 retl, reth;
741 
742     TCGV_UNUSED_I64(retl);
743     TCGV_UNUSED_I64(reth);
744     if (sizemask != 0) {
745         TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2);
746         for (i = real_args = 0; i < nargs; ++i) {
747             int is_64bit = sizemask & (1 << (i+1)*2);
748             if (is_64bit) {
749                 TCGv_i64 orig = MAKE_TCGV_I64(args[i]);
750                 TCGv_i32 h = tcg_temp_new_i32();
751                 TCGv_i32 l = tcg_temp_new_i32();
752                 tcg_gen_extr_i64_i32(l, h, orig);
753                 split_args[real_args++] = GET_TCGV_I32(h);
754                 split_args[real_args++] = GET_TCGV_I32(l);
755             } else {
756                 split_args[real_args++] = args[i];
757             }
758         }
759         nargs = real_args;
760         args = split_args;
761         sizemask = 0;
762     }
763 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
764     for (i = 0; i < nargs; ++i) {
765         int is_64bit = sizemask & (1 << (i+1)*2);
766         int is_signed = sizemask & (2 << (i+1)*2);
767         if (!is_64bit) {
768             TCGv_i64 temp = tcg_temp_new_i64();
769             TCGv_i64 orig = MAKE_TCGV_I64(args[i]);
770             if (is_signed) {
771                 tcg_gen_ext32s_i64(temp, orig);
772             } else {
773                 tcg_gen_ext32u_i64(temp, orig);
774             }
775             args[i] = GET_TCGV_I64(temp);
776         }
777     }
778 #endif /* TCG_TARGET_EXTEND_ARGS */
779 
780     pi_first = pi = s->gen_next_parm_idx;
781     if (ret != TCG_CALL_DUMMY_ARG) {
782 #if defined(__sparc__) && !defined(__arch64__) \
783     && !defined(CONFIG_TCG_INTERPRETER)
784         if (orig_sizemask & 1) {
785             /* The 32-bit ABI is going to return the 64-bit value in
786                the %o0/%o1 register pair.  Prepare for this by using
787                two return temporaries, and reassemble below.  */
788             retl = tcg_temp_new_i64();
789             reth = tcg_temp_new_i64();
790             s->gen_opparam_buf[pi++] = GET_TCGV_I64(reth);
791             s->gen_opparam_buf[pi++] = GET_TCGV_I64(retl);
792             nb_rets = 2;
793         } else {
794             s->gen_opparam_buf[pi++] = ret;
795             nb_rets = 1;
796         }
797 #else
798         if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) {
799 #ifdef HOST_WORDS_BIGENDIAN
800             s->gen_opparam_buf[pi++] = ret + 1;
801             s->gen_opparam_buf[pi++] = ret;
802 #else
803             s->gen_opparam_buf[pi++] = ret;
804             s->gen_opparam_buf[pi++] = ret + 1;
805 #endif
806             nb_rets = 2;
807         } else {
808             s->gen_opparam_buf[pi++] = ret;
809             nb_rets = 1;
810         }
811 #endif
812     } else {
813         nb_rets = 0;
814     }
815     real_args = 0;
816     for (i = 0; i < nargs; i++) {
817         int is_64bit = sizemask & (1 << (i+1)*2);
818         if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
819 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
820             /* some targets want aligned 64 bit args */
821             if (real_args & 1) {
822                 s->gen_opparam_buf[pi++] = TCG_CALL_DUMMY_ARG;
823                 real_args++;
824             }
825 #endif
826 	    /* If stack grows up, then we will be placing successive
827 	       arguments at lower addresses, which means we need to
828 	       reverse the order compared to how we would normally
829 	       treat either big or little-endian.  For those arguments
830 	       that will wind up in registers, this still works for
831 	       HPPA (the only current STACK_GROWSUP target) since the
832 	       argument registers are *also* allocated in decreasing
833 	       order.  If another such target is added, this logic may
834 	       have to get more complicated to differentiate between
835 	       stack arguments and register arguments.  */
836 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
837             s->gen_opparam_buf[pi++] = args[i] + 1;
838             s->gen_opparam_buf[pi++] = args[i];
839 #else
840             s->gen_opparam_buf[pi++] = args[i];
841             s->gen_opparam_buf[pi++] = args[i] + 1;
842 #endif
843             real_args += 2;
844             continue;
845         }
846 
847         s->gen_opparam_buf[pi++] = args[i];
848         real_args++;
849     }
850     s->gen_opparam_buf[pi++] = (uintptr_t)func;
851     s->gen_opparam_buf[pi++] = flags;
852 
853     i = s->gen_next_op_idx;
854     tcg_debug_assert(i < OPC_BUF_SIZE);
855     tcg_debug_assert(pi <= OPPARAM_BUF_SIZE);
856 
857     /* Set links for sequential allocation during translation.  */
858     s->gen_op_buf[i] = (TCGOp){
859         .opc = INDEX_op_call,
860         .callo = nb_rets,
861         .calli = real_args,
862         .args = pi_first,
863         .prev = i - 1,
864         .next = i + 1
865     };
866 
867     /* Make sure the calli field didn't overflow.  */
868     tcg_debug_assert(s->gen_op_buf[i].calli == real_args);
869 
870     s->gen_last_op_idx = i;
871     s->gen_next_op_idx = i + 1;
872     s->gen_next_parm_idx = pi;
873 
874 #if defined(__sparc__) && !defined(__arch64__) \
875     && !defined(CONFIG_TCG_INTERPRETER)
876     /* Free all of the parts we allocated above.  */
877     for (i = real_args = 0; i < orig_nargs; ++i) {
878         int is_64bit = orig_sizemask & (1 << (i+1)*2);
879         if (is_64bit) {
880             TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]);
881             TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]);
882             tcg_temp_free_i32(h);
883             tcg_temp_free_i32(l);
884         } else {
885             real_args++;
886         }
887     }
888     if (orig_sizemask & 1) {
889         /* The 32-bit ABI returned two 32-bit pieces.  Re-assemble them.
890            Note that describing these as TCGv_i64 eliminates an unnecessary
891            zero-extension that tcg_gen_concat_i32_i64 would create.  */
892         tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth);
893         tcg_temp_free_i64(retl);
894         tcg_temp_free_i64(reth);
895     }
896 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
897     for (i = 0; i < nargs; ++i) {
898         int is_64bit = sizemask & (1 << (i+1)*2);
899         if (!is_64bit) {
900             TCGv_i64 temp = MAKE_TCGV_I64(args[i]);
901             tcg_temp_free_i64(temp);
902         }
903     }
904 #endif /* TCG_TARGET_EXTEND_ARGS */
905 }
906 
907 static void tcg_reg_alloc_start(TCGContext *s)
908 {
909     int i;
910     TCGTemp *ts;
911     for(i = 0; i < s->nb_globals; i++) {
912         ts = &s->temps[i];
913         if (ts->fixed_reg) {
914             ts->val_type = TEMP_VAL_REG;
915         } else {
916             ts->val_type = TEMP_VAL_MEM;
917         }
918     }
919     for(i = s->nb_globals; i < s->nb_temps; i++) {
920         ts = &s->temps[i];
921         if (ts->temp_local) {
922             ts->val_type = TEMP_VAL_MEM;
923         } else {
924             ts->val_type = TEMP_VAL_DEAD;
925         }
926         ts->mem_allocated = 0;
927         ts->fixed_reg = 0;
928     }
929 
930     memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
931 }
932 
933 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
934                                  TCGTemp *ts)
935 {
936     int idx = temp_idx(s, ts);
937 
938     if (idx < s->nb_globals) {
939         pstrcpy(buf, buf_size, ts->name);
940     } else if (ts->temp_local) {
941         snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
942     } else {
943         snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
944     }
945     return buf;
946 }
947 
948 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf,
949                                  int buf_size, int idx)
950 {
951     assert(idx >= 0 && idx < s->nb_temps);
952     return tcg_get_arg_str_ptr(s, buf, buf_size, &s->temps[idx]);
953 }
954 
955 /* Find helper name.  */
956 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
957 {
958     const char *ret = NULL;
959     if (s->helpers) {
960         TCGHelperInfo *info = g_hash_table_lookup(s->helpers, (gpointer)val);
961         if (info) {
962             ret = info->name;
963         }
964     }
965     return ret;
966 }
967 
968 static const char * const cond_name[] =
969 {
970     [TCG_COND_NEVER] = "never",
971     [TCG_COND_ALWAYS] = "always",
972     [TCG_COND_EQ] = "eq",
973     [TCG_COND_NE] = "ne",
974     [TCG_COND_LT] = "lt",
975     [TCG_COND_GE] = "ge",
976     [TCG_COND_LE] = "le",
977     [TCG_COND_GT] = "gt",
978     [TCG_COND_LTU] = "ltu",
979     [TCG_COND_GEU] = "geu",
980     [TCG_COND_LEU] = "leu",
981     [TCG_COND_GTU] = "gtu"
982 };
983 
984 static const char * const ldst_name[] =
985 {
986     [MO_UB]   = "ub",
987     [MO_SB]   = "sb",
988     [MO_LEUW] = "leuw",
989     [MO_LESW] = "lesw",
990     [MO_LEUL] = "leul",
991     [MO_LESL] = "lesl",
992     [MO_LEQ]  = "leq",
993     [MO_BEUW] = "beuw",
994     [MO_BESW] = "besw",
995     [MO_BEUL] = "beul",
996     [MO_BESL] = "besl",
997     [MO_BEQ]  = "beq",
998 };
999 
1000 void tcg_dump_ops(TCGContext *s)
1001 {
1002     char buf[128];
1003     TCGOp *op;
1004     int oi;
1005 
1006     for (oi = s->gen_first_op_idx; oi >= 0; oi = op->next) {
1007         int i, k, nb_oargs, nb_iargs, nb_cargs;
1008         const TCGOpDef *def;
1009         const TCGArg *args;
1010         TCGOpcode c;
1011 
1012         op = &s->gen_op_buf[oi];
1013         c = op->opc;
1014         def = &tcg_op_defs[c];
1015         args = &s->gen_opparam_buf[op->args];
1016 
1017         if (c == INDEX_op_insn_start) {
1018             qemu_log("%s ----", oi != s->gen_first_op_idx ? "\n" : "");
1019 
1020             for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1021                 target_ulong a;
1022 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1023                 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
1024 #else
1025                 a = args[i];
1026 #endif
1027                 qemu_log(" " TARGET_FMT_lx, a);
1028             }
1029         } else if (c == INDEX_op_call) {
1030             /* variable number of arguments */
1031             nb_oargs = op->callo;
1032             nb_iargs = op->calli;
1033             nb_cargs = def->nb_cargs;
1034 
1035             /* function name, flags, out args */
1036             qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
1037                      tcg_find_helper(s, args[nb_oargs + nb_iargs]),
1038                      args[nb_oargs + nb_iargs + 1], nb_oargs);
1039             for (i = 0; i < nb_oargs; i++) {
1040                 qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1041                                                    args[i]));
1042             }
1043             for (i = 0; i < nb_iargs; i++) {
1044                 TCGArg arg = args[nb_oargs + i];
1045                 const char *t = "<dummy>";
1046                 if (arg != TCG_CALL_DUMMY_ARG) {
1047                     t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg);
1048                 }
1049                 qemu_log(",%s", t);
1050             }
1051         } else {
1052             qemu_log(" %s ", def->name);
1053 
1054             nb_oargs = def->nb_oargs;
1055             nb_iargs = def->nb_iargs;
1056             nb_cargs = def->nb_cargs;
1057 
1058             k = 0;
1059             for (i = 0; i < nb_oargs; i++) {
1060                 if (k != 0) {
1061                     qemu_log(",");
1062                 }
1063                 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1064                                                    args[k++]));
1065             }
1066             for (i = 0; i < nb_iargs; i++) {
1067                 if (k != 0) {
1068                     qemu_log(",");
1069                 }
1070                 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1071                                                    args[k++]));
1072             }
1073             switch (c) {
1074             case INDEX_op_brcond_i32:
1075             case INDEX_op_setcond_i32:
1076             case INDEX_op_movcond_i32:
1077             case INDEX_op_brcond2_i32:
1078             case INDEX_op_setcond2_i32:
1079             case INDEX_op_brcond_i64:
1080             case INDEX_op_setcond_i64:
1081             case INDEX_op_movcond_i64:
1082                 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) {
1083                     qemu_log(",%s", cond_name[args[k++]]);
1084                 } else {
1085                     qemu_log(",$0x%" TCG_PRIlx, args[k++]);
1086                 }
1087                 i = 1;
1088                 break;
1089             case INDEX_op_qemu_ld_i32:
1090             case INDEX_op_qemu_st_i32:
1091             case INDEX_op_qemu_ld_i64:
1092             case INDEX_op_qemu_st_i64:
1093                 {
1094                     TCGMemOpIdx oi = args[k++];
1095                     TCGMemOp op = get_memop(oi);
1096                     unsigned ix = get_mmuidx(oi);
1097 
1098                     if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
1099                         qemu_log(",$0x%x,%u", op, ix);
1100                     } else {
1101                         const char *s_al = "", *s_op;
1102                         if (op & MO_AMASK) {
1103                             if ((op & MO_AMASK) == MO_ALIGN) {
1104                                 s_al = "al+";
1105                             } else {
1106                                 s_al = "un+";
1107                             }
1108                         }
1109                         s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
1110                         qemu_log(",%s%s,%u", s_al, s_op, ix);
1111                     }
1112                     i = 1;
1113                 }
1114                 break;
1115             default:
1116                 i = 0;
1117                 break;
1118             }
1119             switch (c) {
1120             case INDEX_op_set_label:
1121             case INDEX_op_br:
1122             case INDEX_op_brcond_i32:
1123             case INDEX_op_brcond_i64:
1124             case INDEX_op_brcond2_i32:
1125                 qemu_log("%s$L%d", k ? "," : "", arg_label(args[k])->id);
1126                 i++, k++;
1127                 break;
1128             default:
1129                 break;
1130             }
1131             for (; i < nb_cargs; i++, k++) {
1132                 qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", args[k]);
1133             }
1134         }
1135         qemu_log("\n");
1136     }
1137 }
1138 
1139 /* we give more priority to constraints with less registers */
1140 static int get_constraint_priority(const TCGOpDef *def, int k)
1141 {
1142     const TCGArgConstraint *arg_ct;
1143 
1144     int i, n;
1145     arg_ct = &def->args_ct[k];
1146     if (arg_ct->ct & TCG_CT_ALIAS) {
1147         /* an alias is equivalent to a single register */
1148         n = 1;
1149     } else {
1150         if (!(arg_ct->ct & TCG_CT_REG))
1151             return 0;
1152         n = 0;
1153         for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
1154             if (tcg_regset_test_reg(arg_ct->u.regs, i))
1155                 n++;
1156         }
1157     }
1158     return TCG_TARGET_NB_REGS - n + 1;
1159 }
1160 
1161 /* sort from highest priority to lowest */
1162 static void sort_constraints(TCGOpDef *def, int start, int n)
1163 {
1164     int i, j, p1, p2, tmp;
1165 
1166     for(i = 0; i < n; i++)
1167         def->sorted_args[start + i] = start + i;
1168     if (n <= 1)
1169         return;
1170     for(i = 0; i < n - 1; i++) {
1171         for(j = i + 1; j < n; j++) {
1172             p1 = get_constraint_priority(def, def->sorted_args[start + i]);
1173             p2 = get_constraint_priority(def, def->sorted_args[start + j]);
1174             if (p1 < p2) {
1175                 tmp = def->sorted_args[start + i];
1176                 def->sorted_args[start + i] = def->sorted_args[start + j];
1177                 def->sorted_args[start + j] = tmp;
1178             }
1179         }
1180     }
1181 }
1182 
1183 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
1184 {
1185     TCGOpcode op;
1186     TCGOpDef *def;
1187     const char *ct_str;
1188     int i, nb_args;
1189 
1190     for(;;) {
1191         if (tdefs->op == (TCGOpcode)-1)
1192             break;
1193         op = tdefs->op;
1194         assert((unsigned)op < NB_OPS);
1195         def = &tcg_op_defs[op];
1196 #if defined(CONFIG_DEBUG_TCG)
1197         /* Duplicate entry in op definitions? */
1198         assert(!def->used);
1199         def->used = 1;
1200 #endif
1201         nb_args = def->nb_iargs + def->nb_oargs;
1202         for(i = 0; i < nb_args; i++) {
1203             ct_str = tdefs->args_ct_str[i];
1204             /* Incomplete TCGTargetOpDef entry? */
1205             assert(ct_str != NULL);
1206             tcg_regset_clear(def->args_ct[i].u.regs);
1207             def->args_ct[i].ct = 0;
1208             if (ct_str[0] >= '0' && ct_str[0] <= '9') {
1209                 int oarg;
1210                 oarg = ct_str[0] - '0';
1211                 assert(oarg < def->nb_oargs);
1212                 assert(def->args_ct[oarg].ct & TCG_CT_REG);
1213                 /* TCG_CT_ALIAS is for the output arguments. The input
1214                    argument is tagged with TCG_CT_IALIAS. */
1215                 def->args_ct[i] = def->args_ct[oarg];
1216                 def->args_ct[oarg].ct = TCG_CT_ALIAS;
1217                 def->args_ct[oarg].alias_index = i;
1218                 def->args_ct[i].ct |= TCG_CT_IALIAS;
1219                 def->args_ct[i].alias_index = oarg;
1220             } else {
1221                 for(;;) {
1222                     if (*ct_str == '\0')
1223                         break;
1224                     switch(*ct_str) {
1225                     case 'i':
1226                         def->args_ct[i].ct |= TCG_CT_CONST;
1227                         ct_str++;
1228                         break;
1229                     default:
1230                         if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) {
1231                             fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n",
1232                                     ct_str, i, def->name);
1233                             exit(1);
1234                         }
1235                     }
1236                 }
1237             }
1238         }
1239 
1240         /* TCGTargetOpDef entry with too much information? */
1241         assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
1242 
1243         /* sort the constraints (XXX: this is just an heuristic) */
1244         sort_constraints(def, 0, def->nb_oargs);
1245         sort_constraints(def, def->nb_oargs, def->nb_iargs);
1246 
1247 #if 0
1248         {
1249             int i;
1250 
1251             printf("%s: sorted=", def->name);
1252             for(i = 0; i < def->nb_oargs + def->nb_iargs; i++)
1253                 printf(" %d", def->sorted_args[i]);
1254             printf("\n");
1255         }
1256 #endif
1257         tdefs++;
1258     }
1259 
1260 #if defined(CONFIG_DEBUG_TCG)
1261     i = 0;
1262     for (op = 0; op < tcg_op_defs_max; op++) {
1263         const TCGOpDef *def = &tcg_op_defs[op];
1264         if (def->flags & TCG_OPF_NOT_PRESENT) {
1265             /* Wrong entry in op definitions? */
1266             if (def->used) {
1267                 fprintf(stderr, "Invalid op definition for %s\n", def->name);
1268                 i = 1;
1269             }
1270         } else {
1271             /* Missing entry in op definitions? */
1272             if (!def->used) {
1273                 fprintf(stderr, "Missing op definition for %s\n", def->name);
1274                 i = 1;
1275             }
1276         }
1277     }
1278     if (i == 1) {
1279         tcg_abort();
1280     }
1281 #endif
1282 }
1283 
1284 void tcg_op_remove(TCGContext *s, TCGOp *op)
1285 {
1286     int next = op->next;
1287     int prev = op->prev;
1288 
1289     if (next >= 0) {
1290         s->gen_op_buf[next].prev = prev;
1291     } else {
1292         s->gen_last_op_idx = prev;
1293     }
1294     if (prev >= 0) {
1295         s->gen_op_buf[prev].next = next;
1296     } else {
1297         s->gen_first_op_idx = next;
1298     }
1299 
1300     memset(op, -1, sizeof(*op));
1301 
1302 #ifdef CONFIG_PROFILER
1303     s->del_op_count++;
1304 #endif
1305 }
1306 
1307 #ifdef USE_LIVENESS_ANALYSIS
1308 /* liveness analysis: end of function: all temps are dead, and globals
1309    should be in memory. */
1310 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps,
1311                                    uint8_t *mem_temps)
1312 {
1313     memset(dead_temps, 1, s->nb_temps);
1314     memset(mem_temps, 1, s->nb_globals);
1315     memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals);
1316 }
1317 
1318 /* liveness analysis: end of basic block: all temps are dead, globals
1319    and local temps should be in memory. */
1320 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps,
1321                                  uint8_t *mem_temps)
1322 {
1323     int i;
1324 
1325     memset(dead_temps, 1, s->nb_temps);
1326     memset(mem_temps, 1, s->nb_globals);
1327     for(i = s->nb_globals; i < s->nb_temps; i++) {
1328         mem_temps[i] = s->temps[i].temp_local;
1329     }
1330 }
1331 
1332 /* Liveness analysis : update the opc_dead_args array to tell if a
1333    given input arguments is dead. Instructions updating dead
1334    temporaries are removed. */
1335 static void tcg_liveness_analysis(TCGContext *s)
1336 {
1337     uint8_t *dead_temps, *mem_temps;
1338     int oi, oi_prev, nb_ops;
1339 
1340     nb_ops = s->gen_next_op_idx;
1341     s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
1342     s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
1343 
1344     dead_temps = tcg_malloc(s->nb_temps);
1345     mem_temps = tcg_malloc(s->nb_temps);
1346     tcg_la_func_end(s, dead_temps, mem_temps);
1347 
1348     for (oi = s->gen_last_op_idx; oi >= 0; oi = oi_prev) {
1349         int i, nb_iargs, nb_oargs;
1350         TCGOpcode opc_new, opc_new2;
1351         bool have_opc_new2;
1352         uint16_t dead_args;
1353         uint8_t sync_args;
1354         TCGArg arg;
1355 
1356         TCGOp * const op = &s->gen_op_buf[oi];
1357         TCGArg * const args = &s->gen_opparam_buf[op->args];
1358         TCGOpcode opc = op->opc;
1359         const TCGOpDef *def = &tcg_op_defs[opc];
1360 
1361         oi_prev = op->prev;
1362 
1363         switch (opc) {
1364         case INDEX_op_call:
1365             {
1366                 int call_flags;
1367 
1368                 nb_oargs = op->callo;
1369                 nb_iargs = op->calli;
1370                 call_flags = args[nb_oargs + nb_iargs + 1];
1371 
1372                 /* pure functions can be removed if their result is unused */
1373                 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
1374                     for (i = 0; i < nb_oargs; i++) {
1375                         arg = args[i];
1376                         if (!dead_temps[arg] || mem_temps[arg]) {
1377                             goto do_not_remove_call;
1378                         }
1379                     }
1380                     goto do_remove;
1381                 } else {
1382                 do_not_remove_call:
1383 
1384                     /* output args are dead */
1385                     dead_args = 0;
1386                     sync_args = 0;
1387                     for (i = 0; i < nb_oargs; i++) {
1388                         arg = args[i];
1389                         if (dead_temps[arg]) {
1390                             dead_args |= (1 << i);
1391                         }
1392                         if (mem_temps[arg]) {
1393                             sync_args |= (1 << i);
1394                         }
1395                         dead_temps[arg] = 1;
1396                         mem_temps[arg] = 0;
1397                     }
1398 
1399                     if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
1400                         /* globals should be synced to memory */
1401                         memset(mem_temps, 1, s->nb_globals);
1402                     }
1403                     if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
1404                                         TCG_CALL_NO_READ_GLOBALS))) {
1405                         /* globals should go back to memory */
1406                         memset(dead_temps, 1, s->nb_globals);
1407                     }
1408 
1409                     /* record arguments that die in this helper */
1410                     for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
1411                         arg = args[i];
1412                         if (arg != TCG_CALL_DUMMY_ARG) {
1413                             if (dead_temps[arg]) {
1414                                 dead_args |= (1 << i);
1415                             }
1416                         }
1417                     }
1418                     /* input arguments are live for preceding opcodes */
1419                     for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
1420                         arg = args[i];
1421                         dead_temps[arg] = 0;
1422                     }
1423                     s->op_dead_args[oi] = dead_args;
1424                     s->op_sync_args[oi] = sync_args;
1425                 }
1426             }
1427             break;
1428         case INDEX_op_insn_start:
1429             break;
1430         case INDEX_op_discard:
1431             /* mark the temporary as dead */
1432             dead_temps[args[0]] = 1;
1433             mem_temps[args[0]] = 0;
1434             break;
1435 
1436         case INDEX_op_add2_i32:
1437             opc_new = INDEX_op_add_i32;
1438             goto do_addsub2;
1439         case INDEX_op_sub2_i32:
1440             opc_new = INDEX_op_sub_i32;
1441             goto do_addsub2;
1442         case INDEX_op_add2_i64:
1443             opc_new = INDEX_op_add_i64;
1444             goto do_addsub2;
1445         case INDEX_op_sub2_i64:
1446             opc_new = INDEX_op_sub_i64;
1447         do_addsub2:
1448             nb_iargs = 4;
1449             nb_oargs = 2;
1450             /* Test if the high part of the operation is dead, but not
1451                the low part.  The result can be optimized to a simple
1452                add or sub.  This happens often for x86_64 guest when the
1453                cpu mode is set to 32 bit.  */
1454             if (dead_temps[args[1]] && !mem_temps[args[1]]) {
1455                 if (dead_temps[args[0]] && !mem_temps[args[0]]) {
1456                     goto do_remove;
1457                 }
1458                 /* Replace the opcode and adjust the args in place,
1459                    leaving 3 unused args at the end.  */
1460                 op->opc = opc = opc_new;
1461                 args[1] = args[2];
1462                 args[2] = args[4];
1463                 /* Fall through and mark the single-word operation live.  */
1464                 nb_iargs = 2;
1465                 nb_oargs = 1;
1466             }
1467             goto do_not_remove;
1468 
1469         case INDEX_op_mulu2_i32:
1470             opc_new = INDEX_op_mul_i32;
1471             opc_new2 = INDEX_op_muluh_i32;
1472             have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
1473             goto do_mul2;
1474         case INDEX_op_muls2_i32:
1475             opc_new = INDEX_op_mul_i32;
1476             opc_new2 = INDEX_op_mulsh_i32;
1477             have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
1478             goto do_mul2;
1479         case INDEX_op_mulu2_i64:
1480             opc_new = INDEX_op_mul_i64;
1481             opc_new2 = INDEX_op_muluh_i64;
1482             have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
1483             goto do_mul2;
1484         case INDEX_op_muls2_i64:
1485             opc_new = INDEX_op_mul_i64;
1486             opc_new2 = INDEX_op_mulsh_i64;
1487             have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
1488             goto do_mul2;
1489         do_mul2:
1490             nb_iargs = 2;
1491             nb_oargs = 2;
1492             if (dead_temps[args[1]] && !mem_temps[args[1]]) {
1493                 if (dead_temps[args[0]] && !mem_temps[args[0]]) {
1494                     /* Both parts of the operation are dead.  */
1495                     goto do_remove;
1496                 }
1497                 /* The high part of the operation is dead; generate the low. */
1498                 op->opc = opc = opc_new;
1499                 args[1] = args[2];
1500                 args[2] = args[3];
1501             } else if (have_opc_new2 && dead_temps[args[0]]
1502                        && !mem_temps[args[0]]) {
1503                 /* The low part of the operation is dead; generate the high. */
1504                 op->opc = opc = opc_new2;
1505                 args[0] = args[1];
1506                 args[1] = args[2];
1507                 args[2] = args[3];
1508             } else {
1509                 goto do_not_remove;
1510             }
1511             /* Mark the single-word operation live.  */
1512             nb_oargs = 1;
1513             goto do_not_remove;
1514 
1515         default:
1516             /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
1517             nb_iargs = def->nb_iargs;
1518             nb_oargs = def->nb_oargs;
1519 
1520             /* Test if the operation can be removed because all
1521                its outputs are dead. We assume that nb_oargs == 0
1522                implies side effects */
1523             if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
1524                 for (i = 0; i < nb_oargs; i++) {
1525                     arg = args[i];
1526                     if (!dead_temps[arg] || mem_temps[arg]) {
1527                         goto do_not_remove;
1528                     }
1529                 }
1530             do_remove:
1531                 tcg_op_remove(s, op);
1532             } else {
1533             do_not_remove:
1534                 /* output args are dead */
1535                 dead_args = 0;
1536                 sync_args = 0;
1537                 for (i = 0; i < nb_oargs; i++) {
1538                     arg = args[i];
1539                     if (dead_temps[arg]) {
1540                         dead_args |= (1 << i);
1541                     }
1542                     if (mem_temps[arg]) {
1543                         sync_args |= (1 << i);
1544                     }
1545                     dead_temps[arg] = 1;
1546                     mem_temps[arg] = 0;
1547                 }
1548 
1549                 /* if end of basic block, update */
1550                 if (def->flags & TCG_OPF_BB_END) {
1551                     tcg_la_bb_end(s, dead_temps, mem_temps);
1552                 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
1553                     /* globals should be synced to memory */
1554                     memset(mem_temps, 1, s->nb_globals);
1555                 }
1556 
1557                 /* record arguments that die in this opcode */
1558                 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
1559                     arg = args[i];
1560                     if (dead_temps[arg]) {
1561                         dead_args |= (1 << i);
1562                     }
1563                 }
1564                 /* input arguments are live for preceding opcodes */
1565                 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
1566                     arg = args[i];
1567                     dead_temps[arg] = 0;
1568                 }
1569                 s->op_dead_args[oi] = dead_args;
1570                 s->op_sync_args[oi] = sync_args;
1571             }
1572             break;
1573         }
1574     }
1575 }
1576 #else
1577 /* dummy liveness analysis */
1578 static void tcg_liveness_analysis(TCGContext *s)
1579 {
1580     int nb_ops = s->gen_next_op_idx;
1581 
1582     s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
1583     memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
1584     s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
1585     memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t));
1586 }
1587 #endif
1588 
1589 #ifndef NDEBUG
1590 static void dump_regs(TCGContext *s)
1591 {
1592     TCGTemp *ts;
1593     int i;
1594     char buf[64];
1595 
1596     for(i = 0; i < s->nb_temps; i++) {
1597         ts = &s->temps[i];
1598         printf("  %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i));
1599         switch(ts->val_type) {
1600         case TEMP_VAL_REG:
1601             printf("%s", tcg_target_reg_names[ts->reg]);
1602             break;
1603         case TEMP_VAL_MEM:
1604             printf("%d(%s)", (int)ts->mem_offset,
1605                    tcg_target_reg_names[ts->mem_base->reg]);
1606             break;
1607         case TEMP_VAL_CONST:
1608             printf("$0x%" TCG_PRIlx, ts->val);
1609             break;
1610         case TEMP_VAL_DEAD:
1611             printf("D");
1612             break;
1613         default:
1614             printf("???");
1615             break;
1616         }
1617         printf("\n");
1618     }
1619 
1620     for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
1621         if (s->reg_to_temp[i] != NULL) {
1622             printf("%s: %s\n",
1623                    tcg_target_reg_names[i],
1624                    tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
1625         }
1626     }
1627 }
1628 
1629 static void check_regs(TCGContext *s)
1630 {
1631     int reg;
1632     int k;
1633     TCGTemp *ts;
1634     char buf[64];
1635 
1636     for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
1637         ts = s->reg_to_temp[reg];
1638         if (ts != NULL) {
1639             if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
1640                 printf("Inconsistency for register %s:\n",
1641                        tcg_target_reg_names[reg]);
1642                 goto fail;
1643             }
1644         }
1645     }
1646     for (k = 0; k < s->nb_temps; k++) {
1647         ts = &s->temps[k];
1648         if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg
1649             && s->reg_to_temp[ts->reg] != ts) {
1650             printf("Inconsistency for temp %s:\n",
1651                    tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
1652         fail:
1653             printf("reg state:\n");
1654             dump_regs(s);
1655             tcg_abort();
1656         }
1657     }
1658 }
1659 #endif
1660 
1661 static void temp_allocate_frame(TCGContext *s, int temp)
1662 {
1663     TCGTemp *ts;
1664     ts = &s->temps[temp];
1665 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
1666     /* Sparc64 stack is accessed with offset of 2047 */
1667     s->current_frame_offset = (s->current_frame_offset +
1668                                (tcg_target_long)sizeof(tcg_target_long) - 1) &
1669         ~(sizeof(tcg_target_long) - 1);
1670 #endif
1671     if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
1672         s->frame_end) {
1673         tcg_abort();
1674     }
1675     ts->mem_offset = s->current_frame_offset;
1676     ts->mem_base = s->frame_temp;
1677     ts->mem_allocated = 1;
1678     s->current_frame_offset += sizeof(tcg_target_long);
1679 }
1680 
1681 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet);
1682 
1683 /* sync register 'reg' by saving it to the corresponding temporary */
1684 static void tcg_reg_sync(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
1685 {
1686     TCGTemp *ts = s->reg_to_temp[reg];
1687 
1688     assert(ts->val_type == TEMP_VAL_REG);
1689     if (!ts->mem_coherent && !ts->fixed_reg) {
1690         if (!ts->mem_allocated) {
1691             temp_allocate_frame(s, temp_idx(s, ts));
1692         } else if (ts->indirect_reg) {
1693             tcg_regset_set_reg(allocated_regs, ts->reg);
1694             temp_load(s, ts->mem_base,
1695                       tcg_target_available_regs[TCG_TYPE_PTR],
1696                       allocated_regs);
1697         }
1698         tcg_out_st(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
1699     }
1700     ts->mem_coherent = 1;
1701 }
1702 
1703 /* free register 'reg' by spilling the corresponding temporary if necessary */
1704 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
1705 {
1706     TCGTemp *ts = s->reg_to_temp[reg];
1707 
1708     if (ts != NULL) {
1709         tcg_reg_sync(s, reg, allocated_regs);
1710         ts->val_type = TEMP_VAL_MEM;
1711         s->reg_to_temp[reg] = NULL;
1712     }
1713 }
1714 
1715 /* Allocate a register belonging to reg1 & ~reg2 */
1716 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs,
1717                             TCGRegSet allocated_regs, bool rev)
1718 {
1719     int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
1720     const int *order;
1721     TCGReg reg;
1722     TCGRegSet reg_ct;
1723 
1724     tcg_regset_andnot(reg_ct, desired_regs, allocated_regs);
1725     order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
1726 
1727     /* first try free registers */
1728     for(i = 0; i < n; i++) {
1729         reg = order[i];
1730         if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL)
1731             return reg;
1732     }
1733 
1734     /* XXX: do better spill choice */
1735     for(i = 0; i < n; i++) {
1736         reg = order[i];
1737         if (tcg_regset_test_reg(reg_ct, reg)) {
1738             tcg_reg_free(s, reg, allocated_regs);
1739             return reg;
1740         }
1741     }
1742 
1743     tcg_abort();
1744 }
1745 
1746 /* Make sure the temporary is in a register.  If needed, allocate the register
1747    from DESIRED while avoiding ALLOCATED.  */
1748 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
1749                       TCGRegSet allocated_regs)
1750 {
1751     TCGReg reg;
1752 
1753     switch (ts->val_type) {
1754     case TEMP_VAL_REG:
1755         return;
1756     case TEMP_VAL_CONST:
1757         reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
1758         tcg_out_movi(s, ts->type, reg, ts->val);
1759         ts->mem_coherent = 0;
1760         break;
1761     case TEMP_VAL_MEM:
1762         reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
1763         if (ts->indirect_reg) {
1764             tcg_regset_set_reg(allocated_regs, reg);
1765             temp_load(s, ts->mem_base,
1766                       tcg_target_available_regs[TCG_TYPE_PTR],
1767                       allocated_regs);
1768         }
1769         tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
1770         ts->mem_coherent = 1;
1771         break;
1772     case TEMP_VAL_DEAD:
1773     default:
1774         tcg_abort();
1775     }
1776     ts->reg = reg;
1777     ts->val_type = TEMP_VAL_REG;
1778     s->reg_to_temp[reg] = ts;
1779 }
1780 
1781 /* mark a temporary as dead. */
1782 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
1783 {
1784     if (ts->fixed_reg) {
1785         return;
1786     }
1787     if (ts->val_type == TEMP_VAL_REG) {
1788         s->reg_to_temp[ts->reg] = NULL;
1789     }
1790     ts->val_type = (temp_idx(s, ts) < s->nb_globals || ts->temp_local
1791                     ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
1792 }
1793 
1794 /* sync a temporary to memory. 'allocated_regs' is used in case a
1795    temporary registers needs to be allocated to store a constant. */
1796 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1797 {
1798     if (ts->fixed_reg) {
1799         return;
1800     }
1801     switch (ts->val_type) {
1802     case TEMP_VAL_CONST:
1803         temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs);
1804         /* fallthrough */
1805     case TEMP_VAL_REG:
1806         tcg_reg_sync(s, ts->reg, allocated_regs);
1807         break;
1808     case TEMP_VAL_DEAD:
1809     case TEMP_VAL_MEM:
1810         break;
1811     default:
1812         tcg_abort();
1813     }
1814 }
1815 
1816 /* save a temporary to memory. 'allocated_regs' is used in case a
1817    temporary registers needs to be allocated to store a constant. */
1818 static inline void temp_save(TCGContext *s, TCGTemp *ts,
1819                              TCGRegSet allocated_regs)
1820 {
1821 #ifdef USE_LIVENESS_ANALYSIS
1822     /* ??? Liveness does not yet incorporate indirect bases.  */
1823     if (!ts->indirect_base) {
1824         /* The liveness analysis already ensures that globals are back
1825            in memory. Keep an assert for safety. */
1826         tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg);
1827         return;
1828     }
1829 #endif
1830     temp_sync(s, ts, allocated_regs);
1831     temp_dead(s, ts);
1832 }
1833 
1834 /* save globals to their canonical location and assume they can be
1835    modified be the following code. 'allocated_regs' is used in case a
1836    temporary registers needs to be allocated to store a constant. */
1837 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
1838 {
1839     int i;
1840 
1841     for (i = 0; i < s->nb_globals; i++) {
1842         temp_save(s, &s->temps[i], allocated_regs);
1843     }
1844 }
1845 
1846 /* sync globals to their canonical location and assume they can be
1847    read by the following code. 'allocated_regs' is used in case a
1848    temporary registers needs to be allocated to store a constant. */
1849 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
1850 {
1851     int i;
1852 
1853     for (i = 0; i < s->nb_globals; i++) {
1854         TCGTemp *ts = &s->temps[i];
1855 #ifdef USE_LIVENESS_ANALYSIS
1856         /* ??? Liveness does not yet incorporate indirect bases.  */
1857         if (!ts->indirect_base) {
1858             tcg_debug_assert(ts->val_type != TEMP_VAL_REG
1859                              || ts->fixed_reg
1860                              || ts->mem_coherent);
1861             continue;
1862         }
1863 #endif
1864         temp_sync(s, ts, allocated_regs);
1865     }
1866 }
1867 
1868 /* at the end of a basic block, we assume all temporaries are dead and
1869    all globals are stored at their canonical location. */
1870 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
1871 {
1872     int i;
1873 
1874     for (i = s->nb_globals; i < s->nb_temps; i++) {
1875         TCGTemp *ts = &s->temps[i];
1876         if (ts->temp_local) {
1877             temp_save(s, ts, allocated_regs);
1878         } else {
1879 #ifdef USE_LIVENESS_ANALYSIS
1880             /* ??? Liveness does not yet incorporate indirect bases.  */
1881             if (!ts->indirect_base) {
1882                 /* The liveness analysis already ensures that temps are dead.
1883                    Keep an assert for safety. */
1884                 assert(ts->val_type == TEMP_VAL_DEAD);
1885                 continue;
1886             }
1887 #endif
1888             temp_dead(s, ts);
1889         }
1890     }
1891 
1892     save_globals(s, allocated_regs);
1893 }
1894 
1895 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1)
1896 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1)
1897 
1898 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
1899                                uint16_t dead_args, uint8_t sync_args)
1900 {
1901     TCGTemp *ots;
1902     tcg_target_ulong val;
1903 
1904     ots = &s->temps[args[0]];
1905     val = args[1];
1906 
1907     if (ots->fixed_reg) {
1908         /* for fixed registers, we do not do any constant
1909            propagation */
1910         tcg_out_movi(s, ots->type, ots->reg, val);
1911     } else {
1912         /* The movi is not explicitly generated here */
1913         if (ots->val_type == TEMP_VAL_REG) {
1914             s->reg_to_temp[ots->reg] = NULL;
1915         }
1916         ots->val_type = TEMP_VAL_CONST;
1917         ots->val = val;
1918     }
1919     if (NEED_SYNC_ARG(0)) {
1920         temp_sync(s, ots, s->reserved_regs);
1921     }
1922     if (IS_DEAD_ARG(0)) {
1923         temp_dead(s, ots);
1924     }
1925 }
1926 
1927 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
1928                               const TCGArg *args, uint16_t dead_args,
1929                               uint8_t sync_args)
1930 {
1931     TCGRegSet allocated_regs;
1932     TCGTemp *ts, *ots;
1933     TCGType otype, itype;
1934 
1935     tcg_regset_set(allocated_regs, s->reserved_regs);
1936     ots = &s->temps[args[0]];
1937     ts = &s->temps[args[1]];
1938 
1939     /* Note that otype != itype for no-op truncation.  */
1940     otype = ots->type;
1941     itype = ts->type;
1942 
1943     /* If the source value is not in a register, and we're going to be
1944        forced to have it in a register in order to perform the copy,
1945        then copy the SOURCE value into its own register first.  That way
1946        we don't have to reload SOURCE the next time it is used. */
1947     if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG)
1948         || ts->val_type == TEMP_VAL_MEM) {
1949         temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs);
1950     }
1951 
1952     if (IS_DEAD_ARG(0) && !ots->fixed_reg) {
1953         /* mov to a non-saved dead register makes no sense (even with
1954            liveness analysis disabled). */
1955         assert(NEED_SYNC_ARG(0));
1956         /* The code above should have moved the temp to a register. */
1957         assert(ts->val_type == TEMP_VAL_REG);
1958         if (!ots->mem_allocated) {
1959             temp_allocate_frame(s, args[0]);
1960         }
1961         if (ots->indirect_reg) {
1962             tcg_regset_set_reg(allocated_regs, ts->reg);
1963             temp_load(s, ots->mem_base,
1964                       tcg_target_available_regs[TCG_TYPE_PTR],
1965                       allocated_regs);
1966         }
1967         tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
1968         if (IS_DEAD_ARG(1)) {
1969             temp_dead(s, ts);
1970         }
1971         temp_dead(s, ots);
1972     } else if (ts->val_type == TEMP_VAL_CONST) {
1973         /* propagate constant */
1974         if (ots->val_type == TEMP_VAL_REG) {
1975             s->reg_to_temp[ots->reg] = NULL;
1976         }
1977         ots->val_type = TEMP_VAL_CONST;
1978         ots->val = ts->val;
1979         if (IS_DEAD_ARG(1)) {
1980             temp_dead(s, ts);
1981         }
1982     } else {
1983         /* The code in the first if block should have moved the
1984            temp to a register. */
1985         assert(ts->val_type == TEMP_VAL_REG);
1986         if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) {
1987             /* the mov can be suppressed */
1988             if (ots->val_type == TEMP_VAL_REG) {
1989                 s->reg_to_temp[ots->reg] = NULL;
1990             }
1991             ots->reg = ts->reg;
1992             temp_dead(s, ts);
1993         } else {
1994             if (ots->val_type != TEMP_VAL_REG) {
1995                 /* When allocating a new register, make sure to not spill the
1996                    input one. */
1997                 tcg_regset_set_reg(allocated_regs, ts->reg);
1998                 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
1999                                          allocated_regs, ots->indirect_base);
2000             }
2001             tcg_out_mov(s, otype, ots->reg, ts->reg);
2002         }
2003         ots->val_type = TEMP_VAL_REG;
2004         ots->mem_coherent = 0;
2005         s->reg_to_temp[ots->reg] = ots;
2006         if (NEED_SYNC_ARG(0)) {
2007             tcg_reg_sync(s, ots->reg, allocated_regs);
2008         }
2009     }
2010 }
2011 
2012 static void tcg_reg_alloc_op(TCGContext *s,
2013                              const TCGOpDef *def, TCGOpcode opc,
2014                              const TCGArg *args, uint16_t dead_args,
2015                              uint8_t sync_args)
2016 {
2017     TCGRegSet allocated_regs;
2018     int i, k, nb_iargs, nb_oargs;
2019     TCGReg reg;
2020     TCGArg arg;
2021     const TCGArgConstraint *arg_ct;
2022     TCGTemp *ts;
2023     TCGArg new_args[TCG_MAX_OP_ARGS];
2024     int const_args[TCG_MAX_OP_ARGS];
2025 
2026     nb_oargs = def->nb_oargs;
2027     nb_iargs = def->nb_iargs;
2028 
2029     /* copy constants */
2030     memcpy(new_args + nb_oargs + nb_iargs,
2031            args + nb_oargs + nb_iargs,
2032            sizeof(TCGArg) * def->nb_cargs);
2033 
2034     /* satisfy input constraints */
2035     tcg_regset_set(allocated_regs, s->reserved_regs);
2036     for(k = 0; k < nb_iargs; k++) {
2037         i = def->sorted_args[nb_oargs + k];
2038         arg = args[i];
2039         arg_ct = &def->args_ct[i];
2040         ts = &s->temps[arg];
2041 
2042         if (ts->val_type == TEMP_VAL_CONST
2043             && tcg_target_const_match(ts->val, ts->type, arg_ct)) {
2044             /* constant is OK for instruction */
2045             const_args[i] = 1;
2046             new_args[i] = ts->val;
2047             goto iarg_end;
2048         }
2049 
2050         temp_load(s, ts, arg_ct->u.regs, allocated_regs);
2051 
2052         if (arg_ct->ct & TCG_CT_IALIAS) {
2053             if (ts->fixed_reg) {
2054                 /* if fixed register, we must allocate a new register
2055                    if the alias is not the same register */
2056                 if (arg != args[arg_ct->alias_index])
2057                     goto allocate_in_reg;
2058             } else {
2059                 /* if the input is aliased to an output and if it is
2060                    not dead after the instruction, we must allocate
2061                    a new register and move it */
2062                 if (!IS_DEAD_ARG(i)) {
2063                     goto allocate_in_reg;
2064                 }
2065                 /* check if the current register has already been allocated
2066                    for another input aliased to an output */
2067                 int k2, i2;
2068                 for (k2 = 0 ; k2 < k ; k2++) {
2069                     i2 = def->sorted_args[nb_oargs + k2];
2070                     if ((def->args_ct[i2].ct & TCG_CT_IALIAS) &&
2071                         (new_args[i2] == ts->reg)) {
2072                         goto allocate_in_reg;
2073                     }
2074                 }
2075             }
2076         }
2077         reg = ts->reg;
2078         if (tcg_regset_test_reg(arg_ct->u.regs, reg)) {
2079             /* nothing to do : the constraint is satisfied */
2080         } else {
2081         allocate_in_reg:
2082             /* allocate a new register matching the constraint
2083                and move the temporary register into it */
2084             reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
2085                                 ts->indirect_base);
2086             tcg_out_mov(s, ts->type, reg, ts->reg);
2087         }
2088         new_args[i] = reg;
2089         const_args[i] = 0;
2090         tcg_regset_set_reg(allocated_regs, reg);
2091     iarg_end: ;
2092     }
2093 
2094     /* mark dead temporaries and free the associated registers */
2095     for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2096         if (IS_DEAD_ARG(i)) {
2097             temp_dead(s, &s->temps[args[i]]);
2098         }
2099     }
2100 
2101     if (def->flags & TCG_OPF_BB_END) {
2102         tcg_reg_alloc_bb_end(s, allocated_regs);
2103     } else {
2104         if (def->flags & TCG_OPF_CALL_CLOBBER) {
2105             /* XXX: permit generic clobber register list ? */
2106             for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
2107                 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
2108                     tcg_reg_free(s, i, allocated_regs);
2109                 }
2110             }
2111         }
2112         if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2113             /* sync globals if the op has side effects and might trigger
2114                an exception. */
2115             sync_globals(s, allocated_regs);
2116         }
2117 
2118         /* satisfy the output constraints */
2119         tcg_regset_set(allocated_regs, s->reserved_regs);
2120         for(k = 0; k < nb_oargs; k++) {
2121             i = def->sorted_args[k];
2122             arg = args[i];
2123             arg_ct = &def->args_ct[i];
2124             ts = &s->temps[arg];
2125             if (arg_ct->ct & TCG_CT_ALIAS) {
2126                 reg = new_args[arg_ct->alias_index];
2127             } else {
2128                 /* if fixed register, we try to use it */
2129                 reg = ts->reg;
2130                 if (ts->fixed_reg &&
2131                     tcg_regset_test_reg(arg_ct->u.regs, reg)) {
2132                     goto oarg_end;
2133                 }
2134                 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
2135                                     ts->indirect_base);
2136             }
2137             tcg_regset_set_reg(allocated_regs, reg);
2138             /* if a fixed register is used, then a move will be done afterwards */
2139             if (!ts->fixed_reg) {
2140                 if (ts->val_type == TEMP_VAL_REG) {
2141                     s->reg_to_temp[ts->reg] = NULL;
2142                 }
2143                 ts->val_type = TEMP_VAL_REG;
2144                 ts->reg = reg;
2145                 /* temp value is modified, so the value kept in memory is
2146                    potentially not the same */
2147                 ts->mem_coherent = 0;
2148                 s->reg_to_temp[reg] = ts;
2149             }
2150         oarg_end:
2151             new_args[i] = reg;
2152         }
2153     }
2154 
2155     /* emit instruction */
2156     tcg_out_op(s, opc, new_args, const_args);
2157 
2158     /* move the outputs in the correct register if needed */
2159     for(i = 0; i < nb_oargs; i++) {
2160         ts = &s->temps[args[i]];
2161         reg = new_args[i];
2162         if (ts->fixed_reg && ts->reg != reg) {
2163             tcg_out_mov(s, ts->type, ts->reg, reg);
2164         }
2165         if (NEED_SYNC_ARG(i)) {
2166             tcg_reg_sync(s, reg, allocated_regs);
2167         }
2168         if (IS_DEAD_ARG(i)) {
2169             temp_dead(s, ts);
2170         }
2171     }
2172 }
2173 
2174 #ifdef TCG_TARGET_STACK_GROWSUP
2175 #define STACK_DIR(x) (-(x))
2176 #else
2177 #define STACK_DIR(x) (x)
2178 #endif
2179 
2180 static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
2181                                const TCGArg * const args, uint16_t dead_args,
2182                                uint8_t sync_args)
2183 {
2184     int flags, nb_regs, i;
2185     TCGReg reg;
2186     TCGArg arg;
2187     TCGTemp *ts;
2188     intptr_t stack_offset;
2189     size_t call_stack_size;
2190     tcg_insn_unit *func_addr;
2191     int allocate_args;
2192     TCGRegSet allocated_regs;
2193 
2194     func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs];
2195     flags = args[nb_oargs + nb_iargs + 1];
2196 
2197     nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2198     if (nb_regs > nb_iargs) {
2199         nb_regs = nb_iargs;
2200     }
2201 
2202     /* assign stack slots first */
2203     call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
2204     call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
2205         ~(TCG_TARGET_STACK_ALIGN - 1);
2206     allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
2207     if (allocate_args) {
2208         /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
2209            preallocate call stack */
2210         tcg_abort();
2211     }
2212 
2213     stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
2214     for(i = nb_regs; i < nb_iargs; i++) {
2215         arg = args[nb_oargs + i];
2216 #ifdef TCG_TARGET_STACK_GROWSUP
2217         stack_offset -= sizeof(tcg_target_long);
2218 #endif
2219         if (arg != TCG_CALL_DUMMY_ARG) {
2220             ts = &s->temps[arg];
2221             temp_load(s, ts, tcg_target_available_regs[ts->type],
2222                       s->reserved_regs);
2223             tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
2224         }
2225 #ifndef TCG_TARGET_STACK_GROWSUP
2226         stack_offset += sizeof(tcg_target_long);
2227 #endif
2228     }
2229 
2230     /* assign input registers */
2231     tcg_regset_set(allocated_regs, s->reserved_regs);
2232     for(i = 0; i < nb_regs; i++) {
2233         arg = args[nb_oargs + i];
2234         if (arg != TCG_CALL_DUMMY_ARG) {
2235             ts = &s->temps[arg];
2236             reg = tcg_target_call_iarg_regs[i];
2237             tcg_reg_free(s, reg, allocated_regs);
2238 
2239             if (ts->val_type == TEMP_VAL_REG) {
2240                 if (ts->reg != reg) {
2241                     tcg_out_mov(s, ts->type, reg, ts->reg);
2242                 }
2243             } else {
2244                 TCGRegSet arg_set;
2245 
2246                 tcg_regset_clear(arg_set);
2247                 tcg_regset_set_reg(arg_set, reg);
2248                 temp_load(s, ts, arg_set, allocated_regs);
2249             }
2250 
2251             tcg_regset_set_reg(allocated_regs, reg);
2252         }
2253     }
2254 
2255     /* mark dead temporaries and free the associated registers */
2256     for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2257         if (IS_DEAD_ARG(i)) {
2258             temp_dead(s, &s->temps[args[i]]);
2259         }
2260     }
2261 
2262     /* clobber call registers */
2263     for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
2264         if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
2265             tcg_reg_free(s, i, allocated_regs);
2266         }
2267     }
2268 
2269     /* Save globals if they might be written by the helper, sync them if
2270        they might be read. */
2271     if (flags & TCG_CALL_NO_READ_GLOBALS) {
2272         /* Nothing to do */
2273     } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
2274         sync_globals(s, allocated_regs);
2275     } else {
2276         save_globals(s, allocated_regs);
2277     }
2278 
2279     tcg_out_call(s, func_addr);
2280 
2281     /* assign output registers and emit moves if needed */
2282     for(i = 0; i < nb_oargs; i++) {
2283         arg = args[i];
2284         ts = &s->temps[arg];
2285         reg = tcg_target_call_oarg_regs[i];
2286         assert(s->reg_to_temp[reg] == NULL);
2287 
2288         if (ts->fixed_reg) {
2289             if (ts->reg != reg) {
2290                 tcg_out_mov(s, ts->type, ts->reg, reg);
2291             }
2292         } else {
2293             if (ts->val_type == TEMP_VAL_REG) {
2294                 s->reg_to_temp[ts->reg] = NULL;
2295             }
2296             ts->val_type = TEMP_VAL_REG;
2297             ts->reg = reg;
2298             ts->mem_coherent = 0;
2299             s->reg_to_temp[reg] = ts;
2300             if (NEED_SYNC_ARG(i)) {
2301                 tcg_reg_sync(s, reg, allocated_regs);
2302             }
2303             if (IS_DEAD_ARG(i)) {
2304                 temp_dead(s, ts);
2305             }
2306         }
2307     }
2308 }
2309 
2310 #ifdef CONFIG_PROFILER
2311 
2312 static int64_t tcg_table_op_count[NB_OPS];
2313 
2314 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
2315 {
2316     int i;
2317 
2318     for (i = 0; i < NB_OPS; i++) {
2319         cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name,
2320                     tcg_table_op_count[i]);
2321     }
2322 }
2323 #else
2324 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
2325 {
2326     cpu_fprintf(f, "[TCG profiler not compiled]\n");
2327 }
2328 #endif
2329 
2330 
2331 int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
2332 {
2333     int i, oi, oi_next, num_insns;
2334 
2335 #ifdef CONFIG_PROFILER
2336     {
2337         int n;
2338 
2339         n = s->gen_last_op_idx + 1;
2340         s->op_count += n;
2341         if (n > s->op_count_max) {
2342             s->op_count_max = n;
2343         }
2344 
2345         n = s->nb_temps;
2346         s->temp_count += n;
2347         if (n > s->temp_count_max) {
2348             s->temp_count_max = n;
2349         }
2350     }
2351 #endif
2352 
2353 #ifdef DEBUG_DISAS
2354     if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
2355                  && qemu_log_in_addr_range(tb->pc))) {
2356         qemu_log("OP:\n");
2357         tcg_dump_ops(s);
2358         qemu_log("\n");
2359     }
2360 #endif
2361 
2362 #ifdef CONFIG_PROFILER
2363     s->opt_time -= profile_getclock();
2364 #endif
2365 
2366 #ifdef USE_TCG_OPTIMIZATIONS
2367     tcg_optimize(s);
2368 #endif
2369 
2370 #ifdef CONFIG_PROFILER
2371     s->opt_time += profile_getclock();
2372     s->la_time -= profile_getclock();
2373 #endif
2374 
2375     tcg_liveness_analysis(s);
2376 
2377 #ifdef CONFIG_PROFILER
2378     s->la_time += profile_getclock();
2379 #endif
2380 
2381 #ifdef DEBUG_DISAS
2382     if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
2383                  && qemu_log_in_addr_range(tb->pc))) {
2384         qemu_log("OP after optimization and liveness analysis:\n");
2385         tcg_dump_ops(s);
2386         qemu_log("\n");
2387     }
2388 #endif
2389 
2390     tcg_reg_alloc_start(s);
2391 
2392     s->code_buf = tb->tc_ptr;
2393     s->code_ptr = tb->tc_ptr;
2394 
2395     tcg_out_tb_init(s);
2396 
2397     num_insns = -1;
2398     for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) {
2399         TCGOp * const op = &s->gen_op_buf[oi];
2400         TCGArg * const args = &s->gen_opparam_buf[op->args];
2401         TCGOpcode opc = op->opc;
2402         const TCGOpDef *def = &tcg_op_defs[opc];
2403         uint16_t dead_args = s->op_dead_args[oi];
2404         uint8_t sync_args = s->op_sync_args[oi];
2405 
2406         oi_next = op->next;
2407 #ifdef CONFIG_PROFILER
2408         tcg_table_op_count[opc]++;
2409 #endif
2410 
2411         switch (opc) {
2412         case INDEX_op_mov_i32:
2413         case INDEX_op_mov_i64:
2414             tcg_reg_alloc_mov(s, def, args, dead_args, sync_args);
2415             break;
2416         case INDEX_op_movi_i32:
2417         case INDEX_op_movi_i64:
2418             tcg_reg_alloc_movi(s, args, dead_args, sync_args);
2419             break;
2420         case INDEX_op_insn_start:
2421             if (num_insns >= 0) {
2422                 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
2423             }
2424             num_insns++;
2425             for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
2426                 target_ulong a;
2427 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2428                 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
2429 #else
2430                 a = args[i];
2431 #endif
2432                 s->gen_insn_data[num_insns][i] = a;
2433             }
2434             break;
2435         case INDEX_op_discard:
2436             temp_dead(s, &s->temps[args[0]]);
2437             break;
2438         case INDEX_op_set_label:
2439             tcg_reg_alloc_bb_end(s, s->reserved_regs);
2440             tcg_out_label(s, arg_label(args[0]), s->code_ptr);
2441             break;
2442         case INDEX_op_call:
2443             tcg_reg_alloc_call(s, op->callo, op->calli, args,
2444                                dead_args, sync_args);
2445             break;
2446         default:
2447             /* Sanity check that we've not introduced any unhandled opcodes. */
2448             if (def->flags & TCG_OPF_NOT_PRESENT) {
2449                 tcg_abort();
2450             }
2451             /* Note: in order to speed up the code, it would be much
2452                faster to have specialized register allocator functions for
2453                some common argument patterns */
2454             tcg_reg_alloc_op(s, def, opc, args, dead_args, sync_args);
2455             break;
2456         }
2457 #ifndef NDEBUG
2458         check_regs(s);
2459 #endif
2460         /* Test for (pending) buffer overflow.  The assumption is that any
2461            one operation beginning below the high water mark cannot overrun
2462            the buffer completely.  Thus we can test for overflow after
2463            generating code without having to check during generation.  */
2464         if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
2465             return -1;
2466         }
2467     }
2468     tcg_debug_assert(num_insns >= 0);
2469     s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
2470 
2471     /* Generate TB finalization at the end of block */
2472     if (!tcg_out_tb_finalize(s)) {
2473         return -1;
2474     }
2475 
2476     /* flush instruction cache */
2477     flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
2478 
2479     return tcg_current_code_size(s);
2480 }
2481 
2482 #ifdef CONFIG_PROFILER
2483 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
2484 {
2485     TCGContext *s = &tcg_ctx;
2486     int64_t tb_count = s->tb_count;
2487     int64_t tb_div_count = tb_count ? tb_count : 1;
2488     int64_t tot = s->interm_time + s->code_time;
2489 
2490     cpu_fprintf(f, "JIT cycles          %" PRId64 " (%0.3f s at 2.4 GHz)\n",
2491                 tot, tot / 2.4e9);
2492     cpu_fprintf(f, "translated TBs      %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n",
2493                 tb_count, s->tb_count1 - tb_count,
2494                 (double)(s->tb_count1 - s->tb_count)
2495                 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
2496     cpu_fprintf(f, "avg ops/TB          %0.1f max=%d\n",
2497                 (double)s->op_count / tb_div_count, s->op_count_max);
2498     cpu_fprintf(f, "deleted ops/TB      %0.2f\n",
2499                 (double)s->del_op_count / tb_div_count);
2500     cpu_fprintf(f, "avg temps/TB        %0.2f max=%d\n",
2501                 (double)s->temp_count / tb_div_count, s->temp_count_max);
2502     cpu_fprintf(f, "avg host code/TB    %0.1f\n",
2503                 (double)s->code_out_len / tb_div_count);
2504     cpu_fprintf(f, "avg search data/TB  %0.1f\n",
2505                 (double)s->search_out_len / tb_div_count);
2506 
2507     cpu_fprintf(f, "cycles/op           %0.1f\n",
2508                 s->op_count ? (double)tot / s->op_count : 0);
2509     cpu_fprintf(f, "cycles/in byte      %0.1f\n",
2510                 s->code_in_len ? (double)tot / s->code_in_len : 0);
2511     cpu_fprintf(f, "cycles/out byte     %0.1f\n",
2512                 s->code_out_len ? (double)tot / s->code_out_len : 0);
2513     cpu_fprintf(f, "cycles/search byte     %0.1f\n",
2514                 s->search_out_len ? (double)tot / s->search_out_len : 0);
2515     if (tot == 0) {
2516         tot = 1;
2517     }
2518     cpu_fprintf(f, "  gen_interm time   %0.1f%%\n",
2519                 (double)s->interm_time / tot * 100.0);
2520     cpu_fprintf(f, "  gen_code time     %0.1f%%\n",
2521                 (double)s->code_time / tot * 100.0);
2522     cpu_fprintf(f, "optim./code time    %0.1f%%\n",
2523                 (double)s->opt_time / (s->code_time ? s->code_time : 1)
2524                 * 100.0);
2525     cpu_fprintf(f, "liveness/code time  %0.1f%%\n",
2526                 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
2527     cpu_fprintf(f, "cpu_restore count   %" PRId64 "\n",
2528                 s->restore_count);
2529     cpu_fprintf(f, "  avg cycles        %0.1f\n",
2530                 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
2531 }
2532 #else
2533 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
2534 {
2535     cpu_fprintf(f, "[TCG profiler not compiled]\n");
2536 }
2537 #endif
2538 
2539 #ifdef ELF_HOST_MACHINE
2540 /* In order to use this feature, the backend needs to do three things:
2541 
2542    (1) Define ELF_HOST_MACHINE to indicate both what value to
2543        put into the ELF image and to indicate support for the feature.
2544 
2545    (2) Define tcg_register_jit.  This should create a buffer containing
2546        the contents of a .debug_frame section that describes the post-
2547        prologue unwind info for the tcg machine.
2548 
2549    (3) Call tcg_register_jit_int, with the constructed .debug_frame.
2550 */
2551 
2552 /* Begin GDB interface.  THE FOLLOWING MUST MATCH GDB DOCS.  */
2553 typedef enum {
2554     JIT_NOACTION = 0,
2555     JIT_REGISTER_FN,
2556     JIT_UNREGISTER_FN
2557 } jit_actions_t;
2558 
2559 struct jit_code_entry {
2560     struct jit_code_entry *next_entry;
2561     struct jit_code_entry *prev_entry;
2562     const void *symfile_addr;
2563     uint64_t symfile_size;
2564 };
2565 
2566 struct jit_descriptor {
2567     uint32_t version;
2568     uint32_t action_flag;
2569     struct jit_code_entry *relevant_entry;
2570     struct jit_code_entry *first_entry;
2571 };
2572 
2573 void __jit_debug_register_code(void) __attribute__((noinline));
2574 void __jit_debug_register_code(void)
2575 {
2576     asm("");
2577 }
2578 
2579 /* Must statically initialize the version, because GDB may check
2580    the version before we can set it.  */
2581 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
2582 
2583 /* End GDB interface.  */
2584 
2585 static int find_string(const char *strtab, const char *str)
2586 {
2587     const char *p = strtab + 1;
2588 
2589     while (1) {
2590         if (strcmp(p, str) == 0) {
2591             return p - strtab;
2592         }
2593         p += strlen(p) + 1;
2594     }
2595 }
2596 
2597 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size,
2598                                  const void *debug_frame,
2599                                  size_t debug_frame_size)
2600 {
2601     struct __attribute__((packed)) DebugInfo {
2602         uint32_t  len;
2603         uint16_t  version;
2604         uint32_t  abbrev;
2605         uint8_t   ptr_size;
2606         uint8_t   cu_die;
2607         uint16_t  cu_lang;
2608         uintptr_t cu_low_pc;
2609         uintptr_t cu_high_pc;
2610         uint8_t   fn_die;
2611         char      fn_name[16];
2612         uintptr_t fn_low_pc;
2613         uintptr_t fn_high_pc;
2614         uint8_t   cu_eoc;
2615     };
2616 
2617     struct ElfImage {
2618         ElfW(Ehdr) ehdr;
2619         ElfW(Phdr) phdr;
2620         ElfW(Shdr) shdr[7];
2621         ElfW(Sym)  sym[2];
2622         struct DebugInfo di;
2623         uint8_t    da[24];
2624         char       str[80];
2625     };
2626 
2627     struct ElfImage *img;
2628 
2629     static const struct ElfImage img_template = {
2630         .ehdr = {
2631             .e_ident[EI_MAG0] = ELFMAG0,
2632             .e_ident[EI_MAG1] = ELFMAG1,
2633             .e_ident[EI_MAG2] = ELFMAG2,
2634             .e_ident[EI_MAG3] = ELFMAG3,
2635             .e_ident[EI_CLASS] = ELF_CLASS,
2636             .e_ident[EI_DATA] = ELF_DATA,
2637             .e_ident[EI_VERSION] = EV_CURRENT,
2638             .e_type = ET_EXEC,
2639             .e_machine = ELF_HOST_MACHINE,
2640             .e_version = EV_CURRENT,
2641             .e_phoff = offsetof(struct ElfImage, phdr),
2642             .e_shoff = offsetof(struct ElfImage, shdr),
2643             .e_ehsize = sizeof(ElfW(Shdr)),
2644             .e_phentsize = sizeof(ElfW(Phdr)),
2645             .e_phnum = 1,
2646             .e_shentsize = sizeof(ElfW(Shdr)),
2647             .e_shnum = ARRAY_SIZE(img->shdr),
2648             .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
2649 #ifdef ELF_HOST_FLAGS
2650             .e_flags = ELF_HOST_FLAGS,
2651 #endif
2652 #ifdef ELF_OSABI
2653             .e_ident[EI_OSABI] = ELF_OSABI,
2654 #endif
2655         },
2656         .phdr = {
2657             .p_type = PT_LOAD,
2658             .p_flags = PF_X,
2659         },
2660         .shdr = {
2661             [0] = { .sh_type = SHT_NULL },
2662             /* Trick: The contents of code_gen_buffer are not present in
2663                this fake ELF file; that got allocated elsewhere.  Therefore
2664                we mark .text as SHT_NOBITS (similar to .bss) so that readers
2665                will not look for contents.  We can record any address.  */
2666             [1] = { /* .text */
2667                 .sh_type = SHT_NOBITS,
2668                 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
2669             },
2670             [2] = { /* .debug_info */
2671                 .sh_type = SHT_PROGBITS,
2672                 .sh_offset = offsetof(struct ElfImage, di),
2673                 .sh_size = sizeof(struct DebugInfo),
2674             },
2675             [3] = { /* .debug_abbrev */
2676                 .sh_type = SHT_PROGBITS,
2677                 .sh_offset = offsetof(struct ElfImage, da),
2678                 .sh_size = sizeof(img->da),
2679             },
2680             [4] = { /* .debug_frame */
2681                 .sh_type = SHT_PROGBITS,
2682                 .sh_offset = sizeof(struct ElfImage),
2683             },
2684             [5] = { /* .symtab */
2685                 .sh_type = SHT_SYMTAB,
2686                 .sh_offset = offsetof(struct ElfImage, sym),
2687                 .sh_size = sizeof(img->sym),
2688                 .sh_info = 1,
2689                 .sh_link = ARRAY_SIZE(img->shdr) - 1,
2690                 .sh_entsize = sizeof(ElfW(Sym)),
2691             },
2692             [6] = { /* .strtab */
2693                 .sh_type = SHT_STRTAB,
2694                 .sh_offset = offsetof(struct ElfImage, str),
2695                 .sh_size = sizeof(img->str),
2696             }
2697         },
2698         .sym = {
2699             [1] = { /* code_gen_buffer */
2700                 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
2701                 .st_shndx = 1,
2702             }
2703         },
2704         .di = {
2705             .len = sizeof(struct DebugInfo) - 4,
2706             .version = 2,
2707             .ptr_size = sizeof(void *),
2708             .cu_die = 1,
2709             .cu_lang = 0x8001,  /* DW_LANG_Mips_Assembler */
2710             .fn_die = 2,
2711             .fn_name = "code_gen_buffer"
2712         },
2713         .da = {
2714             1,          /* abbrev number (the cu) */
2715             0x11, 1,    /* DW_TAG_compile_unit, has children */
2716             0x13, 0x5,  /* DW_AT_language, DW_FORM_data2 */
2717             0x11, 0x1,  /* DW_AT_low_pc, DW_FORM_addr */
2718             0x12, 0x1,  /* DW_AT_high_pc, DW_FORM_addr */
2719             0, 0,       /* end of abbrev */
2720             2,          /* abbrev number (the fn) */
2721             0x2e, 0,    /* DW_TAG_subprogram, no children */
2722             0x3, 0x8,   /* DW_AT_name, DW_FORM_string */
2723             0x11, 0x1,  /* DW_AT_low_pc, DW_FORM_addr */
2724             0x12, 0x1,  /* DW_AT_high_pc, DW_FORM_addr */
2725             0, 0,       /* end of abbrev */
2726             0           /* no more abbrev */
2727         },
2728         .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
2729                ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
2730     };
2731 
2732     /* We only need a single jit entry; statically allocate it.  */
2733     static struct jit_code_entry one_entry;
2734 
2735     uintptr_t buf = (uintptr_t)buf_ptr;
2736     size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2737     DebugFrameHeader *dfh;
2738 
2739     img = g_malloc(img_size);
2740     *img = img_template;
2741 
2742     img->phdr.p_vaddr = buf;
2743     img->phdr.p_paddr = buf;
2744     img->phdr.p_memsz = buf_size;
2745 
2746     img->shdr[1].sh_name = find_string(img->str, ".text");
2747     img->shdr[1].sh_addr = buf;
2748     img->shdr[1].sh_size = buf_size;
2749 
2750     img->shdr[2].sh_name = find_string(img->str, ".debug_info");
2751     img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
2752 
2753     img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
2754     img->shdr[4].sh_size = debug_frame_size;
2755 
2756     img->shdr[5].sh_name = find_string(img->str, ".symtab");
2757     img->shdr[6].sh_name = find_string(img->str, ".strtab");
2758 
2759     img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
2760     img->sym[1].st_value = buf;
2761     img->sym[1].st_size = buf_size;
2762 
2763     img->di.cu_low_pc = buf;
2764     img->di.cu_high_pc = buf + buf_size;
2765     img->di.fn_low_pc = buf;
2766     img->di.fn_high_pc = buf + buf_size;
2767 
2768     dfh = (DebugFrameHeader *)(img + 1);
2769     memcpy(dfh, debug_frame, debug_frame_size);
2770     dfh->fde.func_start = buf;
2771     dfh->fde.func_len = buf_size;
2772 
2773 #ifdef DEBUG_JIT
2774     /* Enable this block to be able to debug the ELF image file creation.
2775        One can use readelf, objdump, or other inspection utilities.  */
2776     {
2777         FILE *f = fopen("/tmp/qemu.jit", "w+b");
2778         if (f) {
2779             if (fwrite(img, img_size, 1, f) != img_size) {
2780                 /* Avoid stupid unused return value warning for fwrite.  */
2781             }
2782             fclose(f);
2783         }
2784     }
2785 #endif
2786 
2787     one_entry.symfile_addr = img;
2788     one_entry.symfile_size = img_size;
2789 
2790     __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
2791     __jit_debug_descriptor.relevant_entry = &one_entry;
2792     __jit_debug_descriptor.first_entry = &one_entry;
2793     __jit_debug_register_code();
2794 }
2795 #else
2796 /* No support for the feature.  Provide the entry point expected by exec.c,
2797    and implement the internal function we declared earlier.  */
2798 
2799 static void tcg_register_jit_int(void *buf, size_t size,
2800                                  const void *debug_frame,
2801                                  size_t debug_frame_size)
2802 {
2803 }
2804 
2805 void tcg_register_jit(void *buf, size_t buf_size)
2806 {
2807 }
2808 #endif /* ELF_HOST_MACHINE */
2809