1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/error-report.h" 34 #include "qemu/cutils.h" 35 #include "qemu/host-utils.h" 36 #include "qemu/qemu-print.h" 37 #include "qemu/timer.h" 38 #include "qemu/cacheflush.h" 39 40 /* Note: the long term plan is to reduce the dependencies on the QEMU 41 CPU definitions. Currently they are used for qemu_ld/st 42 instructions */ 43 #define NO_CPU_IO_DEFS 44 45 #include "exec/exec-all.h" 46 #include "tcg/tcg-op.h" 47 48 #if UINTPTR_MAX == UINT32_MAX 49 # define ELF_CLASS ELFCLASS32 50 #else 51 # define ELF_CLASS ELFCLASS64 52 #endif 53 #ifdef HOST_WORDS_BIGENDIAN 54 # define ELF_DATA ELFDATA2MSB 55 #else 56 # define ELF_DATA ELFDATA2LSB 57 #endif 58 59 #include "elf.h" 60 #include "exec/log.h" 61 #include "tcg-internal.h" 62 63 #ifdef CONFIG_TCG_INTERPRETER 64 #include <ffi.h> 65 #endif 66 67 /* Forward declarations for functions declared in tcg-target.c.inc and 68 used here. */ 69 static void tcg_target_init(TCGContext *s); 70 static void tcg_target_qemu_prologue(TCGContext *s); 71 static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 72 intptr_t value, intptr_t addend); 73 74 /* The CIE and FDE header definitions will be common to all hosts. */ 75 typedef struct { 76 uint32_t len __attribute__((aligned((sizeof(void *))))); 77 uint32_t id; 78 uint8_t version; 79 char augmentation[1]; 80 uint8_t code_align; 81 uint8_t data_align; 82 uint8_t return_column; 83 } DebugFrameCIE; 84 85 typedef struct QEMU_PACKED { 86 uint32_t len __attribute__((aligned((sizeof(void *))))); 87 uint32_t cie_offset; 88 uintptr_t func_start; 89 uintptr_t func_len; 90 } DebugFrameFDEHeader; 91 92 typedef struct QEMU_PACKED { 93 DebugFrameCIE cie; 94 DebugFrameFDEHeader fde; 95 } DebugFrameHeader; 96 97 static void tcg_register_jit_int(const void *buf, size_t size, 98 const void *debug_frame, 99 size_t debug_frame_size) 100 __attribute__((unused)); 101 102 /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 103 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 104 intptr_t arg2); 105 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 106 static void tcg_out_movi(TCGContext *s, TCGType type, 107 TCGReg ret, tcg_target_long arg); 108 static void tcg_out_op(TCGContext *s, TCGOpcode opc, 109 const TCGArg args[TCG_MAX_OP_ARGS], 110 const int const_args[TCG_MAX_OP_ARGS]); 111 #if TCG_TARGET_MAYBE_vec 112 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 113 TCGReg dst, TCGReg src); 114 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 115 TCGReg dst, TCGReg base, intptr_t offset); 116 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 117 TCGReg dst, int64_t arg); 118 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 119 unsigned vecl, unsigned vece, 120 const TCGArg args[TCG_MAX_OP_ARGS], 121 const int const_args[TCG_MAX_OP_ARGS]); 122 #else 123 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 124 TCGReg dst, TCGReg src) 125 { 126 g_assert_not_reached(); 127 } 128 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 129 TCGReg dst, TCGReg base, intptr_t offset) 130 { 131 g_assert_not_reached(); 132 } 133 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 134 TCGReg dst, int64_t arg) 135 { 136 g_assert_not_reached(); 137 } 138 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 139 unsigned vecl, unsigned vece, 140 const TCGArg args[TCG_MAX_OP_ARGS], 141 const int const_args[TCG_MAX_OP_ARGS]) 142 { 143 g_assert_not_reached(); 144 } 145 #endif 146 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 147 intptr_t arg2); 148 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 149 TCGReg base, intptr_t ofs); 150 #ifdef CONFIG_TCG_INTERPRETER 151 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 152 ffi_cif *cif); 153 #else 154 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target); 155 #endif 156 static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 157 #ifdef TCG_TARGET_NEED_LDST_LABELS 158 static int tcg_out_ldst_finalize(TCGContext *s); 159 #endif 160 161 TCGContext tcg_init_ctx; 162 __thread TCGContext *tcg_ctx; 163 164 TCGContext **tcg_ctxs; 165 unsigned int tcg_cur_ctxs; 166 unsigned int tcg_max_ctxs; 167 TCGv_env cpu_env = 0; 168 const void *tcg_code_gen_epilogue; 169 uintptr_t tcg_splitwx_diff; 170 171 #ifndef CONFIG_TCG_INTERPRETER 172 tcg_prologue_fn *tcg_qemu_tb_exec; 173 #endif 174 175 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 176 static TCGRegSet tcg_target_call_clobber_regs; 177 178 #if TCG_TARGET_INSN_UNIT_SIZE == 1 179 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 180 { 181 *s->code_ptr++ = v; 182 } 183 184 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 185 uint8_t v) 186 { 187 *p = v; 188 } 189 #endif 190 191 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 192 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 193 { 194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 195 *s->code_ptr++ = v; 196 } else { 197 tcg_insn_unit *p = s->code_ptr; 198 memcpy(p, &v, sizeof(v)); 199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 200 } 201 } 202 203 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 204 uint16_t v) 205 { 206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 207 *p = v; 208 } else { 209 memcpy(p, &v, sizeof(v)); 210 } 211 } 212 #endif 213 214 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 215 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 216 { 217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 218 *s->code_ptr++ = v; 219 } else { 220 tcg_insn_unit *p = s->code_ptr; 221 memcpy(p, &v, sizeof(v)); 222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 223 } 224 } 225 226 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 227 uint32_t v) 228 { 229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 230 *p = v; 231 } else { 232 memcpy(p, &v, sizeof(v)); 233 } 234 } 235 #endif 236 237 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 238 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 239 { 240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 241 *s->code_ptr++ = v; 242 } else { 243 tcg_insn_unit *p = s->code_ptr; 244 memcpy(p, &v, sizeof(v)); 245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 246 } 247 } 248 249 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 250 uint64_t v) 251 { 252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 253 *p = v; 254 } else { 255 memcpy(p, &v, sizeof(v)); 256 } 257 } 258 #endif 259 260 /* label relocation processing */ 261 262 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 263 TCGLabel *l, intptr_t addend) 264 { 265 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 266 267 r->type = type; 268 r->ptr = code_ptr; 269 r->addend = addend; 270 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 271 } 272 273 static void tcg_out_label(TCGContext *s, TCGLabel *l) 274 { 275 tcg_debug_assert(!l->has_value); 276 l->has_value = 1; 277 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 278 } 279 280 TCGLabel *gen_new_label(void) 281 { 282 TCGContext *s = tcg_ctx; 283 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 284 285 memset(l, 0, sizeof(TCGLabel)); 286 l->id = s->nb_labels++; 287 QSIMPLEQ_INIT(&l->relocs); 288 289 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 290 291 return l; 292 } 293 294 static bool tcg_resolve_relocs(TCGContext *s) 295 { 296 TCGLabel *l; 297 298 QSIMPLEQ_FOREACH(l, &s->labels, next) { 299 TCGRelocation *r; 300 uintptr_t value = l->u.value; 301 302 QSIMPLEQ_FOREACH(r, &l->relocs, next) { 303 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 304 return false; 305 } 306 } 307 } 308 return true; 309 } 310 311 static void set_jmp_reset_offset(TCGContext *s, int which) 312 { 313 /* 314 * We will check for overflow at the end of the opcode loop in 315 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 316 */ 317 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); 318 } 319 320 /* Signal overflow, starting over with fewer guest insns. */ 321 static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s) 322 { 323 siglongjmp(s->jmp_trans, -2); 324 } 325 326 #define C_PFX1(P, A) P##A 327 #define C_PFX2(P, A, B) P##A##_##B 328 #define C_PFX3(P, A, B, C) P##A##_##B##_##C 329 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 330 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 331 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 332 333 /* Define an enumeration for the various combinations. */ 334 335 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 336 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 337 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 338 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 339 340 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 341 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 342 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 343 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 344 345 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 346 347 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 348 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 349 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 350 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 351 352 typedef enum { 353 #include "tcg-target-con-set.h" 354 } TCGConstraintSetIndex; 355 356 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 357 358 #undef C_O0_I1 359 #undef C_O0_I2 360 #undef C_O0_I3 361 #undef C_O0_I4 362 #undef C_O1_I1 363 #undef C_O1_I2 364 #undef C_O1_I3 365 #undef C_O1_I4 366 #undef C_N1_I2 367 #undef C_O2_I1 368 #undef C_O2_I2 369 #undef C_O2_I3 370 #undef C_O2_I4 371 372 /* Put all of the constraint sets into an array, indexed by the enum. */ 373 374 #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 375 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 376 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 377 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 378 379 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 380 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 381 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 382 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 383 384 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 385 386 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 387 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 388 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 389 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 390 391 static const TCGTargetOpDef constraint_sets[] = { 392 #include "tcg-target-con-set.h" 393 }; 394 395 396 #undef C_O0_I1 397 #undef C_O0_I2 398 #undef C_O0_I3 399 #undef C_O0_I4 400 #undef C_O1_I1 401 #undef C_O1_I2 402 #undef C_O1_I3 403 #undef C_O1_I4 404 #undef C_N1_I2 405 #undef C_O2_I1 406 #undef C_O2_I2 407 #undef C_O2_I3 408 #undef C_O2_I4 409 410 /* Expand the enumerator to be returned from tcg_target_op_def(). */ 411 412 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 413 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 414 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 415 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 416 417 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 418 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 419 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 420 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 421 422 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 423 424 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 425 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 426 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 427 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 428 429 #include "tcg-target.c.inc" 430 431 static void alloc_tcg_plugin_context(TCGContext *s) 432 { 433 #ifdef CONFIG_PLUGIN 434 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 435 s->plugin_tb->insns = 436 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 437 #endif 438 } 439 440 /* 441 * All TCG threads except the parent (i.e. the one that called tcg_context_init 442 * and registered the target's TCG globals) must register with this function 443 * before initiating translation. 444 * 445 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 446 * of tcg_region_init() for the reasoning behind this. 447 * 448 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 449 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 450 * is not used anymore for translation once this function is called. 451 * 452 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 453 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 454 */ 455 #ifdef CONFIG_USER_ONLY 456 void tcg_register_thread(void) 457 { 458 tcg_ctx = &tcg_init_ctx; 459 } 460 #else 461 void tcg_register_thread(void) 462 { 463 TCGContext *s = g_malloc(sizeof(*s)); 464 unsigned int i, n; 465 466 *s = tcg_init_ctx; 467 468 /* Relink mem_base. */ 469 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 470 if (tcg_init_ctx.temps[i].mem_base) { 471 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 472 tcg_debug_assert(b >= 0 && b < n); 473 s->temps[i].mem_base = &s->temps[b]; 474 } 475 } 476 477 /* Claim an entry in tcg_ctxs */ 478 n = qatomic_fetch_inc(&tcg_cur_ctxs); 479 g_assert(n < tcg_max_ctxs); 480 qatomic_set(&tcg_ctxs[n], s); 481 482 if (n > 0) { 483 alloc_tcg_plugin_context(s); 484 tcg_region_initial_alloc(s); 485 } 486 487 tcg_ctx = s; 488 } 489 #endif /* !CONFIG_USER_ONLY */ 490 491 /* pool based memory allocation */ 492 void *tcg_malloc_internal(TCGContext *s, int size) 493 { 494 TCGPool *p; 495 int pool_size; 496 497 if (size > TCG_POOL_CHUNK_SIZE) { 498 /* big malloc: insert a new pool (XXX: could optimize) */ 499 p = g_malloc(sizeof(TCGPool) + size); 500 p->size = size; 501 p->next = s->pool_first_large; 502 s->pool_first_large = p; 503 return p->data; 504 } else { 505 p = s->pool_current; 506 if (!p) { 507 p = s->pool_first; 508 if (!p) 509 goto new_pool; 510 } else { 511 if (!p->next) { 512 new_pool: 513 pool_size = TCG_POOL_CHUNK_SIZE; 514 p = g_malloc(sizeof(TCGPool) + pool_size); 515 p->size = pool_size; 516 p->next = NULL; 517 if (s->pool_current) 518 s->pool_current->next = p; 519 else 520 s->pool_first = p; 521 } else { 522 p = p->next; 523 } 524 } 525 } 526 s->pool_current = p; 527 s->pool_cur = p->data + size; 528 s->pool_end = p->data + p->size; 529 return p->data; 530 } 531 532 void tcg_pool_reset(TCGContext *s) 533 { 534 TCGPool *p, *t; 535 for (p = s->pool_first_large; p; p = t) { 536 t = p->next; 537 g_free(p); 538 } 539 s->pool_first_large = NULL; 540 s->pool_cur = s->pool_end = NULL; 541 s->pool_current = NULL; 542 } 543 544 #include "exec/helper-proto.h" 545 546 static const TCGHelperInfo all_helpers[] = { 547 #include "exec/helper-tcg.h" 548 }; 549 static GHashTable *helper_table; 550 551 #ifdef CONFIG_TCG_INTERPRETER 552 static GHashTable *ffi_table; 553 554 static ffi_type * const typecode_to_ffi[8] = { 555 [dh_typecode_void] = &ffi_type_void, 556 [dh_typecode_i32] = &ffi_type_uint32, 557 [dh_typecode_s32] = &ffi_type_sint32, 558 [dh_typecode_i64] = &ffi_type_uint64, 559 [dh_typecode_s64] = &ffi_type_sint64, 560 [dh_typecode_ptr] = &ffi_type_pointer, 561 }; 562 #endif 563 564 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 565 static void process_op_defs(TCGContext *s); 566 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 567 TCGReg reg, const char *name); 568 569 static void tcg_context_init(unsigned max_cpus) 570 { 571 TCGContext *s = &tcg_init_ctx; 572 int op, total_args, n, i; 573 TCGOpDef *def; 574 TCGArgConstraint *args_ct; 575 TCGTemp *ts; 576 577 memset(s, 0, sizeof(*s)); 578 s->nb_globals = 0; 579 580 /* Count total number of arguments and allocate the corresponding 581 space */ 582 total_args = 0; 583 for(op = 0; op < NB_OPS; op++) { 584 def = &tcg_op_defs[op]; 585 n = def->nb_iargs + def->nb_oargs; 586 total_args += n; 587 } 588 589 args_ct = g_new0(TCGArgConstraint, total_args); 590 591 for(op = 0; op < NB_OPS; op++) { 592 def = &tcg_op_defs[op]; 593 def->args_ct = args_ct; 594 n = def->nb_iargs + def->nb_oargs; 595 args_ct += n; 596 } 597 598 /* Register helpers. */ 599 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 600 helper_table = g_hash_table_new(NULL, NULL); 601 602 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 603 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 604 (gpointer)&all_helpers[i]); 605 } 606 607 #ifdef CONFIG_TCG_INTERPRETER 608 /* g_direct_hash/equal for direct comparisons on uint32_t. */ 609 ffi_table = g_hash_table_new(NULL, NULL); 610 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 611 struct { 612 ffi_cif cif; 613 ffi_type *args[]; 614 } *ca; 615 uint32_t typemask = all_helpers[i].typemask; 616 gpointer hash = (gpointer)(uintptr_t)typemask; 617 ffi_status status; 618 int nargs; 619 620 if (g_hash_table_lookup(ffi_table, hash)) { 621 continue; 622 } 623 624 /* Ignoring the return type, find the last non-zero field. */ 625 nargs = 32 - clz32(typemask >> 3); 626 nargs = DIV_ROUND_UP(nargs, 3); 627 628 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 629 ca->cif.rtype = typecode_to_ffi[typemask & 7]; 630 ca->cif.nargs = nargs; 631 632 if (nargs != 0) { 633 ca->cif.arg_types = ca->args; 634 for (i = 0; i < nargs; ++i) { 635 int typecode = extract32(typemask, (i + 1) * 3, 3); 636 ca->args[i] = typecode_to_ffi[typecode]; 637 } 638 } 639 640 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 641 ca->cif.rtype, ca->cif.arg_types); 642 assert(status == FFI_OK); 643 644 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); 645 } 646 #endif 647 648 tcg_target_init(s); 649 process_op_defs(s); 650 651 /* Reverse the order of the saved registers, assuming they're all at 652 the start of tcg_target_reg_alloc_order. */ 653 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 654 int r = tcg_target_reg_alloc_order[n]; 655 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 656 break; 657 } 658 } 659 for (i = 0; i < n; ++i) { 660 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 661 } 662 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 663 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 664 } 665 666 alloc_tcg_plugin_context(s); 667 668 tcg_ctx = s; 669 /* 670 * In user-mode we simply share the init context among threads, since we 671 * use a single region. See the documentation tcg_region_init() for the 672 * reasoning behind this. 673 * In softmmu we will have at most max_cpus TCG threads. 674 */ 675 #ifdef CONFIG_USER_ONLY 676 tcg_ctxs = &tcg_ctx; 677 tcg_cur_ctxs = 1; 678 tcg_max_ctxs = 1; 679 #else 680 tcg_max_ctxs = max_cpus; 681 tcg_ctxs = g_new0(TCGContext *, max_cpus); 682 #endif 683 684 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 685 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 686 cpu_env = temp_tcgv_ptr(ts); 687 } 688 689 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 690 { 691 tcg_context_init(max_cpus); 692 tcg_region_init(tb_size, splitwx, max_cpus); 693 } 694 695 /* 696 * Allocate TBs right before their corresponding translated code, making 697 * sure that TBs and code are on different cache lines. 698 */ 699 TranslationBlock *tcg_tb_alloc(TCGContext *s) 700 { 701 uintptr_t align = qemu_icache_linesize; 702 TranslationBlock *tb; 703 void *next; 704 705 retry: 706 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 707 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 708 709 if (unlikely(next > s->code_gen_highwater)) { 710 if (tcg_region_alloc(s)) { 711 return NULL; 712 } 713 goto retry; 714 } 715 qatomic_set(&s->code_gen_ptr, next); 716 s->data_gen_ptr = NULL; 717 return tb; 718 } 719 720 void tcg_prologue_init(TCGContext *s) 721 { 722 size_t prologue_size; 723 724 s->code_ptr = s->code_gen_ptr; 725 s->code_buf = s->code_gen_ptr; 726 s->data_gen_ptr = NULL; 727 728 #ifndef CONFIG_TCG_INTERPRETER 729 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 730 #endif 731 732 #ifdef TCG_TARGET_NEED_POOL_LABELS 733 s->pool_labels = NULL; 734 #endif 735 736 qemu_thread_jit_write(); 737 /* Generate the prologue. */ 738 tcg_target_qemu_prologue(s); 739 740 #ifdef TCG_TARGET_NEED_POOL_LABELS 741 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 742 { 743 int result = tcg_out_pool_finalize(s); 744 tcg_debug_assert(result == 0); 745 } 746 #endif 747 748 prologue_size = tcg_current_code_size(s); 749 750 #ifndef CONFIG_TCG_INTERPRETER 751 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 752 (uintptr_t)s->code_buf, prologue_size); 753 #endif 754 755 tcg_region_prologue_set(s); 756 757 #ifdef DEBUG_DISAS 758 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 759 FILE *logfile = qemu_log_lock(); 760 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 761 if (s->data_gen_ptr) { 762 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 763 size_t data_size = prologue_size - code_size; 764 size_t i; 765 766 log_disas(s->code_gen_ptr, code_size); 767 768 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 769 if (sizeof(tcg_target_ulong) == 8) { 770 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 771 (uintptr_t)s->data_gen_ptr + i, 772 *(uint64_t *)(s->data_gen_ptr + i)); 773 } else { 774 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 775 (uintptr_t)s->data_gen_ptr + i, 776 *(uint32_t *)(s->data_gen_ptr + i)); 777 } 778 } 779 } else { 780 log_disas(s->code_gen_ptr, prologue_size); 781 } 782 qemu_log("\n"); 783 qemu_log_flush(); 784 qemu_log_unlock(logfile); 785 } 786 #endif 787 788 #ifndef CONFIG_TCG_INTERPRETER 789 /* 790 * Assert that goto_ptr is implemented completely, setting an epilogue. 791 * For tci, we use NULL as the signal to return from the interpreter, 792 * so skip this check. 793 */ 794 if (TCG_TARGET_HAS_goto_ptr) { 795 tcg_debug_assert(tcg_code_gen_epilogue != NULL); 796 } 797 #endif 798 } 799 800 void tcg_func_start(TCGContext *s) 801 { 802 tcg_pool_reset(s); 803 s->nb_temps = s->nb_globals; 804 805 /* No temps have been previously allocated for size or locality. */ 806 memset(s->free_temps, 0, sizeof(s->free_temps)); 807 808 /* No constant temps have been previously allocated. */ 809 for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 810 if (s->const_table[i]) { 811 g_hash_table_remove_all(s->const_table[i]); 812 } 813 } 814 815 s->nb_ops = 0; 816 s->nb_labels = 0; 817 s->current_frame_offset = s->frame_start; 818 819 #ifdef CONFIG_DEBUG_TCG 820 s->goto_tb_issue_mask = 0; 821 #endif 822 823 QTAILQ_INIT(&s->ops); 824 QTAILQ_INIT(&s->free_ops); 825 QSIMPLEQ_INIT(&s->labels); 826 } 827 828 static TCGTemp *tcg_temp_alloc(TCGContext *s) 829 { 830 int n = s->nb_temps++; 831 832 if (n >= TCG_MAX_TEMPS) { 833 tcg_raise_tb_overflow(s); 834 } 835 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 836 } 837 838 static TCGTemp *tcg_global_alloc(TCGContext *s) 839 { 840 TCGTemp *ts; 841 842 tcg_debug_assert(s->nb_globals == s->nb_temps); 843 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 844 s->nb_globals++; 845 ts = tcg_temp_alloc(s); 846 ts->kind = TEMP_GLOBAL; 847 848 return ts; 849 } 850 851 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 852 TCGReg reg, const char *name) 853 { 854 TCGTemp *ts; 855 856 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 857 tcg_abort(); 858 } 859 860 ts = tcg_global_alloc(s); 861 ts->base_type = type; 862 ts->type = type; 863 ts->kind = TEMP_FIXED; 864 ts->reg = reg; 865 ts->name = name; 866 tcg_regset_set_reg(s->reserved_regs, reg); 867 868 return ts; 869 } 870 871 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 872 { 873 s->frame_start = start; 874 s->frame_end = start + size; 875 s->frame_temp 876 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 877 } 878 879 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 880 intptr_t offset, const char *name) 881 { 882 TCGContext *s = tcg_ctx; 883 TCGTemp *base_ts = tcgv_ptr_temp(base); 884 TCGTemp *ts = tcg_global_alloc(s); 885 int indirect_reg = 0, bigendian = 0; 886 #ifdef HOST_WORDS_BIGENDIAN 887 bigendian = 1; 888 #endif 889 890 switch (base_ts->kind) { 891 case TEMP_FIXED: 892 break; 893 case TEMP_GLOBAL: 894 /* We do not support double-indirect registers. */ 895 tcg_debug_assert(!base_ts->indirect_reg); 896 base_ts->indirect_base = 1; 897 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 898 ? 2 : 1); 899 indirect_reg = 1; 900 break; 901 default: 902 g_assert_not_reached(); 903 } 904 905 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 906 TCGTemp *ts2 = tcg_global_alloc(s); 907 char buf[64]; 908 909 ts->base_type = TCG_TYPE_I64; 910 ts->type = TCG_TYPE_I32; 911 ts->indirect_reg = indirect_reg; 912 ts->mem_allocated = 1; 913 ts->mem_base = base_ts; 914 ts->mem_offset = offset + bigendian * 4; 915 pstrcpy(buf, sizeof(buf), name); 916 pstrcat(buf, sizeof(buf), "_0"); 917 ts->name = strdup(buf); 918 919 tcg_debug_assert(ts2 == ts + 1); 920 ts2->base_type = TCG_TYPE_I64; 921 ts2->type = TCG_TYPE_I32; 922 ts2->indirect_reg = indirect_reg; 923 ts2->mem_allocated = 1; 924 ts2->mem_base = base_ts; 925 ts2->mem_offset = offset + (1 - bigendian) * 4; 926 pstrcpy(buf, sizeof(buf), name); 927 pstrcat(buf, sizeof(buf), "_1"); 928 ts2->name = strdup(buf); 929 } else { 930 ts->base_type = type; 931 ts->type = type; 932 ts->indirect_reg = indirect_reg; 933 ts->mem_allocated = 1; 934 ts->mem_base = base_ts; 935 ts->mem_offset = offset; 936 ts->name = name; 937 } 938 return ts; 939 } 940 941 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) 942 { 943 TCGContext *s = tcg_ctx; 944 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL; 945 TCGTemp *ts; 946 int idx, k; 947 948 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 949 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 950 if (idx < TCG_MAX_TEMPS) { 951 /* There is already an available temp with the right type. */ 952 clear_bit(idx, s->free_temps[k].l); 953 954 ts = &s->temps[idx]; 955 ts->temp_allocated = 1; 956 tcg_debug_assert(ts->base_type == type); 957 tcg_debug_assert(ts->kind == kind); 958 } else { 959 ts = tcg_temp_alloc(s); 960 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 961 TCGTemp *ts2 = tcg_temp_alloc(s); 962 963 ts->base_type = type; 964 ts->type = TCG_TYPE_I32; 965 ts->temp_allocated = 1; 966 ts->kind = kind; 967 968 tcg_debug_assert(ts2 == ts + 1); 969 ts2->base_type = TCG_TYPE_I64; 970 ts2->type = TCG_TYPE_I32; 971 ts2->temp_allocated = 1; 972 ts2->kind = kind; 973 } else { 974 ts->base_type = type; 975 ts->type = type; 976 ts->temp_allocated = 1; 977 ts->kind = kind; 978 } 979 } 980 981 #if defined(CONFIG_DEBUG_TCG) 982 s->temps_in_use++; 983 #endif 984 return ts; 985 } 986 987 TCGv_vec tcg_temp_new_vec(TCGType type) 988 { 989 TCGTemp *t; 990 991 #ifdef CONFIG_DEBUG_TCG 992 switch (type) { 993 case TCG_TYPE_V64: 994 assert(TCG_TARGET_HAS_v64); 995 break; 996 case TCG_TYPE_V128: 997 assert(TCG_TARGET_HAS_v128); 998 break; 999 case TCG_TYPE_V256: 1000 assert(TCG_TARGET_HAS_v256); 1001 break; 1002 default: 1003 g_assert_not_reached(); 1004 } 1005 #endif 1006 1007 t = tcg_temp_new_internal(type, 0); 1008 return temp_tcgv_vec(t); 1009 } 1010 1011 /* Create a new temp of the same type as an existing temp. */ 1012 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1013 { 1014 TCGTemp *t = tcgv_vec_temp(match); 1015 1016 tcg_debug_assert(t->temp_allocated != 0); 1017 1018 t = tcg_temp_new_internal(t->base_type, 0); 1019 return temp_tcgv_vec(t); 1020 } 1021 1022 void tcg_temp_free_internal(TCGTemp *ts) 1023 { 1024 TCGContext *s = tcg_ctx; 1025 int k, idx; 1026 1027 /* In order to simplify users of tcg_constant_*, silently ignore free. */ 1028 if (ts->kind == TEMP_CONST) { 1029 return; 1030 } 1031 1032 #if defined(CONFIG_DEBUG_TCG) 1033 s->temps_in_use--; 1034 if (s->temps_in_use < 0) { 1035 fprintf(stderr, "More temporaries freed than allocated!\n"); 1036 } 1037 #endif 1038 1039 tcg_debug_assert(ts->kind < TEMP_GLOBAL); 1040 tcg_debug_assert(ts->temp_allocated != 0); 1041 ts->temp_allocated = 0; 1042 1043 idx = temp_idx(ts); 1044 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT); 1045 set_bit(idx, s->free_temps[k].l); 1046 } 1047 1048 TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1049 { 1050 TCGContext *s = tcg_ctx; 1051 GHashTable *h = s->const_table[type]; 1052 TCGTemp *ts; 1053 1054 if (h == NULL) { 1055 h = g_hash_table_new(g_int64_hash, g_int64_equal); 1056 s->const_table[type] = h; 1057 } 1058 1059 ts = g_hash_table_lookup(h, &val); 1060 if (ts == NULL) { 1061 ts = tcg_temp_alloc(s); 1062 1063 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1064 TCGTemp *ts2 = tcg_temp_alloc(s); 1065 1066 ts->base_type = TCG_TYPE_I64; 1067 ts->type = TCG_TYPE_I32; 1068 ts->kind = TEMP_CONST; 1069 ts->temp_allocated = 1; 1070 /* 1071 * Retain the full value of the 64-bit constant in the low 1072 * part, so that the hash table works. Actual uses will 1073 * truncate the value to the low part. 1074 */ 1075 ts->val = val; 1076 1077 tcg_debug_assert(ts2 == ts + 1); 1078 ts2->base_type = TCG_TYPE_I64; 1079 ts2->type = TCG_TYPE_I32; 1080 ts2->kind = TEMP_CONST; 1081 ts2->temp_allocated = 1; 1082 ts2->val = val >> 32; 1083 } else { 1084 ts->base_type = type; 1085 ts->type = type; 1086 ts->kind = TEMP_CONST; 1087 ts->temp_allocated = 1; 1088 ts->val = val; 1089 } 1090 g_hash_table_insert(h, &ts->val, ts); 1091 } 1092 1093 return ts; 1094 } 1095 1096 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1097 { 1098 val = dup_const(vece, val); 1099 return temp_tcgv_vec(tcg_constant_internal(type, val)); 1100 } 1101 1102 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 1103 { 1104 TCGTemp *t = tcgv_vec_temp(match); 1105 1106 tcg_debug_assert(t->temp_allocated != 0); 1107 return tcg_constant_vec(t->base_type, vece, val); 1108 } 1109 1110 TCGv_i32 tcg_const_i32(int32_t val) 1111 { 1112 TCGv_i32 t0; 1113 t0 = tcg_temp_new_i32(); 1114 tcg_gen_movi_i32(t0, val); 1115 return t0; 1116 } 1117 1118 TCGv_i64 tcg_const_i64(int64_t val) 1119 { 1120 TCGv_i64 t0; 1121 t0 = tcg_temp_new_i64(); 1122 tcg_gen_movi_i64(t0, val); 1123 return t0; 1124 } 1125 1126 TCGv_i32 tcg_const_local_i32(int32_t val) 1127 { 1128 TCGv_i32 t0; 1129 t0 = tcg_temp_local_new_i32(); 1130 tcg_gen_movi_i32(t0, val); 1131 return t0; 1132 } 1133 1134 TCGv_i64 tcg_const_local_i64(int64_t val) 1135 { 1136 TCGv_i64 t0; 1137 t0 = tcg_temp_local_new_i64(); 1138 tcg_gen_movi_i64(t0, val); 1139 return t0; 1140 } 1141 1142 #if defined(CONFIG_DEBUG_TCG) 1143 void tcg_clear_temp_count(void) 1144 { 1145 TCGContext *s = tcg_ctx; 1146 s->temps_in_use = 0; 1147 } 1148 1149 int tcg_check_temp_count(void) 1150 { 1151 TCGContext *s = tcg_ctx; 1152 if (s->temps_in_use) { 1153 /* Clear the count so that we don't give another 1154 * warning immediately next time around. 1155 */ 1156 s->temps_in_use = 0; 1157 return 1; 1158 } 1159 return 0; 1160 } 1161 #endif 1162 1163 /* Return true if OP may appear in the opcode stream. 1164 Test the runtime variable that controls each opcode. */ 1165 bool tcg_op_supported(TCGOpcode op) 1166 { 1167 const bool have_vec 1168 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1169 1170 switch (op) { 1171 case INDEX_op_discard: 1172 case INDEX_op_set_label: 1173 case INDEX_op_call: 1174 case INDEX_op_br: 1175 case INDEX_op_mb: 1176 case INDEX_op_insn_start: 1177 case INDEX_op_exit_tb: 1178 case INDEX_op_goto_tb: 1179 case INDEX_op_qemu_ld_i32: 1180 case INDEX_op_qemu_st_i32: 1181 case INDEX_op_qemu_ld_i64: 1182 case INDEX_op_qemu_st_i64: 1183 return true; 1184 1185 case INDEX_op_qemu_st8_i32: 1186 return TCG_TARGET_HAS_qemu_st8_i32; 1187 1188 case INDEX_op_goto_ptr: 1189 return TCG_TARGET_HAS_goto_ptr; 1190 1191 case INDEX_op_mov_i32: 1192 case INDEX_op_setcond_i32: 1193 case INDEX_op_brcond_i32: 1194 case INDEX_op_ld8u_i32: 1195 case INDEX_op_ld8s_i32: 1196 case INDEX_op_ld16u_i32: 1197 case INDEX_op_ld16s_i32: 1198 case INDEX_op_ld_i32: 1199 case INDEX_op_st8_i32: 1200 case INDEX_op_st16_i32: 1201 case INDEX_op_st_i32: 1202 case INDEX_op_add_i32: 1203 case INDEX_op_sub_i32: 1204 case INDEX_op_mul_i32: 1205 case INDEX_op_and_i32: 1206 case INDEX_op_or_i32: 1207 case INDEX_op_xor_i32: 1208 case INDEX_op_shl_i32: 1209 case INDEX_op_shr_i32: 1210 case INDEX_op_sar_i32: 1211 return true; 1212 1213 case INDEX_op_movcond_i32: 1214 return TCG_TARGET_HAS_movcond_i32; 1215 case INDEX_op_div_i32: 1216 case INDEX_op_divu_i32: 1217 return TCG_TARGET_HAS_div_i32; 1218 case INDEX_op_rem_i32: 1219 case INDEX_op_remu_i32: 1220 return TCG_TARGET_HAS_rem_i32; 1221 case INDEX_op_div2_i32: 1222 case INDEX_op_divu2_i32: 1223 return TCG_TARGET_HAS_div2_i32; 1224 case INDEX_op_rotl_i32: 1225 case INDEX_op_rotr_i32: 1226 return TCG_TARGET_HAS_rot_i32; 1227 case INDEX_op_deposit_i32: 1228 return TCG_TARGET_HAS_deposit_i32; 1229 case INDEX_op_extract_i32: 1230 return TCG_TARGET_HAS_extract_i32; 1231 case INDEX_op_sextract_i32: 1232 return TCG_TARGET_HAS_sextract_i32; 1233 case INDEX_op_extract2_i32: 1234 return TCG_TARGET_HAS_extract2_i32; 1235 case INDEX_op_add2_i32: 1236 return TCG_TARGET_HAS_add2_i32; 1237 case INDEX_op_sub2_i32: 1238 return TCG_TARGET_HAS_sub2_i32; 1239 case INDEX_op_mulu2_i32: 1240 return TCG_TARGET_HAS_mulu2_i32; 1241 case INDEX_op_muls2_i32: 1242 return TCG_TARGET_HAS_muls2_i32; 1243 case INDEX_op_muluh_i32: 1244 return TCG_TARGET_HAS_muluh_i32; 1245 case INDEX_op_mulsh_i32: 1246 return TCG_TARGET_HAS_mulsh_i32; 1247 case INDEX_op_ext8s_i32: 1248 return TCG_TARGET_HAS_ext8s_i32; 1249 case INDEX_op_ext16s_i32: 1250 return TCG_TARGET_HAS_ext16s_i32; 1251 case INDEX_op_ext8u_i32: 1252 return TCG_TARGET_HAS_ext8u_i32; 1253 case INDEX_op_ext16u_i32: 1254 return TCG_TARGET_HAS_ext16u_i32; 1255 case INDEX_op_bswap16_i32: 1256 return TCG_TARGET_HAS_bswap16_i32; 1257 case INDEX_op_bswap32_i32: 1258 return TCG_TARGET_HAS_bswap32_i32; 1259 case INDEX_op_not_i32: 1260 return TCG_TARGET_HAS_not_i32; 1261 case INDEX_op_neg_i32: 1262 return TCG_TARGET_HAS_neg_i32; 1263 case INDEX_op_andc_i32: 1264 return TCG_TARGET_HAS_andc_i32; 1265 case INDEX_op_orc_i32: 1266 return TCG_TARGET_HAS_orc_i32; 1267 case INDEX_op_eqv_i32: 1268 return TCG_TARGET_HAS_eqv_i32; 1269 case INDEX_op_nand_i32: 1270 return TCG_TARGET_HAS_nand_i32; 1271 case INDEX_op_nor_i32: 1272 return TCG_TARGET_HAS_nor_i32; 1273 case INDEX_op_clz_i32: 1274 return TCG_TARGET_HAS_clz_i32; 1275 case INDEX_op_ctz_i32: 1276 return TCG_TARGET_HAS_ctz_i32; 1277 case INDEX_op_ctpop_i32: 1278 return TCG_TARGET_HAS_ctpop_i32; 1279 1280 case INDEX_op_brcond2_i32: 1281 case INDEX_op_setcond2_i32: 1282 return TCG_TARGET_REG_BITS == 32; 1283 1284 case INDEX_op_mov_i64: 1285 case INDEX_op_setcond_i64: 1286 case INDEX_op_brcond_i64: 1287 case INDEX_op_ld8u_i64: 1288 case INDEX_op_ld8s_i64: 1289 case INDEX_op_ld16u_i64: 1290 case INDEX_op_ld16s_i64: 1291 case INDEX_op_ld32u_i64: 1292 case INDEX_op_ld32s_i64: 1293 case INDEX_op_ld_i64: 1294 case INDEX_op_st8_i64: 1295 case INDEX_op_st16_i64: 1296 case INDEX_op_st32_i64: 1297 case INDEX_op_st_i64: 1298 case INDEX_op_add_i64: 1299 case INDEX_op_sub_i64: 1300 case INDEX_op_mul_i64: 1301 case INDEX_op_and_i64: 1302 case INDEX_op_or_i64: 1303 case INDEX_op_xor_i64: 1304 case INDEX_op_shl_i64: 1305 case INDEX_op_shr_i64: 1306 case INDEX_op_sar_i64: 1307 case INDEX_op_ext_i32_i64: 1308 case INDEX_op_extu_i32_i64: 1309 return TCG_TARGET_REG_BITS == 64; 1310 1311 case INDEX_op_movcond_i64: 1312 return TCG_TARGET_HAS_movcond_i64; 1313 case INDEX_op_div_i64: 1314 case INDEX_op_divu_i64: 1315 return TCG_TARGET_HAS_div_i64; 1316 case INDEX_op_rem_i64: 1317 case INDEX_op_remu_i64: 1318 return TCG_TARGET_HAS_rem_i64; 1319 case INDEX_op_div2_i64: 1320 case INDEX_op_divu2_i64: 1321 return TCG_TARGET_HAS_div2_i64; 1322 case INDEX_op_rotl_i64: 1323 case INDEX_op_rotr_i64: 1324 return TCG_TARGET_HAS_rot_i64; 1325 case INDEX_op_deposit_i64: 1326 return TCG_TARGET_HAS_deposit_i64; 1327 case INDEX_op_extract_i64: 1328 return TCG_TARGET_HAS_extract_i64; 1329 case INDEX_op_sextract_i64: 1330 return TCG_TARGET_HAS_sextract_i64; 1331 case INDEX_op_extract2_i64: 1332 return TCG_TARGET_HAS_extract2_i64; 1333 case INDEX_op_extrl_i64_i32: 1334 return TCG_TARGET_HAS_extrl_i64_i32; 1335 case INDEX_op_extrh_i64_i32: 1336 return TCG_TARGET_HAS_extrh_i64_i32; 1337 case INDEX_op_ext8s_i64: 1338 return TCG_TARGET_HAS_ext8s_i64; 1339 case INDEX_op_ext16s_i64: 1340 return TCG_TARGET_HAS_ext16s_i64; 1341 case INDEX_op_ext32s_i64: 1342 return TCG_TARGET_HAS_ext32s_i64; 1343 case INDEX_op_ext8u_i64: 1344 return TCG_TARGET_HAS_ext8u_i64; 1345 case INDEX_op_ext16u_i64: 1346 return TCG_TARGET_HAS_ext16u_i64; 1347 case INDEX_op_ext32u_i64: 1348 return TCG_TARGET_HAS_ext32u_i64; 1349 case INDEX_op_bswap16_i64: 1350 return TCG_TARGET_HAS_bswap16_i64; 1351 case INDEX_op_bswap32_i64: 1352 return TCG_TARGET_HAS_bswap32_i64; 1353 case INDEX_op_bswap64_i64: 1354 return TCG_TARGET_HAS_bswap64_i64; 1355 case INDEX_op_not_i64: 1356 return TCG_TARGET_HAS_not_i64; 1357 case INDEX_op_neg_i64: 1358 return TCG_TARGET_HAS_neg_i64; 1359 case INDEX_op_andc_i64: 1360 return TCG_TARGET_HAS_andc_i64; 1361 case INDEX_op_orc_i64: 1362 return TCG_TARGET_HAS_orc_i64; 1363 case INDEX_op_eqv_i64: 1364 return TCG_TARGET_HAS_eqv_i64; 1365 case INDEX_op_nand_i64: 1366 return TCG_TARGET_HAS_nand_i64; 1367 case INDEX_op_nor_i64: 1368 return TCG_TARGET_HAS_nor_i64; 1369 case INDEX_op_clz_i64: 1370 return TCG_TARGET_HAS_clz_i64; 1371 case INDEX_op_ctz_i64: 1372 return TCG_TARGET_HAS_ctz_i64; 1373 case INDEX_op_ctpop_i64: 1374 return TCG_TARGET_HAS_ctpop_i64; 1375 case INDEX_op_add2_i64: 1376 return TCG_TARGET_HAS_add2_i64; 1377 case INDEX_op_sub2_i64: 1378 return TCG_TARGET_HAS_sub2_i64; 1379 case INDEX_op_mulu2_i64: 1380 return TCG_TARGET_HAS_mulu2_i64; 1381 case INDEX_op_muls2_i64: 1382 return TCG_TARGET_HAS_muls2_i64; 1383 case INDEX_op_muluh_i64: 1384 return TCG_TARGET_HAS_muluh_i64; 1385 case INDEX_op_mulsh_i64: 1386 return TCG_TARGET_HAS_mulsh_i64; 1387 1388 case INDEX_op_mov_vec: 1389 case INDEX_op_dup_vec: 1390 case INDEX_op_dupm_vec: 1391 case INDEX_op_ld_vec: 1392 case INDEX_op_st_vec: 1393 case INDEX_op_add_vec: 1394 case INDEX_op_sub_vec: 1395 case INDEX_op_and_vec: 1396 case INDEX_op_or_vec: 1397 case INDEX_op_xor_vec: 1398 case INDEX_op_cmp_vec: 1399 return have_vec; 1400 case INDEX_op_dup2_vec: 1401 return have_vec && TCG_TARGET_REG_BITS == 32; 1402 case INDEX_op_not_vec: 1403 return have_vec && TCG_TARGET_HAS_not_vec; 1404 case INDEX_op_neg_vec: 1405 return have_vec && TCG_TARGET_HAS_neg_vec; 1406 case INDEX_op_abs_vec: 1407 return have_vec && TCG_TARGET_HAS_abs_vec; 1408 case INDEX_op_andc_vec: 1409 return have_vec && TCG_TARGET_HAS_andc_vec; 1410 case INDEX_op_orc_vec: 1411 return have_vec && TCG_TARGET_HAS_orc_vec; 1412 case INDEX_op_mul_vec: 1413 return have_vec && TCG_TARGET_HAS_mul_vec; 1414 case INDEX_op_shli_vec: 1415 case INDEX_op_shri_vec: 1416 case INDEX_op_sari_vec: 1417 return have_vec && TCG_TARGET_HAS_shi_vec; 1418 case INDEX_op_shls_vec: 1419 case INDEX_op_shrs_vec: 1420 case INDEX_op_sars_vec: 1421 return have_vec && TCG_TARGET_HAS_shs_vec; 1422 case INDEX_op_shlv_vec: 1423 case INDEX_op_shrv_vec: 1424 case INDEX_op_sarv_vec: 1425 return have_vec && TCG_TARGET_HAS_shv_vec; 1426 case INDEX_op_rotli_vec: 1427 return have_vec && TCG_TARGET_HAS_roti_vec; 1428 case INDEX_op_rotls_vec: 1429 return have_vec && TCG_TARGET_HAS_rots_vec; 1430 case INDEX_op_rotlv_vec: 1431 case INDEX_op_rotrv_vec: 1432 return have_vec && TCG_TARGET_HAS_rotv_vec; 1433 case INDEX_op_ssadd_vec: 1434 case INDEX_op_usadd_vec: 1435 case INDEX_op_sssub_vec: 1436 case INDEX_op_ussub_vec: 1437 return have_vec && TCG_TARGET_HAS_sat_vec; 1438 case INDEX_op_smin_vec: 1439 case INDEX_op_umin_vec: 1440 case INDEX_op_smax_vec: 1441 case INDEX_op_umax_vec: 1442 return have_vec && TCG_TARGET_HAS_minmax_vec; 1443 case INDEX_op_bitsel_vec: 1444 return have_vec && TCG_TARGET_HAS_bitsel_vec; 1445 case INDEX_op_cmpsel_vec: 1446 return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1447 1448 default: 1449 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1450 return true; 1451 } 1452 } 1453 1454 /* Note: we convert the 64 bit args to 32 bit and do some alignment 1455 and endian swap. Maybe it would be better to do the alignment 1456 and endian swap in tcg_reg_alloc_call(). */ 1457 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1458 { 1459 int i, real_args, nb_rets, pi; 1460 unsigned typemask; 1461 const TCGHelperInfo *info; 1462 TCGOp *op; 1463 1464 info = g_hash_table_lookup(helper_table, (gpointer)func); 1465 typemask = info->typemask; 1466 1467 #ifdef CONFIG_PLUGIN 1468 /* detect non-plugin helpers */ 1469 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) { 1470 tcg_ctx->plugin_insn->calls_helpers = true; 1471 } 1472 #endif 1473 1474 #if defined(__sparc__) && !defined(__arch64__) \ 1475 && !defined(CONFIG_TCG_INTERPRETER) 1476 /* We have 64-bit values in one register, but need to pass as two 1477 separate parameters. Split them. */ 1478 int orig_typemask = typemask; 1479 int orig_nargs = nargs; 1480 TCGv_i64 retl, reth; 1481 TCGTemp *split_args[MAX_OPC_PARAM]; 1482 1483 retl = NULL; 1484 reth = NULL; 1485 typemask = 0; 1486 for (i = real_args = 0; i < nargs; ++i) { 1487 int argtype = extract32(orig_typemask, (i + 1) * 3, 3); 1488 bool is_64bit = (argtype & ~1) == dh_typecode_i64; 1489 1490 if (is_64bit) { 1491 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1492 TCGv_i32 h = tcg_temp_new_i32(); 1493 TCGv_i32 l = tcg_temp_new_i32(); 1494 tcg_gen_extr_i64_i32(l, h, orig); 1495 split_args[real_args++] = tcgv_i32_temp(h); 1496 typemask |= dh_typecode_i32 << (real_args * 3); 1497 split_args[real_args++] = tcgv_i32_temp(l); 1498 typemask |= dh_typecode_i32 << (real_args * 3); 1499 } else { 1500 split_args[real_args++] = args[i]; 1501 typemask |= argtype << (real_args * 3); 1502 } 1503 } 1504 nargs = real_args; 1505 args = split_args; 1506 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1507 for (i = 0; i < nargs; ++i) { 1508 int argtype = extract32(typemask, (i + 1) * 3, 3); 1509 bool is_32bit = (argtype & ~1) == dh_typecode_i32; 1510 bool is_signed = argtype & 1; 1511 1512 if (is_32bit) { 1513 TCGv_i64 temp = tcg_temp_new_i64(); 1514 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1515 if (is_signed) { 1516 tcg_gen_ext32s_i64(temp, orig); 1517 } else { 1518 tcg_gen_ext32u_i64(temp, orig); 1519 } 1520 args[i] = tcgv_i64_temp(temp); 1521 } 1522 } 1523 #endif /* TCG_TARGET_EXTEND_ARGS */ 1524 1525 op = tcg_emit_op(INDEX_op_call); 1526 1527 pi = 0; 1528 if (ret != NULL) { 1529 #if defined(__sparc__) && !defined(__arch64__) \ 1530 && !defined(CONFIG_TCG_INTERPRETER) 1531 if ((typemask & 6) == dh_typecode_i64) { 1532 /* The 32-bit ABI is going to return the 64-bit value in 1533 the %o0/%o1 register pair. Prepare for this by using 1534 two return temporaries, and reassemble below. */ 1535 retl = tcg_temp_new_i64(); 1536 reth = tcg_temp_new_i64(); 1537 op->args[pi++] = tcgv_i64_arg(reth); 1538 op->args[pi++] = tcgv_i64_arg(retl); 1539 nb_rets = 2; 1540 } else { 1541 op->args[pi++] = temp_arg(ret); 1542 nb_rets = 1; 1543 } 1544 #else 1545 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { 1546 #ifdef HOST_WORDS_BIGENDIAN 1547 op->args[pi++] = temp_arg(ret + 1); 1548 op->args[pi++] = temp_arg(ret); 1549 #else 1550 op->args[pi++] = temp_arg(ret); 1551 op->args[pi++] = temp_arg(ret + 1); 1552 #endif 1553 nb_rets = 2; 1554 } else { 1555 op->args[pi++] = temp_arg(ret); 1556 nb_rets = 1; 1557 } 1558 #endif 1559 } else { 1560 nb_rets = 0; 1561 } 1562 TCGOP_CALLO(op) = nb_rets; 1563 1564 real_args = 0; 1565 for (i = 0; i < nargs; i++) { 1566 int argtype = extract32(typemask, (i + 1) * 3, 3); 1567 bool is_64bit = (argtype & ~1) == dh_typecode_i64; 1568 bool want_align = false; 1569 1570 #if defined(CONFIG_TCG_INTERPRETER) 1571 /* 1572 * Align all arguments, so that they land in predictable places 1573 * for passing off to ffi_call. 1574 */ 1575 want_align = true; 1576 #elif defined(TCG_TARGET_CALL_ALIGN_ARGS) 1577 /* Some targets want aligned 64 bit args */ 1578 want_align = is_64bit; 1579 #endif 1580 1581 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) { 1582 op->args[pi++] = TCG_CALL_DUMMY_ARG; 1583 real_args++; 1584 } 1585 1586 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 1587 /* 1588 * If stack grows up, then we will be placing successive 1589 * arguments at lower addresses, which means we need to 1590 * reverse the order compared to how we would normally 1591 * treat either big or little-endian. For those arguments 1592 * that will wind up in registers, this still works for 1593 * HPPA (the only current STACK_GROWSUP target) since the 1594 * argument registers are *also* allocated in decreasing 1595 * order. If another such target is added, this logic may 1596 * have to get more complicated to differentiate between 1597 * stack arguments and register arguments. 1598 */ 1599 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 1600 op->args[pi++] = temp_arg(args[i] + 1); 1601 op->args[pi++] = temp_arg(args[i]); 1602 #else 1603 op->args[pi++] = temp_arg(args[i]); 1604 op->args[pi++] = temp_arg(args[i] + 1); 1605 #endif 1606 real_args += 2; 1607 continue; 1608 } 1609 1610 op->args[pi++] = temp_arg(args[i]); 1611 real_args++; 1612 } 1613 op->args[pi++] = (uintptr_t)func; 1614 op->args[pi++] = (uintptr_t)info; 1615 TCGOP_CALLI(op) = real_args; 1616 1617 /* Make sure the fields didn't overflow. */ 1618 tcg_debug_assert(TCGOP_CALLI(op) == real_args); 1619 tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); 1620 1621 #if defined(__sparc__) && !defined(__arch64__) \ 1622 && !defined(CONFIG_TCG_INTERPRETER) 1623 /* Free all of the parts we allocated above. */ 1624 for (i = real_args = 0; i < orig_nargs; ++i) { 1625 int argtype = extract32(orig_typemask, (i + 1) * 3, 3); 1626 bool is_64bit = (argtype & ~1) == dh_typecode_i64; 1627 1628 if (is_64bit) { 1629 tcg_temp_free_internal(args[real_args++]); 1630 tcg_temp_free_internal(args[real_args++]); 1631 } else { 1632 real_args++; 1633 } 1634 } 1635 if ((orig_typemask & 6) == dh_typecode_i64) { 1636 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 1637 Note that describing these as TCGv_i64 eliminates an unnecessary 1638 zero-extension that tcg_gen_concat_i32_i64 would create. */ 1639 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth); 1640 tcg_temp_free_i64(retl); 1641 tcg_temp_free_i64(reth); 1642 } 1643 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1644 for (i = 0; i < nargs; ++i) { 1645 int argtype = extract32(typemask, (i + 1) * 3, 3); 1646 bool is_32bit = (argtype & ~1) == dh_typecode_i32; 1647 1648 if (is_32bit) { 1649 tcg_temp_free_internal(args[i]); 1650 } 1651 } 1652 #endif /* TCG_TARGET_EXTEND_ARGS */ 1653 } 1654 1655 static void tcg_reg_alloc_start(TCGContext *s) 1656 { 1657 int i, n; 1658 1659 for (i = 0, n = s->nb_temps; i < n; i++) { 1660 TCGTemp *ts = &s->temps[i]; 1661 TCGTempVal val = TEMP_VAL_MEM; 1662 1663 switch (ts->kind) { 1664 case TEMP_CONST: 1665 val = TEMP_VAL_CONST; 1666 break; 1667 case TEMP_FIXED: 1668 val = TEMP_VAL_REG; 1669 break; 1670 case TEMP_GLOBAL: 1671 break; 1672 case TEMP_NORMAL: 1673 val = TEMP_VAL_DEAD; 1674 /* fall through */ 1675 case TEMP_LOCAL: 1676 ts->mem_allocated = 0; 1677 break; 1678 default: 1679 g_assert_not_reached(); 1680 } 1681 ts->val_type = val; 1682 } 1683 1684 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1685 } 1686 1687 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1688 TCGTemp *ts) 1689 { 1690 int idx = temp_idx(ts); 1691 1692 switch (ts->kind) { 1693 case TEMP_FIXED: 1694 case TEMP_GLOBAL: 1695 pstrcpy(buf, buf_size, ts->name); 1696 break; 1697 case TEMP_LOCAL: 1698 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1699 break; 1700 case TEMP_NORMAL: 1701 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1702 break; 1703 case TEMP_CONST: 1704 switch (ts->type) { 1705 case TCG_TYPE_I32: 1706 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1707 break; 1708 #if TCG_TARGET_REG_BITS > 32 1709 case TCG_TYPE_I64: 1710 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1711 break; 1712 #endif 1713 case TCG_TYPE_V64: 1714 case TCG_TYPE_V128: 1715 case TCG_TYPE_V256: 1716 snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1717 64 << (ts->type - TCG_TYPE_V64), ts->val); 1718 break; 1719 default: 1720 g_assert_not_reached(); 1721 } 1722 break; 1723 } 1724 return buf; 1725 } 1726 1727 static char *tcg_get_arg_str(TCGContext *s, char *buf, 1728 int buf_size, TCGArg arg) 1729 { 1730 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1731 } 1732 1733 static const char * const cond_name[] = 1734 { 1735 [TCG_COND_NEVER] = "never", 1736 [TCG_COND_ALWAYS] = "always", 1737 [TCG_COND_EQ] = "eq", 1738 [TCG_COND_NE] = "ne", 1739 [TCG_COND_LT] = "lt", 1740 [TCG_COND_GE] = "ge", 1741 [TCG_COND_LE] = "le", 1742 [TCG_COND_GT] = "gt", 1743 [TCG_COND_LTU] = "ltu", 1744 [TCG_COND_GEU] = "geu", 1745 [TCG_COND_LEU] = "leu", 1746 [TCG_COND_GTU] = "gtu" 1747 }; 1748 1749 static const char * const ldst_name[] = 1750 { 1751 [MO_UB] = "ub", 1752 [MO_SB] = "sb", 1753 [MO_LEUW] = "leuw", 1754 [MO_LESW] = "lesw", 1755 [MO_LEUL] = "leul", 1756 [MO_LESL] = "lesl", 1757 [MO_LEQ] = "leq", 1758 [MO_BEUW] = "beuw", 1759 [MO_BESW] = "besw", 1760 [MO_BEUL] = "beul", 1761 [MO_BESL] = "besl", 1762 [MO_BEQ] = "beq", 1763 }; 1764 1765 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1766 #ifdef TARGET_ALIGNED_ONLY 1767 [MO_UNALN >> MO_ASHIFT] = "un+", 1768 [MO_ALIGN >> MO_ASHIFT] = "", 1769 #else 1770 [MO_UNALN >> MO_ASHIFT] = "", 1771 [MO_ALIGN >> MO_ASHIFT] = "al+", 1772 #endif 1773 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1774 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1775 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1776 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1777 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1778 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1779 }; 1780 1781 static const char bswap_flag_name[][6] = { 1782 [TCG_BSWAP_IZ] = "iz", 1783 [TCG_BSWAP_OZ] = "oz", 1784 [TCG_BSWAP_OS] = "os", 1785 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1786 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1787 }; 1788 1789 static inline bool tcg_regset_single(TCGRegSet d) 1790 { 1791 return (d & (d - 1)) == 0; 1792 } 1793 1794 static inline TCGReg tcg_regset_first(TCGRegSet d) 1795 { 1796 if (TCG_TARGET_NB_REGS <= 32) { 1797 return ctz32(d); 1798 } else { 1799 return ctz64(d); 1800 } 1801 } 1802 1803 static void tcg_dump_ops(TCGContext *s, bool have_prefs) 1804 { 1805 char buf[128]; 1806 TCGOp *op; 1807 1808 QTAILQ_FOREACH(op, &s->ops, link) { 1809 int i, k, nb_oargs, nb_iargs, nb_cargs; 1810 const TCGOpDef *def; 1811 TCGOpcode c; 1812 int col = 0; 1813 1814 c = op->opc; 1815 def = &tcg_op_defs[c]; 1816 1817 if (c == INDEX_op_insn_start) { 1818 nb_oargs = 0; 1819 col += qemu_log("\n ----"); 1820 1821 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1822 target_ulong a; 1823 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1824 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 1825 #else 1826 a = op->args[i]; 1827 #endif 1828 col += qemu_log(" " TARGET_FMT_lx, a); 1829 } 1830 } else if (c == INDEX_op_call) { 1831 const TCGHelperInfo *info = tcg_call_info(op); 1832 void *func = tcg_call_func(op); 1833 1834 /* variable number of arguments */ 1835 nb_oargs = TCGOP_CALLO(op); 1836 nb_iargs = TCGOP_CALLI(op); 1837 nb_cargs = def->nb_cargs; 1838 1839 col += qemu_log(" %s ", def->name); 1840 1841 /* 1842 * Print the function name from TCGHelperInfo, if available. 1843 * Note that plugins have a template function for the info, 1844 * but the actual function pointer comes from the plugin. 1845 */ 1846 if (func == info->func) { 1847 col += qemu_log("%s", info->name); 1848 } else { 1849 col += qemu_log("plugin(%p)", func); 1850 } 1851 1852 col += qemu_log("$0x%x,$%d", info->flags, nb_oargs); 1853 for (i = 0; i < nb_oargs; i++) { 1854 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf), 1855 op->args[i])); 1856 } 1857 for (i = 0; i < nb_iargs; i++) { 1858 TCGArg arg = op->args[nb_oargs + i]; 1859 const char *t = "<dummy>"; 1860 if (arg != TCG_CALL_DUMMY_ARG) { 1861 t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 1862 } 1863 col += qemu_log(",%s", t); 1864 } 1865 } else { 1866 col += qemu_log(" %s ", def->name); 1867 1868 nb_oargs = def->nb_oargs; 1869 nb_iargs = def->nb_iargs; 1870 nb_cargs = def->nb_cargs; 1871 1872 if (def->flags & TCG_OPF_VECTOR) { 1873 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op), 1874 8 << TCGOP_VECE(op)); 1875 } 1876 1877 k = 0; 1878 for (i = 0; i < nb_oargs; i++) { 1879 if (k != 0) { 1880 col += qemu_log(","); 1881 } 1882 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1883 op->args[k++])); 1884 } 1885 for (i = 0; i < nb_iargs; i++) { 1886 if (k != 0) { 1887 col += qemu_log(","); 1888 } 1889 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1890 op->args[k++])); 1891 } 1892 switch (c) { 1893 case INDEX_op_brcond_i32: 1894 case INDEX_op_setcond_i32: 1895 case INDEX_op_movcond_i32: 1896 case INDEX_op_brcond2_i32: 1897 case INDEX_op_setcond2_i32: 1898 case INDEX_op_brcond_i64: 1899 case INDEX_op_setcond_i64: 1900 case INDEX_op_movcond_i64: 1901 case INDEX_op_cmp_vec: 1902 case INDEX_op_cmpsel_vec: 1903 if (op->args[k] < ARRAY_SIZE(cond_name) 1904 && cond_name[op->args[k]]) { 1905 col += qemu_log(",%s", cond_name[op->args[k++]]); 1906 } else { 1907 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]); 1908 } 1909 i = 1; 1910 break; 1911 case INDEX_op_qemu_ld_i32: 1912 case INDEX_op_qemu_st_i32: 1913 case INDEX_op_qemu_st8_i32: 1914 case INDEX_op_qemu_ld_i64: 1915 case INDEX_op_qemu_st_i64: 1916 { 1917 TCGMemOpIdx oi = op->args[k++]; 1918 MemOp op = get_memop(oi); 1919 unsigned ix = get_mmuidx(oi); 1920 1921 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1922 col += qemu_log(",$0x%x,%u", op, ix); 1923 } else { 1924 const char *s_al, *s_op; 1925 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1926 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1927 col += qemu_log(",%s%s,%u", s_al, s_op, ix); 1928 } 1929 i = 1; 1930 } 1931 break; 1932 case INDEX_op_bswap16_i32: 1933 case INDEX_op_bswap16_i64: 1934 case INDEX_op_bswap32_i32: 1935 case INDEX_op_bswap32_i64: 1936 case INDEX_op_bswap64_i64: 1937 { 1938 TCGArg flags = op->args[k]; 1939 const char *name = NULL; 1940 1941 if (flags < ARRAY_SIZE(bswap_flag_name)) { 1942 name = bswap_flag_name[flags]; 1943 } 1944 if (name) { 1945 col += qemu_log(",%s", name); 1946 } else { 1947 col += qemu_log(",$0x%" TCG_PRIlx, flags); 1948 } 1949 i = k = 1; 1950 } 1951 break; 1952 default: 1953 i = 0; 1954 break; 1955 } 1956 switch (c) { 1957 case INDEX_op_set_label: 1958 case INDEX_op_br: 1959 case INDEX_op_brcond_i32: 1960 case INDEX_op_brcond_i64: 1961 case INDEX_op_brcond2_i32: 1962 col += qemu_log("%s$L%d", k ? "," : "", 1963 arg_label(op->args[k])->id); 1964 i++, k++; 1965 break; 1966 default: 1967 break; 1968 } 1969 for (; i < nb_cargs; i++, k++) { 1970 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]); 1971 } 1972 } 1973 1974 if (have_prefs || op->life) { 1975 1976 QemuLogFile *logfile; 1977 1978 rcu_read_lock(); 1979 logfile = qatomic_rcu_read(&qemu_logfile); 1980 if (logfile) { 1981 for (; col < 40; ++col) { 1982 putc(' ', logfile->fd); 1983 } 1984 } 1985 rcu_read_unlock(); 1986 } 1987 1988 if (op->life) { 1989 unsigned life = op->life; 1990 1991 if (life & (SYNC_ARG * 3)) { 1992 qemu_log(" sync:"); 1993 for (i = 0; i < 2; ++i) { 1994 if (life & (SYNC_ARG << i)) { 1995 qemu_log(" %d", i); 1996 } 1997 } 1998 } 1999 life /= DEAD_ARG; 2000 if (life) { 2001 qemu_log(" dead:"); 2002 for (i = 0; life; ++i, life >>= 1) { 2003 if (life & 1) { 2004 qemu_log(" %d", i); 2005 } 2006 } 2007 } 2008 } 2009 2010 if (have_prefs) { 2011 for (i = 0; i < nb_oargs; ++i) { 2012 TCGRegSet set = op->output_pref[i]; 2013 2014 if (i == 0) { 2015 qemu_log(" pref="); 2016 } else { 2017 qemu_log(","); 2018 } 2019 if (set == 0) { 2020 qemu_log("none"); 2021 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 2022 qemu_log("all"); 2023 #ifdef CONFIG_DEBUG_TCG 2024 } else if (tcg_regset_single(set)) { 2025 TCGReg reg = tcg_regset_first(set); 2026 qemu_log("%s", tcg_target_reg_names[reg]); 2027 #endif 2028 } else if (TCG_TARGET_NB_REGS <= 32) { 2029 qemu_log("%#x", (uint32_t)set); 2030 } else { 2031 qemu_log("%#" PRIx64, (uint64_t)set); 2032 } 2033 } 2034 } 2035 2036 qemu_log("\n"); 2037 } 2038 } 2039 2040 /* we give more priority to constraints with less registers */ 2041 static int get_constraint_priority(const TCGOpDef *def, int k) 2042 { 2043 const TCGArgConstraint *arg_ct = &def->args_ct[k]; 2044 int n; 2045 2046 if (arg_ct->oalias) { 2047 /* an alias is equivalent to a single register */ 2048 n = 1; 2049 } else { 2050 n = ctpop64(arg_ct->regs); 2051 } 2052 return TCG_TARGET_NB_REGS - n + 1; 2053 } 2054 2055 /* sort from highest priority to lowest */ 2056 static void sort_constraints(TCGOpDef *def, int start, int n) 2057 { 2058 int i, j; 2059 TCGArgConstraint *a = def->args_ct; 2060 2061 for (i = 0; i < n; i++) { 2062 a[start + i].sort_index = start + i; 2063 } 2064 if (n <= 1) { 2065 return; 2066 } 2067 for (i = 0; i < n - 1; i++) { 2068 for (j = i + 1; j < n; j++) { 2069 int p1 = get_constraint_priority(def, a[start + i].sort_index); 2070 int p2 = get_constraint_priority(def, a[start + j].sort_index); 2071 if (p1 < p2) { 2072 int tmp = a[start + i].sort_index; 2073 a[start + i].sort_index = a[start + j].sort_index; 2074 a[start + j].sort_index = tmp; 2075 } 2076 } 2077 } 2078 } 2079 2080 static void process_op_defs(TCGContext *s) 2081 { 2082 TCGOpcode op; 2083 2084 for (op = 0; op < NB_OPS; op++) { 2085 TCGOpDef *def = &tcg_op_defs[op]; 2086 const TCGTargetOpDef *tdefs; 2087 int i, nb_args; 2088 2089 if (def->flags & TCG_OPF_NOT_PRESENT) { 2090 continue; 2091 } 2092 2093 nb_args = def->nb_iargs + def->nb_oargs; 2094 if (nb_args == 0) { 2095 continue; 2096 } 2097 2098 /* 2099 * Macro magic should make it impossible, but double-check that 2100 * the array index is in range. Since the signness of an enum 2101 * is implementation defined, force the result to unsigned. 2102 */ 2103 unsigned con_set = tcg_target_op_def(op); 2104 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 2105 tdefs = &constraint_sets[con_set]; 2106 2107 for (i = 0; i < nb_args; i++) { 2108 const char *ct_str = tdefs->args_ct_str[i]; 2109 /* Incomplete TCGTargetOpDef entry. */ 2110 tcg_debug_assert(ct_str != NULL); 2111 2112 while (*ct_str != '\0') { 2113 switch(*ct_str) { 2114 case '0' ... '9': 2115 { 2116 int oarg = *ct_str - '0'; 2117 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]); 2118 tcg_debug_assert(oarg < def->nb_oargs); 2119 tcg_debug_assert(def->args_ct[oarg].regs != 0); 2120 def->args_ct[i] = def->args_ct[oarg]; 2121 /* The output sets oalias. */ 2122 def->args_ct[oarg].oalias = true; 2123 def->args_ct[oarg].alias_index = i; 2124 /* The input sets ialias. */ 2125 def->args_ct[i].ialias = true; 2126 def->args_ct[i].alias_index = oarg; 2127 } 2128 ct_str++; 2129 break; 2130 case '&': 2131 def->args_ct[i].newreg = true; 2132 ct_str++; 2133 break; 2134 case 'i': 2135 def->args_ct[i].ct |= TCG_CT_CONST; 2136 ct_str++; 2137 break; 2138 2139 /* Include all of the target-specific constraints. */ 2140 2141 #undef CONST 2142 #define CONST(CASE, MASK) \ 2143 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break; 2144 #define REGS(CASE, MASK) \ 2145 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break; 2146 2147 #include "tcg-target-con-str.h" 2148 2149 #undef REGS 2150 #undef CONST 2151 default: 2152 /* Typo in TCGTargetOpDef constraint. */ 2153 g_assert_not_reached(); 2154 } 2155 } 2156 } 2157 2158 /* TCGTargetOpDef entry with too much information? */ 2159 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2160 2161 /* sort the constraints (XXX: this is just an heuristic) */ 2162 sort_constraints(def, 0, def->nb_oargs); 2163 sort_constraints(def, def->nb_oargs, def->nb_iargs); 2164 } 2165 } 2166 2167 void tcg_op_remove(TCGContext *s, TCGOp *op) 2168 { 2169 TCGLabel *label; 2170 2171 switch (op->opc) { 2172 case INDEX_op_br: 2173 label = arg_label(op->args[0]); 2174 label->refs--; 2175 break; 2176 case INDEX_op_brcond_i32: 2177 case INDEX_op_brcond_i64: 2178 label = arg_label(op->args[3]); 2179 label->refs--; 2180 break; 2181 case INDEX_op_brcond2_i32: 2182 label = arg_label(op->args[5]); 2183 label->refs--; 2184 break; 2185 default: 2186 break; 2187 } 2188 2189 QTAILQ_REMOVE(&s->ops, op, link); 2190 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2191 s->nb_ops--; 2192 2193 #ifdef CONFIG_PROFILER 2194 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 2195 #endif 2196 } 2197 2198 void tcg_remove_ops_after(TCGOp *op) 2199 { 2200 TCGContext *s = tcg_ctx; 2201 2202 while (true) { 2203 TCGOp *last = tcg_last_op(); 2204 if (last == op) { 2205 return; 2206 } 2207 tcg_op_remove(s, last); 2208 } 2209 } 2210 2211 static TCGOp *tcg_op_alloc(TCGOpcode opc) 2212 { 2213 TCGContext *s = tcg_ctx; 2214 TCGOp *op; 2215 2216 if (likely(QTAILQ_EMPTY(&s->free_ops))) { 2217 op = tcg_malloc(sizeof(TCGOp)); 2218 } else { 2219 op = QTAILQ_FIRST(&s->free_ops); 2220 QTAILQ_REMOVE(&s->free_ops, op, link); 2221 } 2222 memset(op, 0, offsetof(TCGOp, link)); 2223 op->opc = opc; 2224 s->nb_ops++; 2225 2226 return op; 2227 } 2228 2229 TCGOp *tcg_emit_op(TCGOpcode opc) 2230 { 2231 TCGOp *op = tcg_op_alloc(opc); 2232 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2233 return op; 2234 } 2235 2236 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 2237 { 2238 TCGOp *new_op = tcg_op_alloc(opc); 2239 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 2240 return new_op; 2241 } 2242 2243 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 2244 { 2245 TCGOp *new_op = tcg_op_alloc(opc); 2246 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 2247 return new_op; 2248 } 2249 2250 /* Reachable analysis : remove unreachable code. */ 2251 static void reachable_code_pass(TCGContext *s) 2252 { 2253 TCGOp *op, *op_next; 2254 bool dead = false; 2255 2256 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2257 bool remove = dead; 2258 TCGLabel *label; 2259 2260 switch (op->opc) { 2261 case INDEX_op_set_label: 2262 label = arg_label(op->args[0]); 2263 if (label->refs == 0) { 2264 /* 2265 * While there is an occasional backward branch, virtually 2266 * all branches generated by the translators are forward. 2267 * Which means that generally we will have already removed 2268 * all references to the label that will be, and there is 2269 * little to be gained by iterating. 2270 */ 2271 remove = true; 2272 } else { 2273 /* Once we see a label, insns become live again. */ 2274 dead = false; 2275 remove = false; 2276 2277 /* 2278 * Optimization can fold conditional branches to unconditional. 2279 * If we find a label with one reference which is preceded by 2280 * an unconditional branch to it, remove both. This needed to 2281 * wait until the dead code in between them was removed. 2282 */ 2283 if (label->refs == 1) { 2284 TCGOp *op_prev = QTAILQ_PREV(op, link); 2285 if (op_prev->opc == INDEX_op_br && 2286 label == arg_label(op_prev->args[0])) { 2287 tcg_op_remove(s, op_prev); 2288 remove = true; 2289 } 2290 } 2291 } 2292 break; 2293 2294 case INDEX_op_br: 2295 case INDEX_op_exit_tb: 2296 case INDEX_op_goto_ptr: 2297 /* Unconditional branches; everything following is dead. */ 2298 dead = true; 2299 break; 2300 2301 case INDEX_op_call: 2302 /* Notice noreturn helper calls, raising exceptions. */ 2303 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2304 dead = true; 2305 } 2306 break; 2307 2308 case INDEX_op_insn_start: 2309 /* Never remove -- we need to keep these for unwind. */ 2310 remove = false; 2311 break; 2312 2313 default: 2314 break; 2315 } 2316 2317 if (remove) { 2318 tcg_op_remove(s, op); 2319 } 2320 } 2321 } 2322 2323 #define TS_DEAD 1 2324 #define TS_MEM 2 2325 2326 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 2327 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 2328 2329 /* For liveness_pass_1, the register preferences for a given temp. */ 2330 static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 2331 { 2332 return ts->state_ptr; 2333 } 2334 2335 /* For liveness_pass_1, reset the preferences for a given temp to the 2336 * maximal regset for its type. 2337 */ 2338 static inline void la_reset_pref(TCGTemp *ts) 2339 { 2340 *la_temp_pref(ts) 2341 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 2342 } 2343 2344 /* liveness analysis: end of function: all temps are dead, and globals 2345 should be in memory. */ 2346 static void la_func_end(TCGContext *s, int ng, int nt) 2347 { 2348 int i; 2349 2350 for (i = 0; i < ng; ++i) { 2351 s->temps[i].state = TS_DEAD | TS_MEM; 2352 la_reset_pref(&s->temps[i]); 2353 } 2354 for (i = ng; i < nt; ++i) { 2355 s->temps[i].state = TS_DEAD; 2356 la_reset_pref(&s->temps[i]); 2357 } 2358 } 2359 2360 /* liveness analysis: end of basic block: all temps are dead, globals 2361 and local temps should be in memory. */ 2362 static void la_bb_end(TCGContext *s, int ng, int nt) 2363 { 2364 int i; 2365 2366 for (i = 0; i < nt; ++i) { 2367 TCGTemp *ts = &s->temps[i]; 2368 int state; 2369 2370 switch (ts->kind) { 2371 case TEMP_FIXED: 2372 case TEMP_GLOBAL: 2373 case TEMP_LOCAL: 2374 state = TS_DEAD | TS_MEM; 2375 break; 2376 case TEMP_NORMAL: 2377 case TEMP_CONST: 2378 state = TS_DEAD; 2379 break; 2380 default: 2381 g_assert_not_reached(); 2382 } 2383 ts->state = state; 2384 la_reset_pref(ts); 2385 } 2386 } 2387 2388 /* liveness analysis: sync globals back to memory. */ 2389 static void la_global_sync(TCGContext *s, int ng) 2390 { 2391 int i; 2392 2393 for (i = 0; i < ng; ++i) { 2394 int state = s->temps[i].state; 2395 s->temps[i].state = state | TS_MEM; 2396 if (state == TS_DEAD) { 2397 /* If the global was previously dead, reset prefs. */ 2398 la_reset_pref(&s->temps[i]); 2399 } 2400 } 2401 } 2402 2403 /* 2404 * liveness analysis: conditional branch: all temps are dead, 2405 * globals and local temps should be synced. 2406 */ 2407 static void la_bb_sync(TCGContext *s, int ng, int nt) 2408 { 2409 la_global_sync(s, ng); 2410 2411 for (int i = ng; i < nt; ++i) { 2412 TCGTemp *ts = &s->temps[i]; 2413 int state; 2414 2415 switch (ts->kind) { 2416 case TEMP_LOCAL: 2417 state = ts->state; 2418 ts->state = state | TS_MEM; 2419 if (state != TS_DEAD) { 2420 continue; 2421 } 2422 break; 2423 case TEMP_NORMAL: 2424 s->temps[i].state = TS_DEAD; 2425 break; 2426 case TEMP_CONST: 2427 continue; 2428 default: 2429 g_assert_not_reached(); 2430 } 2431 la_reset_pref(&s->temps[i]); 2432 } 2433 } 2434 2435 /* liveness analysis: sync globals back to memory and kill. */ 2436 static void la_global_kill(TCGContext *s, int ng) 2437 { 2438 int i; 2439 2440 for (i = 0; i < ng; i++) { 2441 s->temps[i].state = TS_DEAD | TS_MEM; 2442 la_reset_pref(&s->temps[i]); 2443 } 2444 } 2445 2446 /* liveness analysis: note live globals crossing calls. */ 2447 static void la_cross_call(TCGContext *s, int nt) 2448 { 2449 TCGRegSet mask = ~tcg_target_call_clobber_regs; 2450 int i; 2451 2452 for (i = 0; i < nt; i++) { 2453 TCGTemp *ts = &s->temps[i]; 2454 if (!(ts->state & TS_DEAD)) { 2455 TCGRegSet *pset = la_temp_pref(ts); 2456 TCGRegSet set = *pset; 2457 2458 set &= mask; 2459 /* If the combination is not possible, restart. */ 2460 if (set == 0) { 2461 set = tcg_target_available_regs[ts->type] & mask; 2462 } 2463 *pset = set; 2464 } 2465 } 2466 } 2467 2468 /* Liveness analysis : update the opc_arg_life array to tell if a 2469 given input arguments is dead. Instructions updating dead 2470 temporaries are removed. */ 2471 static void liveness_pass_1(TCGContext *s) 2472 { 2473 int nb_globals = s->nb_globals; 2474 int nb_temps = s->nb_temps; 2475 TCGOp *op, *op_prev; 2476 TCGRegSet *prefs; 2477 int i; 2478 2479 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 2480 for (i = 0; i < nb_temps; ++i) { 2481 s->temps[i].state_ptr = prefs + i; 2482 } 2483 2484 /* ??? Should be redundant with the exit_tb that ends the TB. */ 2485 la_func_end(s, nb_globals, nb_temps); 2486 2487 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 2488 int nb_iargs, nb_oargs; 2489 TCGOpcode opc_new, opc_new2; 2490 bool have_opc_new2; 2491 TCGLifeData arg_life = 0; 2492 TCGTemp *ts; 2493 TCGOpcode opc = op->opc; 2494 const TCGOpDef *def = &tcg_op_defs[opc]; 2495 2496 switch (opc) { 2497 case INDEX_op_call: 2498 { 2499 int call_flags; 2500 int nb_call_regs; 2501 2502 nb_oargs = TCGOP_CALLO(op); 2503 nb_iargs = TCGOP_CALLI(op); 2504 call_flags = tcg_call_flags(op); 2505 2506 /* pure functions can be removed if their result is unused */ 2507 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 2508 for (i = 0; i < nb_oargs; i++) { 2509 ts = arg_temp(op->args[i]); 2510 if (ts->state != TS_DEAD) { 2511 goto do_not_remove_call; 2512 } 2513 } 2514 goto do_remove; 2515 } 2516 do_not_remove_call: 2517 2518 /* Output args are dead. */ 2519 for (i = 0; i < nb_oargs; i++) { 2520 ts = arg_temp(op->args[i]); 2521 if (ts->state & TS_DEAD) { 2522 arg_life |= DEAD_ARG << i; 2523 } 2524 if (ts->state & TS_MEM) { 2525 arg_life |= SYNC_ARG << i; 2526 } 2527 ts->state = TS_DEAD; 2528 la_reset_pref(ts); 2529 2530 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */ 2531 op->output_pref[i] = 0; 2532 } 2533 2534 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 2535 TCG_CALL_NO_READ_GLOBALS))) { 2536 la_global_kill(s, nb_globals); 2537 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 2538 la_global_sync(s, nb_globals); 2539 } 2540 2541 /* Record arguments that die in this helper. */ 2542 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2543 ts = arg_temp(op->args[i]); 2544 if (ts && ts->state & TS_DEAD) { 2545 arg_life |= DEAD_ARG << i; 2546 } 2547 } 2548 2549 /* For all live registers, remove call-clobbered prefs. */ 2550 la_cross_call(s, nb_temps); 2551 2552 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2553 2554 /* Input arguments are live for preceding opcodes. */ 2555 for (i = 0; i < nb_iargs; i++) { 2556 ts = arg_temp(op->args[i + nb_oargs]); 2557 if (ts && ts->state & TS_DEAD) { 2558 /* For those arguments that die, and will be allocated 2559 * in registers, clear the register set for that arg, 2560 * to be filled in below. For args that will be on 2561 * the stack, reset to any available reg. 2562 */ 2563 *la_temp_pref(ts) 2564 = (i < nb_call_regs ? 0 : 2565 tcg_target_available_regs[ts->type]); 2566 ts->state &= ~TS_DEAD; 2567 } 2568 } 2569 2570 /* For each input argument, add its input register to prefs. 2571 If a temp is used once, this produces a single set bit. */ 2572 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) { 2573 ts = arg_temp(op->args[i + nb_oargs]); 2574 if (ts) { 2575 tcg_regset_set_reg(*la_temp_pref(ts), 2576 tcg_target_call_iarg_regs[i]); 2577 } 2578 } 2579 } 2580 break; 2581 case INDEX_op_insn_start: 2582 break; 2583 case INDEX_op_discard: 2584 /* mark the temporary as dead */ 2585 ts = arg_temp(op->args[0]); 2586 ts->state = TS_DEAD; 2587 la_reset_pref(ts); 2588 break; 2589 2590 case INDEX_op_add2_i32: 2591 opc_new = INDEX_op_add_i32; 2592 goto do_addsub2; 2593 case INDEX_op_sub2_i32: 2594 opc_new = INDEX_op_sub_i32; 2595 goto do_addsub2; 2596 case INDEX_op_add2_i64: 2597 opc_new = INDEX_op_add_i64; 2598 goto do_addsub2; 2599 case INDEX_op_sub2_i64: 2600 opc_new = INDEX_op_sub_i64; 2601 do_addsub2: 2602 nb_iargs = 4; 2603 nb_oargs = 2; 2604 /* Test if the high part of the operation is dead, but not 2605 the low part. The result can be optimized to a simple 2606 add or sub. This happens often for x86_64 guest when the 2607 cpu mode is set to 32 bit. */ 2608 if (arg_temp(op->args[1])->state == TS_DEAD) { 2609 if (arg_temp(op->args[0])->state == TS_DEAD) { 2610 goto do_remove; 2611 } 2612 /* Replace the opcode and adjust the args in place, 2613 leaving 3 unused args at the end. */ 2614 op->opc = opc = opc_new; 2615 op->args[1] = op->args[2]; 2616 op->args[2] = op->args[4]; 2617 /* Fall through and mark the single-word operation live. */ 2618 nb_iargs = 2; 2619 nb_oargs = 1; 2620 } 2621 goto do_not_remove; 2622 2623 case INDEX_op_mulu2_i32: 2624 opc_new = INDEX_op_mul_i32; 2625 opc_new2 = INDEX_op_muluh_i32; 2626 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 2627 goto do_mul2; 2628 case INDEX_op_muls2_i32: 2629 opc_new = INDEX_op_mul_i32; 2630 opc_new2 = INDEX_op_mulsh_i32; 2631 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 2632 goto do_mul2; 2633 case INDEX_op_mulu2_i64: 2634 opc_new = INDEX_op_mul_i64; 2635 opc_new2 = INDEX_op_muluh_i64; 2636 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 2637 goto do_mul2; 2638 case INDEX_op_muls2_i64: 2639 opc_new = INDEX_op_mul_i64; 2640 opc_new2 = INDEX_op_mulsh_i64; 2641 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 2642 goto do_mul2; 2643 do_mul2: 2644 nb_iargs = 2; 2645 nb_oargs = 2; 2646 if (arg_temp(op->args[1])->state == TS_DEAD) { 2647 if (arg_temp(op->args[0])->state == TS_DEAD) { 2648 /* Both parts of the operation are dead. */ 2649 goto do_remove; 2650 } 2651 /* The high part of the operation is dead; generate the low. */ 2652 op->opc = opc = opc_new; 2653 op->args[1] = op->args[2]; 2654 op->args[2] = op->args[3]; 2655 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 2656 /* The low part of the operation is dead; generate the high. */ 2657 op->opc = opc = opc_new2; 2658 op->args[0] = op->args[1]; 2659 op->args[1] = op->args[2]; 2660 op->args[2] = op->args[3]; 2661 } else { 2662 goto do_not_remove; 2663 } 2664 /* Mark the single-word operation live. */ 2665 nb_oargs = 1; 2666 goto do_not_remove; 2667 2668 default: 2669 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 2670 nb_iargs = def->nb_iargs; 2671 nb_oargs = def->nb_oargs; 2672 2673 /* Test if the operation can be removed because all 2674 its outputs are dead. We assume that nb_oargs == 0 2675 implies side effects */ 2676 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 2677 for (i = 0; i < nb_oargs; i++) { 2678 if (arg_temp(op->args[i])->state != TS_DEAD) { 2679 goto do_not_remove; 2680 } 2681 } 2682 goto do_remove; 2683 } 2684 goto do_not_remove; 2685 2686 do_remove: 2687 tcg_op_remove(s, op); 2688 break; 2689 2690 do_not_remove: 2691 for (i = 0; i < nb_oargs; i++) { 2692 ts = arg_temp(op->args[i]); 2693 2694 /* Remember the preference of the uses that followed. */ 2695 op->output_pref[i] = *la_temp_pref(ts); 2696 2697 /* Output args are dead. */ 2698 if (ts->state & TS_DEAD) { 2699 arg_life |= DEAD_ARG << i; 2700 } 2701 if (ts->state & TS_MEM) { 2702 arg_life |= SYNC_ARG << i; 2703 } 2704 ts->state = TS_DEAD; 2705 la_reset_pref(ts); 2706 } 2707 2708 /* If end of basic block, update. */ 2709 if (def->flags & TCG_OPF_BB_EXIT) { 2710 la_func_end(s, nb_globals, nb_temps); 2711 } else if (def->flags & TCG_OPF_COND_BRANCH) { 2712 la_bb_sync(s, nb_globals, nb_temps); 2713 } else if (def->flags & TCG_OPF_BB_END) { 2714 la_bb_end(s, nb_globals, nb_temps); 2715 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2716 la_global_sync(s, nb_globals); 2717 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2718 la_cross_call(s, nb_temps); 2719 } 2720 } 2721 2722 /* Record arguments that die in this opcode. */ 2723 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2724 ts = arg_temp(op->args[i]); 2725 if (ts->state & TS_DEAD) { 2726 arg_life |= DEAD_ARG << i; 2727 } 2728 } 2729 2730 /* Input arguments are live for preceding opcodes. */ 2731 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2732 ts = arg_temp(op->args[i]); 2733 if (ts->state & TS_DEAD) { 2734 /* For operands that were dead, initially allow 2735 all regs for the type. */ 2736 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 2737 ts->state &= ~TS_DEAD; 2738 } 2739 } 2740 2741 /* Incorporate constraints for this operand. */ 2742 switch (opc) { 2743 case INDEX_op_mov_i32: 2744 case INDEX_op_mov_i64: 2745 /* Note that these are TCG_OPF_NOT_PRESENT and do not 2746 have proper constraints. That said, special case 2747 moves to propagate preferences backward. */ 2748 if (IS_DEAD_ARG(1)) { 2749 *la_temp_pref(arg_temp(op->args[0])) 2750 = *la_temp_pref(arg_temp(op->args[1])); 2751 } 2752 break; 2753 2754 default: 2755 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2756 const TCGArgConstraint *ct = &def->args_ct[i]; 2757 TCGRegSet set, *pset; 2758 2759 ts = arg_temp(op->args[i]); 2760 pset = la_temp_pref(ts); 2761 set = *pset; 2762 2763 set &= ct->regs; 2764 if (ct->ialias) { 2765 set &= op->output_pref[ct->alias_index]; 2766 } 2767 /* If the combination is not possible, restart. */ 2768 if (set == 0) { 2769 set = ct->regs; 2770 } 2771 *pset = set; 2772 } 2773 break; 2774 } 2775 break; 2776 } 2777 op->life = arg_life; 2778 } 2779 } 2780 2781 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 2782 static bool liveness_pass_2(TCGContext *s) 2783 { 2784 int nb_globals = s->nb_globals; 2785 int nb_temps, i; 2786 bool changes = false; 2787 TCGOp *op, *op_next; 2788 2789 /* Create a temporary for each indirect global. */ 2790 for (i = 0; i < nb_globals; ++i) { 2791 TCGTemp *its = &s->temps[i]; 2792 if (its->indirect_reg) { 2793 TCGTemp *dts = tcg_temp_alloc(s); 2794 dts->type = its->type; 2795 dts->base_type = its->base_type; 2796 its->state_ptr = dts; 2797 } else { 2798 its->state_ptr = NULL; 2799 } 2800 /* All globals begin dead. */ 2801 its->state = TS_DEAD; 2802 } 2803 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 2804 TCGTemp *its = &s->temps[i]; 2805 its->state_ptr = NULL; 2806 its->state = TS_DEAD; 2807 } 2808 2809 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2810 TCGOpcode opc = op->opc; 2811 const TCGOpDef *def = &tcg_op_defs[opc]; 2812 TCGLifeData arg_life = op->life; 2813 int nb_iargs, nb_oargs, call_flags; 2814 TCGTemp *arg_ts, *dir_ts; 2815 2816 if (opc == INDEX_op_call) { 2817 nb_oargs = TCGOP_CALLO(op); 2818 nb_iargs = TCGOP_CALLI(op); 2819 call_flags = tcg_call_flags(op); 2820 } else { 2821 nb_iargs = def->nb_iargs; 2822 nb_oargs = def->nb_oargs; 2823 2824 /* Set flags similar to how calls require. */ 2825 if (def->flags & TCG_OPF_COND_BRANCH) { 2826 /* Like reading globals: sync_globals */ 2827 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2828 } else if (def->flags & TCG_OPF_BB_END) { 2829 /* Like writing globals: save_globals */ 2830 call_flags = 0; 2831 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2832 /* Like reading globals: sync_globals */ 2833 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2834 } else { 2835 /* No effect on globals. */ 2836 call_flags = (TCG_CALL_NO_READ_GLOBALS | 2837 TCG_CALL_NO_WRITE_GLOBALS); 2838 } 2839 } 2840 2841 /* Make sure that input arguments are available. */ 2842 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2843 arg_ts = arg_temp(op->args[i]); 2844 if (arg_ts) { 2845 dir_ts = arg_ts->state_ptr; 2846 if (dir_ts && arg_ts->state == TS_DEAD) { 2847 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 2848 ? INDEX_op_ld_i32 2849 : INDEX_op_ld_i64); 2850 TCGOp *lop = tcg_op_insert_before(s, op, lopc); 2851 2852 lop->args[0] = temp_arg(dir_ts); 2853 lop->args[1] = temp_arg(arg_ts->mem_base); 2854 lop->args[2] = arg_ts->mem_offset; 2855 2856 /* Loaded, but synced with memory. */ 2857 arg_ts->state = TS_MEM; 2858 } 2859 } 2860 } 2861 2862 /* Perform input replacement, and mark inputs that became dead. 2863 No action is required except keeping temp_state up to date 2864 so that we reload when needed. */ 2865 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2866 arg_ts = arg_temp(op->args[i]); 2867 if (arg_ts) { 2868 dir_ts = arg_ts->state_ptr; 2869 if (dir_ts) { 2870 op->args[i] = temp_arg(dir_ts); 2871 changes = true; 2872 if (IS_DEAD_ARG(i)) { 2873 arg_ts->state = TS_DEAD; 2874 } 2875 } 2876 } 2877 } 2878 2879 /* Liveness analysis should ensure that the following are 2880 all correct, for call sites and basic block end points. */ 2881 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 2882 /* Nothing to do */ 2883 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 2884 for (i = 0; i < nb_globals; ++i) { 2885 /* Liveness should see that globals are synced back, 2886 that is, either TS_DEAD or TS_MEM. */ 2887 arg_ts = &s->temps[i]; 2888 tcg_debug_assert(arg_ts->state_ptr == 0 2889 || arg_ts->state != 0); 2890 } 2891 } else { 2892 for (i = 0; i < nb_globals; ++i) { 2893 /* Liveness should see that globals are saved back, 2894 that is, TS_DEAD, waiting to be reloaded. */ 2895 arg_ts = &s->temps[i]; 2896 tcg_debug_assert(arg_ts->state_ptr == 0 2897 || arg_ts->state == TS_DEAD); 2898 } 2899 } 2900 2901 /* Outputs become available. */ 2902 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 2903 arg_ts = arg_temp(op->args[0]); 2904 dir_ts = arg_ts->state_ptr; 2905 if (dir_ts) { 2906 op->args[0] = temp_arg(dir_ts); 2907 changes = true; 2908 2909 /* The output is now live and modified. */ 2910 arg_ts->state = 0; 2911 2912 if (NEED_SYNC_ARG(0)) { 2913 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 2914 ? INDEX_op_st_i32 2915 : INDEX_op_st_i64); 2916 TCGOp *sop = tcg_op_insert_after(s, op, sopc); 2917 TCGTemp *out_ts = dir_ts; 2918 2919 if (IS_DEAD_ARG(0)) { 2920 out_ts = arg_temp(op->args[1]); 2921 arg_ts->state = TS_DEAD; 2922 tcg_op_remove(s, op); 2923 } else { 2924 arg_ts->state = TS_MEM; 2925 } 2926 2927 sop->args[0] = temp_arg(out_ts); 2928 sop->args[1] = temp_arg(arg_ts->mem_base); 2929 sop->args[2] = arg_ts->mem_offset; 2930 } else { 2931 tcg_debug_assert(!IS_DEAD_ARG(0)); 2932 } 2933 } 2934 } else { 2935 for (i = 0; i < nb_oargs; i++) { 2936 arg_ts = arg_temp(op->args[i]); 2937 dir_ts = arg_ts->state_ptr; 2938 if (!dir_ts) { 2939 continue; 2940 } 2941 op->args[i] = temp_arg(dir_ts); 2942 changes = true; 2943 2944 /* The output is now live and modified. */ 2945 arg_ts->state = 0; 2946 2947 /* Sync outputs upon their last write. */ 2948 if (NEED_SYNC_ARG(i)) { 2949 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 2950 ? INDEX_op_st_i32 2951 : INDEX_op_st_i64); 2952 TCGOp *sop = tcg_op_insert_after(s, op, sopc); 2953 2954 sop->args[0] = temp_arg(dir_ts); 2955 sop->args[1] = temp_arg(arg_ts->mem_base); 2956 sop->args[2] = arg_ts->mem_offset; 2957 2958 arg_ts->state = TS_MEM; 2959 } 2960 /* Drop outputs that are dead. */ 2961 if (IS_DEAD_ARG(i)) { 2962 arg_ts->state = TS_DEAD; 2963 } 2964 } 2965 } 2966 } 2967 2968 return changes; 2969 } 2970 2971 #ifdef CONFIG_DEBUG_TCG 2972 static void dump_regs(TCGContext *s) 2973 { 2974 TCGTemp *ts; 2975 int i; 2976 char buf[64]; 2977 2978 for(i = 0; i < s->nb_temps; i++) { 2979 ts = &s->temps[i]; 2980 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2981 switch(ts->val_type) { 2982 case TEMP_VAL_REG: 2983 printf("%s", tcg_target_reg_names[ts->reg]); 2984 break; 2985 case TEMP_VAL_MEM: 2986 printf("%d(%s)", (int)ts->mem_offset, 2987 tcg_target_reg_names[ts->mem_base->reg]); 2988 break; 2989 case TEMP_VAL_CONST: 2990 printf("$0x%" PRIx64, ts->val); 2991 break; 2992 case TEMP_VAL_DEAD: 2993 printf("D"); 2994 break; 2995 default: 2996 printf("???"); 2997 break; 2998 } 2999 printf("\n"); 3000 } 3001 3002 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 3003 if (s->reg_to_temp[i] != NULL) { 3004 printf("%s: %s\n", 3005 tcg_target_reg_names[i], 3006 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 3007 } 3008 } 3009 } 3010 3011 static void check_regs(TCGContext *s) 3012 { 3013 int reg; 3014 int k; 3015 TCGTemp *ts; 3016 char buf[64]; 3017 3018 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 3019 ts = s->reg_to_temp[reg]; 3020 if (ts != NULL) { 3021 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 3022 printf("Inconsistency for register %s:\n", 3023 tcg_target_reg_names[reg]); 3024 goto fail; 3025 } 3026 } 3027 } 3028 for (k = 0; k < s->nb_temps; k++) { 3029 ts = &s->temps[k]; 3030 if (ts->val_type == TEMP_VAL_REG 3031 && ts->kind != TEMP_FIXED 3032 && s->reg_to_temp[ts->reg] != ts) { 3033 printf("Inconsistency for temp %s:\n", 3034 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 3035 fail: 3036 printf("reg state:\n"); 3037 dump_regs(s); 3038 tcg_abort(); 3039 } 3040 } 3041 } 3042 #endif 3043 3044 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3045 { 3046 intptr_t off, size, align; 3047 3048 switch (ts->type) { 3049 case TCG_TYPE_I32: 3050 size = align = 4; 3051 break; 3052 case TCG_TYPE_I64: 3053 case TCG_TYPE_V64: 3054 size = align = 8; 3055 break; 3056 case TCG_TYPE_V128: 3057 size = align = 16; 3058 break; 3059 case TCG_TYPE_V256: 3060 /* Note that we do not require aligned storage for V256. */ 3061 size = 32, align = 16; 3062 break; 3063 default: 3064 g_assert_not_reached(); 3065 } 3066 3067 assert(align <= TCG_TARGET_STACK_ALIGN); 3068 off = ROUND_UP(s->current_frame_offset, align); 3069 3070 /* If we've exhausted the stack frame, restart with a smaller TB. */ 3071 if (off + size > s->frame_end) { 3072 tcg_raise_tb_overflow(s); 3073 } 3074 s->current_frame_offset = off + size; 3075 3076 ts->mem_offset = off; 3077 #if defined(__sparc__) 3078 ts->mem_offset += TCG_TARGET_STACK_BIAS; 3079 #endif 3080 ts->mem_base = s->frame_temp; 3081 ts->mem_allocated = 1; 3082 } 3083 3084 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3085 3086 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 3087 mark it free; otherwise mark it dead. */ 3088 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3089 { 3090 TCGTempVal new_type; 3091 3092 switch (ts->kind) { 3093 case TEMP_FIXED: 3094 return; 3095 case TEMP_GLOBAL: 3096 case TEMP_LOCAL: 3097 new_type = TEMP_VAL_MEM; 3098 break; 3099 case TEMP_NORMAL: 3100 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3101 break; 3102 case TEMP_CONST: 3103 new_type = TEMP_VAL_CONST; 3104 break; 3105 default: 3106 g_assert_not_reached(); 3107 } 3108 if (ts->val_type == TEMP_VAL_REG) { 3109 s->reg_to_temp[ts->reg] = NULL; 3110 } 3111 ts->val_type = new_type; 3112 } 3113 3114 /* Mark a temporary as dead. */ 3115 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 3116 { 3117 temp_free_or_dead(s, ts, 1); 3118 } 3119 3120 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 3121 registers needs to be allocated to store a constant. If 'free_or_dead' 3122 is non-zero, subsequently release the temporary; if it is positive, the 3123 temp is dead; if it is negative, the temp is free. */ 3124 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 3125 TCGRegSet preferred_regs, int free_or_dead) 3126 { 3127 if (!temp_readonly(ts) && !ts->mem_coherent) { 3128 if (!ts->mem_allocated) { 3129 temp_allocate_frame(s, ts); 3130 } 3131 switch (ts->val_type) { 3132 case TEMP_VAL_CONST: 3133 /* If we're going to free the temp immediately, then we won't 3134 require it later in a register, so attempt to store the 3135 constant to memory directly. */ 3136 if (free_or_dead 3137 && tcg_out_sti(s, ts->type, ts->val, 3138 ts->mem_base->reg, ts->mem_offset)) { 3139 break; 3140 } 3141 temp_load(s, ts, tcg_target_available_regs[ts->type], 3142 allocated_regs, preferred_regs); 3143 /* fallthrough */ 3144 3145 case TEMP_VAL_REG: 3146 tcg_out_st(s, ts->type, ts->reg, 3147 ts->mem_base->reg, ts->mem_offset); 3148 break; 3149 3150 case TEMP_VAL_MEM: 3151 break; 3152 3153 case TEMP_VAL_DEAD: 3154 default: 3155 tcg_abort(); 3156 } 3157 ts->mem_coherent = 1; 3158 } 3159 if (free_or_dead) { 3160 temp_free_or_dead(s, ts, free_or_dead); 3161 } 3162 } 3163 3164 /* free register 'reg' by spilling the corresponding temporary if necessary */ 3165 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 3166 { 3167 TCGTemp *ts = s->reg_to_temp[reg]; 3168 if (ts != NULL) { 3169 temp_sync(s, ts, allocated_regs, 0, -1); 3170 } 3171 } 3172 3173 /** 3174 * tcg_reg_alloc: 3175 * @required_regs: Set of registers in which we must allocate. 3176 * @allocated_regs: Set of registers which must be avoided. 3177 * @preferred_regs: Set of registers we should prefer. 3178 * @rev: True if we search the registers in "indirect" order. 3179 * 3180 * The allocated register must be in @required_regs & ~@allocated_regs, 3181 * but if we can put it in @preferred_regs we may save a move later. 3182 */ 3183 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3184 TCGRegSet allocated_regs, 3185 TCGRegSet preferred_regs, bool rev) 3186 { 3187 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3188 TCGRegSet reg_ct[2]; 3189 const int *order; 3190 3191 reg_ct[1] = required_regs & ~allocated_regs; 3192 tcg_debug_assert(reg_ct[1] != 0); 3193 reg_ct[0] = reg_ct[1] & preferred_regs; 3194 3195 /* Skip the preferred_regs option if it cannot be satisfied, 3196 or if the preference made no difference. */ 3197 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3198 3199 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3200 3201 /* Try free registers, preferences first. */ 3202 for (j = f; j < 2; j++) { 3203 TCGRegSet set = reg_ct[j]; 3204 3205 if (tcg_regset_single(set)) { 3206 /* One register in the set. */ 3207 TCGReg reg = tcg_regset_first(set); 3208 if (s->reg_to_temp[reg] == NULL) { 3209 return reg; 3210 } 3211 } else { 3212 for (i = 0; i < n; i++) { 3213 TCGReg reg = order[i]; 3214 if (s->reg_to_temp[reg] == NULL && 3215 tcg_regset_test_reg(set, reg)) { 3216 return reg; 3217 } 3218 } 3219 } 3220 } 3221 3222 /* We must spill something. */ 3223 for (j = f; j < 2; j++) { 3224 TCGRegSet set = reg_ct[j]; 3225 3226 if (tcg_regset_single(set)) { 3227 /* One register in the set. */ 3228 TCGReg reg = tcg_regset_first(set); 3229 tcg_reg_free(s, reg, allocated_regs); 3230 return reg; 3231 } else { 3232 for (i = 0; i < n; i++) { 3233 TCGReg reg = order[i]; 3234 if (tcg_regset_test_reg(set, reg)) { 3235 tcg_reg_free(s, reg, allocated_regs); 3236 return reg; 3237 } 3238 } 3239 } 3240 } 3241 3242 tcg_abort(); 3243 } 3244 3245 /* Make sure the temporary is in a register. If needed, allocate the register 3246 from DESIRED while avoiding ALLOCATED. */ 3247 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3248 TCGRegSet allocated_regs, TCGRegSet preferred_regs) 3249 { 3250 TCGReg reg; 3251 3252 switch (ts->val_type) { 3253 case TEMP_VAL_REG: 3254 return; 3255 case TEMP_VAL_CONST: 3256 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3257 preferred_regs, ts->indirect_base); 3258 if (ts->type <= TCG_TYPE_I64) { 3259 tcg_out_movi(s, ts->type, reg, ts->val); 3260 } else { 3261 uint64_t val = ts->val; 3262 MemOp vece = MO_64; 3263 3264 /* 3265 * Find the minimal vector element that matches the constant. 3266 * The targets will, in general, have to do this search anyway, 3267 * do this generically. 3268 */ 3269 if (val == dup_const(MO_8, val)) { 3270 vece = MO_8; 3271 } else if (val == dup_const(MO_16, val)) { 3272 vece = MO_16; 3273 } else if (val == dup_const(MO_32, val)) { 3274 vece = MO_32; 3275 } 3276 3277 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 3278 } 3279 ts->mem_coherent = 0; 3280 break; 3281 case TEMP_VAL_MEM: 3282 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3283 preferred_regs, ts->indirect_base); 3284 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 3285 ts->mem_coherent = 1; 3286 break; 3287 case TEMP_VAL_DEAD: 3288 default: 3289 tcg_abort(); 3290 } 3291 ts->reg = reg; 3292 ts->val_type = TEMP_VAL_REG; 3293 s->reg_to_temp[reg] = ts; 3294 } 3295 3296 /* Save a temporary to memory. 'allocated_regs' is used in case a 3297 temporary registers needs to be allocated to store a constant. */ 3298 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 3299 { 3300 /* The liveness analysis already ensures that globals are back 3301 in memory. Keep an tcg_debug_assert for safety. */ 3302 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 3303 } 3304 3305 /* save globals to their canonical location and assume they can be 3306 modified be the following code. 'allocated_regs' is used in case a 3307 temporary registers needs to be allocated to store a constant. */ 3308 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3309 { 3310 int i, n; 3311 3312 for (i = 0, n = s->nb_globals; i < n; i++) { 3313 temp_save(s, &s->temps[i], allocated_regs); 3314 } 3315 } 3316 3317 /* sync globals to their canonical location and assume they can be 3318 read by the following code. 'allocated_regs' is used in case a 3319 temporary registers needs to be allocated to store a constant. */ 3320 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 3321 { 3322 int i, n; 3323 3324 for (i = 0, n = s->nb_globals; i < n; i++) { 3325 TCGTemp *ts = &s->temps[i]; 3326 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3327 || ts->kind == TEMP_FIXED 3328 || ts->mem_coherent); 3329 } 3330 } 3331 3332 /* at the end of a basic block, we assume all temporaries are dead and 3333 all globals are stored at their canonical location. */ 3334 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3335 { 3336 int i; 3337 3338 for (i = s->nb_globals; i < s->nb_temps; i++) { 3339 TCGTemp *ts = &s->temps[i]; 3340 3341 switch (ts->kind) { 3342 case TEMP_LOCAL: 3343 temp_save(s, ts, allocated_regs); 3344 break; 3345 case TEMP_NORMAL: 3346 /* The liveness analysis already ensures that temps are dead. 3347 Keep an tcg_debug_assert for safety. */ 3348 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3349 break; 3350 case TEMP_CONST: 3351 /* Similarly, we should have freed any allocated register. */ 3352 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3353 break; 3354 default: 3355 g_assert_not_reached(); 3356 } 3357 } 3358 3359 save_globals(s, allocated_regs); 3360 } 3361 3362 /* 3363 * At a conditional branch, we assume all temporaries are dead and 3364 * all globals and local temps are synced to their location. 3365 */ 3366 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3367 { 3368 sync_globals(s, allocated_regs); 3369 3370 for (int i = s->nb_globals; i < s->nb_temps; i++) { 3371 TCGTemp *ts = &s->temps[i]; 3372 /* 3373 * The liveness analysis already ensures that temps are dead. 3374 * Keep tcg_debug_asserts for safety. 3375 */ 3376 switch (ts->kind) { 3377 case TEMP_LOCAL: 3378 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3379 break; 3380 case TEMP_NORMAL: 3381 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3382 break; 3383 case TEMP_CONST: 3384 break; 3385 default: 3386 g_assert_not_reached(); 3387 } 3388 } 3389 } 3390 3391 /* 3392 * Specialized code generation for INDEX_op_mov_* with a constant. 3393 */ 3394 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3395 tcg_target_ulong val, TCGLifeData arg_life, 3396 TCGRegSet preferred_regs) 3397 { 3398 /* ENV should not be modified. */ 3399 tcg_debug_assert(!temp_readonly(ots)); 3400 3401 /* The movi is not explicitly generated here. */ 3402 if (ots->val_type == TEMP_VAL_REG) { 3403 s->reg_to_temp[ots->reg] = NULL; 3404 } 3405 ots->val_type = TEMP_VAL_CONST; 3406 ots->val = val; 3407 ots->mem_coherent = 0; 3408 if (NEED_SYNC_ARG(0)) { 3409 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 3410 } else if (IS_DEAD_ARG(0)) { 3411 temp_dead(s, ots); 3412 } 3413 } 3414 3415 /* 3416 * Specialized code generation for INDEX_op_mov_*. 3417 */ 3418 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 3419 { 3420 const TCGLifeData arg_life = op->life; 3421 TCGRegSet allocated_regs, preferred_regs; 3422 TCGTemp *ts, *ots; 3423 TCGType otype, itype; 3424 3425 allocated_regs = s->reserved_regs; 3426 preferred_regs = op->output_pref[0]; 3427 ots = arg_temp(op->args[0]); 3428 ts = arg_temp(op->args[1]); 3429 3430 /* ENV should not be modified. */ 3431 tcg_debug_assert(!temp_readonly(ots)); 3432 3433 /* Note that otype != itype for no-op truncation. */ 3434 otype = ots->type; 3435 itype = ts->type; 3436 3437 if (ts->val_type == TEMP_VAL_CONST) { 3438 /* propagate constant or generate sti */ 3439 tcg_target_ulong val = ts->val; 3440 if (IS_DEAD_ARG(1)) { 3441 temp_dead(s, ts); 3442 } 3443 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 3444 return; 3445 } 3446 3447 /* If the source value is in memory we're going to be forced 3448 to have it in a register in order to perform the copy. Copy 3449 the SOURCE value into its own register first, that way we 3450 don't have to reload SOURCE the next time it is used. */ 3451 if (ts->val_type == TEMP_VAL_MEM) { 3452 temp_load(s, ts, tcg_target_available_regs[itype], 3453 allocated_regs, preferred_regs); 3454 } 3455 3456 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 3457 if (IS_DEAD_ARG(0)) { 3458 /* mov to a non-saved dead register makes no sense (even with 3459 liveness analysis disabled). */ 3460 tcg_debug_assert(NEED_SYNC_ARG(0)); 3461 if (!ots->mem_allocated) { 3462 temp_allocate_frame(s, ots); 3463 } 3464 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 3465 if (IS_DEAD_ARG(1)) { 3466 temp_dead(s, ts); 3467 } 3468 temp_dead(s, ots); 3469 } else { 3470 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 3471 /* the mov can be suppressed */ 3472 if (ots->val_type == TEMP_VAL_REG) { 3473 s->reg_to_temp[ots->reg] = NULL; 3474 } 3475 ots->reg = ts->reg; 3476 temp_dead(s, ts); 3477 } else { 3478 if (ots->val_type != TEMP_VAL_REG) { 3479 /* When allocating a new register, make sure to not spill the 3480 input one. */ 3481 tcg_regset_set_reg(allocated_regs, ts->reg); 3482 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 3483 allocated_regs, preferred_regs, 3484 ots->indirect_base); 3485 } 3486 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) { 3487 /* 3488 * Cross register class move not supported. 3489 * Store the source register into the destination slot 3490 * and leave the destination temp as TEMP_VAL_MEM. 3491 */ 3492 assert(!temp_readonly(ots)); 3493 if (!ts->mem_allocated) { 3494 temp_allocate_frame(s, ots); 3495 } 3496 tcg_out_st(s, ts->type, ts->reg, 3497 ots->mem_base->reg, ots->mem_offset); 3498 ots->mem_coherent = 1; 3499 temp_free_or_dead(s, ots, -1); 3500 return; 3501 } 3502 } 3503 ots->val_type = TEMP_VAL_REG; 3504 ots->mem_coherent = 0; 3505 s->reg_to_temp[ots->reg] = ots; 3506 if (NEED_SYNC_ARG(0)) { 3507 temp_sync(s, ots, allocated_regs, 0, 0); 3508 } 3509 } 3510 } 3511 3512 /* 3513 * Specialized code generation for INDEX_op_dup_vec. 3514 */ 3515 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 3516 { 3517 const TCGLifeData arg_life = op->life; 3518 TCGRegSet dup_out_regs, dup_in_regs; 3519 TCGTemp *its, *ots; 3520 TCGType itype, vtype; 3521 intptr_t endian_fixup; 3522 unsigned vece; 3523 bool ok; 3524 3525 ots = arg_temp(op->args[0]); 3526 its = arg_temp(op->args[1]); 3527 3528 /* ENV should not be modified. */ 3529 tcg_debug_assert(!temp_readonly(ots)); 3530 3531 itype = its->type; 3532 vece = TCGOP_VECE(op); 3533 vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3534 3535 if (its->val_type == TEMP_VAL_CONST) { 3536 /* Propagate constant via movi -> dupi. */ 3537 tcg_target_ulong val = its->val; 3538 if (IS_DEAD_ARG(1)) { 3539 temp_dead(s, its); 3540 } 3541 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]); 3542 return; 3543 } 3544 3545 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 3546 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 3547 3548 /* Allocate the output register now. */ 3549 if (ots->val_type != TEMP_VAL_REG) { 3550 TCGRegSet allocated_regs = s->reserved_regs; 3551 3552 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 3553 /* Make sure to not spill the input register. */ 3554 tcg_regset_set_reg(allocated_regs, its->reg); 3555 } 3556 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3557 op->output_pref[0], ots->indirect_base); 3558 ots->val_type = TEMP_VAL_REG; 3559 ots->mem_coherent = 0; 3560 s->reg_to_temp[ots->reg] = ots; 3561 } 3562 3563 switch (its->val_type) { 3564 case TEMP_VAL_REG: 3565 /* 3566 * The dup constriaints must be broad, covering all possible VECE. 3567 * However, tcg_op_dup_vec() gets to see the VECE and we allow it 3568 * to fail, indicating that extra moves are required for that case. 3569 */ 3570 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 3571 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 3572 goto done; 3573 } 3574 /* Try again from memory or a vector input register. */ 3575 } 3576 if (!its->mem_coherent) { 3577 /* 3578 * The input register is not synced, and so an extra store 3579 * would be required to use memory. Attempt an integer-vector 3580 * register move first. We do not have a TCGRegSet for this. 3581 */ 3582 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 3583 break; 3584 } 3585 /* Sync the temp back to its slot and load from there. */ 3586 temp_sync(s, its, s->reserved_regs, 0, 0); 3587 } 3588 /* fall through */ 3589 3590 case TEMP_VAL_MEM: 3591 #ifdef HOST_WORDS_BIGENDIAN 3592 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8; 3593 endian_fixup -= 1 << vece; 3594 #else 3595 endian_fixup = 0; 3596 #endif 3597 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 3598 its->mem_offset + endian_fixup)) { 3599 goto done; 3600 } 3601 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 3602 break; 3603 3604 default: 3605 g_assert_not_reached(); 3606 } 3607 3608 /* We now have a vector input register, so dup must succeed. */ 3609 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 3610 tcg_debug_assert(ok); 3611 3612 done: 3613 if (IS_DEAD_ARG(1)) { 3614 temp_dead(s, its); 3615 } 3616 if (NEED_SYNC_ARG(0)) { 3617 temp_sync(s, ots, s->reserved_regs, 0, 0); 3618 } 3619 if (IS_DEAD_ARG(0)) { 3620 temp_dead(s, ots); 3621 } 3622 } 3623 3624 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 3625 { 3626 const TCGLifeData arg_life = op->life; 3627 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 3628 TCGRegSet i_allocated_regs; 3629 TCGRegSet o_allocated_regs; 3630 int i, k, nb_iargs, nb_oargs; 3631 TCGReg reg; 3632 TCGArg arg; 3633 const TCGArgConstraint *arg_ct; 3634 TCGTemp *ts; 3635 TCGArg new_args[TCG_MAX_OP_ARGS]; 3636 int const_args[TCG_MAX_OP_ARGS]; 3637 3638 nb_oargs = def->nb_oargs; 3639 nb_iargs = def->nb_iargs; 3640 3641 /* copy constants */ 3642 memcpy(new_args + nb_oargs + nb_iargs, 3643 op->args + nb_oargs + nb_iargs, 3644 sizeof(TCGArg) * def->nb_cargs); 3645 3646 i_allocated_regs = s->reserved_regs; 3647 o_allocated_regs = s->reserved_regs; 3648 3649 /* satisfy input constraints */ 3650 for (k = 0; k < nb_iargs; k++) { 3651 TCGRegSet i_preferred_regs, o_preferred_regs; 3652 3653 i = def->args_ct[nb_oargs + k].sort_index; 3654 arg = op->args[i]; 3655 arg_ct = &def->args_ct[i]; 3656 ts = arg_temp(arg); 3657 3658 if (ts->val_type == TEMP_VAL_CONST 3659 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 3660 /* constant is OK for instruction */ 3661 const_args[i] = 1; 3662 new_args[i] = ts->val; 3663 continue; 3664 } 3665 3666 i_preferred_regs = o_preferred_regs = 0; 3667 if (arg_ct->ialias) { 3668 o_preferred_regs = op->output_pref[arg_ct->alias_index]; 3669 3670 /* 3671 * If the input is readonly, then it cannot also be an 3672 * output and aliased to itself. If the input is not 3673 * dead after the instruction, we must allocate a new 3674 * register and move it. 3675 */ 3676 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 3677 goto allocate_in_reg; 3678 } 3679 3680 /* 3681 * Check if the current register has already been allocated 3682 * for another input aliased to an output. 3683 */ 3684 if (ts->val_type == TEMP_VAL_REG) { 3685 reg = ts->reg; 3686 for (int k2 = 0; k2 < k; k2++) { 3687 int i2 = def->args_ct[nb_oargs + k2].sort_index; 3688 if (def->args_ct[i2].ialias && reg == new_args[i2]) { 3689 goto allocate_in_reg; 3690 } 3691 } 3692 } 3693 i_preferred_regs = o_preferred_regs; 3694 } 3695 3696 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs); 3697 reg = ts->reg; 3698 3699 if (!tcg_regset_test_reg(arg_ct->regs, reg)) { 3700 allocate_in_reg: 3701 /* 3702 * Allocate a new register matching the constraint 3703 * and move the temporary register into it. 3704 */ 3705 temp_load(s, ts, tcg_target_available_regs[ts->type], 3706 i_allocated_regs, 0); 3707 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs, 3708 o_preferred_regs, ts->indirect_base); 3709 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 3710 /* 3711 * Cross register class move not supported. Sync the 3712 * temp back to its slot and load from there. 3713 */ 3714 temp_sync(s, ts, i_allocated_regs, 0, 0); 3715 tcg_out_ld(s, ts->type, reg, 3716 ts->mem_base->reg, ts->mem_offset); 3717 } 3718 } 3719 new_args[i] = reg; 3720 const_args[i] = 0; 3721 tcg_regset_set_reg(i_allocated_regs, reg); 3722 } 3723 3724 /* mark dead temporaries and free the associated registers */ 3725 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3726 if (IS_DEAD_ARG(i)) { 3727 temp_dead(s, arg_temp(op->args[i])); 3728 } 3729 } 3730 3731 if (def->flags & TCG_OPF_COND_BRANCH) { 3732 tcg_reg_alloc_cbranch(s, i_allocated_regs); 3733 } else if (def->flags & TCG_OPF_BB_END) { 3734 tcg_reg_alloc_bb_end(s, i_allocated_regs); 3735 } else { 3736 if (def->flags & TCG_OPF_CALL_CLOBBER) { 3737 /* XXX: permit generic clobber register list ? */ 3738 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3739 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3740 tcg_reg_free(s, i, i_allocated_regs); 3741 } 3742 } 3743 } 3744 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3745 /* sync globals if the op has side effects and might trigger 3746 an exception. */ 3747 sync_globals(s, i_allocated_regs); 3748 } 3749 3750 /* satisfy the output constraints */ 3751 for(k = 0; k < nb_oargs; k++) { 3752 i = def->args_ct[k].sort_index; 3753 arg = op->args[i]; 3754 arg_ct = &def->args_ct[i]; 3755 ts = arg_temp(arg); 3756 3757 /* ENV should not be modified. */ 3758 tcg_debug_assert(!temp_readonly(ts)); 3759 3760 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 3761 reg = new_args[arg_ct->alias_index]; 3762 } else if (arg_ct->newreg) { 3763 reg = tcg_reg_alloc(s, arg_ct->regs, 3764 i_allocated_regs | o_allocated_regs, 3765 op->output_pref[k], ts->indirect_base); 3766 } else { 3767 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 3768 op->output_pref[k], ts->indirect_base); 3769 } 3770 tcg_regset_set_reg(o_allocated_regs, reg); 3771 if (ts->val_type == TEMP_VAL_REG) { 3772 s->reg_to_temp[ts->reg] = NULL; 3773 } 3774 ts->val_type = TEMP_VAL_REG; 3775 ts->reg = reg; 3776 /* 3777 * Temp value is modified, so the value kept in memory is 3778 * potentially not the same. 3779 */ 3780 ts->mem_coherent = 0; 3781 s->reg_to_temp[reg] = ts; 3782 new_args[i] = reg; 3783 } 3784 } 3785 3786 /* emit instruction */ 3787 if (def->flags & TCG_OPF_VECTOR) { 3788 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 3789 new_args, const_args); 3790 } else { 3791 tcg_out_op(s, op->opc, new_args, const_args); 3792 } 3793 3794 /* move the outputs in the correct register if needed */ 3795 for(i = 0; i < nb_oargs; i++) { 3796 ts = arg_temp(op->args[i]); 3797 3798 /* ENV should not be modified. */ 3799 tcg_debug_assert(!temp_readonly(ts)); 3800 3801 if (NEED_SYNC_ARG(i)) { 3802 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 3803 } else if (IS_DEAD_ARG(i)) { 3804 temp_dead(s, ts); 3805 } 3806 } 3807 } 3808 3809 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 3810 { 3811 const TCGLifeData arg_life = op->life; 3812 TCGTemp *ots, *itsl, *itsh; 3813 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3814 3815 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 3816 tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 3817 tcg_debug_assert(TCGOP_VECE(op) == MO_64); 3818 3819 ots = arg_temp(op->args[0]); 3820 itsl = arg_temp(op->args[1]); 3821 itsh = arg_temp(op->args[2]); 3822 3823 /* ENV should not be modified. */ 3824 tcg_debug_assert(!temp_readonly(ots)); 3825 3826 /* Allocate the output register now. */ 3827 if (ots->val_type != TEMP_VAL_REG) { 3828 TCGRegSet allocated_regs = s->reserved_regs; 3829 TCGRegSet dup_out_regs = 3830 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 3831 3832 /* Make sure to not spill the input registers. */ 3833 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 3834 tcg_regset_set_reg(allocated_regs, itsl->reg); 3835 } 3836 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 3837 tcg_regset_set_reg(allocated_regs, itsh->reg); 3838 } 3839 3840 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3841 op->output_pref[0], ots->indirect_base); 3842 ots->val_type = TEMP_VAL_REG; 3843 ots->mem_coherent = 0; 3844 s->reg_to_temp[ots->reg] = ots; 3845 } 3846 3847 /* Promote dup2 of immediates to dupi_vec. */ 3848 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 3849 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 3850 MemOp vece = MO_64; 3851 3852 if (val == dup_const(MO_8, val)) { 3853 vece = MO_8; 3854 } else if (val == dup_const(MO_16, val)) { 3855 vece = MO_16; 3856 } else if (val == dup_const(MO_32, val)) { 3857 vece = MO_32; 3858 } 3859 3860 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 3861 goto done; 3862 } 3863 3864 /* If the two inputs form one 64-bit value, try dupm_vec. */ 3865 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) { 3866 if (!itsl->mem_coherent) { 3867 temp_sync(s, itsl, s->reserved_regs, 0, 0); 3868 } 3869 if (!itsh->mem_coherent) { 3870 temp_sync(s, itsh, s->reserved_regs, 0, 0); 3871 } 3872 #ifdef HOST_WORDS_BIGENDIAN 3873 TCGTemp *its = itsh; 3874 #else 3875 TCGTemp *its = itsl; 3876 #endif 3877 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 3878 its->mem_base->reg, its->mem_offset)) { 3879 goto done; 3880 } 3881 } 3882 3883 /* Fall back to generic expansion. */ 3884 return false; 3885 3886 done: 3887 if (IS_DEAD_ARG(1)) { 3888 temp_dead(s, itsl); 3889 } 3890 if (IS_DEAD_ARG(2)) { 3891 temp_dead(s, itsh); 3892 } 3893 if (NEED_SYNC_ARG(0)) { 3894 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 3895 } else if (IS_DEAD_ARG(0)) { 3896 temp_dead(s, ots); 3897 } 3898 return true; 3899 } 3900 3901 #ifdef TCG_TARGET_STACK_GROWSUP 3902 #define STACK_DIR(x) (-(x)) 3903 #else 3904 #define STACK_DIR(x) (x) 3905 #endif 3906 3907 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 3908 { 3909 const int nb_oargs = TCGOP_CALLO(op); 3910 const int nb_iargs = TCGOP_CALLI(op); 3911 const TCGLifeData arg_life = op->life; 3912 const TCGHelperInfo *info; 3913 int flags, nb_regs, i; 3914 TCGReg reg; 3915 TCGArg arg; 3916 TCGTemp *ts; 3917 intptr_t stack_offset; 3918 size_t call_stack_size; 3919 tcg_insn_unit *func_addr; 3920 int allocate_args; 3921 TCGRegSet allocated_regs; 3922 3923 func_addr = tcg_call_func(op); 3924 info = tcg_call_info(op); 3925 flags = info->flags; 3926 3927 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 3928 if (nb_regs > nb_iargs) { 3929 nb_regs = nb_iargs; 3930 } 3931 3932 /* assign stack slots first */ 3933 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 3934 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 3935 ~(TCG_TARGET_STACK_ALIGN - 1); 3936 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 3937 if (allocate_args) { 3938 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 3939 preallocate call stack */ 3940 tcg_abort(); 3941 } 3942 3943 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 3944 for (i = nb_regs; i < nb_iargs; i++) { 3945 arg = op->args[nb_oargs + i]; 3946 #ifdef TCG_TARGET_STACK_GROWSUP 3947 stack_offset -= sizeof(tcg_target_long); 3948 #endif 3949 if (arg != TCG_CALL_DUMMY_ARG) { 3950 ts = arg_temp(arg); 3951 temp_load(s, ts, tcg_target_available_regs[ts->type], 3952 s->reserved_regs, 0); 3953 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 3954 } 3955 #ifndef TCG_TARGET_STACK_GROWSUP 3956 stack_offset += sizeof(tcg_target_long); 3957 #endif 3958 } 3959 3960 /* assign input registers */ 3961 allocated_regs = s->reserved_regs; 3962 for (i = 0; i < nb_regs; i++) { 3963 arg = op->args[nb_oargs + i]; 3964 if (arg != TCG_CALL_DUMMY_ARG) { 3965 ts = arg_temp(arg); 3966 reg = tcg_target_call_iarg_regs[i]; 3967 3968 if (ts->val_type == TEMP_VAL_REG) { 3969 if (ts->reg != reg) { 3970 tcg_reg_free(s, reg, allocated_regs); 3971 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 3972 /* 3973 * Cross register class move not supported. Sync the 3974 * temp back to its slot and load from there. 3975 */ 3976 temp_sync(s, ts, allocated_regs, 0, 0); 3977 tcg_out_ld(s, ts->type, reg, 3978 ts->mem_base->reg, ts->mem_offset); 3979 } 3980 } 3981 } else { 3982 TCGRegSet arg_set = 0; 3983 3984 tcg_reg_free(s, reg, allocated_regs); 3985 tcg_regset_set_reg(arg_set, reg); 3986 temp_load(s, ts, arg_set, allocated_regs, 0); 3987 } 3988 3989 tcg_regset_set_reg(allocated_regs, reg); 3990 } 3991 } 3992 3993 /* mark dead temporaries and free the associated registers */ 3994 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3995 if (IS_DEAD_ARG(i)) { 3996 temp_dead(s, arg_temp(op->args[i])); 3997 } 3998 } 3999 4000 /* clobber call registers */ 4001 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4002 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4003 tcg_reg_free(s, i, allocated_regs); 4004 } 4005 } 4006 4007 /* Save globals if they might be written by the helper, sync them if 4008 they might be read. */ 4009 if (flags & TCG_CALL_NO_READ_GLOBALS) { 4010 /* Nothing to do */ 4011 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 4012 sync_globals(s, allocated_regs); 4013 } else { 4014 save_globals(s, allocated_regs); 4015 } 4016 4017 #ifdef CONFIG_TCG_INTERPRETER 4018 { 4019 gpointer hash = (gpointer)(uintptr_t)info->typemask; 4020 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash); 4021 assert(cif != NULL); 4022 tcg_out_call(s, func_addr, cif); 4023 } 4024 #else 4025 tcg_out_call(s, func_addr); 4026 #endif 4027 4028 /* assign output registers and emit moves if needed */ 4029 for(i = 0; i < nb_oargs; i++) { 4030 arg = op->args[i]; 4031 ts = arg_temp(arg); 4032 4033 /* ENV should not be modified. */ 4034 tcg_debug_assert(!temp_readonly(ts)); 4035 4036 reg = tcg_target_call_oarg_regs[i]; 4037 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 4038 if (ts->val_type == TEMP_VAL_REG) { 4039 s->reg_to_temp[ts->reg] = NULL; 4040 } 4041 ts->val_type = TEMP_VAL_REG; 4042 ts->reg = reg; 4043 ts->mem_coherent = 0; 4044 s->reg_to_temp[reg] = ts; 4045 if (NEED_SYNC_ARG(i)) { 4046 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i)); 4047 } else if (IS_DEAD_ARG(i)) { 4048 temp_dead(s, ts); 4049 } 4050 } 4051 } 4052 4053 #ifdef CONFIG_PROFILER 4054 4055 /* avoid copy/paste errors */ 4056 #define PROF_ADD(to, from, field) \ 4057 do { \ 4058 (to)->field += qatomic_read(&((from)->field)); \ 4059 } while (0) 4060 4061 #define PROF_MAX(to, from, field) \ 4062 do { \ 4063 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4064 if (val__ > (to)->field) { \ 4065 (to)->field = val__; \ 4066 } \ 4067 } while (0) 4068 4069 /* Pass in a zero'ed @prof */ 4070 static inline 4071 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4072 { 4073 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4074 unsigned int i; 4075 4076 for (i = 0; i < n_ctxs; i++) { 4077 TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4078 const TCGProfile *orig = &s->prof; 4079 4080 if (counters) { 4081 PROF_ADD(prof, orig, cpu_exec_time); 4082 PROF_ADD(prof, orig, tb_count1); 4083 PROF_ADD(prof, orig, tb_count); 4084 PROF_ADD(prof, orig, op_count); 4085 PROF_MAX(prof, orig, op_count_max); 4086 PROF_ADD(prof, orig, temp_count); 4087 PROF_MAX(prof, orig, temp_count_max); 4088 PROF_ADD(prof, orig, del_op_count); 4089 PROF_ADD(prof, orig, code_in_len); 4090 PROF_ADD(prof, orig, code_out_len); 4091 PROF_ADD(prof, orig, search_out_len); 4092 PROF_ADD(prof, orig, interm_time); 4093 PROF_ADD(prof, orig, code_time); 4094 PROF_ADD(prof, orig, la_time); 4095 PROF_ADD(prof, orig, opt_time); 4096 PROF_ADD(prof, orig, restore_count); 4097 PROF_ADD(prof, orig, restore_time); 4098 } 4099 if (table) { 4100 int i; 4101 4102 for (i = 0; i < NB_OPS; i++) { 4103 PROF_ADD(prof, orig, table_op_count[i]); 4104 } 4105 } 4106 } 4107 } 4108 4109 #undef PROF_ADD 4110 #undef PROF_MAX 4111 4112 static void tcg_profile_snapshot_counters(TCGProfile *prof) 4113 { 4114 tcg_profile_snapshot(prof, true, false); 4115 } 4116 4117 static void tcg_profile_snapshot_table(TCGProfile *prof) 4118 { 4119 tcg_profile_snapshot(prof, false, true); 4120 } 4121 4122 void tcg_dump_op_count(void) 4123 { 4124 TCGProfile prof = {}; 4125 int i; 4126 4127 tcg_profile_snapshot_table(&prof); 4128 for (i = 0; i < NB_OPS; i++) { 4129 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name, 4130 prof.table_op_count[i]); 4131 } 4132 } 4133 4134 int64_t tcg_cpu_exec_time(void) 4135 { 4136 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4137 unsigned int i; 4138 int64_t ret = 0; 4139 4140 for (i = 0; i < n_ctxs; i++) { 4141 const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4142 const TCGProfile *prof = &s->prof; 4143 4144 ret += qatomic_read(&prof->cpu_exec_time); 4145 } 4146 return ret; 4147 } 4148 #else 4149 void tcg_dump_op_count(void) 4150 { 4151 qemu_printf("[TCG profiler not compiled]\n"); 4152 } 4153 4154 int64_t tcg_cpu_exec_time(void) 4155 { 4156 error_report("%s: TCG profiler not compiled", __func__); 4157 exit(EXIT_FAILURE); 4158 } 4159 #endif 4160 4161 4162 int tcg_gen_code(TCGContext *s, TranslationBlock *tb) 4163 { 4164 #ifdef CONFIG_PROFILER 4165 TCGProfile *prof = &s->prof; 4166 #endif 4167 int i, num_insns; 4168 TCGOp *op; 4169 4170 #ifdef CONFIG_PROFILER 4171 { 4172 int n = 0; 4173 4174 QTAILQ_FOREACH(op, &s->ops, link) { 4175 n++; 4176 } 4177 qatomic_set(&prof->op_count, prof->op_count + n); 4178 if (n > prof->op_count_max) { 4179 qatomic_set(&prof->op_count_max, n); 4180 } 4181 4182 n = s->nb_temps; 4183 qatomic_set(&prof->temp_count, prof->temp_count + n); 4184 if (n > prof->temp_count_max) { 4185 qatomic_set(&prof->temp_count_max, n); 4186 } 4187 } 4188 #endif 4189 4190 #ifdef DEBUG_DISAS 4191 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4192 && qemu_log_in_addr_range(tb->pc))) { 4193 FILE *logfile = qemu_log_lock(); 4194 qemu_log("OP:\n"); 4195 tcg_dump_ops(s, false); 4196 qemu_log("\n"); 4197 qemu_log_unlock(logfile); 4198 } 4199 #endif 4200 4201 #ifdef CONFIG_DEBUG_TCG 4202 /* Ensure all labels referenced have been emitted. */ 4203 { 4204 TCGLabel *l; 4205 bool error = false; 4206 4207 QSIMPLEQ_FOREACH(l, &s->labels, next) { 4208 if (unlikely(!l->present) && l->refs) { 4209 qemu_log_mask(CPU_LOG_TB_OP, 4210 "$L%d referenced but not present.\n", l->id); 4211 error = true; 4212 } 4213 } 4214 assert(!error); 4215 } 4216 #endif 4217 4218 #ifdef CONFIG_PROFILER 4219 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 4220 #endif 4221 4222 #ifdef USE_TCG_OPTIMIZATIONS 4223 tcg_optimize(s); 4224 #endif 4225 4226 #ifdef CONFIG_PROFILER 4227 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 4228 qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 4229 #endif 4230 4231 reachable_code_pass(s); 4232 liveness_pass_1(s); 4233 4234 if (s->nb_indirects > 0) { 4235 #ifdef DEBUG_DISAS 4236 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 4237 && qemu_log_in_addr_range(tb->pc))) { 4238 FILE *logfile = qemu_log_lock(); 4239 qemu_log("OP before indirect lowering:\n"); 4240 tcg_dump_ops(s, false); 4241 qemu_log("\n"); 4242 qemu_log_unlock(logfile); 4243 } 4244 #endif 4245 /* Replace indirect temps with direct temps. */ 4246 if (liveness_pass_2(s)) { 4247 /* If changes were made, re-run liveness. */ 4248 liveness_pass_1(s); 4249 } 4250 } 4251 4252 #ifdef CONFIG_PROFILER 4253 qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 4254 #endif 4255 4256 #ifdef DEBUG_DISAS 4257 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 4258 && qemu_log_in_addr_range(tb->pc))) { 4259 FILE *logfile = qemu_log_lock(); 4260 qemu_log("OP after optimization and liveness analysis:\n"); 4261 tcg_dump_ops(s, true); 4262 qemu_log("\n"); 4263 qemu_log_unlock(logfile); 4264 } 4265 #endif 4266 4267 tcg_reg_alloc_start(s); 4268 4269 /* 4270 * Reset the buffer pointers when restarting after overflow. 4271 * TODO: Move this into translate-all.c with the rest of the 4272 * buffer management. Having only this done here is confusing. 4273 */ 4274 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 4275 s->code_ptr = s->code_buf; 4276 4277 #ifdef TCG_TARGET_NEED_LDST_LABELS 4278 QSIMPLEQ_INIT(&s->ldst_labels); 4279 #endif 4280 #ifdef TCG_TARGET_NEED_POOL_LABELS 4281 s->pool_labels = NULL; 4282 #endif 4283 4284 num_insns = -1; 4285 QTAILQ_FOREACH(op, &s->ops, link) { 4286 TCGOpcode opc = op->opc; 4287 4288 #ifdef CONFIG_PROFILER 4289 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 4290 #endif 4291 4292 switch (opc) { 4293 case INDEX_op_mov_i32: 4294 case INDEX_op_mov_i64: 4295 case INDEX_op_mov_vec: 4296 tcg_reg_alloc_mov(s, op); 4297 break; 4298 case INDEX_op_dup_vec: 4299 tcg_reg_alloc_dup(s, op); 4300 break; 4301 case INDEX_op_insn_start: 4302 if (num_insns >= 0) { 4303 size_t off = tcg_current_code_size(s); 4304 s->gen_insn_end_off[num_insns] = off; 4305 /* Assert that we do not overflow our stored offset. */ 4306 assert(s->gen_insn_end_off[num_insns] == off); 4307 } 4308 num_insns++; 4309 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 4310 target_ulong a; 4311 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 4312 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 4313 #else 4314 a = op->args[i]; 4315 #endif 4316 s->gen_insn_data[num_insns][i] = a; 4317 } 4318 break; 4319 case INDEX_op_discard: 4320 temp_dead(s, arg_temp(op->args[0])); 4321 break; 4322 case INDEX_op_set_label: 4323 tcg_reg_alloc_bb_end(s, s->reserved_regs); 4324 tcg_out_label(s, arg_label(op->args[0])); 4325 break; 4326 case INDEX_op_call: 4327 tcg_reg_alloc_call(s, op); 4328 break; 4329 case INDEX_op_dup2_vec: 4330 if (tcg_reg_alloc_dup2(s, op)) { 4331 break; 4332 } 4333 /* fall through */ 4334 default: 4335 /* Sanity check that we've not introduced any unhandled opcodes. */ 4336 tcg_debug_assert(tcg_op_supported(opc)); 4337 /* Note: in order to speed up the code, it would be much 4338 faster to have specialized register allocator functions for 4339 some common argument patterns */ 4340 tcg_reg_alloc_op(s, op); 4341 break; 4342 } 4343 #ifdef CONFIG_DEBUG_TCG 4344 check_regs(s); 4345 #endif 4346 /* Test for (pending) buffer overflow. The assumption is that any 4347 one operation beginning below the high water mark cannot overrun 4348 the buffer completely. Thus we can test for overflow after 4349 generating code without having to check during generation. */ 4350 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 4351 return -1; 4352 } 4353 /* Test for TB overflow, as seen by gen_insn_end_off. */ 4354 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 4355 return -2; 4356 } 4357 } 4358 tcg_debug_assert(num_insns >= 0); 4359 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 4360 4361 /* Generate TB finalization at the end of block */ 4362 #ifdef TCG_TARGET_NEED_LDST_LABELS 4363 i = tcg_out_ldst_finalize(s); 4364 if (i < 0) { 4365 return i; 4366 } 4367 #endif 4368 #ifdef TCG_TARGET_NEED_POOL_LABELS 4369 i = tcg_out_pool_finalize(s); 4370 if (i < 0) { 4371 return i; 4372 } 4373 #endif 4374 if (!tcg_resolve_relocs(s)) { 4375 return -2; 4376 } 4377 4378 #ifndef CONFIG_TCG_INTERPRETER 4379 /* flush instruction cache */ 4380 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 4381 (uintptr_t)s->code_buf, 4382 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 4383 #endif 4384 4385 return tcg_current_code_size(s); 4386 } 4387 4388 #ifdef CONFIG_PROFILER 4389 void tcg_dump_info(void) 4390 { 4391 TCGProfile prof = {}; 4392 const TCGProfile *s; 4393 int64_t tb_count; 4394 int64_t tb_div_count; 4395 int64_t tot; 4396 4397 tcg_profile_snapshot_counters(&prof); 4398 s = &prof; 4399 tb_count = s->tb_count; 4400 tb_div_count = tb_count ? tb_count : 1; 4401 tot = s->interm_time + s->code_time; 4402 4403 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 4404 tot, tot / 2.4e9); 4405 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64 4406 " %0.1f%%)\n", 4407 tb_count, s->tb_count1 - tb_count, 4408 (double)(s->tb_count1 - s->tb_count) 4409 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 4410 qemu_printf("avg ops/TB %0.1f max=%d\n", 4411 (double)s->op_count / tb_div_count, s->op_count_max); 4412 qemu_printf("deleted ops/TB %0.2f\n", 4413 (double)s->del_op_count / tb_div_count); 4414 qemu_printf("avg temps/TB %0.2f max=%d\n", 4415 (double)s->temp_count / tb_div_count, s->temp_count_max); 4416 qemu_printf("avg host code/TB %0.1f\n", 4417 (double)s->code_out_len / tb_div_count); 4418 qemu_printf("avg search data/TB %0.1f\n", 4419 (double)s->search_out_len / tb_div_count); 4420 4421 qemu_printf("cycles/op %0.1f\n", 4422 s->op_count ? (double)tot / s->op_count : 0); 4423 qemu_printf("cycles/in byte %0.1f\n", 4424 s->code_in_len ? (double)tot / s->code_in_len : 0); 4425 qemu_printf("cycles/out byte %0.1f\n", 4426 s->code_out_len ? (double)tot / s->code_out_len : 0); 4427 qemu_printf("cycles/search byte %0.1f\n", 4428 s->search_out_len ? (double)tot / s->search_out_len : 0); 4429 if (tot == 0) { 4430 tot = 1; 4431 } 4432 qemu_printf(" gen_interm time %0.1f%%\n", 4433 (double)s->interm_time / tot * 100.0); 4434 qemu_printf(" gen_code time %0.1f%%\n", 4435 (double)s->code_time / tot * 100.0); 4436 qemu_printf("optim./code time %0.1f%%\n", 4437 (double)s->opt_time / (s->code_time ? s->code_time : 1) 4438 * 100.0); 4439 qemu_printf("liveness/code time %0.1f%%\n", 4440 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 4441 qemu_printf("cpu_restore count %" PRId64 "\n", 4442 s->restore_count); 4443 qemu_printf(" avg cycles %0.1f\n", 4444 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 4445 } 4446 #else 4447 void tcg_dump_info(void) 4448 { 4449 qemu_printf("[TCG profiler not compiled]\n"); 4450 } 4451 #endif 4452 4453 #ifdef ELF_HOST_MACHINE 4454 /* In order to use this feature, the backend needs to do three things: 4455 4456 (1) Define ELF_HOST_MACHINE to indicate both what value to 4457 put into the ELF image and to indicate support for the feature. 4458 4459 (2) Define tcg_register_jit. This should create a buffer containing 4460 the contents of a .debug_frame section that describes the post- 4461 prologue unwind info for the tcg machine. 4462 4463 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 4464 */ 4465 4466 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 4467 typedef enum { 4468 JIT_NOACTION = 0, 4469 JIT_REGISTER_FN, 4470 JIT_UNREGISTER_FN 4471 } jit_actions_t; 4472 4473 struct jit_code_entry { 4474 struct jit_code_entry *next_entry; 4475 struct jit_code_entry *prev_entry; 4476 const void *symfile_addr; 4477 uint64_t symfile_size; 4478 }; 4479 4480 struct jit_descriptor { 4481 uint32_t version; 4482 uint32_t action_flag; 4483 struct jit_code_entry *relevant_entry; 4484 struct jit_code_entry *first_entry; 4485 }; 4486 4487 void __jit_debug_register_code(void) __attribute__((noinline)); 4488 void __jit_debug_register_code(void) 4489 { 4490 asm(""); 4491 } 4492 4493 /* Must statically initialize the version, because GDB may check 4494 the version before we can set it. */ 4495 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 4496 4497 /* End GDB interface. */ 4498 4499 static int find_string(const char *strtab, const char *str) 4500 { 4501 const char *p = strtab + 1; 4502 4503 while (1) { 4504 if (strcmp(p, str) == 0) { 4505 return p - strtab; 4506 } 4507 p += strlen(p) + 1; 4508 } 4509 } 4510 4511 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 4512 const void *debug_frame, 4513 size_t debug_frame_size) 4514 { 4515 struct __attribute__((packed)) DebugInfo { 4516 uint32_t len; 4517 uint16_t version; 4518 uint32_t abbrev; 4519 uint8_t ptr_size; 4520 uint8_t cu_die; 4521 uint16_t cu_lang; 4522 uintptr_t cu_low_pc; 4523 uintptr_t cu_high_pc; 4524 uint8_t fn_die; 4525 char fn_name[16]; 4526 uintptr_t fn_low_pc; 4527 uintptr_t fn_high_pc; 4528 uint8_t cu_eoc; 4529 }; 4530 4531 struct ElfImage { 4532 ElfW(Ehdr) ehdr; 4533 ElfW(Phdr) phdr; 4534 ElfW(Shdr) shdr[7]; 4535 ElfW(Sym) sym[2]; 4536 struct DebugInfo di; 4537 uint8_t da[24]; 4538 char str[80]; 4539 }; 4540 4541 struct ElfImage *img; 4542 4543 static const struct ElfImage img_template = { 4544 .ehdr = { 4545 .e_ident[EI_MAG0] = ELFMAG0, 4546 .e_ident[EI_MAG1] = ELFMAG1, 4547 .e_ident[EI_MAG2] = ELFMAG2, 4548 .e_ident[EI_MAG3] = ELFMAG3, 4549 .e_ident[EI_CLASS] = ELF_CLASS, 4550 .e_ident[EI_DATA] = ELF_DATA, 4551 .e_ident[EI_VERSION] = EV_CURRENT, 4552 .e_type = ET_EXEC, 4553 .e_machine = ELF_HOST_MACHINE, 4554 .e_version = EV_CURRENT, 4555 .e_phoff = offsetof(struct ElfImage, phdr), 4556 .e_shoff = offsetof(struct ElfImage, shdr), 4557 .e_ehsize = sizeof(ElfW(Shdr)), 4558 .e_phentsize = sizeof(ElfW(Phdr)), 4559 .e_phnum = 1, 4560 .e_shentsize = sizeof(ElfW(Shdr)), 4561 .e_shnum = ARRAY_SIZE(img->shdr), 4562 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 4563 #ifdef ELF_HOST_FLAGS 4564 .e_flags = ELF_HOST_FLAGS, 4565 #endif 4566 #ifdef ELF_OSABI 4567 .e_ident[EI_OSABI] = ELF_OSABI, 4568 #endif 4569 }, 4570 .phdr = { 4571 .p_type = PT_LOAD, 4572 .p_flags = PF_X, 4573 }, 4574 .shdr = { 4575 [0] = { .sh_type = SHT_NULL }, 4576 /* Trick: The contents of code_gen_buffer are not present in 4577 this fake ELF file; that got allocated elsewhere. Therefore 4578 we mark .text as SHT_NOBITS (similar to .bss) so that readers 4579 will not look for contents. We can record any address. */ 4580 [1] = { /* .text */ 4581 .sh_type = SHT_NOBITS, 4582 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 4583 }, 4584 [2] = { /* .debug_info */ 4585 .sh_type = SHT_PROGBITS, 4586 .sh_offset = offsetof(struct ElfImage, di), 4587 .sh_size = sizeof(struct DebugInfo), 4588 }, 4589 [3] = { /* .debug_abbrev */ 4590 .sh_type = SHT_PROGBITS, 4591 .sh_offset = offsetof(struct ElfImage, da), 4592 .sh_size = sizeof(img->da), 4593 }, 4594 [4] = { /* .debug_frame */ 4595 .sh_type = SHT_PROGBITS, 4596 .sh_offset = sizeof(struct ElfImage), 4597 }, 4598 [5] = { /* .symtab */ 4599 .sh_type = SHT_SYMTAB, 4600 .sh_offset = offsetof(struct ElfImage, sym), 4601 .sh_size = sizeof(img->sym), 4602 .sh_info = 1, 4603 .sh_link = ARRAY_SIZE(img->shdr) - 1, 4604 .sh_entsize = sizeof(ElfW(Sym)), 4605 }, 4606 [6] = { /* .strtab */ 4607 .sh_type = SHT_STRTAB, 4608 .sh_offset = offsetof(struct ElfImage, str), 4609 .sh_size = sizeof(img->str), 4610 } 4611 }, 4612 .sym = { 4613 [1] = { /* code_gen_buffer */ 4614 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 4615 .st_shndx = 1, 4616 } 4617 }, 4618 .di = { 4619 .len = sizeof(struct DebugInfo) - 4, 4620 .version = 2, 4621 .ptr_size = sizeof(void *), 4622 .cu_die = 1, 4623 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 4624 .fn_die = 2, 4625 .fn_name = "code_gen_buffer" 4626 }, 4627 .da = { 4628 1, /* abbrev number (the cu) */ 4629 0x11, 1, /* DW_TAG_compile_unit, has children */ 4630 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 4631 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 4632 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 4633 0, 0, /* end of abbrev */ 4634 2, /* abbrev number (the fn) */ 4635 0x2e, 0, /* DW_TAG_subprogram, no children */ 4636 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 4637 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 4638 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 4639 0, 0, /* end of abbrev */ 4640 0 /* no more abbrev */ 4641 }, 4642 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 4643 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 4644 }; 4645 4646 /* We only need a single jit entry; statically allocate it. */ 4647 static struct jit_code_entry one_entry; 4648 4649 uintptr_t buf = (uintptr_t)buf_ptr; 4650 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 4651 DebugFrameHeader *dfh; 4652 4653 img = g_malloc(img_size); 4654 *img = img_template; 4655 4656 img->phdr.p_vaddr = buf; 4657 img->phdr.p_paddr = buf; 4658 img->phdr.p_memsz = buf_size; 4659 4660 img->shdr[1].sh_name = find_string(img->str, ".text"); 4661 img->shdr[1].sh_addr = buf; 4662 img->shdr[1].sh_size = buf_size; 4663 4664 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 4665 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 4666 4667 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 4668 img->shdr[4].sh_size = debug_frame_size; 4669 4670 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 4671 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 4672 4673 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 4674 img->sym[1].st_value = buf; 4675 img->sym[1].st_size = buf_size; 4676 4677 img->di.cu_low_pc = buf; 4678 img->di.cu_high_pc = buf + buf_size; 4679 img->di.fn_low_pc = buf; 4680 img->di.fn_high_pc = buf + buf_size; 4681 4682 dfh = (DebugFrameHeader *)(img + 1); 4683 memcpy(dfh, debug_frame, debug_frame_size); 4684 dfh->fde.func_start = buf; 4685 dfh->fde.func_len = buf_size; 4686 4687 #ifdef DEBUG_JIT 4688 /* Enable this block to be able to debug the ELF image file creation. 4689 One can use readelf, objdump, or other inspection utilities. */ 4690 { 4691 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 4692 if (f) { 4693 if (fwrite(img, img_size, 1, f) != img_size) { 4694 /* Avoid stupid unused return value warning for fwrite. */ 4695 } 4696 fclose(f); 4697 } 4698 } 4699 #endif 4700 4701 one_entry.symfile_addr = img; 4702 one_entry.symfile_size = img_size; 4703 4704 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 4705 __jit_debug_descriptor.relevant_entry = &one_entry; 4706 __jit_debug_descriptor.first_entry = &one_entry; 4707 __jit_debug_register_code(); 4708 } 4709 #else 4710 /* No support for the feature. Provide the entry point expected by exec.c, 4711 and implement the internal function we declared earlier. */ 4712 4713 static void tcg_register_jit_int(const void *buf, size_t size, 4714 const void *debug_frame, 4715 size_t debug_frame_size) 4716 { 4717 } 4718 4719 void tcg_register_jit(const void *buf, size_t buf_size) 4720 { 4721 } 4722 #endif /* ELF_HOST_MACHINE */ 4723 4724 #if !TCG_TARGET_MAYBE_vec 4725 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 4726 { 4727 g_assert_not_reached(); 4728 } 4729 #endif 4730