1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/error-report.h" 34 #include "qemu/cutils.h" 35 #include "qemu/host-utils.h" 36 #include "qemu/qemu-print.h" 37 #include "qemu/timer.h" 38 #include "qemu/cacheflush.h" 39 #include "qemu/cacheinfo.h" 40 41 /* Note: the long term plan is to reduce the dependencies on the QEMU 42 CPU definitions. Currently they are used for qemu_ld/st 43 instructions */ 44 #define NO_CPU_IO_DEFS 45 46 #include "exec/exec-all.h" 47 #include "tcg/tcg-op.h" 48 49 #if UINTPTR_MAX == UINT32_MAX 50 # define ELF_CLASS ELFCLASS32 51 #else 52 # define ELF_CLASS ELFCLASS64 53 #endif 54 #if HOST_BIG_ENDIAN 55 # define ELF_DATA ELFDATA2MSB 56 #else 57 # define ELF_DATA ELFDATA2LSB 58 #endif 59 60 #include "elf.h" 61 #include "exec/log.h" 62 #include "tcg/tcg-ldst.h" 63 #include "tcg-internal.h" 64 65 #ifdef CONFIG_TCG_INTERPRETER 66 #include <ffi.h> 67 #endif 68 69 /* Forward declarations for functions declared in tcg-target.c.inc and 70 used here. */ 71 static void tcg_target_init(TCGContext *s); 72 static void tcg_target_qemu_prologue(TCGContext *s); 73 static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 74 intptr_t value, intptr_t addend); 75 76 /* The CIE and FDE header definitions will be common to all hosts. */ 77 typedef struct { 78 uint32_t len __attribute__((aligned((sizeof(void *))))); 79 uint32_t id; 80 uint8_t version; 81 char augmentation[1]; 82 uint8_t code_align; 83 uint8_t data_align; 84 uint8_t return_column; 85 } DebugFrameCIE; 86 87 typedef struct QEMU_PACKED { 88 uint32_t len __attribute__((aligned((sizeof(void *))))); 89 uint32_t cie_offset; 90 uintptr_t func_start; 91 uintptr_t func_len; 92 } DebugFrameFDEHeader; 93 94 typedef struct QEMU_PACKED { 95 DebugFrameCIE cie; 96 DebugFrameFDEHeader fde; 97 } DebugFrameHeader; 98 99 static void tcg_register_jit_int(const void *buf, size_t size, 100 const void *debug_frame, 101 size_t debug_frame_size) 102 __attribute__((unused)); 103 104 /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 105 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 106 intptr_t arg2); 107 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 108 static void tcg_out_movi(TCGContext *s, TCGType type, 109 TCGReg ret, tcg_target_long arg); 110 static void tcg_out_op(TCGContext *s, TCGOpcode opc, 111 const TCGArg args[TCG_MAX_OP_ARGS], 112 const int const_args[TCG_MAX_OP_ARGS]); 113 #if TCG_TARGET_MAYBE_vec 114 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 115 TCGReg dst, TCGReg src); 116 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 117 TCGReg dst, TCGReg base, intptr_t offset); 118 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 119 TCGReg dst, int64_t arg); 120 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 121 unsigned vecl, unsigned vece, 122 const TCGArg args[TCG_MAX_OP_ARGS], 123 const int const_args[TCG_MAX_OP_ARGS]); 124 #else 125 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 126 TCGReg dst, TCGReg src) 127 { 128 g_assert_not_reached(); 129 } 130 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 131 TCGReg dst, TCGReg base, intptr_t offset) 132 { 133 g_assert_not_reached(); 134 } 135 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 136 TCGReg dst, int64_t arg) 137 { 138 g_assert_not_reached(); 139 } 140 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 141 unsigned vecl, unsigned vece, 142 const TCGArg args[TCG_MAX_OP_ARGS], 143 const int const_args[TCG_MAX_OP_ARGS]) 144 { 145 g_assert_not_reached(); 146 } 147 #endif 148 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 149 intptr_t arg2); 150 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 151 TCGReg base, intptr_t ofs); 152 #ifdef CONFIG_TCG_INTERPRETER 153 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 154 ffi_cif *cif); 155 #else 156 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target); 157 #endif 158 static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 159 #ifdef TCG_TARGET_NEED_LDST_LABELS 160 static int tcg_out_ldst_finalize(TCGContext *s); 161 #endif 162 163 TCGContext tcg_init_ctx; 164 __thread TCGContext *tcg_ctx; 165 166 TCGContext **tcg_ctxs; 167 unsigned int tcg_cur_ctxs; 168 unsigned int tcg_max_ctxs; 169 TCGv_env cpu_env = 0; 170 const void *tcg_code_gen_epilogue; 171 uintptr_t tcg_splitwx_diff; 172 173 #ifndef CONFIG_TCG_INTERPRETER 174 tcg_prologue_fn *tcg_qemu_tb_exec; 175 #endif 176 177 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 178 static TCGRegSet tcg_target_call_clobber_regs; 179 180 #if TCG_TARGET_INSN_UNIT_SIZE == 1 181 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 182 { 183 *s->code_ptr++ = v; 184 } 185 186 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 187 uint8_t v) 188 { 189 *p = v; 190 } 191 #endif 192 193 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 194 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 195 { 196 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 197 *s->code_ptr++ = v; 198 } else { 199 tcg_insn_unit *p = s->code_ptr; 200 memcpy(p, &v, sizeof(v)); 201 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 202 } 203 } 204 205 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 206 uint16_t v) 207 { 208 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 209 *p = v; 210 } else { 211 memcpy(p, &v, sizeof(v)); 212 } 213 } 214 #endif 215 216 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 217 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 218 { 219 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 220 *s->code_ptr++ = v; 221 } else { 222 tcg_insn_unit *p = s->code_ptr; 223 memcpy(p, &v, sizeof(v)); 224 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 225 } 226 } 227 228 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 229 uint32_t v) 230 { 231 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 232 *p = v; 233 } else { 234 memcpy(p, &v, sizeof(v)); 235 } 236 } 237 #endif 238 239 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 240 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 241 { 242 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 243 *s->code_ptr++ = v; 244 } else { 245 tcg_insn_unit *p = s->code_ptr; 246 memcpy(p, &v, sizeof(v)); 247 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 248 } 249 } 250 251 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 252 uint64_t v) 253 { 254 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 255 *p = v; 256 } else { 257 memcpy(p, &v, sizeof(v)); 258 } 259 } 260 #endif 261 262 /* label relocation processing */ 263 264 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 265 TCGLabel *l, intptr_t addend) 266 { 267 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 268 269 r->type = type; 270 r->ptr = code_ptr; 271 r->addend = addend; 272 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 273 } 274 275 static void tcg_out_label(TCGContext *s, TCGLabel *l) 276 { 277 tcg_debug_assert(!l->has_value); 278 l->has_value = 1; 279 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 280 } 281 282 TCGLabel *gen_new_label(void) 283 { 284 TCGContext *s = tcg_ctx; 285 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 286 287 memset(l, 0, sizeof(TCGLabel)); 288 l->id = s->nb_labels++; 289 QSIMPLEQ_INIT(&l->relocs); 290 291 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 292 293 return l; 294 } 295 296 static bool tcg_resolve_relocs(TCGContext *s) 297 { 298 TCGLabel *l; 299 300 QSIMPLEQ_FOREACH(l, &s->labels, next) { 301 TCGRelocation *r; 302 uintptr_t value = l->u.value; 303 304 QSIMPLEQ_FOREACH(r, &l->relocs, next) { 305 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 306 return false; 307 } 308 } 309 } 310 return true; 311 } 312 313 static void set_jmp_reset_offset(TCGContext *s, int which) 314 { 315 /* 316 * We will check for overflow at the end of the opcode loop in 317 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 318 */ 319 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); 320 } 321 322 /* Signal overflow, starting over with fewer guest insns. */ 323 static G_NORETURN 324 void tcg_raise_tb_overflow(TCGContext *s) 325 { 326 siglongjmp(s->jmp_trans, -2); 327 } 328 329 #define C_PFX1(P, A) P##A 330 #define C_PFX2(P, A, B) P##A##_##B 331 #define C_PFX3(P, A, B, C) P##A##_##B##_##C 332 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 333 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 334 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 335 336 /* Define an enumeration for the various combinations. */ 337 338 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 339 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 340 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 341 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 342 343 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 344 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 345 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 346 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 347 348 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 349 350 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 351 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 352 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 353 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 354 355 typedef enum { 356 #include "tcg-target-con-set.h" 357 } TCGConstraintSetIndex; 358 359 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 360 361 #undef C_O0_I1 362 #undef C_O0_I2 363 #undef C_O0_I3 364 #undef C_O0_I4 365 #undef C_O1_I1 366 #undef C_O1_I2 367 #undef C_O1_I3 368 #undef C_O1_I4 369 #undef C_N1_I2 370 #undef C_O2_I1 371 #undef C_O2_I2 372 #undef C_O2_I3 373 #undef C_O2_I4 374 375 /* Put all of the constraint sets into an array, indexed by the enum. */ 376 377 #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 378 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 379 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 380 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 381 382 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 383 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 384 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 385 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 386 387 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 388 389 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 390 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 391 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 392 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 393 394 static const TCGTargetOpDef constraint_sets[] = { 395 #include "tcg-target-con-set.h" 396 }; 397 398 399 #undef C_O0_I1 400 #undef C_O0_I2 401 #undef C_O0_I3 402 #undef C_O0_I4 403 #undef C_O1_I1 404 #undef C_O1_I2 405 #undef C_O1_I3 406 #undef C_O1_I4 407 #undef C_N1_I2 408 #undef C_O2_I1 409 #undef C_O2_I2 410 #undef C_O2_I3 411 #undef C_O2_I4 412 413 /* Expand the enumerator to be returned from tcg_target_op_def(). */ 414 415 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 416 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 417 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 418 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 419 420 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 421 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 422 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 423 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 424 425 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 426 427 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 428 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 429 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 430 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 431 432 #include "tcg-target.c.inc" 433 434 static void alloc_tcg_plugin_context(TCGContext *s) 435 { 436 #ifdef CONFIG_PLUGIN 437 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 438 s->plugin_tb->insns = 439 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 440 #endif 441 } 442 443 /* 444 * All TCG threads except the parent (i.e. the one that called tcg_context_init 445 * and registered the target's TCG globals) must register with this function 446 * before initiating translation. 447 * 448 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 449 * of tcg_region_init() for the reasoning behind this. 450 * 451 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 452 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 453 * is not used anymore for translation once this function is called. 454 * 455 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 456 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 457 */ 458 #ifdef CONFIG_USER_ONLY 459 void tcg_register_thread(void) 460 { 461 tcg_ctx = &tcg_init_ctx; 462 } 463 #else 464 void tcg_register_thread(void) 465 { 466 TCGContext *s = g_malloc(sizeof(*s)); 467 unsigned int i, n; 468 469 *s = tcg_init_ctx; 470 471 /* Relink mem_base. */ 472 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 473 if (tcg_init_ctx.temps[i].mem_base) { 474 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 475 tcg_debug_assert(b >= 0 && b < n); 476 s->temps[i].mem_base = &s->temps[b]; 477 } 478 } 479 480 /* Claim an entry in tcg_ctxs */ 481 n = qatomic_fetch_inc(&tcg_cur_ctxs); 482 g_assert(n < tcg_max_ctxs); 483 qatomic_set(&tcg_ctxs[n], s); 484 485 if (n > 0) { 486 alloc_tcg_plugin_context(s); 487 tcg_region_initial_alloc(s); 488 } 489 490 tcg_ctx = s; 491 } 492 #endif /* !CONFIG_USER_ONLY */ 493 494 /* pool based memory allocation */ 495 void *tcg_malloc_internal(TCGContext *s, int size) 496 { 497 TCGPool *p; 498 int pool_size; 499 500 if (size > TCG_POOL_CHUNK_SIZE) { 501 /* big malloc: insert a new pool (XXX: could optimize) */ 502 p = g_malloc(sizeof(TCGPool) + size); 503 p->size = size; 504 p->next = s->pool_first_large; 505 s->pool_first_large = p; 506 return p->data; 507 } else { 508 p = s->pool_current; 509 if (!p) { 510 p = s->pool_first; 511 if (!p) 512 goto new_pool; 513 } else { 514 if (!p->next) { 515 new_pool: 516 pool_size = TCG_POOL_CHUNK_SIZE; 517 p = g_malloc(sizeof(TCGPool) + pool_size); 518 p->size = pool_size; 519 p->next = NULL; 520 if (s->pool_current) { 521 s->pool_current->next = p; 522 } else { 523 s->pool_first = p; 524 } 525 } else { 526 p = p->next; 527 } 528 } 529 } 530 s->pool_current = p; 531 s->pool_cur = p->data + size; 532 s->pool_end = p->data + p->size; 533 return p->data; 534 } 535 536 void tcg_pool_reset(TCGContext *s) 537 { 538 TCGPool *p, *t; 539 for (p = s->pool_first_large; p; p = t) { 540 t = p->next; 541 g_free(p); 542 } 543 s->pool_first_large = NULL; 544 s->pool_cur = s->pool_end = NULL; 545 s->pool_current = NULL; 546 } 547 548 #include "exec/helper-proto.h" 549 550 static const TCGHelperInfo all_helpers[] = { 551 #include "exec/helper-tcg.h" 552 }; 553 static GHashTable *helper_table; 554 555 #ifdef CONFIG_TCG_INTERPRETER 556 static GHashTable *ffi_table; 557 558 static ffi_type * const typecode_to_ffi[8] = { 559 [dh_typecode_void] = &ffi_type_void, 560 [dh_typecode_i32] = &ffi_type_uint32, 561 [dh_typecode_s32] = &ffi_type_sint32, 562 [dh_typecode_i64] = &ffi_type_uint64, 563 [dh_typecode_s64] = &ffi_type_sint64, 564 [dh_typecode_ptr] = &ffi_type_pointer, 565 }; 566 #endif 567 568 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 569 static void process_op_defs(TCGContext *s); 570 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 571 TCGReg reg, const char *name); 572 573 static void tcg_context_init(unsigned max_cpus) 574 { 575 TCGContext *s = &tcg_init_ctx; 576 int op, total_args, n, i; 577 TCGOpDef *def; 578 TCGArgConstraint *args_ct; 579 TCGTemp *ts; 580 581 memset(s, 0, sizeof(*s)); 582 s->nb_globals = 0; 583 584 /* Count total number of arguments and allocate the corresponding 585 space */ 586 total_args = 0; 587 for(op = 0; op < NB_OPS; op++) { 588 def = &tcg_op_defs[op]; 589 n = def->nb_iargs + def->nb_oargs; 590 total_args += n; 591 } 592 593 args_ct = g_new0(TCGArgConstraint, total_args); 594 595 for(op = 0; op < NB_OPS; op++) { 596 def = &tcg_op_defs[op]; 597 def->args_ct = args_ct; 598 n = def->nb_iargs + def->nb_oargs; 599 args_ct += n; 600 } 601 602 /* Register helpers. */ 603 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 604 helper_table = g_hash_table_new(NULL, NULL); 605 606 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 607 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 608 (gpointer)&all_helpers[i]); 609 } 610 611 #ifdef CONFIG_TCG_INTERPRETER 612 /* g_direct_hash/equal for direct comparisons on uint32_t. */ 613 ffi_table = g_hash_table_new(NULL, NULL); 614 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 615 struct { 616 ffi_cif cif; 617 ffi_type *args[]; 618 } *ca; 619 uint32_t typemask = all_helpers[i].typemask; 620 gpointer hash = (gpointer)(uintptr_t)typemask; 621 ffi_status status; 622 int nargs; 623 624 if (g_hash_table_lookup(ffi_table, hash)) { 625 continue; 626 } 627 628 /* Ignoring the return type, find the last non-zero field. */ 629 nargs = 32 - clz32(typemask >> 3); 630 nargs = DIV_ROUND_UP(nargs, 3); 631 632 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 633 ca->cif.rtype = typecode_to_ffi[typemask & 7]; 634 ca->cif.nargs = nargs; 635 636 if (nargs != 0) { 637 ca->cif.arg_types = ca->args; 638 for (int j = 0; j < nargs; ++j) { 639 int typecode = extract32(typemask, (j + 1) * 3, 3); 640 ca->args[j] = typecode_to_ffi[typecode]; 641 } 642 } 643 644 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 645 ca->cif.rtype, ca->cif.arg_types); 646 assert(status == FFI_OK); 647 648 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); 649 } 650 #endif 651 652 tcg_target_init(s); 653 process_op_defs(s); 654 655 /* Reverse the order of the saved registers, assuming they're all at 656 the start of tcg_target_reg_alloc_order. */ 657 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 658 int r = tcg_target_reg_alloc_order[n]; 659 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 660 break; 661 } 662 } 663 for (i = 0; i < n; ++i) { 664 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 665 } 666 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 667 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 668 } 669 670 alloc_tcg_plugin_context(s); 671 672 tcg_ctx = s; 673 /* 674 * In user-mode we simply share the init context among threads, since we 675 * use a single region. See the documentation tcg_region_init() for the 676 * reasoning behind this. 677 * In softmmu we will have at most max_cpus TCG threads. 678 */ 679 #ifdef CONFIG_USER_ONLY 680 tcg_ctxs = &tcg_ctx; 681 tcg_cur_ctxs = 1; 682 tcg_max_ctxs = 1; 683 #else 684 tcg_max_ctxs = max_cpus; 685 tcg_ctxs = g_new0(TCGContext *, max_cpus); 686 #endif 687 688 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 689 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 690 cpu_env = temp_tcgv_ptr(ts); 691 } 692 693 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 694 { 695 tcg_context_init(max_cpus); 696 tcg_region_init(tb_size, splitwx, max_cpus); 697 } 698 699 /* 700 * Allocate TBs right before their corresponding translated code, making 701 * sure that TBs and code are on different cache lines. 702 */ 703 TranslationBlock *tcg_tb_alloc(TCGContext *s) 704 { 705 uintptr_t align = qemu_icache_linesize; 706 TranslationBlock *tb; 707 void *next; 708 709 retry: 710 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 711 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 712 713 if (unlikely(next > s->code_gen_highwater)) { 714 if (tcg_region_alloc(s)) { 715 return NULL; 716 } 717 goto retry; 718 } 719 qatomic_set(&s->code_gen_ptr, next); 720 s->data_gen_ptr = NULL; 721 return tb; 722 } 723 724 void tcg_prologue_init(TCGContext *s) 725 { 726 size_t prologue_size; 727 728 s->code_ptr = s->code_gen_ptr; 729 s->code_buf = s->code_gen_ptr; 730 s->data_gen_ptr = NULL; 731 732 #ifndef CONFIG_TCG_INTERPRETER 733 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 734 #endif 735 736 #ifdef TCG_TARGET_NEED_POOL_LABELS 737 s->pool_labels = NULL; 738 #endif 739 740 qemu_thread_jit_write(); 741 /* Generate the prologue. */ 742 tcg_target_qemu_prologue(s); 743 744 #ifdef TCG_TARGET_NEED_POOL_LABELS 745 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 746 { 747 int result = tcg_out_pool_finalize(s); 748 tcg_debug_assert(result == 0); 749 } 750 #endif 751 752 prologue_size = tcg_current_code_size(s); 753 754 #ifndef CONFIG_TCG_INTERPRETER 755 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 756 (uintptr_t)s->code_buf, prologue_size); 757 #endif 758 759 #ifdef DEBUG_DISAS 760 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 761 FILE *logfile = qemu_log_trylock(); 762 if (logfile) { 763 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 764 if (s->data_gen_ptr) { 765 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 766 size_t data_size = prologue_size - code_size; 767 size_t i; 768 769 disas(logfile, s->code_gen_ptr, code_size); 770 771 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 772 if (sizeof(tcg_target_ulong) == 8) { 773 fprintf(logfile, 774 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 775 (uintptr_t)s->data_gen_ptr + i, 776 *(uint64_t *)(s->data_gen_ptr + i)); 777 } else { 778 fprintf(logfile, 779 "0x%08" PRIxPTR ": .long 0x%08x\n", 780 (uintptr_t)s->data_gen_ptr + i, 781 *(uint32_t *)(s->data_gen_ptr + i)); 782 } 783 } 784 } else { 785 disas(logfile, s->code_gen_ptr, prologue_size); 786 } 787 fprintf(logfile, "\n"); 788 qemu_log_unlock(logfile); 789 } 790 } 791 #endif 792 793 #ifndef CONFIG_TCG_INTERPRETER 794 /* 795 * Assert that goto_ptr is implemented completely, setting an epilogue. 796 * For tci, we use NULL as the signal to return from the interpreter, 797 * so skip this check. 798 */ 799 tcg_debug_assert(tcg_code_gen_epilogue != NULL); 800 #endif 801 802 tcg_region_prologue_set(s); 803 } 804 805 void tcg_func_start(TCGContext *s) 806 { 807 tcg_pool_reset(s); 808 s->nb_temps = s->nb_globals; 809 810 /* No temps have been previously allocated for size or locality. */ 811 memset(s->free_temps, 0, sizeof(s->free_temps)); 812 813 /* No constant temps have been previously allocated. */ 814 for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 815 if (s->const_table[i]) { 816 g_hash_table_remove_all(s->const_table[i]); 817 } 818 } 819 820 s->nb_ops = 0; 821 s->nb_labels = 0; 822 s->current_frame_offset = s->frame_start; 823 824 #ifdef CONFIG_DEBUG_TCG 825 s->goto_tb_issue_mask = 0; 826 #endif 827 828 QTAILQ_INIT(&s->ops); 829 QTAILQ_INIT(&s->free_ops); 830 QSIMPLEQ_INIT(&s->labels); 831 } 832 833 static TCGTemp *tcg_temp_alloc(TCGContext *s) 834 { 835 int n = s->nb_temps++; 836 837 if (n >= TCG_MAX_TEMPS) { 838 tcg_raise_tb_overflow(s); 839 } 840 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 841 } 842 843 static TCGTemp *tcg_global_alloc(TCGContext *s) 844 { 845 TCGTemp *ts; 846 847 tcg_debug_assert(s->nb_globals == s->nb_temps); 848 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 849 s->nb_globals++; 850 ts = tcg_temp_alloc(s); 851 ts->kind = TEMP_GLOBAL; 852 853 return ts; 854 } 855 856 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 857 TCGReg reg, const char *name) 858 { 859 TCGTemp *ts; 860 861 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 862 tcg_abort(); 863 } 864 865 ts = tcg_global_alloc(s); 866 ts->base_type = type; 867 ts->type = type; 868 ts->kind = TEMP_FIXED; 869 ts->reg = reg; 870 ts->name = name; 871 tcg_regset_set_reg(s->reserved_regs, reg); 872 873 return ts; 874 } 875 876 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 877 { 878 s->frame_start = start; 879 s->frame_end = start + size; 880 s->frame_temp 881 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 882 } 883 884 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 885 intptr_t offset, const char *name) 886 { 887 TCGContext *s = tcg_ctx; 888 TCGTemp *base_ts = tcgv_ptr_temp(base); 889 TCGTemp *ts = tcg_global_alloc(s); 890 int indirect_reg = 0, bigendian = 0; 891 #if HOST_BIG_ENDIAN 892 bigendian = 1; 893 #endif 894 895 switch (base_ts->kind) { 896 case TEMP_FIXED: 897 break; 898 case TEMP_GLOBAL: 899 /* We do not support double-indirect registers. */ 900 tcg_debug_assert(!base_ts->indirect_reg); 901 base_ts->indirect_base = 1; 902 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 903 ? 2 : 1); 904 indirect_reg = 1; 905 break; 906 default: 907 g_assert_not_reached(); 908 } 909 910 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 911 TCGTemp *ts2 = tcg_global_alloc(s); 912 char buf[64]; 913 914 ts->base_type = TCG_TYPE_I64; 915 ts->type = TCG_TYPE_I32; 916 ts->indirect_reg = indirect_reg; 917 ts->mem_allocated = 1; 918 ts->mem_base = base_ts; 919 ts->mem_offset = offset + bigendian * 4; 920 pstrcpy(buf, sizeof(buf), name); 921 pstrcat(buf, sizeof(buf), "_0"); 922 ts->name = strdup(buf); 923 924 tcg_debug_assert(ts2 == ts + 1); 925 ts2->base_type = TCG_TYPE_I64; 926 ts2->type = TCG_TYPE_I32; 927 ts2->indirect_reg = indirect_reg; 928 ts2->mem_allocated = 1; 929 ts2->mem_base = base_ts; 930 ts2->mem_offset = offset + (1 - bigendian) * 4; 931 pstrcpy(buf, sizeof(buf), name); 932 pstrcat(buf, sizeof(buf), "_1"); 933 ts2->name = strdup(buf); 934 } else { 935 ts->base_type = type; 936 ts->type = type; 937 ts->indirect_reg = indirect_reg; 938 ts->mem_allocated = 1; 939 ts->mem_base = base_ts; 940 ts->mem_offset = offset; 941 ts->name = name; 942 } 943 return ts; 944 } 945 946 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) 947 { 948 TCGContext *s = tcg_ctx; 949 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL; 950 TCGTemp *ts; 951 int idx, k; 952 953 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 954 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 955 if (idx < TCG_MAX_TEMPS) { 956 /* There is already an available temp with the right type. */ 957 clear_bit(idx, s->free_temps[k].l); 958 959 ts = &s->temps[idx]; 960 ts->temp_allocated = 1; 961 tcg_debug_assert(ts->base_type == type); 962 tcg_debug_assert(ts->kind == kind); 963 } else { 964 ts = tcg_temp_alloc(s); 965 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 966 TCGTemp *ts2 = tcg_temp_alloc(s); 967 968 ts->base_type = type; 969 ts->type = TCG_TYPE_I32; 970 ts->temp_allocated = 1; 971 ts->kind = kind; 972 973 tcg_debug_assert(ts2 == ts + 1); 974 ts2->base_type = TCG_TYPE_I64; 975 ts2->type = TCG_TYPE_I32; 976 ts2->temp_allocated = 1; 977 ts2->kind = kind; 978 } else { 979 ts->base_type = type; 980 ts->type = type; 981 ts->temp_allocated = 1; 982 ts->kind = kind; 983 } 984 } 985 986 #if defined(CONFIG_DEBUG_TCG) 987 s->temps_in_use++; 988 #endif 989 return ts; 990 } 991 992 TCGv_vec tcg_temp_new_vec(TCGType type) 993 { 994 TCGTemp *t; 995 996 #ifdef CONFIG_DEBUG_TCG 997 switch (type) { 998 case TCG_TYPE_V64: 999 assert(TCG_TARGET_HAS_v64); 1000 break; 1001 case TCG_TYPE_V128: 1002 assert(TCG_TARGET_HAS_v128); 1003 break; 1004 case TCG_TYPE_V256: 1005 assert(TCG_TARGET_HAS_v256); 1006 break; 1007 default: 1008 g_assert_not_reached(); 1009 } 1010 #endif 1011 1012 t = tcg_temp_new_internal(type, 0); 1013 return temp_tcgv_vec(t); 1014 } 1015 1016 /* Create a new temp of the same type as an existing temp. */ 1017 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1018 { 1019 TCGTemp *t = tcgv_vec_temp(match); 1020 1021 tcg_debug_assert(t->temp_allocated != 0); 1022 1023 t = tcg_temp_new_internal(t->base_type, 0); 1024 return temp_tcgv_vec(t); 1025 } 1026 1027 void tcg_temp_free_internal(TCGTemp *ts) 1028 { 1029 TCGContext *s = tcg_ctx; 1030 int k, idx; 1031 1032 switch (ts->kind) { 1033 case TEMP_CONST: 1034 /* 1035 * In order to simplify users of tcg_constant_*, 1036 * silently ignore free. 1037 */ 1038 return; 1039 case TEMP_NORMAL: 1040 case TEMP_LOCAL: 1041 break; 1042 default: 1043 g_assert_not_reached(); 1044 } 1045 1046 #if defined(CONFIG_DEBUG_TCG) 1047 s->temps_in_use--; 1048 if (s->temps_in_use < 0) { 1049 fprintf(stderr, "More temporaries freed than allocated!\n"); 1050 } 1051 #endif 1052 1053 tcg_debug_assert(ts->temp_allocated != 0); 1054 ts->temp_allocated = 0; 1055 1056 idx = temp_idx(ts); 1057 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT); 1058 set_bit(idx, s->free_temps[k].l); 1059 } 1060 1061 TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1062 { 1063 TCGContext *s = tcg_ctx; 1064 GHashTable *h = s->const_table[type]; 1065 TCGTemp *ts; 1066 1067 if (h == NULL) { 1068 h = g_hash_table_new(g_int64_hash, g_int64_equal); 1069 s->const_table[type] = h; 1070 } 1071 1072 ts = g_hash_table_lookup(h, &val); 1073 if (ts == NULL) { 1074 ts = tcg_temp_alloc(s); 1075 1076 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1077 TCGTemp *ts2 = tcg_temp_alloc(s); 1078 1079 ts->base_type = TCG_TYPE_I64; 1080 ts->type = TCG_TYPE_I32; 1081 ts->kind = TEMP_CONST; 1082 ts->temp_allocated = 1; 1083 /* 1084 * Retain the full value of the 64-bit constant in the low 1085 * part, so that the hash table works. Actual uses will 1086 * truncate the value to the low part. 1087 */ 1088 ts->val = val; 1089 1090 tcg_debug_assert(ts2 == ts + 1); 1091 ts2->base_type = TCG_TYPE_I64; 1092 ts2->type = TCG_TYPE_I32; 1093 ts2->kind = TEMP_CONST; 1094 ts2->temp_allocated = 1; 1095 ts2->val = val >> 32; 1096 } else { 1097 ts->base_type = type; 1098 ts->type = type; 1099 ts->kind = TEMP_CONST; 1100 ts->temp_allocated = 1; 1101 ts->val = val; 1102 } 1103 g_hash_table_insert(h, &ts->val, ts); 1104 } 1105 1106 return ts; 1107 } 1108 1109 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1110 { 1111 val = dup_const(vece, val); 1112 return temp_tcgv_vec(tcg_constant_internal(type, val)); 1113 } 1114 1115 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 1116 { 1117 TCGTemp *t = tcgv_vec_temp(match); 1118 1119 tcg_debug_assert(t->temp_allocated != 0); 1120 return tcg_constant_vec(t->base_type, vece, val); 1121 } 1122 1123 TCGv_i32 tcg_const_i32(int32_t val) 1124 { 1125 TCGv_i32 t0; 1126 t0 = tcg_temp_new_i32(); 1127 tcg_gen_movi_i32(t0, val); 1128 return t0; 1129 } 1130 1131 TCGv_i64 tcg_const_i64(int64_t val) 1132 { 1133 TCGv_i64 t0; 1134 t0 = tcg_temp_new_i64(); 1135 tcg_gen_movi_i64(t0, val); 1136 return t0; 1137 } 1138 1139 TCGv_i32 tcg_const_local_i32(int32_t val) 1140 { 1141 TCGv_i32 t0; 1142 t0 = tcg_temp_local_new_i32(); 1143 tcg_gen_movi_i32(t0, val); 1144 return t0; 1145 } 1146 1147 TCGv_i64 tcg_const_local_i64(int64_t val) 1148 { 1149 TCGv_i64 t0; 1150 t0 = tcg_temp_local_new_i64(); 1151 tcg_gen_movi_i64(t0, val); 1152 return t0; 1153 } 1154 1155 #if defined(CONFIG_DEBUG_TCG) 1156 void tcg_clear_temp_count(void) 1157 { 1158 TCGContext *s = tcg_ctx; 1159 s->temps_in_use = 0; 1160 } 1161 1162 int tcg_check_temp_count(void) 1163 { 1164 TCGContext *s = tcg_ctx; 1165 if (s->temps_in_use) { 1166 /* Clear the count so that we don't give another 1167 * warning immediately next time around. 1168 */ 1169 s->temps_in_use = 0; 1170 return 1; 1171 } 1172 return 0; 1173 } 1174 #endif 1175 1176 /* Return true if OP may appear in the opcode stream. 1177 Test the runtime variable that controls each opcode. */ 1178 bool tcg_op_supported(TCGOpcode op) 1179 { 1180 const bool have_vec 1181 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1182 1183 switch (op) { 1184 case INDEX_op_discard: 1185 case INDEX_op_set_label: 1186 case INDEX_op_call: 1187 case INDEX_op_br: 1188 case INDEX_op_mb: 1189 case INDEX_op_insn_start: 1190 case INDEX_op_exit_tb: 1191 case INDEX_op_goto_tb: 1192 case INDEX_op_goto_ptr: 1193 case INDEX_op_qemu_ld_i32: 1194 case INDEX_op_qemu_st_i32: 1195 case INDEX_op_qemu_ld_i64: 1196 case INDEX_op_qemu_st_i64: 1197 return true; 1198 1199 case INDEX_op_qemu_st8_i32: 1200 return TCG_TARGET_HAS_qemu_st8_i32; 1201 1202 case INDEX_op_mov_i32: 1203 case INDEX_op_setcond_i32: 1204 case INDEX_op_brcond_i32: 1205 case INDEX_op_ld8u_i32: 1206 case INDEX_op_ld8s_i32: 1207 case INDEX_op_ld16u_i32: 1208 case INDEX_op_ld16s_i32: 1209 case INDEX_op_ld_i32: 1210 case INDEX_op_st8_i32: 1211 case INDEX_op_st16_i32: 1212 case INDEX_op_st_i32: 1213 case INDEX_op_add_i32: 1214 case INDEX_op_sub_i32: 1215 case INDEX_op_mul_i32: 1216 case INDEX_op_and_i32: 1217 case INDEX_op_or_i32: 1218 case INDEX_op_xor_i32: 1219 case INDEX_op_shl_i32: 1220 case INDEX_op_shr_i32: 1221 case INDEX_op_sar_i32: 1222 return true; 1223 1224 case INDEX_op_movcond_i32: 1225 return TCG_TARGET_HAS_movcond_i32; 1226 case INDEX_op_div_i32: 1227 case INDEX_op_divu_i32: 1228 return TCG_TARGET_HAS_div_i32; 1229 case INDEX_op_rem_i32: 1230 case INDEX_op_remu_i32: 1231 return TCG_TARGET_HAS_rem_i32; 1232 case INDEX_op_div2_i32: 1233 case INDEX_op_divu2_i32: 1234 return TCG_TARGET_HAS_div2_i32; 1235 case INDEX_op_rotl_i32: 1236 case INDEX_op_rotr_i32: 1237 return TCG_TARGET_HAS_rot_i32; 1238 case INDEX_op_deposit_i32: 1239 return TCG_TARGET_HAS_deposit_i32; 1240 case INDEX_op_extract_i32: 1241 return TCG_TARGET_HAS_extract_i32; 1242 case INDEX_op_sextract_i32: 1243 return TCG_TARGET_HAS_sextract_i32; 1244 case INDEX_op_extract2_i32: 1245 return TCG_TARGET_HAS_extract2_i32; 1246 case INDEX_op_add2_i32: 1247 return TCG_TARGET_HAS_add2_i32; 1248 case INDEX_op_sub2_i32: 1249 return TCG_TARGET_HAS_sub2_i32; 1250 case INDEX_op_mulu2_i32: 1251 return TCG_TARGET_HAS_mulu2_i32; 1252 case INDEX_op_muls2_i32: 1253 return TCG_TARGET_HAS_muls2_i32; 1254 case INDEX_op_muluh_i32: 1255 return TCG_TARGET_HAS_muluh_i32; 1256 case INDEX_op_mulsh_i32: 1257 return TCG_TARGET_HAS_mulsh_i32; 1258 case INDEX_op_ext8s_i32: 1259 return TCG_TARGET_HAS_ext8s_i32; 1260 case INDEX_op_ext16s_i32: 1261 return TCG_TARGET_HAS_ext16s_i32; 1262 case INDEX_op_ext8u_i32: 1263 return TCG_TARGET_HAS_ext8u_i32; 1264 case INDEX_op_ext16u_i32: 1265 return TCG_TARGET_HAS_ext16u_i32; 1266 case INDEX_op_bswap16_i32: 1267 return TCG_TARGET_HAS_bswap16_i32; 1268 case INDEX_op_bswap32_i32: 1269 return TCG_TARGET_HAS_bswap32_i32; 1270 case INDEX_op_not_i32: 1271 return TCG_TARGET_HAS_not_i32; 1272 case INDEX_op_neg_i32: 1273 return TCG_TARGET_HAS_neg_i32; 1274 case INDEX_op_andc_i32: 1275 return TCG_TARGET_HAS_andc_i32; 1276 case INDEX_op_orc_i32: 1277 return TCG_TARGET_HAS_orc_i32; 1278 case INDEX_op_eqv_i32: 1279 return TCG_TARGET_HAS_eqv_i32; 1280 case INDEX_op_nand_i32: 1281 return TCG_TARGET_HAS_nand_i32; 1282 case INDEX_op_nor_i32: 1283 return TCG_TARGET_HAS_nor_i32; 1284 case INDEX_op_clz_i32: 1285 return TCG_TARGET_HAS_clz_i32; 1286 case INDEX_op_ctz_i32: 1287 return TCG_TARGET_HAS_ctz_i32; 1288 case INDEX_op_ctpop_i32: 1289 return TCG_TARGET_HAS_ctpop_i32; 1290 1291 case INDEX_op_brcond2_i32: 1292 case INDEX_op_setcond2_i32: 1293 return TCG_TARGET_REG_BITS == 32; 1294 1295 case INDEX_op_mov_i64: 1296 case INDEX_op_setcond_i64: 1297 case INDEX_op_brcond_i64: 1298 case INDEX_op_ld8u_i64: 1299 case INDEX_op_ld8s_i64: 1300 case INDEX_op_ld16u_i64: 1301 case INDEX_op_ld16s_i64: 1302 case INDEX_op_ld32u_i64: 1303 case INDEX_op_ld32s_i64: 1304 case INDEX_op_ld_i64: 1305 case INDEX_op_st8_i64: 1306 case INDEX_op_st16_i64: 1307 case INDEX_op_st32_i64: 1308 case INDEX_op_st_i64: 1309 case INDEX_op_add_i64: 1310 case INDEX_op_sub_i64: 1311 case INDEX_op_mul_i64: 1312 case INDEX_op_and_i64: 1313 case INDEX_op_or_i64: 1314 case INDEX_op_xor_i64: 1315 case INDEX_op_shl_i64: 1316 case INDEX_op_shr_i64: 1317 case INDEX_op_sar_i64: 1318 case INDEX_op_ext_i32_i64: 1319 case INDEX_op_extu_i32_i64: 1320 return TCG_TARGET_REG_BITS == 64; 1321 1322 case INDEX_op_movcond_i64: 1323 return TCG_TARGET_HAS_movcond_i64; 1324 case INDEX_op_div_i64: 1325 case INDEX_op_divu_i64: 1326 return TCG_TARGET_HAS_div_i64; 1327 case INDEX_op_rem_i64: 1328 case INDEX_op_remu_i64: 1329 return TCG_TARGET_HAS_rem_i64; 1330 case INDEX_op_div2_i64: 1331 case INDEX_op_divu2_i64: 1332 return TCG_TARGET_HAS_div2_i64; 1333 case INDEX_op_rotl_i64: 1334 case INDEX_op_rotr_i64: 1335 return TCG_TARGET_HAS_rot_i64; 1336 case INDEX_op_deposit_i64: 1337 return TCG_TARGET_HAS_deposit_i64; 1338 case INDEX_op_extract_i64: 1339 return TCG_TARGET_HAS_extract_i64; 1340 case INDEX_op_sextract_i64: 1341 return TCG_TARGET_HAS_sextract_i64; 1342 case INDEX_op_extract2_i64: 1343 return TCG_TARGET_HAS_extract2_i64; 1344 case INDEX_op_extrl_i64_i32: 1345 return TCG_TARGET_HAS_extrl_i64_i32; 1346 case INDEX_op_extrh_i64_i32: 1347 return TCG_TARGET_HAS_extrh_i64_i32; 1348 case INDEX_op_ext8s_i64: 1349 return TCG_TARGET_HAS_ext8s_i64; 1350 case INDEX_op_ext16s_i64: 1351 return TCG_TARGET_HAS_ext16s_i64; 1352 case INDEX_op_ext32s_i64: 1353 return TCG_TARGET_HAS_ext32s_i64; 1354 case INDEX_op_ext8u_i64: 1355 return TCG_TARGET_HAS_ext8u_i64; 1356 case INDEX_op_ext16u_i64: 1357 return TCG_TARGET_HAS_ext16u_i64; 1358 case INDEX_op_ext32u_i64: 1359 return TCG_TARGET_HAS_ext32u_i64; 1360 case INDEX_op_bswap16_i64: 1361 return TCG_TARGET_HAS_bswap16_i64; 1362 case INDEX_op_bswap32_i64: 1363 return TCG_TARGET_HAS_bswap32_i64; 1364 case INDEX_op_bswap64_i64: 1365 return TCG_TARGET_HAS_bswap64_i64; 1366 case INDEX_op_not_i64: 1367 return TCG_TARGET_HAS_not_i64; 1368 case INDEX_op_neg_i64: 1369 return TCG_TARGET_HAS_neg_i64; 1370 case INDEX_op_andc_i64: 1371 return TCG_TARGET_HAS_andc_i64; 1372 case INDEX_op_orc_i64: 1373 return TCG_TARGET_HAS_orc_i64; 1374 case INDEX_op_eqv_i64: 1375 return TCG_TARGET_HAS_eqv_i64; 1376 case INDEX_op_nand_i64: 1377 return TCG_TARGET_HAS_nand_i64; 1378 case INDEX_op_nor_i64: 1379 return TCG_TARGET_HAS_nor_i64; 1380 case INDEX_op_clz_i64: 1381 return TCG_TARGET_HAS_clz_i64; 1382 case INDEX_op_ctz_i64: 1383 return TCG_TARGET_HAS_ctz_i64; 1384 case INDEX_op_ctpop_i64: 1385 return TCG_TARGET_HAS_ctpop_i64; 1386 case INDEX_op_add2_i64: 1387 return TCG_TARGET_HAS_add2_i64; 1388 case INDEX_op_sub2_i64: 1389 return TCG_TARGET_HAS_sub2_i64; 1390 case INDEX_op_mulu2_i64: 1391 return TCG_TARGET_HAS_mulu2_i64; 1392 case INDEX_op_muls2_i64: 1393 return TCG_TARGET_HAS_muls2_i64; 1394 case INDEX_op_muluh_i64: 1395 return TCG_TARGET_HAS_muluh_i64; 1396 case INDEX_op_mulsh_i64: 1397 return TCG_TARGET_HAS_mulsh_i64; 1398 1399 case INDEX_op_mov_vec: 1400 case INDEX_op_dup_vec: 1401 case INDEX_op_dupm_vec: 1402 case INDEX_op_ld_vec: 1403 case INDEX_op_st_vec: 1404 case INDEX_op_add_vec: 1405 case INDEX_op_sub_vec: 1406 case INDEX_op_and_vec: 1407 case INDEX_op_or_vec: 1408 case INDEX_op_xor_vec: 1409 case INDEX_op_cmp_vec: 1410 return have_vec; 1411 case INDEX_op_dup2_vec: 1412 return have_vec && TCG_TARGET_REG_BITS == 32; 1413 case INDEX_op_not_vec: 1414 return have_vec && TCG_TARGET_HAS_not_vec; 1415 case INDEX_op_neg_vec: 1416 return have_vec && TCG_TARGET_HAS_neg_vec; 1417 case INDEX_op_abs_vec: 1418 return have_vec && TCG_TARGET_HAS_abs_vec; 1419 case INDEX_op_andc_vec: 1420 return have_vec && TCG_TARGET_HAS_andc_vec; 1421 case INDEX_op_orc_vec: 1422 return have_vec && TCG_TARGET_HAS_orc_vec; 1423 case INDEX_op_nand_vec: 1424 return have_vec && TCG_TARGET_HAS_nand_vec; 1425 case INDEX_op_nor_vec: 1426 return have_vec && TCG_TARGET_HAS_nor_vec; 1427 case INDEX_op_eqv_vec: 1428 return have_vec && TCG_TARGET_HAS_eqv_vec; 1429 case INDEX_op_mul_vec: 1430 return have_vec && TCG_TARGET_HAS_mul_vec; 1431 case INDEX_op_shli_vec: 1432 case INDEX_op_shri_vec: 1433 case INDEX_op_sari_vec: 1434 return have_vec && TCG_TARGET_HAS_shi_vec; 1435 case INDEX_op_shls_vec: 1436 case INDEX_op_shrs_vec: 1437 case INDEX_op_sars_vec: 1438 return have_vec && TCG_TARGET_HAS_shs_vec; 1439 case INDEX_op_shlv_vec: 1440 case INDEX_op_shrv_vec: 1441 case INDEX_op_sarv_vec: 1442 return have_vec && TCG_TARGET_HAS_shv_vec; 1443 case INDEX_op_rotli_vec: 1444 return have_vec && TCG_TARGET_HAS_roti_vec; 1445 case INDEX_op_rotls_vec: 1446 return have_vec && TCG_TARGET_HAS_rots_vec; 1447 case INDEX_op_rotlv_vec: 1448 case INDEX_op_rotrv_vec: 1449 return have_vec && TCG_TARGET_HAS_rotv_vec; 1450 case INDEX_op_ssadd_vec: 1451 case INDEX_op_usadd_vec: 1452 case INDEX_op_sssub_vec: 1453 case INDEX_op_ussub_vec: 1454 return have_vec && TCG_TARGET_HAS_sat_vec; 1455 case INDEX_op_smin_vec: 1456 case INDEX_op_umin_vec: 1457 case INDEX_op_smax_vec: 1458 case INDEX_op_umax_vec: 1459 return have_vec && TCG_TARGET_HAS_minmax_vec; 1460 case INDEX_op_bitsel_vec: 1461 return have_vec && TCG_TARGET_HAS_bitsel_vec; 1462 case INDEX_op_cmpsel_vec: 1463 return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1464 1465 default: 1466 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1467 return true; 1468 } 1469 } 1470 1471 /* Note: we convert the 64 bit args to 32 bit and do some alignment 1472 and endian swap. Maybe it would be better to do the alignment 1473 and endian swap in tcg_reg_alloc_call(). */ 1474 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1475 { 1476 int i, real_args, nb_rets, pi; 1477 unsigned typemask; 1478 const TCGHelperInfo *info; 1479 TCGOp *op; 1480 1481 info = g_hash_table_lookup(helper_table, (gpointer)func); 1482 typemask = info->typemask; 1483 1484 #ifdef CONFIG_PLUGIN 1485 /* detect non-plugin helpers */ 1486 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) { 1487 tcg_ctx->plugin_insn->calls_helpers = true; 1488 } 1489 #endif 1490 1491 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1492 for (i = 0; i < nargs; ++i) { 1493 int argtype = extract32(typemask, (i + 1) * 3, 3); 1494 bool is_32bit = (argtype & ~1) == dh_typecode_i32; 1495 bool is_signed = argtype & 1; 1496 1497 if (is_32bit) { 1498 TCGv_i64 temp = tcg_temp_new_i64(); 1499 TCGv_i32 orig = temp_tcgv_i32(args[i]); 1500 if (is_signed) { 1501 tcg_gen_ext_i32_i64(temp, orig); 1502 } else { 1503 tcg_gen_extu_i32_i64(temp, orig); 1504 } 1505 args[i] = tcgv_i64_temp(temp); 1506 } 1507 } 1508 #endif /* TCG_TARGET_EXTEND_ARGS */ 1509 1510 op = tcg_emit_op(INDEX_op_call); 1511 1512 pi = 0; 1513 if (ret != NULL) { 1514 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { 1515 #if HOST_BIG_ENDIAN 1516 op->args[pi++] = temp_arg(ret + 1); 1517 op->args[pi++] = temp_arg(ret); 1518 #else 1519 op->args[pi++] = temp_arg(ret); 1520 op->args[pi++] = temp_arg(ret + 1); 1521 #endif 1522 nb_rets = 2; 1523 } else { 1524 op->args[pi++] = temp_arg(ret); 1525 nb_rets = 1; 1526 } 1527 } else { 1528 nb_rets = 0; 1529 } 1530 TCGOP_CALLO(op) = nb_rets; 1531 1532 real_args = 0; 1533 for (i = 0; i < nargs; i++) { 1534 int argtype = extract32(typemask, (i + 1) * 3, 3); 1535 bool is_64bit = (argtype & ~1) == dh_typecode_i64; 1536 bool want_align = false; 1537 1538 #if defined(CONFIG_TCG_INTERPRETER) 1539 /* 1540 * Align all arguments, so that they land in predictable places 1541 * for passing off to ffi_call. 1542 */ 1543 want_align = true; 1544 #elif defined(TCG_TARGET_CALL_ALIGN_ARGS) 1545 /* Some targets want aligned 64 bit args */ 1546 want_align = is_64bit; 1547 #endif 1548 1549 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) { 1550 op->args[pi++] = TCG_CALL_DUMMY_ARG; 1551 real_args++; 1552 } 1553 1554 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 1555 op->args[pi++] = temp_arg(args[i] + HOST_BIG_ENDIAN); 1556 op->args[pi++] = temp_arg(args[i] + !HOST_BIG_ENDIAN); 1557 real_args += 2; 1558 continue; 1559 } 1560 1561 op->args[pi++] = temp_arg(args[i]); 1562 real_args++; 1563 } 1564 op->args[pi++] = (uintptr_t)func; 1565 op->args[pi++] = (uintptr_t)info; 1566 TCGOP_CALLI(op) = real_args; 1567 1568 /* Make sure the fields didn't overflow. */ 1569 tcg_debug_assert(TCGOP_CALLI(op) == real_args); 1570 tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); 1571 1572 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1573 for (i = 0; i < nargs; ++i) { 1574 int argtype = extract32(typemask, (i + 1) * 3, 3); 1575 bool is_32bit = (argtype & ~1) == dh_typecode_i32; 1576 1577 if (is_32bit) { 1578 tcg_temp_free_internal(args[i]); 1579 } 1580 } 1581 #endif /* TCG_TARGET_EXTEND_ARGS */ 1582 } 1583 1584 static void tcg_reg_alloc_start(TCGContext *s) 1585 { 1586 int i, n; 1587 1588 for (i = 0, n = s->nb_temps; i < n; i++) { 1589 TCGTemp *ts = &s->temps[i]; 1590 TCGTempVal val = TEMP_VAL_MEM; 1591 1592 switch (ts->kind) { 1593 case TEMP_CONST: 1594 val = TEMP_VAL_CONST; 1595 break; 1596 case TEMP_FIXED: 1597 val = TEMP_VAL_REG; 1598 break; 1599 case TEMP_GLOBAL: 1600 break; 1601 case TEMP_NORMAL: 1602 case TEMP_EBB: 1603 val = TEMP_VAL_DEAD; 1604 /* fall through */ 1605 case TEMP_LOCAL: 1606 ts->mem_allocated = 0; 1607 break; 1608 default: 1609 g_assert_not_reached(); 1610 } 1611 ts->val_type = val; 1612 } 1613 1614 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1615 } 1616 1617 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1618 TCGTemp *ts) 1619 { 1620 int idx = temp_idx(ts); 1621 1622 switch (ts->kind) { 1623 case TEMP_FIXED: 1624 case TEMP_GLOBAL: 1625 pstrcpy(buf, buf_size, ts->name); 1626 break; 1627 case TEMP_LOCAL: 1628 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1629 break; 1630 case TEMP_EBB: 1631 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals); 1632 break; 1633 case TEMP_NORMAL: 1634 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1635 break; 1636 case TEMP_CONST: 1637 switch (ts->type) { 1638 case TCG_TYPE_I32: 1639 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1640 break; 1641 #if TCG_TARGET_REG_BITS > 32 1642 case TCG_TYPE_I64: 1643 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1644 break; 1645 #endif 1646 case TCG_TYPE_V64: 1647 case TCG_TYPE_V128: 1648 case TCG_TYPE_V256: 1649 snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1650 64 << (ts->type - TCG_TYPE_V64), ts->val); 1651 break; 1652 default: 1653 g_assert_not_reached(); 1654 } 1655 break; 1656 } 1657 return buf; 1658 } 1659 1660 static char *tcg_get_arg_str(TCGContext *s, char *buf, 1661 int buf_size, TCGArg arg) 1662 { 1663 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1664 } 1665 1666 static const char * const cond_name[] = 1667 { 1668 [TCG_COND_NEVER] = "never", 1669 [TCG_COND_ALWAYS] = "always", 1670 [TCG_COND_EQ] = "eq", 1671 [TCG_COND_NE] = "ne", 1672 [TCG_COND_LT] = "lt", 1673 [TCG_COND_GE] = "ge", 1674 [TCG_COND_LE] = "le", 1675 [TCG_COND_GT] = "gt", 1676 [TCG_COND_LTU] = "ltu", 1677 [TCG_COND_GEU] = "geu", 1678 [TCG_COND_LEU] = "leu", 1679 [TCG_COND_GTU] = "gtu" 1680 }; 1681 1682 static const char * const ldst_name[] = 1683 { 1684 [MO_UB] = "ub", 1685 [MO_SB] = "sb", 1686 [MO_LEUW] = "leuw", 1687 [MO_LESW] = "lesw", 1688 [MO_LEUL] = "leul", 1689 [MO_LESL] = "lesl", 1690 [MO_LEUQ] = "leq", 1691 [MO_BEUW] = "beuw", 1692 [MO_BESW] = "besw", 1693 [MO_BEUL] = "beul", 1694 [MO_BESL] = "besl", 1695 [MO_BEUQ] = "beq", 1696 }; 1697 1698 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1699 #ifdef TARGET_ALIGNED_ONLY 1700 [MO_UNALN >> MO_ASHIFT] = "un+", 1701 [MO_ALIGN >> MO_ASHIFT] = "", 1702 #else 1703 [MO_UNALN >> MO_ASHIFT] = "", 1704 [MO_ALIGN >> MO_ASHIFT] = "al+", 1705 #endif 1706 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1707 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1708 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1709 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1710 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1711 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1712 }; 1713 1714 static const char bswap_flag_name[][6] = { 1715 [TCG_BSWAP_IZ] = "iz", 1716 [TCG_BSWAP_OZ] = "oz", 1717 [TCG_BSWAP_OS] = "os", 1718 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1719 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1720 }; 1721 1722 static inline bool tcg_regset_single(TCGRegSet d) 1723 { 1724 return (d & (d - 1)) == 0; 1725 } 1726 1727 static inline TCGReg tcg_regset_first(TCGRegSet d) 1728 { 1729 if (TCG_TARGET_NB_REGS <= 32) { 1730 return ctz32(d); 1731 } else { 1732 return ctz64(d); 1733 } 1734 } 1735 1736 /* Return only the number of characters output -- no error return. */ 1737 #define ne_fprintf(...) \ 1738 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 1739 1740 static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 1741 { 1742 char buf[128]; 1743 TCGOp *op; 1744 1745 QTAILQ_FOREACH(op, &s->ops, link) { 1746 int i, k, nb_oargs, nb_iargs, nb_cargs; 1747 const TCGOpDef *def; 1748 TCGOpcode c; 1749 int col = 0; 1750 1751 c = op->opc; 1752 def = &tcg_op_defs[c]; 1753 1754 if (c == INDEX_op_insn_start) { 1755 nb_oargs = 0; 1756 col += ne_fprintf(f, "\n ----"); 1757 1758 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1759 target_ulong a; 1760 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1761 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 1762 #else 1763 a = op->args[i]; 1764 #endif 1765 col += ne_fprintf(f, " " TARGET_FMT_lx, a); 1766 } 1767 } else if (c == INDEX_op_call) { 1768 const TCGHelperInfo *info = tcg_call_info(op); 1769 void *func = tcg_call_func(op); 1770 1771 /* variable number of arguments */ 1772 nb_oargs = TCGOP_CALLO(op); 1773 nb_iargs = TCGOP_CALLI(op); 1774 nb_cargs = def->nb_cargs; 1775 1776 col += ne_fprintf(f, " %s ", def->name); 1777 1778 /* 1779 * Print the function name from TCGHelperInfo, if available. 1780 * Note that plugins have a template function for the info, 1781 * but the actual function pointer comes from the plugin. 1782 */ 1783 if (func == info->func) { 1784 col += ne_fprintf(f, "%s", info->name); 1785 } else { 1786 col += ne_fprintf(f, "plugin(%p)", func); 1787 } 1788 1789 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 1790 for (i = 0; i < nb_oargs; i++) { 1791 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 1792 op->args[i])); 1793 } 1794 for (i = 0; i < nb_iargs; i++) { 1795 TCGArg arg = op->args[nb_oargs + i]; 1796 const char *t = "<dummy>"; 1797 if (arg != TCG_CALL_DUMMY_ARG) { 1798 t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 1799 } 1800 col += ne_fprintf(f, ",%s", t); 1801 } 1802 } else { 1803 col += ne_fprintf(f, " %s ", def->name); 1804 1805 nb_oargs = def->nb_oargs; 1806 nb_iargs = def->nb_iargs; 1807 nb_cargs = def->nb_cargs; 1808 1809 if (def->flags & TCG_OPF_VECTOR) { 1810 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op), 1811 8 << TCGOP_VECE(op)); 1812 } 1813 1814 k = 0; 1815 for (i = 0; i < nb_oargs; i++) { 1816 const char *sep = k ? "," : ""; 1817 col += ne_fprintf(f, "%s%s", sep, 1818 tcg_get_arg_str(s, buf, sizeof(buf), 1819 op->args[k++])); 1820 } 1821 for (i = 0; i < nb_iargs; i++) { 1822 const char *sep = k ? "," : ""; 1823 col += ne_fprintf(f, "%s%s", sep, 1824 tcg_get_arg_str(s, buf, sizeof(buf), 1825 op->args[k++])); 1826 } 1827 switch (c) { 1828 case INDEX_op_brcond_i32: 1829 case INDEX_op_setcond_i32: 1830 case INDEX_op_movcond_i32: 1831 case INDEX_op_brcond2_i32: 1832 case INDEX_op_setcond2_i32: 1833 case INDEX_op_brcond_i64: 1834 case INDEX_op_setcond_i64: 1835 case INDEX_op_movcond_i64: 1836 case INDEX_op_cmp_vec: 1837 case INDEX_op_cmpsel_vec: 1838 if (op->args[k] < ARRAY_SIZE(cond_name) 1839 && cond_name[op->args[k]]) { 1840 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 1841 } else { 1842 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 1843 } 1844 i = 1; 1845 break; 1846 case INDEX_op_qemu_ld_i32: 1847 case INDEX_op_qemu_st_i32: 1848 case INDEX_op_qemu_st8_i32: 1849 case INDEX_op_qemu_ld_i64: 1850 case INDEX_op_qemu_st_i64: 1851 { 1852 MemOpIdx oi = op->args[k++]; 1853 MemOp op = get_memop(oi); 1854 unsigned ix = get_mmuidx(oi); 1855 1856 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1857 col += ne_fprintf(f, ",$0x%x,%u", op, ix); 1858 } else { 1859 const char *s_al, *s_op; 1860 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1861 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1862 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix); 1863 } 1864 i = 1; 1865 } 1866 break; 1867 case INDEX_op_bswap16_i32: 1868 case INDEX_op_bswap16_i64: 1869 case INDEX_op_bswap32_i32: 1870 case INDEX_op_bswap32_i64: 1871 case INDEX_op_bswap64_i64: 1872 { 1873 TCGArg flags = op->args[k]; 1874 const char *name = NULL; 1875 1876 if (flags < ARRAY_SIZE(bswap_flag_name)) { 1877 name = bswap_flag_name[flags]; 1878 } 1879 if (name) { 1880 col += ne_fprintf(f, ",%s", name); 1881 } else { 1882 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 1883 } 1884 i = k = 1; 1885 } 1886 break; 1887 default: 1888 i = 0; 1889 break; 1890 } 1891 switch (c) { 1892 case INDEX_op_set_label: 1893 case INDEX_op_br: 1894 case INDEX_op_brcond_i32: 1895 case INDEX_op_brcond_i64: 1896 case INDEX_op_brcond2_i32: 1897 col += ne_fprintf(f, "%s$L%d", k ? "," : "", 1898 arg_label(op->args[k])->id); 1899 i++, k++; 1900 break; 1901 default: 1902 break; 1903 } 1904 for (; i < nb_cargs; i++, k++) { 1905 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 1906 op->args[k]); 1907 } 1908 } 1909 1910 if (have_prefs || op->life) { 1911 for (; col < 40; ++col) { 1912 putc(' ', f); 1913 } 1914 } 1915 1916 if (op->life) { 1917 unsigned life = op->life; 1918 1919 if (life & (SYNC_ARG * 3)) { 1920 ne_fprintf(f, " sync:"); 1921 for (i = 0; i < 2; ++i) { 1922 if (life & (SYNC_ARG << i)) { 1923 ne_fprintf(f, " %d", i); 1924 } 1925 } 1926 } 1927 life /= DEAD_ARG; 1928 if (life) { 1929 ne_fprintf(f, " dead:"); 1930 for (i = 0; life; ++i, life >>= 1) { 1931 if (life & 1) { 1932 ne_fprintf(f, " %d", i); 1933 } 1934 } 1935 } 1936 } 1937 1938 if (have_prefs) { 1939 for (i = 0; i < nb_oargs; ++i) { 1940 TCGRegSet set = op->output_pref[i]; 1941 1942 if (i == 0) { 1943 ne_fprintf(f, " pref="); 1944 } else { 1945 ne_fprintf(f, ","); 1946 } 1947 if (set == 0) { 1948 ne_fprintf(f, "none"); 1949 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 1950 ne_fprintf(f, "all"); 1951 #ifdef CONFIG_DEBUG_TCG 1952 } else if (tcg_regset_single(set)) { 1953 TCGReg reg = tcg_regset_first(set); 1954 ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 1955 #endif 1956 } else if (TCG_TARGET_NB_REGS <= 32) { 1957 ne_fprintf(f, "0x%x", (uint32_t)set); 1958 } else { 1959 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 1960 } 1961 } 1962 } 1963 1964 putc('\n', f); 1965 } 1966 } 1967 1968 /* we give more priority to constraints with less registers */ 1969 static int get_constraint_priority(const TCGOpDef *def, int k) 1970 { 1971 const TCGArgConstraint *arg_ct = &def->args_ct[k]; 1972 int n = ctpop64(arg_ct->regs); 1973 1974 /* 1975 * Sort constraints of a single register first, which includes output 1976 * aliases (which must exactly match the input already allocated). 1977 */ 1978 if (n == 1 || arg_ct->oalias) { 1979 return INT_MAX; 1980 } 1981 1982 /* 1983 * Sort register pairs next, first then second immediately after. 1984 * Arbitrarily sort multiple pairs by the index of the first reg; 1985 * there shouldn't be many pairs. 1986 */ 1987 switch (arg_ct->pair) { 1988 case 1: 1989 case 3: 1990 return (k + 1) * 2; 1991 case 2: 1992 return (arg_ct->pair_index + 1) * 2 - 1; 1993 } 1994 1995 /* Finally, sort by decreasing register count. */ 1996 assert(n > 1); 1997 return -n; 1998 } 1999 2000 /* sort from highest priority to lowest */ 2001 static void sort_constraints(TCGOpDef *def, int start, int n) 2002 { 2003 int i, j; 2004 TCGArgConstraint *a = def->args_ct; 2005 2006 for (i = 0; i < n; i++) { 2007 a[start + i].sort_index = start + i; 2008 } 2009 if (n <= 1) { 2010 return; 2011 } 2012 for (i = 0; i < n - 1; i++) { 2013 for (j = i + 1; j < n; j++) { 2014 int p1 = get_constraint_priority(def, a[start + i].sort_index); 2015 int p2 = get_constraint_priority(def, a[start + j].sort_index); 2016 if (p1 < p2) { 2017 int tmp = a[start + i].sort_index; 2018 a[start + i].sort_index = a[start + j].sort_index; 2019 a[start + j].sort_index = tmp; 2020 } 2021 } 2022 } 2023 } 2024 2025 static void process_op_defs(TCGContext *s) 2026 { 2027 TCGOpcode op; 2028 2029 for (op = 0; op < NB_OPS; op++) { 2030 TCGOpDef *def = &tcg_op_defs[op]; 2031 const TCGTargetOpDef *tdefs; 2032 bool saw_alias_pair = false; 2033 int i, o, i2, o2, nb_args; 2034 2035 if (def->flags & TCG_OPF_NOT_PRESENT) { 2036 continue; 2037 } 2038 2039 nb_args = def->nb_iargs + def->nb_oargs; 2040 if (nb_args == 0) { 2041 continue; 2042 } 2043 2044 /* 2045 * Macro magic should make it impossible, but double-check that 2046 * the array index is in range. Since the signness of an enum 2047 * is implementation defined, force the result to unsigned. 2048 */ 2049 unsigned con_set = tcg_target_op_def(op); 2050 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 2051 tdefs = &constraint_sets[con_set]; 2052 2053 for (i = 0; i < nb_args; i++) { 2054 const char *ct_str = tdefs->args_ct_str[i]; 2055 bool input_p = i >= def->nb_oargs; 2056 2057 /* Incomplete TCGTargetOpDef entry. */ 2058 tcg_debug_assert(ct_str != NULL); 2059 2060 switch (*ct_str) { 2061 case '0' ... '9': 2062 o = *ct_str - '0'; 2063 tcg_debug_assert(input_p); 2064 tcg_debug_assert(o < def->nb_oargs); 2065 tcg_debug_assert(def->args_ct[o].regs != 0); 2066 tcg_debug_assert(!def->args_ct[o].oalias); 2067 def->args_ct[i] = def->args_ct[o]; 2068 /* The output sets oalias. */ 2069 def->args_ct[o].oalias = 1; 2070 def->args_ct[o].alias_index = i; 2071 /* The input sets ialias. */ 2072 def->args_ct[i].ialias = 1; 2073 def->args_ct[i].alias_index = o; 2074 if (def->args_ct[i].pair) { 2075 saw_alias_pair = true; 2076 } 2077 tcg_debug_assert(ct_str[1] == '\0'); 2078 continue; 2079 2080 case '&': 2081 tcg_debug_assert(!input_p); 2082 def->args_ct[i].newreg = true; 2083 ct_str++; 2084 break; 2085 2086 case 'p': /* plus */ 2087 /* Allocate to the register after the previous. */ 2088 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 2089 o = i - 1; 2090 tcg_debug_assert(!def->args_ct[o].pair); 2091 tcg_debug_assert(!def->args_ct[o].ct); 2092 def->args_ct[i] = (TCGArgConstraint){ 2093 .pair = 2, 2094 .pair_index = o, 2095 .regs = def->args_ct[o].regs << 1, 2096 }; 2097 def->args_ct[o].pair = 1; 2098 def->args_ct[o].pair_index = i; 2099 tcg_debug_assert(ct_str[1] == '\0'); 2100 continue; 2101 2102 case 'm': /* minus */ 2103 /* Allocate to the register before the previous. */ 2104 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 2105 o = i - 1; 2106 tcg_debug_assert(!def->args_ct[o].pair); 2107 tcg_debug_assert(!def->args_ct[o].ct); 2108 def->args_ct[i] = (TCGArgConstraint){ 2109 .pair = 1, 2110 .pair_index = o, 2111 .regs = def->args_ct[o].regs >> 1, 2112 }; 2113 def->args_ct[o].pair = 2; 2114 def->args_ct[o].pair_index = i; 2115 tcg_debug_assert(ct_str[1] == '\0'); 2116 continue; 2117 } 2118 2119 do { 2120 switch (*ct_str) { 2121 case 'i': 2122 def->args_ct[i].ct |= TCG_CT_CONST; 2123 break; 2124 2125 /* Include all of the target-specific constraints. */ 2126 2127 #undef CONST 2128 #define CONST(CASE, MASK) \ 2129 case CASE: def->args_ct[i].ct |= MASK; break; 2130 #define REGS(CASE, MASK) \ 2131 case CASE: def->args_ct[i].regs |= MASK; break; 2132 2133 #include "tcg-target-con-str.h" 2134 2135 #undef REGS 2136 #undef CONST 2137 default: 2138 case '0' ... '9': 2139 case '&': 2140 case 'p': 2141 case 'm': 2142 /* Typo in TCGTargetOpDef constraint. */ 2143 g_assert_not_reached(); 2144 } 2145 } while (*++ct_str != '\0'); 2146 } 2147 2148 /* TCGTargetOpDef entry with too much information? */ 2149 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2150 2151 /* 2152 * Fix up output pairs that are aliased with inputs. 2153 * When we created the alias, we copied pair from the output. 2154 * There are three cases: 2155 * (1a) Pairs of inputs alias pairs of outputs. 2156 * (1b) One input aliases the first of a pair of outputs. 2157 * (2) One input aliases the second of a pair of outputs. 2158 * 2159 * Case 1a is handled by making sure that the pair_index'es are 2160 * properly updated so that they appear the same as a pair of inputs. 2161 * 2162 * Case 1b is handled by setting the pair_index of the input to 2163 * itself, simply so it doesn't point to an unrelated argument. 2164 * Since we don't encounter the "second" during the input allocation 2165 * phase, nothing happens with the second half of the input pair. 2166 * 2167 * Case 2 is handled by setting the second input to pair=3, the 2168 * first output to pair=3, and the pair_index'es to match. 2169 */ 2170 if (saw_alias_pair) { 2171 for (i = def->nb_oargs; i < nb_args; i++) { 2172 /* 2173 * Since [0-9pm] must be alone in the constraint string, 2174 * the only way they can both be set is if the pair comes 2175 * from the output alias. 2176 */ 2177 if (!def->args_ct[i].ialias) { 2178 continue; 2179 } 2180 switch (def->args_ct[i].pair) { 2181 case 0: 2182 break; 2183 case 1: 2184 o = def->args_ct[i].alias_index; 2185 o2 = def->args_ct[o].pair_index; 2186 tcg_debug_assert(def->args_ct[o].pair == 1); 2187 tcg_debug_assert(def->args_ct[o2].pair == 2); 2188 if (def->args_ct[o2].oalias) { 2189 /* Case 1a */ 2190 i2 = def->args_ct[o2].alias_index; 2191 tcg_debug_assert(def->args_ct[i2].pair == 2); 2192 def->args_ct[i2].pair_index = i; 2193 def->args_ct[i].pair_index = i2; 2194 } else { 2195 /* Case 1b */ 2196 def->args_ct[i].pair_index = i; 2197 } 2198 break; 2199 case 2: 2200 o = def->args_ct[i].alias_index; 2201 o2 = def->args_ct[o].pair_index; 2202 tcg_debug_assert(def->args_ct[o].pair == 2); 2203 tcg_debug_assert(def->args_ct[o2].pair == 1); 2204 if (def->args_ct[o2].oalias) { 2205 /* Case 1a */ 2206 i2 = def->args_ct[o2].alias_index; 2207 tcg_debug_assert(def->args_ct[i2].pair == 1); 2208 def->args_ct[i2].pair_index = i; 2209 def->args_ct[i].pair_index = i2; 2210 } else { 2211 /* Case 2 */ 2212 def->args_ct[i].pair = 3; 2213 def->args_ct[o2].pair = 3; 2214 def->args_ct[i].pair_index = o2; 2215 def->args_ct[o2].pair_index = i; 2216 } 2217 break; 2218 default: 2219 g_assert_not_reached(); 2220 } 2221 } 2222 } 2223 2224 /* sort the constraints (XXX: this is just an heuristic) */ 2225 sort_constraints(def, 0, def->nb_oargs); 2226 sort_constraints(def, def->nb_oargs, def->nb_iargs); 2227 } 2228 } 2229 2230 void tcg_op_remove(TCGContext *s, TCGOp *op) 2231 { 2232 TCGLabel *label; 2233 2234 switch (op->opc) { 2235 case INDEX_op_br: 2236 label = arg_label(op->args[0]); 2237 label->refs--; 2238 break; 2239 case INDEX_op_brcond_i32: 2240 case INDEX_op_brcond_i64: 2241 label = arg_label(op->args[3]); 2242 label->refs--; 2243 break; 2244 case INDEX_op_brcond2_i32: 2245 label = arg_label(op->args[5]); 2246 label->refs--; 2247 break; 2248 default: 2249 break; 2250 } 2251 2252 QTAILQ_REMOVE(&s->ops, op, link); 2253 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2254 s->nb_ops--; 2255 2256 #ifdef CONFIG_PROFILER 2257 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 2258 #endif 2259 } 2260 2261 void tcg_remove_ops_after(TCGOp *op) 2262 { 2263 TCGContext *s = tcg_ctx; 2264 2265 while (true) { 2266 TCGOp *last = tcg_last_op(); 2267 if (last == op) { 2268 return; 2269 } 2270 tcg_op_remove(s, last); 2271 } 2272 } 2273 2274 static TCGOp *tcg_op_alloc(TCGOpcode opc) 2275 { 2276 TCGContext *s = tcg_ctx; 2277 TCGOp *op; 2278 2279 if (likely(QTAILQ_EMPTY(&s->free_ops))) { 2280 op = tcg_malloc(sizeof(TCGOp)); 2281 } else { 2282 op = QTAILQ_FIRST(&s->free_ops); 2283 QTAILQ_REMOVE(&s->free_ops, op, link); 2284 } 2285 memset(op, 0, offsetof(TCGOp, link)); 2286 op->opc = opc; 2287 s->nb_ops++; 2288 2289 return op; 2290 } 2291 2292 TCGOp *tcg_emit_op(TCGOpcode opc) 2293 { 2294 TCGOp *op = tcg_op_alloc(opc); 2295 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2296 return op; 2297 } 2298 2299 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 2300 { 2301 TCGOp *new_op = tcg_op_alloc(opc); 2302 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 2303 return new_op; 2304 } 2305 2306 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 2307 { 2308 TCGOp *new_op = tcg_op_alloc(opc); 2309 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 2310 return new_op; 2311 } 2312 2313 /* Reachable analysis : remove unreachable code. */ 2314 static void reachable_code_pass(TCGContext *s) 2315 { 2316 TCGOp *op, *op_next; 2317 bool dead = false; 2318 2319 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2320 bool remove = dead; 2321 TCGLabel *label; 2322 2323 switch (op->opc) { 2324 case INDEX_op_set_label: 2325 label = arg_label(op->args[0]); 2326 if (label->refs == 0) { 2327 /* 2328 * While there is an occasional backward branch, virtually 2329 * all branches generated by the translators are forward. 2330 * Which means that generally we will have already removed 2331 * all references to the label that will be, and there is 2332 * little to be gained by iterating. 2333 */ 2334 remove = true; 2335 } else { 2336 /* Once we see a label, insns become live again. */ 2337 dead = false; 2338 remove = false; 2339 2340 /* 2341 * Optimization can fold conditional branches to unconditional. 2342 * If we find a label with one reference which is preceded by 2343 * an unconditional branch to it, remove both. This needed to 2344 * wait until the dead code in between them was removed. 2345 */ 2346 if (label->refs == 1) { 2347 TCGOp *op_prev = QTAILQ_PREV(op, link); 2348 if (op_prev->opc == INDEX_op_br && 2349 label == arg_label(op_prev->args[0])) { 2350 tcg_op_remove(s, op_prev); 2351 remove = true; 2352 } 2353 } 2354 } 2355 break; 2356 2357 case INDEX_op_br: 2358 case INDEX_op_exit_tb: 2359 case INDEX_op_goto_ptr: 2360 /* Unconditional branches; everything following is dead. */ 2361 dead = true; 2362 break; 2363 2364 case INDEX_op_call: 2365 /* Notice noreturn helper calls, raising exceptions. */ 2366 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2367 dead = true; 2368 } 2369 break; 2370 2371 case INDEX_op_insn_start: 2372 /* Never remove -- we need to keep these for unwind. */ 2373 remove = false; 2374 break; 2375 2376 default: 2377 break; 2378 } 2379 2380 if (remove) { 2381 tcg_op_remove(s, op); 2382 } 2383 } 2384 } 2385 2386 #define TS_DEAD 1 2387 #define TS_MEM 2 2388 2389 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 2390 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 2391 2392 /* For liveness_pass_1, the register preferences for a given temp. */ 2393 static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 2394 { 2395 return ts->state_ptr; 2396 } 2397 2398 /* For liveness_pass_1, reset the preferences for a given temp to the 2399 * maximal regset for its type. 2400 */ 2401 static inline void la_reset_pref(TCGTemp *ts) 2402 { 2403 *la_temp_pref(ts) 2404 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 2405 } 2406 2407 /* liveness analysis: end of function: all temps are dead, and globals 2408 should be in memory. */ 2409 static void la_func_end(TCGContext *s, int ng, int nt) 2410 { 2411 int i; 2412 2413 for (i = 0; i < ng; ++i) { 2414 s->temps[i].state = TS_DEAD | TS_MEM; 2415 la_reset_pref(&s->temps[i]); 2416 } 2417 for (i = ng; i < nt; ++i) { 2418 s->temps[i].state = TS_DEAD; 2419 la_reset_pref(&s->temps[i]); 2420 } 2421 } 2422 2423 /* liveness analysis: end of basic block: all temps are dead, globals 2424 and local temps should be in memory. */ 2425 static void la_bb_end(TCGContext *s, int ng, int nt) 2426 { 2427 int i; 2428 2429 for (i = 0; i < nt; ++i) { 2430 TCGTemp *ts = &s->temps[i]; 2431 int state; 2432 2433 switch (ts->kind) { 2434 case TEMP_FIXED: 2435 case TEMP_GLOBAL: 2436 case TEMP_LOCAL: 2437 state = TS_DEAD | TS_MEM; 2438 break; 2439 case TEMP_NORMAL: 2440 case TEMP_EBB: 2441 case TEMP_CONST: 2442 state = TS_DEAD; 2443 break; 2444 default: 2445 g_assert_not_reached(); 2446 } 2447 ts->state = state; 2448 la_reset_pref(ts); 2449 } 2450 } 2451 2452 /* liveness analysis: sync globals back to memory. */ 2453 static void la_global_sync(TCGContext *s, int ng) 2454 { 2455 int i; 2456 2457 for (i = 0; i < ng; ++i) { 2458 int state = s->temps[i].state; 2459 s->temps[i].state = state | TS_MEM; 2460 if (state == TS_DEAD) { 2461 /* If the global was previously dead, reset prefs. */ 2462 la_reset_pref(&s->temps[i]); 2463 } 2464 } 2465 } 2466 2467 /* 2468 * liveness analysis: conditional branch: all temps are dead unless 2469 * explicitly live-across-conditional-branch, globals and local temps 2470 * should be synced. 2471 */ 2472 static void la_bb_sync(TCGContext *s, int ng, int nt) 2473 { 2474 la_global_sync(s, ng); 2475 2476 for (int i = ng; i < nt; ++i) { 2477 TCGTemp *ts = &s->temps[i]; 2478 int state; 2479 2480 switch (ts->kind) { 2481 case TEMP_LOCAL: 2482 state = ts->state; 2483 ts->state = state | TS_MEM; 2484 if (state != TS_DEAD) { 2485 continue; 2486 } 2487 break; 2488 case TEMP_NORMAL: 2489 s->temps[i].state = TS_DEAD; 2490 break; 2491 case TEMP_EBB: 2492 case TEMP_CONST: 2493 continue; 2494 default: 2495 g_assert_not_reached(); 2496 } 2497 la_reset_pref(&s->temps[i]); 2498 } 2499 } 2500 2501 /* liveness analysis: sync globals back to memory and kill. */ 2502 static void la_global_kill(TCGContext *s, int ng) 2503 { 2504 int i; 2505 2506 for (i = 0; i < ng; i++) { 2507 s->temps[i].state = TS_DEAD | TS_MEM; 2508 la_reset_pref(&s->temps[i]); 2509 } 2510 } 2511 2512 /* liveness analysis: note live globals crossing calls. */ 2513 static void la_cross_call(TCGContext *s, int nt) 2514 { 2515 TCGRegSet mask = ~tcg_target_call_clobber_regs; 2516 int i; 2517 2518 for (i = 0; i < nt; i++) { 2519 TCGTemp *ts = &s->temps[i]; 2520 if (!(ts->state & TS_DEAD)) { 2521 TCGRegSet *pset = la_temp_pref(ts); 2522 TCGRegSet set = *pset; 2523 2524 set &= mask; 2525 /* If the combination is not possible, restart. */ 2526 if (set == 0) { 2527 set = tcg_target_available_regs[ts->type] & mask; 2528 } 2529 *pset = set; 2530 } 2531 } 2532 } 2533 2534 /* Liveness analysis : update the opc_arg_life array to tell if a 2535 given input arguments is dead. Instructions updating dead 2536 temporaries are removed. */ 2537 static void liveness_pass_1(TCGContext *s) 2538 { 2539 int nb_globals = s->nb_globals; 2540 int nb_temps = s->nb_temps; 2541 TCGOp *op, *op_prev; 2542 TCGRegSet *prefs; 2543 int i; 2544 2545 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 2546 for (i = 0; i < nb_temps; ++i) { 2547 s->temps[i].state_ptr = prefs + i; 2548 } 2549 2550 /* ??? Should be redundant with the exit_tb that ends the TB. */ 2551 la_func_end(s, nb_globals, nb_temps); 2552 2553 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 2554 int nb_iargs, nb_oargs; 2555 TCGOpcode opc_new, opc_new2; 2556 bool have_opc_new2; 2557 TCGLifeData arg_life = 0; 2558 TCGTemp *ts; 2559 TCGOpcode opc = op->opc; 2560 const TCGOpDef *def = &tcg_op_defs[opc]; 2561 2562 switch (opc) { 2563 case INDEX_op_call: 2564 { 2565 int call_flags; 2566 int nb_call_regs; 2567 2568 nb_oargs = TCGOP_CALLO(op); 2569 nb_iargs = TCGOP_CALLI(op); 2570 call_flags = tcg_call_flags(op); 2571 2572 /* pure functions can be removed if their result is unused */ 2573 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 2574 for (i = 0; i < nb_oargs; i++) { 2575 ts = arg_temp(op->args[i]); 2576 if (ts->state != TS_DEAD) { 2577 goto do_not_remove_call; 2578 } 2579 } 2580 goto do_remove; 2581 } 2582 do_not_remove_call: 2583 2584 /* Output args are dead. */ 2585 for (i = 0; i < nb_oargs; i++) { 2586 ts = arg_temp(op->args[i]); 2587 if (ts->state & TS_DEAD) { 2588 arg_life |= DEAD_ARG << i; 2589 } 2590 if (ts->state & TS_MEM) { 2591 arg_life |= SYNC_ARG << i; 2592 } 2593 ts->state = TS_DEAD; 2594 la_reset_pref(ts); 2595 2596 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */ 2597 op->output_pref[i] = 0; 2598 } 2599 2600 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 2601 TCG_CALL_NO_READ_GLOBALS))) { 2602 la_global_kill(s, nb_globals); 2603 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 2604 la_global_sync(s, nb_globals); 2605 } 2606 2607 /* Record arguments that die in this helper. */ 2608 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2609 ts = arg_temp(op->args[i]); 2610 if (ts && ts->state & TS_DEAD) { 2611 arg_life |= DEAD_ARG << i; 2612 } 2613 } 2614 2615 /* For all live registers, remove call-clobbered prefs. */ 2616 la_cross_call(s, nb_temps); 2617 2618 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2619 2620 /* Input arguments are live for preceding opcodes. */ 2621 for (i = 0; i < nb_iargs; i++) { 2622 ts = arg_temp(op->args[i + nb_oargs]); 2623 if (ts && ts->state & TS_DEAD) { 2624 /* For those arguments that die, and will be allocated 2625 * in registers, clear the register set for that arg, 2626 * to be filled in below. For args that will be on 2627 * the stack, reset to any available reg. 2628 */ 2629 *la_temp_pref(ts) 2630 = (i < nb_call_regs ? 0 : 2631 tcg_target_available_regs[ts->type]); 2632 ts->state &= ~TS_DEAD; 2633 } 2634 } 2635 2636 /* For each input argument, add its input register to prefs. 2637 If a temp is used once, this produces a single set bit. */ 2638 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) { 2639 ts = arg_temp(op->args[i + nb_oargs]); 2640 if (ts) { 2641 tcg_regset_set_reg(*la_temp_pref(ts), 2642 tcg_target_call_iarg_regs[i]); 2643 } 2644 } 2645 } 2646 break; 2647 case INDEX_op_insn_start: 2648 break; 2649 case INDEX_op_discard: 2650 /* mark the temporary as dead */ 2651 ts = arg_temp(op->args[0]); 2652 ts->state = TS_DEAD; 2653 la_reset_pref(ts); 2654 break; 2655 2656 case INDEX_op_add2_i32: 2657 opc_new = INDEX_op_add_i32; 2658 goto do_addsub2; 2659 case INDEX_op_sub2_i32: 2660 opc_new = INDEX_op_sub_i32; 2661 goto do_addsub2; 2662 case INDEX_op_add2_i64: 2663 opc_new = INDEX_op_add_i64; 2664 goto do_addsub2; 2665 case INDEX_op_sub2_i64: 2666 opc_new = INDEX_op_sub_i64; 2667 do_addsub2: 2668 nb_iargs = 4; 2669 nb_oargs = 2; 2670 /* Test if the high part of the operation is dead, but not 2671 the low part. The result can be optimized to a simple 2672 add or sub. This happens often for x86_64 guest when the 2673 cpu mode is set to 32 bit. */ 2674 if (arg_temp(op->args[1])->state == TS_DEAD) { 2675 if (arg_temp(op->args[0])->state == TS_DEAD) { 2676 goto do_remove; 2677 } 2678 /* Replace the opcode and adjust the args in place, 2679 leaving 3 unused args at the end. */ 2680 op->opc = opc = opc_new; 2681 op->args[1] = op->args[2]; 2682 op->args[2] = op->args[4]; 2683 /* Fall through and mark the single-word operation live. */ 2684 nb_iargs = 2; 2685 nb_oargs = 1; 2686 } 2687 goto do_not_remove; 2688 2689 case INDEX_op_mulu2_i32: 2690 opc_new = INDEX_op_mul_i32; 2691 opc_new2 = INDEX_op_muluh_i32; 2692 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 2693 goto do_mul2; 2694 case INDEX_op_muls2_i32: 2695 opc_new = INDEX_op_mul_i32; 2696 opc_new2 = INDEX_op_mulsh_i32; 2697 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 2698 goto do_mul2; 2699 case INDEX_op_mulu2_i64: 2700 opc_new = INDEX_op_mul_i64; 2701 opc_new2 = INDEX_op_muluh_i64; 2702 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 2703 goto do_mul2; 2704 case INDEX_op_muls2_i64: 2705 opc_new = INDEX_op_mul_i64; 2706 opc_new2 = INDEX_op_mulsh_i64; 2707 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 2708 goto do_mul2; 2709 do_mul2: 2710 nb_iargs = 2; 2711 nb_oargs = 2; 2712 if (arg_temp(op->args[1])->state == TS_DEAD) { 2713 if (arg_temp(op->args[0])->state == TS_DEAD) { 2714 /* Both parts of the operation are dead. */ 2715 goto do_remove; 2716 } 2717 /* The high part of the operation is dead; generate the low. */ 2718 op->opc = opc = opc_new; 2719 op->args[1] = op->args[2]; 2720 op->args[2] = op->args[3]; 2721 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 2722 /* The low part of the operation is dead; generate the high. */ 2723 op->opc = opc = opc_new2; 2724 op->args[0] = op->args[1]; 2725 op->args[1] = op->args[2]; 2726 op->args[2] = op->args[3]; 2727 } else { 2728 goto do_not_remove; 2729 } 2730 /* Mark the single-word operation live. */ 2731 nb_oargs = 1; 2732 goto do_not_remove; 2733 2734 default: 2735 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 2736 nb_iargs = def->nb_iargs; 2737 nb_oargs = def->nb_oargs; 2738 2739 /* Test if the operation can be removed because all 2740 its outputs are dead. We assume that nb_oargs == 0 2741 implies side effects */ 2742 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 2743 for (i = 0; i < nb_oargs; i++) { 2744 if (arg_temp(op->args[i])->state != TS_DEAD) { 2745 goto do_not_remove; 2746 } 2747 } 2748 goto do_remove; 2749 } 2750 goto do_not_remove; 2751 2752 do_remove: 2753 tcg_op_remove(s, op); 2754 break; 2755 2756 do_not_remove: 2757 for (i = 0; i < nb_oargs; i++) { 2758 ts = arg_temp(op->args[i]); 2759 2760 /* Remember the preference of the uses that followed. */ 2761 op->output_pref[i] = *la_temp_pref(ts); 2762 2763 /* Output args are dead. */ 2764 if (ts->state & TS_DEAD) { 2765 arg_life |= DEAD_ARG << i; 2766 } 2767 if (ts->state & TS_MEM) { 2768 arg_life |= SYNC_ARG << i; 2769 } 2770 ts->state = TS_DEAD; 2771 la_reset_pref(ts); 2772 } 2773 2774 /* If end of basic block, update. */ 2775 if (def->flags & TCG_OPF_BB_EXIT) { 2776 la_func_end(s, nb_globals, nb_temps); 2777 } else if (def->flags & TCG_OPF_COND_BRANCH) { 2778 la_bb_sync(s, nb_globals, nb_temps); 2779 } else if (def->flags & TCG_OPF_BB_END) { 2780 la_bb_end(s, nb_globals, nb_temps); 2781 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2782 la_global_sync(s, nb_globals); 2783 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2784 la_cross_call(s, nb_temps); 2785 } 2786 } 2787 2788 /* Record arguments that die in this opcode. */ 2789 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2790 ts = arg_temp(op->args[i]); 2791 if (ts->state & TS_DEAD) { 2792 arg_life |= DEAD_ARG << i; 2793 } 2794 } 2795 2796 /* Input arguments are live for preceding opcodes. */ 2797 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2798 ts = arg_temp(op->args[i]); 2799 if (ts->state & TS_DEAD) { 2800 /* For operands that were dead, initially allow 2801 all regs for the type. */ 2802 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 2803 ts->state &= ~TS_DEAD; 2804 } 2805 } 2806 2807 /* Incorporate constraints for this operand. */ 2808 switch (opc) { 2809 case INDEX_op_mov_i32: 2810 case INDEX_op_mov_i64: 2811 /* Note that these are TCG_OPF_NOT_PRESENT and do not 2812 have proper constraints. That said, special case 2813 moves to propagate preferences backward. */ 2814 if (IS_DEAD_ARG(1)) { 2815 *la_temp_pref(arg_temp(op->args[0])) 2816 = *la_temp_pref(arg_temp(op->args[1])); 2817 } 2818 break; 2819 2820 default: 2821 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2822 const TCGArgConstraint *ct = &def->args_ct[i]; 2823 TCGRegSet set, *pset; 2824 2825 ts = arg_temp(op->args[i]); 2826 pset = la_temp_pref(ts); 2827 set = *pset; 2828 2829 set &= ct->regs; 2830 if (ct->ialias) { 2831 set &= op->output_pref[ct->alias_index]; 2832 } 2833 /* If the combination is not possible, restart. */ 2834 if (set == 0) { 2835 set = ct->regs; 2836 } 2837 *pset = set; 2838 } 2839 break; 2840 } 2841 break; 2842 } 2843 op->life = arg_life; 2844 } 2845 } 2846 2847 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 2848 static bool liveness_pass_2(TCGContext *s) 2849 { 2850 int nb_globals = s->nb_globals; 2851 int nb_temps, i; 2852 bool changes = false; 2853 TCGOp *op, *op_next; 2854 2855 /* Create a temporary for each indirect global. */ 2856 for (i = 0; i < nb_globals; ++i) { 2857 TCGTemp *its = &s->temps[i]; 2858 if (its->indirect_reg) { 2859 TCGTemp *dts = tcg_temp_alloc(s); 2860 dts->type = its->type; 2861 dts->base_type = its->base_type; 2862 dts->kind = TEMP_EBB; 2863 its->state_ptr = dts; 2864 } else { 2865 its->state_ptr = NULL; 2866 } 2867 /* All globals begin dead. */ 2868 its->state = TS_DEAD; 2869 } 2870 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 2871 TCGTemp *its = &s->temps[i]; 2872 its->state_ptr = NULL; 2873 its->state = TS_DEAD; 2874 } 2875 2876 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2877 TCGOpcode opc = op->opc; 2878 const TCGOpDef *def = &tcg_op_defs[opc]; 2879 TCGLifeData arg_life = op->life; 2880 int nb_iargs, nb_oargs, call_flags; 2881 TCGTemp *arg_ts, *dir_ts; 2882 2883 if (opc == INDEX_op_call) { 2884 nb_oargs = TCGOP_CALLO(op); 2885 nb_iargs = TCGOP_CALLI(op); 2886 call_flags = tcg_call_flags(op); 2887 } else { 2888 nb_iargs = def->nb_iargs; 2889 nb_oargs = def->nb_oargs; 2890 2891 /* Set flags similar to how calls require. */ 2892 if (def->flags & TCG_OPF_COND_BRANCH) { 2893 /* Like reading globals: sync_globals */ 2894 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2895 } else if (def->flags & TCG_OPF_BB_END) { 2896 /* Like writing globals: save_globals */ 2897 call_flags = 0; 2898 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2899 /* Like reading globals: sync_globals */ 2900 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2901 } else { 2902 /* No effect on globals. */ 2903 call_flags = (TCG_CALL_NO_READ_GLOBALS | 2904 TCG_CALL_NO_WRITE_GLOBALS); 2905 } 2906 } 2907 2908 /* Make sure that input arguments are available. */ 2909 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2910 arg_ts = arg_temp(op->args[i]); 2911 if (arg_ts) { 2912 dir_ts = arg_ts->state_ptr; 2913 if (dir_ts && arg_ts->state == TS_DEAD) { 2914 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 2915 ? INDEX_op_ld_i32 2916 : INDEX_op_ld_i64); 2917 TCGOp *lop = tcg_op_insert_before(s, op, lopc); 2918 2919 lop->args[0] = temp_arg(dir_ts); 2920 lop->args[1] = temp_arg(arg_ts->mem_base); 2921 lop->args[2] = arg_ts->mem_offset; 2922 2923 /* Loaded, but synced with memory. */ 2924 arg_ts->state = TS_MEM; 2925 } 2926 } 2927 } 2928 2929 /* Perform input replacement, and mark inputs that became dead. 2930 No action is required except keeping temp_state up to date 2931 so that we reload when needed. */ 2932 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2933 arg_ts = arg_temp(op->args[i]); 2934 if (arg_ts) { 2935 dir_ts = arg_ts->state_ptr; 2936 if (dir_ts) { 2937 op->args[i] = temp_arg(dir_ts); 2938 changes = true; 2939 if (IS_DEAD_ARG(i)) { 2940 arg_ts->state = TS_DEAD; 2941 } 2942 } 2943 } 2944 } 2945 2946 /* Liveness analysis should ensure that the following are 2947 all correct, for call sites and basic block end points. */ 2948 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 2949 /* Nothing to do */ 2950 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 2951 for (i = 0; i < nb_globals; ++i) { 2952 /* Liveness should see that globals are synced back, 2953 that is, either TS_DEAD or TS_MEM. */ 2954 arg_ts = &s->temps[i]; 2955 tcg_debug_assert(arg_ts->state_ptr == 0 2956 || arg_ts->state != 0); 2957 } 2958 } else { 2959 for (i = 0; i < nb_globals; ++i) { 2960 /* Liveness should see that globals are saved back, 2961 that is, TS_DEAD, waiting to be reloaded. */ 2962 arg_ts = &s->temps[i]; 2963 tcg_debug_assert(arg_ts->state_ptr == 0 2964 || arg_ts->state == TS_DEAD); 2965 } 2966 } 2967 2968 /* Outputs become available. */ 2969 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 2970 arg_ts = arg_temp(op->args[0]); 2971 dir_ts = arg_ts->state_ptr; 2972 if (dir_ts) { 2973 op->args[0] = temp_arg(dir_ts); 2974 changes = true; 2975 2976 /* The output is now live and modified. */ 2977 arg_ts->state = 0; 2978 2979 if (NEED_SYNC_ARG(0)) { 2980 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 2981 ? INDEX_op_st_i32 2982 : INDEX_op_st_i64); 2983 TCGOp *sop = tcg_op_insert_after(s, op, sopc); 2984 TCGTemp *out_ts = dir_ts; 2985 2986 if (IS_DEAD_ARG(0)) { 2987 out_ts = arg_temp(op->args[1]); 2988 arg_ts->state = TS_DEAD; 2989 tcg_op_remove(s, op); 2990 } else { 2991 arg_ts->state = TS_MEM; 2992 } 2993 2994 sop->args[0] = temp_arg(out_ts); 2995 sop->args[1] = temp_arg(arg_ts->mem_base); 2996 sop->args[2] = arg_ts->mem_offset; 2997 } else { 2998 tcg_debug_assert(!IS_DEAD_ARG(0)); 2999 } 3000 } 3001 } else { 3002 for (i = 0; i < nb_oargs; i++) { 3003 arg_ts = arg_temp(op->args[i]); 3004 dir_ts = arg_ts->state_ptr; 3005 if (!dir_ts) { 3006 continue; 3007 } 3008 op->args[i] = temp_arg(dir_ts); 3009 changes = true; 3010 3011 /* The output is now live and modified. */ 3012 arg_ts->state = 0; 3013 3014 /* Sync outputs upon their last write. */ 3015 if (NEED_SYNC_ARG(i)) { 3016 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 3017 ? INDEX_op_st_i32 3018 : INDEX_op_st_i64); 3019 TCGOp *sop = tcg_op_insert_after(s, op, sopc); 3020 3021 sop->args[0] = temp_arg(dir_ts); 3022 sop->args[1] = temp_arg(arg_ts->mem_base); 3023 sop->args[2] = arg_ts->mem_offset; 3024 3025 arg_ts->state = TS_MEM; 3026 } 3027 /* Drop outputs that are dead. */ 3028 if (IS_DEAD_ARG(i)) { 3029 arg_ts->state = TS_DEAD; 3030 } 3031 } 3032 } 3033 } 3034 3035 return changes; 3036 } 3037 3038 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3039 { 3040 intptr_t off, size, align; 3041 3042 switch (ts->type) { 3043 case TCG_TYPE_I32: 3044 size = align = 4; 3045 break; 3046 case TCG_TYPE_I64: 3047 case TCG_TYPE_V64: 3048 size = align = 8; 3049 break; 3050 case TCG_TYPE_V128: 3051 size = align = 16; 3052 break; 3053 case TCG_TYPE_V256: 3054 /* Note that we do not require aligned storage for V256. */ 3055 size = 32, align = 16; 3056 break; 3057 default: 3058 g_assert_not_reached(); 3059 } 3060 3061 /* 3062 * Assume the stack is sufficiently aligned. 3063 * This affects e.g. ARM NEON, where we have 8 byte stack alignment 3064 * and do not require 16 byte vector alignment. This seems slightly 3065 * easier than fully parameterizing the above switch statement. 3066 */ 3067 align = MIN(TCG_TARGET_STACK_ALIGN, align); 3068 off = ROUND_UP(s->current_frame_offset, align); 3069 3070 /* If we've exhausted the stack frame, restart with a smaller TB. */ 3071 if (off + size > s->frame_end) { 3072 tcg_raise_tb_overflow(s); 3073 } 3074 s->current_frame_offset = off + size; 3075 3076 ts->mem_offset = off; 3077 #if defined(__sparc__) 3078 ts->mem_offset += TCG_TARGET_STACK_BIAS; 3079 #endif 3080 ts->mem_base = s->frame_temp; 3081 ts->mem_allocated = 1; 3082 } 3083 3084 /* Assign @reg to @ts, and update reg_to_temp[]. */ 3085 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 3086 { 3087 if (ts->val_type == TEMP_VAL_REG) { 3088 TCGReg old = ts->reg; 3089 tcg_debug_assert(s->reg_to_temp[old] == ts); 3090 if (old == reg) { 3091 return; 3092 } 3093 s->reg_to_temp[old] = NULL; 3094 } 3095 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3096 s->reg_to_temp[reg] = ts; 3097 ts->val_type = TEMP_VAL_REG; 3098 ts->reg = reg; 3099 } 3100 3101 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 3102 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 3103 { 3104 tcg_debug_assert(type != TEMP_VAL_REG); 3105 if (ts->val_type == TEMP_VAL_REG) { 3106 TCGReg reg = ts->reg; 3107 tcg_debug_assert(s->reg_to_temp[reg] == ts); 3108 s->reg_to_temp[reg] = NULL; 3109 } 3110 ts->val_type = type; 3111 } 3112 3113 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3114 3115 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 3116 mark it free; otherwise mark it dead. */ 3117 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3118 { 3119 TCGTempVal new_type; 3120 3121 switch (ts->kind) { 3122 case TEMP_FIXED: 3123 return; 3124 case TEMP_GLOBAL: 3125 case TEMP_LOCAL: 3126 new_type = TEMP_VAL_MEM; 3127 break; 3128 case TEMP_NORMAL: 3129 case TEMP_EBB: 3130 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3131 break; 3132 case TEMP_CONST: 3133 new_type = TEMP_VAL_CONST; 3134 break; 3135 default: 3136 g_assert_not_reached(); 3137 } 3138 set_temp_val_nonreg(s, ts, new_type); 3139 } 3140 3141 /* Mark a temporary as dead. */ 3142 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 3143 { 3144 temp_free_or_dead(s, ts, 1); 3145 } 3146 3147 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 3148 registers needs to be allocated to store a constant. If 'free_or_dead' 3149 is non-zero, subsequently release the temporary; if it is positive, the 3150 temp is dead; if it is negative, the temp is free. */ 3151 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 3152 TCGRegSet preferred_regs, int free_or_dead) 3153 { 3154 if (!temp_readonly(ts) && !ts->mem_coherent) { 3155 if (!ts->mem_allocated) { 3156 temp_allocate_frame(s, ts); 3157 } 3158 switch (ts->val_type) { 3159 case TEMP_VAL_CONST: 3160 /* If we're going to free the temp immediately, then we won't 3161 require it later in a register, so attempt to store the 3162 constant to memory directly. */ 3163 if (free_or_dead 3164 && tcg_out_sti(s, ts->type, ts->val, 3165 ts->mem_base->reg, ts->mem_offset)) { 3166 break; 3167 } 3168 temp_load(s, ts, tcg_target_available_regs[ts->type], 3169 allocated_regs, preferred_regs); 3170 /* fallthrough */ 3171 3172 case TEMP_VAL_REG: 3173 tcg_out_st(s, ts->type, ts->reg, 3174 ts->mem_base->reg, ts->mem_offset); 3175 break; 3176 3177 case TEMP_VAL_MEM: 3178 break; 3179 3180 case TEMP_VAL_DEAD: 3181 default: 3182 tcg_abort(); 3183 } 3184 ts->mem_coherent = 1; 3185 } 3186 if (free_or_dead) { 3187 temp_free_or_dead(s, ts, free_or_dead); 3188 } 3189 } 3190 3191 /* free register 'reg' by spilling the corresponding temporary if necessary */ 3192 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 3193 { 3194 TCGTemp *ts = s->reg_to_temp[reg]; 3195 if (ts != NULL) { 3196 temp_sync(s, ts, allocated_regs, 0, -1); 3197 } 3198 } 3199 3200 /** 3201 * tcg_reg_alloc: 3202 * @required_regs: Set of registers in which we must allocate. 3203 * @allocated_regs: Set of registers which must be avoided. 3204 * @preferred_regs: Set of registers we should prefer. 3205 * @rev: True if we search the registers in "indirect" order. 3206 * 3207 * The allocated register must be in @required_regs & ~@allocated_regs, 3208 * but if we can put it in @preferred_regs we may save a move later. 3209 */ 3210 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3211 TCGRegSet allocated_regs, 3212 TCGRegSet preferred_regs, bool rev) 3213 { 3214 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3215 TCGRegSet reg_ct[2]; 3216 const int *order; 3217 3218 reg_ct[1] = required_regs & ~allocated_regs; 3219 tcg_debug_assert(reg_ct[1] != 0); 3220 reg_ct[0] = reg_ct[1] & preferred_regs; 3221 3222 /* Skip the preferred_regs option if it cannot be satisfied, 3223 or if the preference made no difference. */ 3224 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3225 3226 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3227 3228 /* Try free registers, preferences first. */ 3229 for (j = f; j < 2; j++) { 3230 TCGRegSet set = reg_ct[j]; 3231 3232 if (tcg_regset_single(set)) { 3233 /* One register in the set. */ 3234 TCGReg reg = tcg_regset_first(set); 3235 if (s->reg_to_temp[reg] == NULL) { 3236 return reg; 3237 } 3238 } else { 3239 for (i = 0; i < n; i++) { 3240 TCGReg reg = order[i]; 3241 if (s->reg_to_temp[reg] == NULL && 3242 tcg_regset_test_reg(set, reg)) { 3243 return reg; 3244 } 3245 } 3246 } 3247 } 3248 3249 /* We must spill something. */ 3250 for (j = f; j < 2; j++) { 3251 TCGRegSet set = reg_ct[j]; 3252 3253 if (tcg_regset_single(set)) { 3254 /* One register in the set. */ 3255 TCGReg reg = tcg_regset_first(set); 3256 tcg_reg_free(s, reg, allocated_regs); 3257 return reg; 3258 } else { 3259 for (i = 0; i < n; i++) { 3260 TCGReg reg = order[i]; 3261 if (tcg_regset_test_reg(set, reg)) { 3262 tcg_reg_free(s, reg, allocated_regs); 3263 return reg; 3264 } 3265 } 3266 } 3267 } 3268 3269 tcg_abort(); 3270 } 3271 3272 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 3273 TCGRegSet allocated_regs, 3274 TCGRegSet preferred_regs, bool rev) 3275 { 3276 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3277 TCGRegSet reg_ct[2]; 3278 const int *order; 3279 3280 /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 3281 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 3282 tcg_debug_assert(reg_ct[1] != 0); 3283 reg_ct[0] = reg_ct[1] & preferred_regs; 3284 3285 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3286 3287 /* 3288 * Skip the preferred_regs option if it cannot be satisfied, 3289 * or if the preference made no difference. 3290 */ 3291 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3292 3293 /* 3294 * Minimize the number of flushes by looking for 2 free registers first, 3295 * then a single flush, then two flushes. 3296 */ 3297 for (fmin = 2; fmin >= 0; fmin--) { 3298 for (j = k; j < 2; j++) { 3299 TCGRegSet set = reg_ct[j]; 3300 3301 for (i = 0; i < n; i++) { 3302 TCGReg reg = order[i]; 3303 3304 if (tcg_regset_test_reg(set, reg)) { 3305 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 3306 if (f >= fmin) { 3307 tcg_reg_free(s, reg, allocated_regs); 3308 tcg_reg_free(s, reg + 1, allocated_regs); 3309 return reg; 3310 } 3311 } 3312 } 3313 } 3314 } 3315 tcg_abort(); 3316 } 3317 3318 /* Make sure the temporary is in a register. If needed, allocate the register 3319 from DESIRED while avoiding ALLOCATED. */ 3320 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3321 TCGRegSet allocated_regs, TCGRegSet preferred_regs) 3322 { 3323 TCGReg reg; 3324 3325 switch (ts->val_type) { 3326 case TEMP_VAL_REG: 3327 return; 3328 case TEMP_VAL_CONST: 3329 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3330 preferred_regs, ts->indirect_base); 3331 if (ts->type <= TCG_TYPE_I64) { 3332 tcg_out_movi(s, ts->type, reg, ts->val); 3333 } else { 3334 uint64_t val = ts->val; 3335 MemOp vece = MO_64; 3336 3337 /* 3338 * Find the minimal vector element that matches the constant. 3339 * The targets will, in general, have to do this search anyway, 3340 * do this generically. 3341 */ 3342 if (val == dup_const(MO_8, val)) { 3343 vece = MO_8; 3344 } else if (val == dup_const(MO_16, val)) { 3345 vece = MO_16; 3346 } else if (val == dup_const(MO_32, val)) { 3347 vece = MO_32; 3348 } 3349 3350 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 3351 } 3352 ts->mem_coherent = 0; 3353 break; 3354 case TEMP_VAL_MEM: 3355 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3356 preferred_regs, ts->indirect_base); 3357 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 3358 ts->mem_coherent = 1; 3359 break; 3360 case TEMP_VAL_DEAD: 3361 default: 3362 tcg_abort(); 3363 } 3364 set_temp_val_reg(s, ts, reg); 3365 } 3366 3367 /* Save a temporary to memory. 'allocated_regs' is used in case a 3368 temporary registers needs to be allocated to store a constant. */ 3369 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 3370 { 3371 /* The liveness analysis already ensures that globals are back 3372 in memory. Keep an tcg_debug_assert for safety. */ 3373 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 3374 } 3375 3376 /* save globals to their canonical location and assume they can be 3377 modified be the following code. 'allocated_regs' is used in case a 3378 temporary registers needs to be allocated to store a constant. */ 3379 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3380 { 3381 int i, n; 3382 3383 for (i = 0, n = s->nb_globals; i < n; i++) { 3384 temp_save(s, &s->temps[i], allocated_regs); 3385 } 3386 } 3387 3388 /* sync globals to their canonical location and assume they can be 3389 read by the following code. 'allocated_regs' is used in case a 3390 temporary registers needs to be allocated to store a constant. */ 3391 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 3392 { 3393 int i, n; 3394 3395 for (i = 0, n = s->nb_globals; i < n; i++) { 3396 TCGTemp *ts = &s->temps[i]; 3397 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3398 || ts->kind == TEMP_FIXED 3399 || ts->mem_coherent); 3400 } 3401 } 3402 3403 /* at the end of a basic block, we assume all temporaries are dead and 3404 all globals are stored at their canonical location. */ 3405 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3406 { 3407 int i; 3408 3409 for (i = s->nb_globals; i < s->nb_temps; i++) { 3410 TCGTemp *ts = &s->temps[i]; 3411 3412 switch (ts->kind) { 3413 case TEMP_LOCAL: 3414 temp_save(s, ts, allocated_regs); 3415 break; 3416 case TEMP_NORMAL: 3417 case TEMP_EBB: 3418 /* The liveness analysis already ensures that temps are dead. 3419 Keep an tcg_debug_assert for safety. */ 3420 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3421 break; 3422 case TEMP_CONST: 3423 /* Similarly, we should have freed any allocated register. */ 3424 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3425 break; 3426 default: 3427 g_assert_not_reached(); 3428 } 3429 } 3430 3431 save_globals(s, allocated_regs); 3432 } 3433 3434 /* 3435 * At a conditional branch, we assume all temporaries are dead unless 3436 * explicitly live-across-conditional-branch; all globals and local 3437 * temps are synced to their location. 3438 */ 3439 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3440 { 3441 sync_globals(s, allocated_regs); 3442 3443 for (int i = s->nb_globals; i < s->nb_temps; i++) { 3444 TCGTemp *ts = &s->temps[i]; 3445 /* 3446 * The liveness analysis already ensures that temps are dead. 3447 * Keep tcg_debug_asserts for safety. 3448 */ 3449 switch (ts->kind) { 3450 case TEMP_LOCAL: 3451 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3452 break; 3453 case TEMP_NORMAL: 3454 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3455 break; 3456 case TEMP_EBB: 3457 case TEMP_CONST: 3458 break; 3459 default: 3460 g_assert_not_reached(); 3461 } 3462 } 3463 } 3464 3465 /* 3466 * Specialized code generation for INDEX_op_mov_* with a constant. 3467 */ 3468 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3469 tcg_target_ulong val, TCGLifeData arg_life, 3470 TCGRegSet preferred_regs) 3471 { 3472 /* ENV should not be modified. */ 3473 tcg_debug_assert(!temp_readonly(ots)); 3474 3475 /* The movi is not explicitly generated here. */ 3476 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 3477 ots->val = val; 3478 ots->mem_coherent = 0; 3479 if (NEED_SYNC_ARG(0)) { 3480 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 3481 } else if (IS_DEAD_ARG(0)) { 3482 temp_dead(s, ots); 3483 } 3484 } 3485 3486 /* 3487 * Specialized code generation for INDEX_op_mov_*. 3488 */ 3489 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 3490 { 3491 const TCGLifeData arg_life = op->life; 3492 TCGRegSet allocated_regs, preferred_regs; 3493 TCGTemp *ts, *ots; 3494 TCGType otype, itype; 3495 TCGReg oreg, ireg; 3496 3497 allocated_regs = s->reserved_regs; 3498 preferred_regs = op->output_pref[0]; 3499 ots = arg_temp(op->args[0]); 3500 ts = arg_temp(op->args[1]); 3501 3502 /* ENV should not be modified. */ 3503 tcg_debug_assert(!temp_readonly(ots)); 3504 3505 /* Note that otype != itype for no-op truncation. */ 3506 otype = ots->type; 3507 itype = ts->type; 3508 3509 if (ts->val_type == TEMP_VAL_CONST) { 3510 /* propagate constant or generate sti */ 3511 tcg_target_ulong val = ts->val; 3512 if (IS_DEAD_ARG(1)) { 3513 temp_dead(s, ts); 3514 } 3515 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 3516 return; 3517 } 3518 3519 /* If the source value is in memory we're going to be forced 3520 to have it in a register in order to perform the copy. Copy 3521 the SOURCE value into its own register first, that way we 3522 don't have to reload SOURCE the next time it is used. */ 3523 if (ts->val_type == TEMP_VAL_MEM) { 3524 temp_load(s, ts, tcg_target_available_regs[itype], 3525 allocated_regs, preferred_regs); 3526 } 3527 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 3528 ireg = ts->reg; 3529 3530 if (IS_DEAD_ARG(0)) { 3531 /* mov to a non-saved dead register makes no sense (even with 3532 liveness analysis disabled). */ 3533 tcg_debug_assert(NEED_SYNC_ARG(0)); 3534 if (!ots->mem_allocated) { 3535 temp_allocate_frame(s, ots); 3536 } 3537 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 3538 if (IS_DEAD_ARG(1)) { 3539 temp_dead(s, ts); 3540 } 3541 temp_dead(s, ots); 3542 return; 3543 } 3544 3545 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 3546 /* 3547 * The mov can be suppressed. Kill input first, so that it 3548 * is unlinked from reg_to_temp, then set the output to the 3549 * reg that we saved from the input. 3550 */ 3551 temp_dead(s, ts); 3552 oreg = ireg; 3553 } else { 3554 if (ots->val_type == TEMP_VAL_REG) { 3555 oreg = ots->reg; 3556 } else { 3557 /* Make sure to not spill the input register during allocation. */ 3558 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 3559 allocated_regs | ((TCGRegSet)1 << ireg), 3560 preferred_regs, ots->indirect_base); 3561 } 3562 if (!tcg_out_mov(s, otype, oreg, ireg)) { 3563 /* 3564 * Cross register class move not supported. 3565 * Store the source register into the destination slot 3566 * and leave the destination temp as TEMP_VAL_MEM. 3567 */ 3568 assert(!temp_readonly(ots)); 3569 if (!ts->mem_allocated) { 3570 temp_allocate_frame(s, ots); 3571 } 3572 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 3573 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 3574 ots->mem_coherent = 1; 3575 return; 3576 } 3577 } 3578 set_temp_val_reg(s, ots, oreg); 3579 ots->mem_coherent = 0; 3580 3581 if (NEED_SYNC_ARG(0)) { 3582 temp_sync(s, ots, allocated_regs, 0, 0); 3583 } 3584 } 3585 3586 /* 3587 * Specialized code generation for INDEX_op_dup_vec. 3588 */ 3589 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 3590 { 3591 const TCGLifeData arg_life = op->life; 3592 TCGRegSet dup_out_regs, dup_in_regs; 3593 TCGTemp *its, *ots; 3594 TCGType itype, vtype; 3595 intptr_t endian_fixup; 3596 unsigned vece; 3597 bool ok; 3598 3599 ots = arg_temp(op->args[0]); 3600 its = arg_temp(op->args[1]); 3601 3602 /* ENV should not be modified. */ 3603 tcg_debug_assert(!temp_readonly(ots)); 3604 3605 itype = its->type; 3606 vece = TCGOP_VECE(op); 3607 vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3608 3609 if (its->val_type == TEMP_VAL_CONST) { 3610 /* Propagate constant via movi -> dupi. */ 3611 tcg_target_ulong val = its->val; 3612 if (IS_DEAD_ARG(1)) { 3613 temp_dead(s, its); 3614 } 3615 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]); 3616 return; 3617 } 3618 3619 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 3620 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 3621 3622 /* Allocate the output register now. */ 3623 if (ots->val_type != TEMP_VAL_REG) { 3624 TCGRegSet allocated_regs = s->reserved_regs; 3625 TCGReg oreg; 3626 3627 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 3628 /* Make sure to not spill the input register. */ 3629 tcg_regset_set_reg(allocated_regs, its->reg); 3630 } 3631 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3632 op->output_pref[0], ots->indirect_base); 3633 set_temp_val_reg(s, ots, oreg); 3634 } 3635 3636 switch (its->val_type) { 3637 case TEMP_VAL_REG: 3638 /* 3639 * The dup constriaints must be broad, covering all possible VECE. 3640 * However, tcg_op_dup_vec() gets to see the VECE and we allow it 3641 * to fail, indicating that extra moves are required for that case. 3642 */ 3643 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 3644 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 3645 goto done; 3646 } 3647 /* Try again from memory or a vector input register. */ 3648 } 3649 if (!its->mem_coherent) { 3650 /* 3651 * The input register is not synced, and so an extra store 3652 * would be required to use memory. Attempt an integer-vector 3653 * register move first. We do not have a TCGRegSet for this. 3654 */ 3655 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 3656 break; 3657 } 3658 /* Sync the temp back to its slot and load from there. */ 3659 temp_sync(s, its, s->reserved_regs, 0, 0); 3660 } 3661 /* fall through */ 3662 3663 case TEMP_VAL_MEM: 3664 #if HOST_BIG_ENDIAN 3665 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8; 3666 endian_fixup -= 1 << vece; 3667 #else 3668 endian_fixup = 0; 3669 #endif 3670 /* Attempt to dup directly from the input memory slot. */ 3671 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 3672 its->mem_offset + endian_fixup)) { 3673 goto done; 3674 } 3675 /* Load the input into the destination vector register. */ 3676 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 3677 break; 3678 3679 default: 3680 g_assert_not_reached(); 3681 } 3682 3683 /* We now have a vector input register, so dup must succeed. */ 3684 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 3685 tcg_debug_assert(ok); 3686 3687 done: 3688 ots->mem_coherent = 0; 3689 if (IS_DEAD_ARG(1)) { 3690 temp_dead(s, its); 3691 } 3692 if (NEED_SYNC_ARG(0)) { 3693 temp_sync(s, ots, s->reserved_regs, 0, 0); 3694 } 3695 if (IS_DEAD_ARG(0)) { 3696 temp_dead(s, ots); 3697 } 3698 } 3699 3700 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 3701 { 3702 const TCGLifeData arg_life = op->life; 3703 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 3704 TCGRegSet i_allocated_regs; 3705 TCGRegSet o_allocated_regs; 3706 int i, k, nb_iargs, nb_oargs; 3707 TCGReg reg; 3708 TCGArg arg; 3709 const TCGArgConstraint *arg_ct; 3710 TCGTemp *ts; 3711 TCGArg new_args[TCG_MAX_OP_ARGS]; 3712 int const_args[TCG_MAX_OP_ARGS]; 3713 3714 nb_oargs = def->nb_oargs; 3715 nb_iargs = def->nb_iargs; 3716 3717 /* copy constants */ 3718 memcpy(new_args + nb_oargs + nb_iargs, 3719 op->args + nb_oargs + nb_iargs, 3720 sizeof(TCGArg) * def->nb_cargs); 3721 3722 i_allocated_regs = s->reserved_regs; 3723 o_allocated_regs = s->reserved_regs; 3724 3725 /* satisfy input constraints */ 3726 for (k = 0; k < nb_iargs; k++) { 3727 TCGRegSet i_preferred_regs, i_required_regs; 3728 bool allocate_new_reg, copyto_new_reg; 3729 TCGTemp *ts2; 3730 int i1, i2; 3731 3732 i = def->args_ct[nb_oargs + k].sort_index; 3733 arg = op->args[i]; 3734 arg_ct = &def->args_ct[i]; 3735 ts = arg_temp(arg); 3736 3737 if (ts->val_type == TEMP_VAL_CONST 3738 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 3739 /* constant is OK for instruction */ 3740 const_args[i] = 1; 3741 new_args[i] = ts->val; 3742 continue; 3743 } 3744 3745 reg = ts->reg; 3746 i_preferred_regs = 0; 3747 i_required_regs = arg_ct->regs; 3748 allocate_new_reg = false; 3749 copyto_new_reg = false; 3750 3751 switch (arg_ct->pair) { 3752 case 0: /* not paired */ 3753 if (arg_ct->ialias) { 3754 i_preferred_regs = op->output_pref[arg_ct->alias_index]; 3755 3756 /* 3757 * If the input is readonly, then it cannot also be an 3758 * output and aliased to itself. If the input is not 3759 * dead after the instruction, we must allocate a new 3760 * register and move it. 3761 */ 3762 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 3763 allocate_new_reg = true; 3764 } else if (ts->val_type == TEMP_VAL_REG) { 3765 /* 3766 * Check if the current register has already been 3767 * allocated for another input. 3768 */ 3769 allocate_new_reg = 3770 tcg_regset_test_reg(i_allocated_regs, reg); 3771 } 3772 } 3773 if (!allocate_new_reg) { 3774 temp_load(s, ts, i_required_regs, i_allocated_regs, 3775 i_preferred_regs); 3776 reg = ts->reg; 3777 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 3778 } 3779 if (allocate_new_reg) { 3780 /* 3781 * Allocate a new register matching the constraint 3782 * and move the temporary register into it. 3783 */ 3784 temp_load(s, ts, tcg_target_available_regs[ts->type], 3785 i_allocated_regs, 0); 3786 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 3787 i_preferred_regs, ts->indirect_base); 3788 copyto_new_reg = true; 3789 } 3790 break; 3791 3792 case 1: 3793 /* First of an input pair; if i1 == i2, the second is an output. */ 3794 i1 = i; 3795 i2 = arg_ct->pair_index; 3796 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 3797 3798 /* 3799 * It is easier to default to allocating a new pair 3800 * and to identify a few cases where it's not required. 3801 */ 3802 if (arg_ct->ialias) { 3803 i_preferred_regs = op->output_pref[arg_ct->alias_index]; 3804 if (IS_DEAD_ARG(i1) && 3805 IS_DEAD_ARG(i2) && 3806 !temp_readonly(ts) && 3807 ts->val_type == TEMP_VAL_REG && 3808 ts->reg < TCG_TARGET_NB_REGS - 1 && 3809 tcg_regset_test_reg(i_required_regs, reg) && 3810 !tcg_regset_test_reg(i_allocated_regs, reg) && 3811 !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 3812 (ts2 3813 ? ts2->val_type == TEMP_VAL_REG && 3814 ts2->reg == reg + 1 && 3815 !temp_readonly(ts2) 3816 : s->reg_to_temp[reg + 1] == NULL)) { 3817 break; 3818 } 3819 } else { 3820 /* Without aliasing, the pair must also be an input. */ 3821 tcg_debug_assert(ts2); 3822 if (ts->val_type == TEMP_VAL_REG && 3823 ts2->val_type == TEMP_VAL_REG && 3824 ts2->reg == reg + 1 && 3825 tcg_regset_test_reg(i_required_regs, reg)) { 3826 break; 3827 } 3828 } 3829 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 3830 0, ts->indirect_base); 3831 goto do_pair; 3832 3833 case 2: /* pair second */ 3834 reg = new_args[arg_ct->pair_index] + 1; 3835 goto do_pair; 3836 3837 case 3: /* ialias with second output, no first input */ 3838 tcg_debug_assert(arg_ct->ialias); 3839 i_preferred_regs = op->output_pref[arg_ct->alias_index]; 3840 3841 if (IS_DEAD_ARG(i) && 3842 !temp_readonly(ts) && 3843 ts->val_type == TEMP_VAL_REG && 3844 reg > 0 && 3845 s->reg_to_temp[reg - 1] == NULL && 3846 tcg_regset_test_reg(i_required_regs, reg) && 3847 !tcg_regset_test_reg(i_allocated_regs, reg) && 3848 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 3849 tcg_regset_set_reg(i_allocated_regs, reg - 1); 3850 break; 3851 } 3852 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 3853 i_allocated_regs, 0, 3854 ts->indirect_base); 3855 tcg_regset_set_reg(i_allocated_regs, reg); 3856 reg += 1; 3857 goto do_pair; 3858 3859 do_pair: 3860 /* 3861 * If an aliased input is not dead after the instruction, 3862 * we must allocate a new register and move it. 3863 */ 3864 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 3865 TCGRegSet t_allocated_regs = i_allocated_regs; 3866 3867 /* 3868 * Because of the alias, and the continued life, make sure 3869 * that the temp is somewhere *other* than the reg pair, 3870 * and we get a copy in reg. 3871 */ 3872 tcg_regset_set_reg(t_allocated_regs, reg); 3873 tcg_regset_set_reg(t_allocated_regs, reg + 1); 3874 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 3875 /* If ts was already in reg, copy it somewhere else. */ 3876 TCGReg nr; 3877 bool ok; 3878 3879 tcg_debug_assert(ts->kind != TEMP_FIXED); 3880 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 3881 t_allocated_regs, 0, ts->indirect_base); 3882 ok = tcg_out_mov(s, ts->type, nr, reg); 3883 tcg_debug_assert(ok); 3884 3885 set_temp_val_reg(s, ts, nr); 3886 } else { 3887 temp_load(s, ts, tcg_target_available_regs[ts->type], 3888 t_allocated_regs, 0); 3889 copyto_new_reg = true; 3890 } 3891 } else { 3892 /* Preferably allocate to reg, otherwise copy. */ 3893 i_required_regs = (TCGRegSet)1 << reg; 3894 temp_load(s, ts, i_required_regs, i_allocated_regs, 3895 i_preferred_regs); 3896 copyto_new_reg = ts->reg != reg; 3897 } 3898 break; 3899 3900 default: 3901 g_assert_not_reached(); 3902 } 3903 3904 if (copyto_new_reg) { 3905 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 3906 /* 3907 * Cross register class move not supported. Sync the 3908 * temp back to its slot and load from there. 3909 */ 3910 temp_sync(s, ts, i_allocated_regs, 0, 0); 3911 tcg_out_ld(s, ts->type, reg, 3912 ts->mem_base->reg, ts->mem_offset); 3913 } 3914 } 3915 new_args[i] = reg; 3916 const_args[i] = 0; 3917 tcg_regset_set_reg(i_allocated_regs, reg); 3918 } 3919 3920 /* mark dead temporaries and free the associated registers */ 3921 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3922 if (IS_DEAD_ARG(i)) { 3923 temp_dead(s, arg_temp(op->args[i])); 3924 } 3925 } 3926 3927 if (def->flags & TCG_OPF_COND_BRANCH) { 3928 tcg_reg_alloc_cbranch(s, i_allocated_regs); 3929 } else if (def->flags & TCG_OPF_BB_END) { 3930 tcg_reg_alloc_bb_end(s, i_allocated_regs); 3931 } else { 3932 if (def->flags & TCG_OPF_CALL_CLOBBER) { 3933 /* XXX: permit generic clobber register list ? */ 3934 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3935 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3936 tcg_reg_free(s, i, i_allocated_regs); 3937 } 3938 } 3939 } 3940 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3941 /* sync globals if the op has side effects and might trigger 3942 an exception. */ 3943 sync_globals(s, i_allocated_regs); 3944 } 3945 3946 /* satisfy the output constraints */ 3947 for(k = 0; k < nb_oargs; k++) { 3948 i = def->args_ct[k].sort_index; 3949 arg = op->args[i]; 3950 arg_ct = &def->args_ct[i]; 3951 ts = arg_temp(arg); 3952 3953 /* ENV should not be modified. */ 3954 tcg_debug_assert(!temp_readonly(ts)); 3955 3956 switch (arg_ct->pair) { 3957 case 0: /* not paired */ 3958 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 3959 reg = new_args[arg_ct->alias_index]; 3960 } else if (arg_ct->newreg) { 3961 reg = tcg_reg_alloc(s, arg_ct->regs, 3962 i_allocated_regs | o_allocated_regs, 3963 op->output_pref[k], ts->indirect_base); 3964 } else { 3965 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 3966 op->output_pref[k], ts->indirect_base); 3967 } 3968 break; 3969 3970 case 1: /* first of pair */ 3971 tcg_debug_assert(!arg_ct->newreg); 3972 if (arg_ct->oalias) { 3973 reg = new_args[arg_ct->alias_index]; 3974 break; 3975 } 3976 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 3977 op->output_pref[k], ts->indirect_base); 3978 break; 3979 3980 case 2: /* second of pair */ 3981 tcg_debug_assert(!arg_ct->newreg); 3982 if (arg_ct->oalias) { 3983 reg = new_args[arg_ct->alias_index]; 3984 } else { 3985 reg = new_args[arg_ct->pair_index] + 1; 3986 } 3987 break; 3988 3989 case 3: /* first of pair, aliasing with a second input */ 3990 tcg_debug_assert(!arg_ct->newreg); 3991 reg = new_args[arg_ct->pair_index] - 1; 3992 break; 3993 3994 default: 3995 g_assert_not_reached(); 3996 } 3997 tcg_regset_set_reg(o_allocated_regs, reg); 3998 set_temp_val_reg(s, ts, reg); 3999 ts->mem_coherent = 0; 4000 new_args[i] = reg; 4001 } 4002 } 4003 4004 /* emit instruction */ 4005 if (def->flags & TCG_OPF_VECTOR) { 4006 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 4007 new_args, const_args); 4008 } else { 4009 tcg_out_op(s, op->opc, new_args, const_args); 4010 } 4011 4012 /* move the outputs in the correct register if needed */ 4013 for(i = 0; i < nb_oargs; i++) { 4014 ts = arg_temp(op->args[i]); 4015 4016 /* ENV should not be modified. */ 4017 tcg_debug_assert(!temp_readonly(ts)); 4018 4019 if (NEED_SYNC_ARG(i)) { 4020 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 4021 } else if (IS_DEAD_ARG(i)) { 4022 temp_dead(s, ts); 4023 } 4024 } 4025 } 4026 4027 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 4028 { 4029 const TCGLifeData arg_life = op->life; 4030 TCGTemp *ots, *itsl, *itsh; 4031 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4032 4033 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 4034 tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 4035 tcg_debug_assert(TCGOP_VECE(op) == MO_64); 4036 4037 ots = arg_temp(op->args[0]); 4038 itsl = arg_temp(op->args[1]); 4039 itsh = arg_temp(op->args[2]); 4040 4041 /* ENV should not be modified. */ 4042 tcg_debug_assert(!temp_readonly(ots)); 4043 4044 /* Allocate the output register now. */ 4045 if (ots->val_type != TEMP_VAL_REG) { 4046 TCGRegSet allocated_regs = s->reserved_regs; 4047 TCGRegSet dup_out_regs = 4048 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 4049 TCGReg oreg; 4050 4051 /* Make sure to not spill the input registers. */ 4052 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 4053 tcg_regset_set_reg(allocated_regs, itsl->reg); 4054 } 4055 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 4056 tcg_regset_set_reg(allocated_regs, itsh->reg); 4057 } 4058 4059 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 4060 op->output_pref[0], ots->indirect_base); 4061 set_temp_val_reg(s, ots, oreg); 4062 } 4063 4064 /* Promote dup2 of immediates to dupi_vec. */ 4065 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 4066 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 4067 MemOp vece = MO_64; 4068 4069 if (val == dup_const(MO_8, val)) { 4070 vece = MO_8; 4071 } else if (val == dup_const(MO_16, val)) { 4072 vece = MO_16; 4073 } else if (val == dup_const(MO_32, val)) { 4074 vece = MO_32; 4075 } 4076 4077 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 4078 goto done; 4079 } 4080 4081 /* If the two inputs form one 64-bit value, try dupm_vec. */ 4082 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) { 4083 if (!itsl->mem_coherent) { 4084 temp_sync(s, itsl, s->reserved_regs, 0, 0); 4085 } 4086 if (!itsh->mem_coherent) { 4087 temp_sync(s, itsh, s->reserved_regs, 0, 0); 4088 } 4089 #if HOST_BIG_ENDIAN 4090 TCGTemp *its = itsh; 4091 #else 4092 TCGTemp *its = itsl; 4093 #endif 4094 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 4095 its->mem_base->reg, its->mem_offset)) { 4096 goto done; 4097 } 4098 } 4099 4100 /* Fall back to generic expansion. */ 4101 return false; 4102 4103 done: 4104 ots->mem_coherent = 0; 4105 if (IS_DEAD_ARG(1)) { 4106 temp_dead(s, itsl); 4107 } 4108 if (IS_DEAD_ARG(2)) { 4109 temp_dead(s, itsh); 4110 } 4111 if (NEED_SYNC_ARG(0)) { 4112 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 4113 } else if (IS_DEAD_ARG(0)) { 4114 temp_dead(s, ots); 4115 } 4116 return true; 4117 } 4118 4119 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 4120 { 4121 const int nb_oargs = TCGOP_CALLO(op); 4122 const int nb_iargs = TCGOP_CALLI(op); 4123 const TCGLifeData arg_life = op->life; 4124 const TCGHelperInfo *info; 4125 int flags, nb_regs, i; 4126 TCGReg reg; 4127 TCGArg arg; 4128 TCGTemp *ts; 4129 intptr_t stack_offset; 4130 size_t call_stack_size; 4131 tcg_insn_unit *func_addr; 4132 int allocate_args; 4133 TCGRegSet allocated_regs; 4134 4135 func_addr = tcg_call_func(op); 4136 info = tcg_call_info(op); 4137 flags = info->flags; 4138 4139 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 4140 if (nb_regs > nb_iargs) { 4141 nb_regs = nb_iargs; 4142 } 4143 4144 /* assign stack slots first */ 4145 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 4146 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 4147 ~(TCG_TARGET_STACK_ALIGN - 1); 4148 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 4149 if (allocate_args) { 4150 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 4151 preallocate call stack */ 4152 tcg_abort(); 4153 } 4154 4155 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 4156 for (i = nb_regs; i < nb_iargs; i++) { 4157 arg = op->args[nb_oargs + i]; 4158 if (arg != TCG_CALL_DUMMY_ARG) { 4159 ts = arg_temp(arg); 4160 temp_load(s, ts, tcg_target_available_regs[ts->type], 4161 s->reserved_regs, 0); 4162 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 4163 } 4164 stack_offset += sizeof(tcg_target_long); 4165 } 4166 4167 /* assign input registers */ 4168 allocated_regs = s->reserved_regs; 4169 for (i = 0; i < nb_regs; i++) { 4170 arg = op->args[nb_oargs + i]; 4171 if (arg != TCG_CALL_DUMMY_ARG) { 4172 ts = arg_temp(arg); 4173 reg = tcg_target_call_iarg_regs[i]; 4174 4175 if (ts->val_type == TEMP_VAL_REG) { 4176 if (ts->reg != reg) { 4177 tcg_reg_free(s, reg, allocated_regs); 4178 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4179 /* 4180 * Cross register class move not supported. Sync the 4181 * temp back to its slot and load from there. 4182 */ 4183 temp_sync(s, ts, allocated_regs, 0, 0); 4184 tcg_out_ld(s, ts->type, reg, 4185 ts->mem_base->reg, ts->mem_offset); 4186 } 4187 } 4188 } else { 4189 TCGRegSet arg_set = 0; 4190 4191 tcg_reg_free(s, reg, allocated_regs); 4192 tcg_regset_set_reg(arg_set, reg); 4193 temp_load(s, ts, arg_set, allocated_regs, 0); 4194 } 4195 4196 tcg_regset_set_reg(allocated_regs, reg); 4197 } 4198 } 4199 4200 /* mark dead temporaries and free the associated registers */ 4201 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4202 if (IS_DEAD_ARG(i)) { 4203 temp_dead(s, arg_temp(op->args[i])); 4204 } 4205 } 4206 4207 /* clobber call registers */ 4208 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4209 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4210 tcg_reg_free(s, i, allocated_regs); 4211 } 4212 } 4213 4214 /* Save globals if they might be written by the helper, sync them if 4215 they might be read. */ 4216 if (flags & TCG_CALL_NO_READ_GLOBALS) { 4217 /* Nothing to do */ 4218 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 4219 sync_globals(s, allocated_regs); 4220 } else { 4221 save_globals(s, allocated_regs); 4222 } 4223 4224 #ifdef CONFIG_TCG_INTERPRETER 4225 { 4226 gpointer hash = (gpointer)(uintptr_t)info->typemask; 4227 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash); 4228 assert(cif != NULL); 4229 tcg_out_call(s, func_addr, cif); 4230 } 4231 #else 4232 tcg_out_call(s, func_addr); 4233 #endif 4234 4235 /* assign output registers and emit moves if needed */ 4236 for(i = 0; i < nb_oargs; i++) { 4237 arg = op->args[i]; 4238 ts = arg_temp(arg); 4239 4240 /* ENV should not be modified. */ 4241 tcg_debug_assert(!temp_readonly(ts)); 4242 4243 reg = tcg_target_call_oarg_regs[i]; 4244 set_temp_val_reg(s, ts, reg); 4245 ts->mem_coherent = 0; 4246 if (NEED_SYNC_ARG(i)) { 4247 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i)); 4248 } else if (IS_DEAD_ARG(i)) { 4249 temp_dead(s, ts); 4250 } 4251 } 4252 } 4253 4254 #ifdef CONFIG_PROFILER 4255 4256 /* avoid copy/paste errors */ 4257 #define PROF_ADD(to, from, field) \ 4258 do { \ 4259 (to)->field += qatomic_read(&((from)->field)); \ 4260 } while (0) 4261 4262 #define PROF_MAX(to, from, field) \ 4263 do { \ 4264 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4265 if (val__ > (to)->field) { \ 4266 (to)->field = val__; \ 4267 } \ 4268 } while (0) 4269 4270 /* Pass in a zero'ed @prof */ 4271 static inline 4272 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4273 { 4274 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4275 unsigned int i; 4276 4277 for (i = 0; i < n_ctxs; i++) { 4278 TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4279 const TCGProfile *orig = &s->prof; 4280 4281 if (counters) { 4282 PROF_ADD(prof, orig, cpu_exec_time); 4283 PROF_ADD(prof, orig, tb_count1); 4284 PROF_ADD(prof, orig, tb_count); 4285 PROF_ADD(prof, orig, op_count); 4286 PROF_MAX(prof, orig, op_count_max); 4287 PROF_ADD(prof, orig, temp_count); 4288 PROF_MAX(prof, orig, temp_count_max); 4289 PROF_ADD(prof, orig, del_op_count); 4290 PROF_ADD(prof, orig, code_in_len); 4291 PROF_ADD(prof, orig, code_out_len); 4292 PROF_ADD(prof, orig, search_out_len); 4293 PROF_ADD(prof, orig, interm_time); 4294 PROF_ADD(prof, orig, code_time); 4295 PROF_ADD(prof, orig, la_time); 4296 PROF_ADD(prof, orig, opt_time); 4297 PROF_ADD(prof, orig, restore_count); 4298 PROF_ADD(prof, orig, restore_time); 4299 } 4300 if (table) { 4301 int i; 4302 4303 for (i = 0; i < NB_OPS; i++) { 4304 PROF_ADD(prof, orig, table_op_count[i]); 4305 } 4306 } 4307 } 4308 } 4309 4310 #undef PROF_ADD 4311 #undef PROF_MAX 4312 4313 static void tcg_profile_snapshot_counters(TCGProfile *prof) 4314 { 4315 tcg_profile_snapshot(prof, true, false); 4316 } 4317 4318 static void tcg_profile_snapshot_table(TCGProfile *prof) 4319 { 4320 tcg_profile_snapshot(prof, false, true); 4321 } 4322 4323 void tcg_dump_op_count(GString *buf) 4324 { 4325 TCGProfile prof = {}; 4326 int i; 4327 4328 tcg_profile_snapshot_table(&prof); 4329 for (i = 0; i < NB_OPS; i++) { 4330 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name, 4331 prof.table_op_count[i]); 4332 } 4333 } 4334 4335 int64_t tcg_cpu_exec_time(void) 4336 { 4337 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4338 unsigned int i; 4339 int64_t ret = 0; 4340 4341 for (i = 0; i < n_ctxs; i++) { 4342 const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4343 const TCGProfile *prof = &s->prof; 4344 4345 ret += qatomic_read(&prof->cpu_exec_time); 4346 } 4347 return ret; 4348 } 4349 #else 4350 void tcg_dump_op_count(GString *buf) 4351 { 4352 g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 4353 } 4354 4355 int64_t tcg_cpu_exec_time(void) 4356 { 4357 error_report("%s: TCG profiler not compiled", __func__); 4358 exit(EXIT_FAILURE); 4359 } 4360 #endif 4361 4362 4363 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) 4364 { 4365 #ifdef CONFIG_PROFILER 4366 TCGProfile *prof = &s->prof; 4367 #endif 4368 int i, num_insns; 4369 TCGOp *op; 4370 4371 #ifdef CONFIG_PROFILER 4372 { 4373 int n = 0; 4374 4375 QTAILQ_FOREACH(op, &s->ops, link) { 4376 n++; 4377 } 4378 qatomic_set(&prof->op_count, prof->op_count + n); 4379 if (n > prof->op_count_max) { 4380 qatomic_set(&prof->op_count_max, n); 4381 } 4382 4383 n = s->nb_temps; 4384 qatomic_set(&prof->temp_count, prof->temp_count + n); 4385 if (n > prof->temp_count_max) { 4386 qatomic_set(&prof->temp_count_max, n); 4387 } 4388 } 4389 #endif 4390 4391 #ifdef DEBUG_DISAS 4392 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4393 && qemu_log_in_addr_range(pc_start))) { 4394 FILE *logfile = qemu_log_trylock(); 4395 if (logfile) { 4396 fprintf(logfile, "OP:\n"); 4397 tcg_dump_ops(s, logfile, false); 4398 fprintf(logfile, "\n"); 4399 qemu_log_unlock(logfile); 4400 } 4401 } 4402 #endif 4403 4404 #ifdef CONFIG_DEBUG_TCG 4405 /* Ensure all labels referenced have been emitted. */ 4406 { 4407 TCGLabel *l; 4408 bool error = false; 4409 4410 QSIMPLEQ_FOREACH(l, &s->labels, next) { 4411 if (unlikely(!l->present) && l->refs) { 4412 qemu_log_mask(CPU_LOG_TB_OP, 4413 "$L%d referenced but not present.\n", l->id); 4414 error = true; 4415 } 4416 } 4417 assert(!error); 4418 } 4419 #endif 4420 4421 #ifdef CONFIG_PROFILER 4422 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 4423 #endif 4424 4425 #ifdef USE_TCG_OPTIMIZATIONS 4426 tcg_optimize(s); 4427 #endif 4428 4429 #ifdef CONFIG_PROFILER 4430 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 4431 qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 4432 #endif 4433 4434 reachable_code_pass(s); 4435 liveness_pass_1(s); 4436 4437 if (s->nb_indirects > 0) { 4438 #ifdef DEBUG_DISAS 4439 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 4440 && qemu_log_in_addr_range(pc_start))) { 4441 FILE *logfile = qemu_log_trylock(); 4442 if (logfile) { 4443 fprintf(logfile, "OP before indirect lowering:\n"); 4444 tcg_dump_ops(s, logfile, false); 4445 fprintf(logfile, "\n"); 4446 qemu_log_unlock(logfile); 4447 } 4448 } 4449 #endif 4450 /* Replace indirect temps with direct temps. */ 4451 if (liveness_pass_2(s)) { 4452 /* If changes were made, re-run liveness. */ 4453 liveness_pass_1(s); 4454 } 4455 } 4456 4457 #ifdef CONFIG_PROFILER 4458 qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 4459 #endif 4460 4461 #ifdef DEBUG_DISAS 4462 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 4463 && qemu_log_in_addr_range(pc_start))) { 4464 FILE *logfile = qemu_log_trylock(); 4465 if (logfile) { 4466 fprintf(logfile, "OP after optimization and liveness analysis:\n"); 4467 tcg_dump_ops(s, logfile, true); 4468 fprintf(logfile, "\n"); 4469 qemu_log_unlock(logfile); 4470 } 4471 } 4472 #endif 4473 4474 /* Initialize goto_tb jump offsets. */ 4475 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 4476 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 4477 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 4478 if (TCG_TARGET_HAS_direct_jump) { 4479 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 4480 tcg_ctx->tb_jmp_target_addr = NULL; 4481 } else { 4482 tcg_ctx->tb_jmp_insn_offset = NULL; 4483 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 4484 } 4485 4486 tcg_reg_alloc_start(s); 4487 4488 /* 4489 * Reset the buffer pointers when restarting after overflow. 4490 * TODO: Move this into translate-all.c with the rest of the 4491 * buffer management. Having only this done here is confusing. 4492 */ 4493 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 4494 s->code_ptr = s->code_buf; 4495 4496 #ifdef TCG_TARGET_NEED_LDST_LABELS 4497 QSIMPLEQ_INIT(&s->ldst_labels); 4498 #endif 4499 #ifdef TCG_TARGET_NEED_POOL_LABELS 4500 s->pool_labels = NULL; 4501 #endif 4502 4503 num_insns = -1; 4504 QTAILQ_FOREACH(op, &s->ops, link) { 4505 TCGOpcode opc = op->opc; 4506 4507 #ifdef CONFIG_PROFILER 4508 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 4509 #endif 4510 4511 switch (opc) { 4512 case INDEX_op_mov_i32: 4513 case INDEX_op_mov_i64: 4514 case INDEX_op_mov_vec: 4515 tcg_reg_alloc_mov(s, op); 4516 break; 4517 case INDEX_op_dup_vec: 4518 tcg_reg_alloc_dup(s, op); 4519 break; 4520 case INDEX_op_insn_start: 4521 if (num_insns >= 0) { 4522 size_t off = tcg_current_code_size(s); 4523 s->gen_insn_end_off[num_insns] = off; 4524 /* Assert that we do not overflow our stored offset. */ 4525 assert(s->gen_insn_end_off[num_insns] == off); 4526 } 4527 num_insns++; 4528 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 4529 target_ulong a; 4530 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 4531 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 4532 #else 4533 a = op->args[i]; 4534 #endif 4535 s->gen_insn_data[num_insns][i] = a; 4536 } 4537 break; 4538 case INDEX_op_discard: 4539 temp_dead(s, arg_temp(op->args[0])); 4540 break; 4541 case INDEX_op_set_label: 4542 tcg_reg_alloc_bb_end(s, s->reserved_regs); 4543 tcg_out_label(s, arg_label(op->args[0])); 4544 break; 4545 case INDEX_op_call: 4546 tcg_reg_alloc_call(s, op); 4547 break; 4548 case INDEX_op_dup2_vec: 4549 if (tcg_reg_alloc_dup2(s, op)) { 4550 break; 4551 } 4552 /* fall through */ 4553 default: 4554 /* Sanity check that we've not introduced any unhandled opcodes. */ 4555 tcg_debug_assert(tcg_op_supported(opc)); 4556 /* Note: in order to speed up the code, it would be much 4557 faster to have specialized register allocator functions for 4558 some common argument patterns */ 4559 tcg_reg_alloc_op(s, op); 4560 break; 4561 } 4562 /* Test for (pending) buffer overflow. The assumption is that any 4563 one operation beginning below the high water mark cannot overrun 4564 the buffer completely. Thus we can test for overflow after 4565 generating code without having to check during generation. */ 4566 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 4567 return -1; 4568 } 4569 /* Test for TB overflow, as seen by gen_insn_end_off. */ 4570 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 4571 return -2; 4572 } 4573 } 4574 tcg_debug_assert(num_insns >= 0); 4575 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 4576 4577 /* Generate TB finalization at the end of block */ 4578 #ifdef TCG_TARGET_NEED_LDST_LABELS 4579 i = tcg_out_ldst_finalize(s); 4580 if (i < 0) { 4581 return i; 4582 } 4583 #endif 4584 #ifdef TCG_TARGET_NEED_POOL_LABELS 4585 i = tcg_out_pool_finalize(s); 4586 if (i < 0) { 4587 return i; 4588 } 4589 #endif 4590 if (!tcg_resolve_relocs(s)) { 4591 return -2; 4592 } 4593 4594 #ifndef CONFIG_TCG_INTERPRETER 4595 /* flush instruction cache */ 4596 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 4597 (uintptr_t)s->code_buf, 4598 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 4599 #endif 4600 4601 return tcg_current_code_size(s); 4602 } 4603 4604 #ifdef CONFIG_PROFILER 4605 void tcg_dump_info(GString *buf) 4606 { 4607 TCGProfile prof = {}; 4608 const TCGProfile *s; 4609 int64_t tb_count; 4610 int64_t tb_div_count; 4611 int64_t tot; 4612 4613 tcg_profile_snapshot_counters(&prof); 4614 s = &prof; 4615 tb_count = s->tb_count; 4616 tb_div_count = tb_count ? tb_count : 1; 4617 tot = s->interm_time + s->code_time; 4618 4619 g_string_append_printf(buf, "JIT cycles %" PRId64 4620 " (%0.3f s at 2.4 GHz)\n", 4621 tot, tot / 2.4e9); 4622 g_string_append_printf(buf, "translated TBs %" PRId64 4623 " (aborted=%" PRId64 " %0.1f%%)\n", 4624 tb_count, s->tb_count1 - tb_count, 4625 (double)(s->tb_count1 - s->tb_count) 4626 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 4627 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n", 4628 (double)s->op_count / tb_div_count, s->op_count_max); 4629 g_string_append_printf(buf, "deleted ops/TB %0.2f\n", 4630 (double)s->del_op_count / tb_div_count); 4631 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n", 4632 (double)s->temp_count / tb_div_count, 4633 s->temp_count_max); 4634 g_string_append_printf(buf, "avg host code/TB %0.1f\n", 4635 (double)s->code_out_len / tb_div_count); 4636 g_string_append_printf(buf, "avg search data/TB %0.1f\n", 4637 (double)s->search_out_len / tb_div_count); 4638 4639 g_string_append_printf(buf, "cycles/op %0.1f\n", 4640 s->op_count ? (double)tot / s->op_count : 0); 4641 g_string_append_printf(buf, "cycles/in byte %0.1f\n", 4642 s->code_in_len ? (double)tot / s->code_in_len : 0); 4643 g_string_append_printf(buf, "cycles/out byte %0.1f\n", 4644 s->code_out_len ? (double)tot / s->code_out_len : 0); 4645 g_string_append_printf(buf, "cycles/search byte %0.1f\n", 4646 s->search_out_len ? 4647 (double)tot / s->search_out_len : 0); 4648 if (tot == 0) { 4649 tot = 1; 4650 } 4651 g_string_append_printf(buf, " gen_interm time %0.1f%%\n", 4652 (double)s->interm_time / tot * 100.0); 4653 g_string_append_printf(buf, " gen_code time %0.1f%%\n", 4654 (double)s->code_time / tot * 100.0); 4655 g_string_append_printf(buf, "optim./code time %0.1f%%\n", 4656 (double)s->opt_time / (s->code_time ? 4657 s->code_time : 1) 4658 * 100.0); 4659 g_string_append_printf(buf, "liveness/code time %0.1f%%\n", 4660 (double)s->la_time / (s->code_time ? 4661 s->code_time : 1) * 100.0); 4662 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n", 4663 s->restore_count); 4664 g_string_append_printf(buf, " avg cycles %0.1f\n", 4665 s->restore_count ? 4666 (double)s->restore_time / s->restore_count : 0); 4667 } 4668 #else 4669 void tcg_dump_info(GString *buf) 4670 { 4671 g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 4672 } 4673 #endif 4674 4675 #ifdef ELF_HOST_MACHINE 4676 /* In order to use this feature, the backend needs to do three things: 4677 4678 (1) Define ELF_HOST_MACHINE to indicate both what value to 4679 put into the ELF image and to indicate support for the feature. 4680 4681 (2) Define tcg_register_jit. This should create a buffer containing 4682 the contents of a .debug_frame section that describes the post- 4683 prologue unwind info for the tcg machine. 4684 4685 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 4686 */ 4687 4688 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 4689 typedef enum { 4690 JIT_NOACTION = 0, 4691 JIT_REGISTER_FN, 4692 JIT_UNREGISTER_FN 4693 } jit_actions_t; 4694 4695 struct jit_code_entry { 4696 struct jit_code_entry *next_entry; 4697 struct jit_code_entry *prev_entry; 4698 const void *symfile_addr; 4699 uint64_t symfile_size; 4700 }; 4701 4702 struct jit_descriptor { 4703 uint32_t version; 4704 uint32_t action_flag; 4705 struct jit_code_entry *relevant_entry; 4706 struct jit_code_entry *first_entry; 4707 }; 4708 4709 void __jit_debug_register_code(void) __attribute__((noinline)); 4710 void __jit_debug_register_code(void) 4711 { 4712 asm(""); 4713 } 4714 4715 /* Must statically initialize the version, because GDB may check 4716 the version before we can set it. */ 4717 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 4718 4719 /* End GDB interface. */ 4720 4721 static int find_string(const char *strtab, const char *str) 4722 { 4723 const char *p = strtab + 1; 4724 4725 while (1) { 4726 if (strcmp(p, str) == 0) { 4727 return p - strtab; 4728 } 4729 p += strlen(p) + 1; 4730 } 4731 } 4732 4733 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 4734 const void *debug_frame, 4735 size_t debug_frame_size) 4736 { 4737 struct __attribute__((packed)) DebugInfo { 4738 uint32_t len; 4739 uint16_t version; 4740 uint32_t abbrev; 4741 uint8_t ptr_size; 4742 uint8_t cu_die; 4743 uint16_t cu_lang; 4744 uintptr_t cu_low_pc; 4745 uintptr_t cu_high_pc; 4746 uint8_t fn_die; 4747 char fn_name[16]; 4748 uintptr_t fn_low_pc; 4749 uintptr_t fn_high_pc; 4750 uint8_t cu_eoc; 4751 }; 4752 4753 struct ElfImage { 4754 ElfW(Ehdr) ehdr; 4755 ElfW(Phdr) phdr; 4756 ElfW(Shdr) shdr[7]; 4757 ElfW(Sym) sym[2]; 4758 struct DebugInfo di; 4759 uint8_t da[24]; 4760 char str[80]; 4761 }; 4762 4763 struct ElfImage *img; 4764 4765 static const struct ElfImage img_template = { 4766 .ehdr = { 4767 .e_ident[EI_MAG0] = ELFMAG0, 4768 .e_ident[EI_MAG1] = ELFMAG1, 4769 .e_ident[EI_MAG2] = ELFMAG2, 4770 .e_ident[EI_MAG3] = ELFMAG3, 4771 .e_ident[EI_CLASS] = ELF_CLASS, 4772 .e_ident[EI_DATA] = ELF_DATA, 4773 .e_ident[EI_VERSION] = EV_CURRENT, 4774 .e_type = ET_EXEC, 4775 .e_machine = ELF_HOST_MACHINE, 4776 .e_version = EV_CURRENT, 4777 .e_phoff = offsetof(struct ElfImage, phdr), 4778 .e_shoff = offsetof(struct ElfImage, shdr), 4779 .e_ehsize = sizeof(ElfW(Shdr)), 4780 .e_phentsize = sizeof(ElfW(Phdr)), 4781 .e_phnum = 1, 4782 .e_shentsize = sizeof(ElfW(Shdr)), 4783 .e_shnum = ARRAY_SIZE(img->shdr), 4784 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 4785 #ifdef ELF_HOST_FLAGS 4786 .e_flags = ELF_HOST_FLAGS, 4787 #endif 4788 #ifdef ELF_OSABI 4789 .e_ident[EI_OSABI] = ELF_OSABI, 4790 #endif 4791 }, 4792 .phdr = { 4793 .p_type = PT_LOAD, 4794 .p_flags = PF_X, 4795 }, 4796 .shdr = { 4797 [0] = { .sh_type = SHT_NULL }, 4798 /* Trick: The contents of code_gen_buffer are not present in 4799 this fake ELF file; that got allocated elsewhere. Therefore 4800 we mark .text as SHT_NOBITS (similar to .bss) so that readers 4801 will not look for contents. We can record any address. */ 4802 [1] = { /* .text */ 4803 .sh_type = SHT_NOBITS, 4804 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 4805 }, 4806 [2] = { /* .debug_info */ 4807 .sh_type = SHT_PROGBITS, 4808 .sh_offset = offsetof(struct ElfImage, di), 4809 .sh_size = sizeof(struct DebugInfo), 4810 }, 4811 [3] = { /* .debug_abbrev */ 4812 .sh_type = SHT_PROGBITS, 4813 .sh_offset = offsetof(struct ElfImage, da), 4814 .sh_size = sizeof(img->da), 4815 }, 4816 [4] = { /* .debug_frame */ 4817 .sh_type = SHT_PROGBITS, 4818 .sh_offset = sizeof(struct ElfImage), 4819 }, 4820 [5] = { /* .symtab */ 4821 .sh_type = SHT_SYMTAB, 4822 .sh_offset = offsetof(struct ElfImage, sym), 4823 .sh_size = sizeof(img->sym), 4824 .sh_info = 1, 4825 .sh_link = ARRAY_SIZE(img->shdr) - 1, 4826 .sh_entsize = sizeof(ElfW(Sym)), 4827 }, 4828 [6] = { /* .strtab */ 4829 .sh_type = SHT_STRTAB, 4830 .sh_offset = offsetof(struct ElfImage, str), 4831 .sh_size = sizeof(img->str), 4832 } 4833 }, 4834 .sym = { 4835 [1] = { /* code_gen_buffer */ 4836 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 4837 .st_shndx = 1, 4838 } 4839 }, 4840 .di = { 4841 .len = sizeof(struct DebugInfo) - 4, 4842 .version = 2, 4843 .ptr_size = sizeof(void *), 4844 .cu_die = 1, 4845 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 4846 .fn_die = 2, 4847 .fn_name = "code_gen_buffer" 4848 }, 4849 .da = { 4850 1, /* abbrev number (the cu) */ 4851 0x11, 1, /* DW_TAG_compile_unit, has children */ 4852 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 4853 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 4854 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 4855 0, 0, /* end of abbrev */ 4856 2, /* abbrev number (the fn) */ 4857 0x2e, 0, /* DW_TAG_subprogram, no children */ 4858 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 4859 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 4860 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 4861 0, 0, /* end of abbrev */ 4862 0 /* no more abbrev */ 4863 }, 4864 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 4865 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 4866 }; 4867 4868 /* We only need a single jit entry; statically allocate it. */ 4869 static struct jit_code_entry one_entry; 4870 4871 uintptr_t buf = (uintptr_t)buf_ptr; 4872 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 4873 DebugFrameHeader *dfh; 4874 4875 img = g_malloc(img_size); 4876 *img = img_template; 4877 4878 img->phdr.p_vaddr = buf; 4879 img->phdr.p_paddr = buf; 4880 img->phdr.p_memsz = buf_size; 4881 4882 img->shdr[1].sh_name = find_string(img->str, ".text"); 4883 img->shdr[1].sh_addr = buf; 4884 img->shdr[1].sh_size = buf_size; 4885 4886 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 4887 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 4888 4889 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 4890 img->shdr[4].sh_size = debug_frame_size; 4891 4892 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 4893 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 4894 4895 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 4896 img->sym[1].st_value = buf; 4897 img->sym[1].st_size = buf_size; 4898 4899 img->di.cu_low_pc = buf; 4900 img->di.cu_high_pc = buf + buf_size; 4901 img->di.fn_low_pc = buf; 4902 img->di.fn_high_pc = buf + buf_size; 4903 4904 dfh = (DebugFrameHeader *)(img + 1); 4905 memcpy(dfh, debug_frame, debug_frame_size); 4906 dfh->fde.func_start = buf; 4907 dfh->fde.func_len = buf_size; 4908 4909 #ifdef DEBUG_JIT 4910 /* Enable this block to be able to debug the ELF image file creation. 4911 One can use readelf, objdump, or other inspection utilities. */ 4912 { 4913 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 4914 FILE *f = fopen(jit, "w+b"); 4915 if (f) { 4916 if (fwrite(img, img_size, 1, f) != img_size) { 4917 /* Avoid stupid unused return value warning for fwrite. */ 4918 } 4919 fclose(f); 4920 } 4921 } 4922 #endif 4923 4924 one_entry.symfile_addr = img; 4925 one_entry.symfile_size = img_size; 4926 4927 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 4928 __jit_debug_descriptor.relevant_entry = &one_entry; 4929 __jit_debug_descriptor.first_entry = &one_entry; 4930 __jit_debug_register_code(); 4931 } 4932 #else 4933 /* No support for the feature. Provide the entry point expected by exec.c, 4934 and implement the internal function we declared earlier. */ 4935 4936 static void tcg_register_jit_int(const void *buf, size_t size, 4937 const void *debug_frame, 4938 size_t debug_frame_size) 4939 { 4940 } 4941 4942 void tcg_register_jit(const void *buf, size_t buf_size) 4943 { 4944 } 4945 #endif /* ELF_HOST_MACHINE */ 4946 4947 #if !TCG_TARGET_MAYBE_vec 4948 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 4949 { 4950 g_assert_not_reached(); 4951 } 4952 #endif 4953