1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/error-report.h" 34 #include "qemu/cutils.h" 35 #include "qemu/host-utils.h" 36 #include "qemu/qemu-print.h" 37 #include "qemu/timer.h" 38 #include "qemu/cacheflush.h" 39 #include "qemu/cacheinfo.h" 40 41 /* Note: the long term plan is to reduce the dependencies on the QEMU 42 CPU definitions. Currently they are used for qemu_ld/st 43 instructions */ 44 #define NO_CPU_IO_DEFS 45 46 #include "exec/exec-all.h" 47 #include "tcg/tcg-op.h" 48 49 #if UINTPTR_MAX == UINT32_MAX 50 # define ELF_CLASS ELFCLASS32 51 #else 52 # define ELF_CLASS ELFCLASS64 53 #endif 54 #if HOST_BIG_ENDIAN 55 # define ELF_DATA ELFDATA2MSB 56 #else 57 # define ELF_DATA ELFDATA2LSB 58 #endif 59 60 #include "elf.h" 61 #include "exec/log.h" 62 #include "tcg/tcg-ldst.h" 63 #include "tcg-internal.h" 64 65 /* Forward declarations for functions declared in tcg-target.c.inc and 66 used here. */ 67 static void tcg_target_init(TCGContext *s); 68 static void tcg_target_qemu_prologue(TCGContext *s); 69 static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 70 intptr_t value, intptr_t addend); 71 72 /* The CIE and FDE header definitions will be common to all hosts. */ 73 typedef struct { 74 uint32_t len __attribute__((aligned((sizeof(void *))))); 75 uint32_t id; 76 uint8_t version; 77 char augmentation[1]; 78 uint8_t code_align; 79 uint8_t data_align; 80 uint8_t return_column; 81 } DebugFrameCIE; 82 83 typedef struct QEMU_PACKED { 84 uint32_t len __attribute__((aligned((sizeof(void *))))); 85 uint32_t cie_offset; 86 uintptr_t func_start; 87 uintptr_t func_len; 88 } DebugFrameFDEHeader; 89 90 typedef struct QEMU_PACKED { 91 DebugFrameCIE cie; 92 DebugFrameFDEHeader fde; 93 } DebugFrameHeader; 94 95 static void tcg_register_jit_int(const void *buf, size_t size, 96 const void *debug_frame, 97 size_t debug_frame_size) 98 __attribute__((unused)); 99 100 /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 101 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 102 intptr_t arg2); 103 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 104 static void tcg_out_movi(TCGContext *s, TCGType type, 105 TCGReg ret, tcg_target_long arg); 106 static void tcg_out_op(TCGContext *s, TCGOpcode opc, 107 const TCGArg args[TCG_MAX_OP_ARGS], 108 const int const_args[TCG_MAX_OP_ARGS]); 109 #if TCG_TARGET_MAYBE_vec 110 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 111 TCGReg dst, TCGReg src); 112 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 113 TCGReg dst, TCGReg base, intptr_t offset); 114 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 115 TCGReg dst, int64_t arg); 116 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 117 unsigned vecl, unsigned vece, 118 const TCGArg args[TCG_MAX_OP_ARGS], 119 const int const_args[TCG_MAX_OP_ARGS]); 120 #else 121 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 122 TCGReg dst, TCGReg src) 123 { 124 g_assert_not_reached(); 125 } 126 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 127 TCGReg dst, TCGReg base, intptr_t offset) 128 { 129 g_assert_not_reached(); 130 } 131 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 132 TCGReg dst, int64_t arg) 133 { 134 g_assert_not_reached(); 135 } 136 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 137 unsigned vecl, unsigned vece, 138 const TCGArg args[TCG_MAX_OP_ARGS], 139 const int const_args[TCG_MAX_OP_ARGS]) 140 { 141 g_assert_not_reached(); 142 } 143 #endif 144 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 145 intptr_t arg2); 146 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 147 TCGReg base, intptr_t ofs); 148 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 149 const TCGHelperInfo *info); 150 static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 151 #ifdef TCG_TARGET_NEED_LDST_LABELS 152 static int tcg_out_ldst_finalize(TCGContext *s); 153 #endif 154 155 TCGContext tcg_init_ctx; 156 __thread TCGContext *tcg_ctx; 157 158 TCGContext **tcg_ctxs; 159 unsigned int tcg_cur_ctxs; 160 unsigned int tcg_max_ctxs; 161 TCGv_env cpu_env = 0; 162 const void *tcg_code_gen_epilogue; 163 uintptr_t tcg_splitwx_diff; 164 165 #ifndef CONFIG_TCG_INTERPRETER 166 tcg_prologue_fn *tcg_qemu_tb_exec; 167 #endif 168 169 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 170 static TCGRegSet tcg_target_call_clobber_regs; 171 172 #if TCG_TARGET_INSN_UNIT_SIZE == 1 173 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 174 { 175 *s->code_ptr++ = v; 176 } 177 178 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 179 uint8_t v) 180 { 181 *p = v; 182 } 183 #endif 184 185 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 186 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 187 { 188 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 189 *s->code_ptr++ = v; 190 } else { 191 tcg_insn_unit *p = s->code_ptr; 192 memcpy(p, &v, sizeof(v)); 193 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 194 } 195 } 196 197 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 198 uint16_t v) 199 { 200 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 201 *p = v; 202 } else { 203 memcpy(p, &v, sizeof(v)); 204 } 205 } 206 #endif 207 208 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 209 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 210 { 211 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 212 *s->code_ptr++ = v; 213 } else { 214 tcg_insn_unit *p = s->code_ptr; 215 memcpy(p, &v, sizeof(v)); 216 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 217 } 218 } 219 220 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 221 uint32_t v) 222 { 223 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 224 *p = v; 225 } else { 226 memcpy(p, &v, sizeof(v)); 227 } 228 } 229 #endif 230 231 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 232 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 233 { 234 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 235 *s->code_ptr++ = v; 236 } else { 237 tcg_insn_unit *p = s->code_ptr; 238 memcpy(p, &v, sizeof(v)); 239 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 240 } 241 } 242 243 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 244 uint64_t v) 245 { 246 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 247 *p = v; 248 } else { 249 memcpy(p, &v, sizeof(v)); 250 } 251 } 252 #endif 253 254 /* label relocation processing */ 255 256 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 257 TCGLabel *l, intptr_t addend) 258 { 259 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 260 261 r->type = type; 262 r->ptr = code_ptr; 263 r->addend = addend; 264 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 265 } 266 267 static void tcg_out_label(TCGContext *s, TCGLabel *l) 268 { 269 tcg_debug_assert(!l->has_value); 270 l->has_value = 1; 271 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 272 } 273 274 TCGLabel *gen_new_label(void) 275 { 276 TCGContext *s = tcg_ctx; 277 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 278 279 memset(l, 0, sizeof(TCGLabel)); 280 l->id = s->nb_labels++; 281 QSIMPLEQ_INIT(&l->relocs); 282 283 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 284 285 return l; 286 } 287 288 static bool tcg_resolve_relocs(TCGContext *s) 289 { 290 TCGLabel *l; 291 292 QSIMPLEQ_FOREACH(l, &s->labels, next) { 293 TCGRelocation *r; 294 uintptr_t value = l->u.value; 295 296 QSIMPLEQ_FOREACH(r, &l->relocs, next) { 297 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 298 return false; 299 } 300 } 301 } 302 return true; 303 } 304 305 static void set_jmp_reset_offset(TCGContext *s, int which) 306 { 307 /* 308 * We will check for overflow at the end of the opcode loop in 309 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 310 */ 311 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); 312 } 313 314 /* Signal overflow, starting over with fewer guest insns. */ 315 static G_NORETURN 316 void tcg_raise_tb_overflow(TCGContext *s) 317 { 318 siglongjmp(s->jmp_trans, -2); 319 } 320 321 #define C_PFX1(P, A) P##A 322 #define C_PFX2(P, A, B) P##A##_##B 323 #define C_PFX3(P, A, B, C) P##A##_##B##_##C 324 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 325 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 326 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 327 328 /* Define an enumeration for the various combinations. */ 329 330 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 331 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 332 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 333 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 334 335 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 336 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 337 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 338 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 339 340 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 341 342 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 343 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 344 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 345 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 346 347 typedef enum { 348 #include "tcg-target-con-set.h" 349 } TCGConstraintSetIndex; 350 351 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 352 353 #undef C_O0_I1 354 #undef C_O0_I2 355 #undef C_O0_I3 356 #undef C_O0_I4 357 #undef C_O1_I1 358 #undef C_O1_I2 359 #undef C_O1_I3 360 #undef C_O1_I4 361 #undef C_N1_I2 362 #undef C_O2_I1 363 #undef C_O2_I2 364 #undef C_O2_I3 365 #undef C_O2_I4 366 367 /* Put all of the constraint sets into an array, indexed by the enum. */ 368 369 #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 370 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 371 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 372 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 373 374 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 375 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 376 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 377 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 378 379 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 380 381 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 382 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 383 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 384 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 385 386 static const TCGTargetOpDef constraint_sets[] = { 387 #include "tcg-target-con-set.h" 388 }; 389 390 391 #undef C_O0_I1 392 #undef C_O0_I2 393 #undef C_O0_I3 394 #undef C_O0_I4 395 #undef C_O1_I1 396 #undef C_O1_I2 397 #undef C_O1_I3 398 #undef C_O1_I4 399 #undef C_N1_I2 400 #undef C_O2_I1 401 #undef C_O2_I2 402 #undef C_O2_I3 403 #undef C_O2_I4 404 405 /* Expand the enumerator to be returned from tcg_target_op_def(). */ 406 407 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 408 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 409 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 410 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 411 412 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 413 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 414 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 415 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 416 417 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 418 419 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 420 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 421 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 422 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 423 424 #include "tcg-target.c.inc" 425 426 static void alloc_tcg_plugin_context(TCGContext *s) 427 { 428 #ifdef CONFIG_PLUGIN 429 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 430 s->plugin_tb->insns = 431 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 432 #endif 433 } 434 435 /* 436 * All TCG threads except the parent (i.e. the one that called tcg_context_init 437 * and registered the target's TCG globals) must register with this function 438 * before initiating translation. 439 * 440 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 441 * of tcg_region_init() for the reasoning behind this. 442 * 443 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 444 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 445 * is not used anymore for translation once this function is called. 446 * 447 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 448 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 449 */ 450 #ifdef CONFIG_USER_ONLY 451 void tcg_register_thread(void) 452 { 453 tcg_ctx = &tcg_init_ctx; 454 } 455 #else 456 void tcg_register_thread(void) 457 { 458 TCGContext *s = g_malloc(sizeof(*s)); 459 unsigned int i, n; 460 461 *s = tcg_init_ctx; 462 463 /* Relink mem_base. */ 464 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 465 if (tcg_init_ctx.temps[i].mem_base) { 466 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 467 tcg_debug_assert(b >= 0 && b < n); 468 s->temps[i].mem_base = &s->temps[b]; 469 } 470 } 471 472 /* Claim an entry in tcg_ctxs */ 473 n = qatomic_fetch_inc(&tcg_cur_ctxs); 474 g_assert(n < tcg_max_ctxs); 475 qatomic_set(&tcg_ctxs[n], s); 476 477 if (n > 0) { 478 alloc_tcg_plugin_context(s); 479 tcg_region_initial_alloc(s); 480 } 481 482 tcg_ctx = s; 483 } 484 #endif /* !CONFIG_USER_ONLY */ 485 486 /* pool based memory allocation */ 487 void *tcg_malloc_internal(TCGContext *s, int size) 488 { 489 TCGPool *p; 490 int pool_size; 491 492 if (size > TCG_POOL_CHUNK_SIZE) { 493 /* big malloc: insert a new pool (XXX: could optimize) */ 494 p = g_malloc(sizeof(TCGPool) + size); 495 p->size = size; 496 p->next = s->pool_first_large; 497 s->pool_first_large = p; 498 return p->data; 499 } else { 500 p = s->pool_current; 501 if (!p) { 502 p = s->pool_first; 503 if (!p) 504 goto new_pool; 505 } else { 506 if (!p->next) { 507 new_pool: 508 pool_size = TCG_POOL_CHUNK_SIZE; 509 p = g_malloc(sizeof(TCGPool) + pool_size); 510 p->size = pool_size; 511 p->next = NULL; 512 if (s->pool_current) { 513 s->pool_current->next = p; 514 } else { 515 s->pool_first = p; 516 } 517 } else { 518 p = p->next; 519 } 520 } 521 } 522 s->pool_current = p; 523 s->pool_cur = p->data + size; 524 s->pool_end = p->data + p->size; 525 return p->data; 526 } 527 528 void tcg_pool_reset(TCGContext *s) 529 { 530 TCGPool *p, *t; 531 for (p = s->pool_first_large; p; p = t) { 532 t = p->next; 533 g_free(p); 534 } 535 s->pool_first_large = NULL; 536 s->pool_cur = s->pool_end = NULL; 537 s->pool_current = NULL; 538 } 539 540 #include "exec/helper-proto.h" 541 542 static TCGHelperInfo all_helpers[] = { 543 #include "exec/helper-tcg.h" 544 }; 545 static GHashTable *helper_table; 546 547 #ifdef CONFIG_TCG_INTERPRETER 548 static ffi_type *typecode_to_ffi(int argmask) 549 { 550 switch (argmask) { 551 case dh_typecode_void: 552 return &ffi_type_void; 553 case dh_typecode_i32: 554 return &ffi_type_uint32; 555 case dh_typecode_s32: 556 return &ffi_type_sint32; 557 case dh_typecode_i64: 558 return &ffi_type_uint64; 559 case dh_typecode_s64: 560 return &ffi_type_sint64; 561 case dh_typecode_ptr: 562 return &ffi_type_pointer; 563 } 564 g_assert_not_reached(); 565 } 566 567 static void init_ffi_layouts(void) 568 { 569 /* g_direct_hash/equal for direct comparisons on uint32_t. */ 570 GHashTable *ffi_table = g_hash_table_new(NULL, NULL); 571 572 for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 573 TCGHelperInfo *info = &all_helpers[i]; 574 unsigned typemask = info->typemask; 575 gpointer hash = (gpointer)(uintptr_t)typemask; 576 struct { 577 ffi_cif cif; 578 ffi_type *args[]; 579 } *ca; 580 ffi_status status; 581 int nargs; 582 ffi_cif *cif; 583 584 cif = g_hash_table_lookup(ffi_table, hash); 585 if (cif) { 586 info->cif = cif; 587 continue; 588 } 589 590 /* Ignoring the return type, find the last non-zero field. */ 591 nargs = 32 - clz32(typemask >> 3); 592 nargs = DIV_ROUND_UP(nargs, 3); 593 594 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 595 ca->cif.rtype = typecode_to_ffi(typemask & 7); 596 ca->cif.nargs = nargs; 597 598 if (nargs != 0) { 599 ca->cif.arg_types = ca->args; 600 for (int j = 0; j < nargs; ++j) { 601 int typecode = extract32(typemask, (j + 1) * 3, 3); 602 ca->args[j] = typecode_to_ffi(typecode); 603 } 604 } 605 606 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 607 ca->cif.rtype, ca->cif.arg_types); 608 assert(status == FFI_OK); 609 610 cif = &ca->cif; 611 info->cif = cif; 612 g_hash_table_insert(ffi_table, hash, (gpointer)cif); 613 } 614 615 g_hash_table_destroy(ffi_table); 616 } 617 #endif /* CONFIG_TCG_INTERPRETER */ 618 619 typedef struct TCGCumulativeArgs { 620 int arg_idx; /* tcg_gen_callN args[] */ 621 int info_in_idx; /* TCGHelperInfo in[] */ 622 int arg_slot; /* regs+stack slot */ 623 int ref_slot; /* stack slots for references */ 624 } TCGCumulativeArgs; 625 626 static void layout_arg_even(TCGCumulativeArgs *cum) 627 { 628 cum->arg_slot += cum->arg_slot & 1; 629 } 630 631 static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, 632 TCGCallArgumentKind kind) 633 { 634 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 635 636 *loc = (TCGCallArgumentLoc){ 637 .kind = kind, 638 .arg_idx = cum->arg_idx, 639 .arg_slot = cum->arg_slot, 640 }; 641 cum->info_in_idx++; 642 cum->arg_slot++; 643 } 644 645 static void layout_arg_normal_n(TCGCumulativeArgs *cum, 646 TCGHelperInfo *info, int n) 647 { 648 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 649 650 for (int i = 0; i < n; ++i) { 651 /* Layout all using the same arg_idx, adjusting the subindex. */ 652 loc[i] = (TCGCallArgumentLoc){ 653 .kind = TCG_CALL_ARG_NORMAL, 654 .arg_idx = cum->arg_idx, 655 .tmp_subindex = i, 656 .arg_slot = cum->arg_slot + i, 657 }; 658 } 659 cum->info_in_idx += n; 660 cum->arg_slot += n; 661 } 662 663 static void init_call_layout(TCGHelperInfo *info) 664 { 665 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); 666 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 667 unsigned typemask = info->typemask; 668 unsigned typecode; 669 TCGCumulativeArgs cum = { }; 670 671 /* 672 * Parse and place any function return value. 673 */ 674 typecode = typemask & 7; 675 switch (typecode) { 676 case dh_typecode_void: 677 info->nr_out = 0; 678 break; 679 case dh_typecode_i32: 680 case dh_typecode_s32: 681 case dh_typecode_ptr: 682 info->nr_out = 1; 683 info->out_kind = TCG_CALL_RET_NORMAL; 684 break; 685 case dh_typecode_i64: 686 case dh_typecode_s64: 687 info->nr_out = 64 / TCG_TARGET_REG_BITS; 688 info->out_kind = TCG_CALL_RET_NORMAL; 689 break; 690 default: 691 g_assert_not_reached(); 692 } 693 assert(info->nr_out <= ARRAY_SIZE(tcg_target_call_oarg_regs)); 694 695 /* 696 * Parse and place function arguments. 697 */ 698 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { 699 TCGCallArgumentKind kind; 700 TCGType type; 701 702 typecode = typemask & 7; 703 switch (typecode) { 704 case dh_typecode_i32: 705 case dh_typecode_s32: 706 type = TCG_TYPE_I32; 707 break; 708 case dh_typecode_i64: 709 case dh_typecode_s64: 710 type = TCG_TYPE_I64; 711 break; 712 case dh_typecode_ptr: 713 type = TCG_TYPE_PTR; 714 break; 715 default: 716 g_assert_not_reached(); 717 } 718 719 switch (type) { 720 case TCG_TYPE_I32: 721 switch (TCG_TARGET_CALL_ARG_I32) { 722 case TCG_CALL_ARG_EVEN: 723 layout_arg_even(&cum); 724 /* fall through */ 725 case TCG_CALL_ARG_NORMAL: 726 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 727 break; 728 case TCG_CALL_ARG_EXTEND: 729 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); 730 layout_arg_1(&cum, info, kind); 731 break; 732 default: 733 qemu_build_not_reached(); 734 } 735 break; 736 737 case TCG_TYPE_I64: 738 switch (TCG_TARGET_CALL_ARG_I64) { 739 case TCG_CALL_ARG_EVEN: 740 layout_arg_even(&cum); 741 /* fall through */ 742 case TCG_CALL_ARG_NORMAL: 743 if (TCG_TARGET_REG_BITS == 32) { 744 layout_arg_normal_n(&cum, info, 2); 745 } else { 746 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 747 } 748 break; 749 default: 750 qemu_build_not_reached(); 751 } 752 break; 753 754 default: 755 g_assert_not_reached(); 756 } 757 } 758 info->nr_in = cum.info_in_idx; 759 760 /* Validate that we didn't overrun the input array. */ 761 assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); 762 /* Validate the backend has enough argument space. */ 763 assert(cum.arg_slot <= max_reg_slots + max_stk_slots); 764 assert(cum.ref_slot <= max_stk_slots); 765 } 766 767 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 768 static void process_op_defs(TCGContext *s); 769 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 770 TCGReg reg, const char *name); 771 772 static void tcg_context_init(unsigned max_cpus) 773 { 774 TCGContext *s = &tcg_init_ctx; 775 int op, total_args, n, i; 776 TCGOpDef *def; 777 TCGArgConstraint *args_ct; 778 TCGTemp *ts; 779 780 memset(s, 0, sizeof(*s)); 781 s->nb_globals = 0; 782 783 /* Count total number of arguments and allocate the corresponding 784 space */ 785 total_args = 0; 786 for(op = 0; op < NB_OPS; op++) { 787 def = &tcg_op_defs[op]; 788 n = def->nb_iargs + def->nb_oargs; 789 total_args += n; 790 } 791 792 args_ct = g_new0(TCGArgConstraint, total_args); 793 794 for(op = 0; op < NB_OPS; op++) { 795 def = &tcg_op_defs[op]; 796 def->args_ct = args_ct; 797 n = def->nb_iargs + def->nb_oargs; 798 args_ct += n; 799 } 800 801 /* Register helpers. */ 802 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 803 helper_table = g_hash_table_new(NULL, NULL); 804 805 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 806 init_call_layout(&all_helpers[i]); 807 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 808 (gpointer)&all_helpers[i]); 809 } 810 811 #ifdef CONFIG_TCG_INTERPRETER 812 init_ffi_layouts(); 813 #endif 814 815 tcg_target_init(s); 816 process_op_defs(s); 817 818 /* Reverse the order of the saved registers, assuming they're all at 819 the start of tcg_target_reg_alloc_order. */ 820 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 821 int r = tcg_target_reg_alloc_order[n]; 822 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 823 break; 824 } 825 } 826 for (i = 0; i < n; ++i) { 827 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 828 } 829 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 830 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 831 } 832 833 alloc_tcg_plugin_context(s); 834 835 tcg_ctx = s; 836 /* 837 * In user-mode we simply share the init context among threads, since we 838 * use a single region. See the documentation tcg_region_init() for the 839 * reasoning behind this. 840 * In softmmu we will have at most max_cpus TCG threads. 841 */ 842 #ifdef CONFIG_USER_ONLY 843 tcg_ctxs = &tcg_ctx; 844 tcg_cur_ctxs = 1; 845 tcg_max_ctxs = 1; 846 #else 847 tcg_max_ctxs = max_cpus; 848 tcg_ctxs = g_new0(TCGContext *, max_cpus); 849 #endif 850 851 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 852 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 853 cpu_env = temp_tcgv_ptr(ts); 854 } 855 856 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 857 { 858 tcg_context_init(max_cpus); 859 tcg_region_init(tb_size, splitwx, max_cpus); 860 } 861 862 /* 863 * Allocate TBs right before their corresponding translated code, making 864 * sure that TBs and code are on different cache lines. 865 */ 866 TranslationBlock *tcg_tb_alloc(TCGContext *s) 867 { 868 uintptr_t align = qemu_icache_linesize; 869 TranslationBlock *tb; 870 void *next; 871 872 retry: 873 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 874 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 875 876 if (unlikely(next > s->code_gen_highwater)) { 877 if (tcg_region_alloc(s)) { 878 return NULL; 879 } 880 goto retry; 881 } 882 qatomic_set(&s->code_gen_ptr, next); 883 s->data_gen_ptr = NULL; 884 return tb; 885 } 886 887 void tcg_prologue_init(TCGContext *s) 888 { 889 size_t prologue_size; 890 891 s->code_ptr = s->code_gen_ptr; 892 s->code_buf = s->code_gen_ptr; 893 s->data_gen_ptr = NULL; 894 895 #ifndef CONFIG_TCG_INTERPRETER 896 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 897 #endif 898 899 #ifdef TCG_TARGET_NEED_POOL_LABELS 900 s->pool_labels = NULL; 901 #endif 902 903 qemu_thread_jit_write(); 904 /* Generate the prologue. */ 905 tcg_target_qemu_prologue(s); 906 907 #ifdef TCG_TARGET_NEED_POOL_LABELS 908 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 909 { 910 int result = tcg_out_pool_finalize(s); 911 tcg_debug_assert(result == 0); 912 } 913 #endif 914 915 prologue_size = tcg_current_code_size(s); 916 917 #ifndef CONFIG_TCG_INTERPRETER 918 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 919 (uintptr_t)s->code_buf, prologue_size); 920 #endif 921 922 #ifdef DEBUG_DISAS 923 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 924 FILE *logfile = qemu_log_trylock(); 925 if (logfile) { 926 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 927 if (s->data_gen_ptr) { 928 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 929 size_t data_size = prologue_size - code_size; 930 size_t i; 931 932 disas(logfile, s->code_gen_ptr, code_size); 933 934 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 935 if (sizeof(tcg_target_ulong) == 8) { 936 fprintf(logfile, 937 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 938 (uintptr_t)s->data_gen_ptr + i, 939 *(uint64_t *)(s->data_gen_ptr + i)); 940 } else { 941 fprintf(logfile, 942 "0x%08" PRIxPTR ": .long 0x%08x\n", 943 (uintptr_t)s->data_gen_ptr + i, 944 *(uint32_t *)(s->data_gen_ptr + i)); 945 } 946 } 947 } else { 948 disas(logfile, s->code_gen_ptr, prologue_size); 949 } 950 fprintf(logfile, "\n"); 951 qemu_log_unlock(logfile); 952 } 953 } 954 #endif 955 956 #ifndef CONFIG_TCG_INTERPRETER 957 /* 958 * Assert that goto_ptr is implemented completely, setting an epilogue. 959 * For tci, we use NULL as the signal to return from the interpreter, 960 * so skip this check. 961 */ 962 tcg_debug_assert(tcg_code_gen_epilogue != NULL); 963 #endif 964 965 tcg_region_prologue_set(s); 966 } 967 968 void tcg_func_start(TCGContext *s) 969 { 970 tcg_pool_reset(s); 971 s->nb_temps = s->nb_globals; 972 973 /* No temps have been previously allocated for size or locality. */ 974 memset(s->free_temps, 0, sizeof(s->free_temps)); 975 976 /* No constant temps have been previously allocated. */ 977 for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 978 if (s->const_table[i]) { 979 g_hash_table_remove_all(s->const_table[i]); 980 } 981 } 982 983 s->nb_ops = 0; 984 s->nb_labels = 0; 985 s->current_frame_offset = s->frame_start; 986 987 #ifdef CONFIG_DEBUG_TCG 988 s->goto_tb_issue_mask = 0; 989 #endif 990 991 QTAILQ_INIT(&s->ops); 992 QTAILQ_INIT(&s->free_ops); 993 QSIMPLEQ_INIT(&s->labels); 994 } 995 996 static TCGTemp *tcg_temp_alloc(TCGContext *s) 997 { 998 int n = s->nb_temps++; 999 1000 if (n >= TCG_MAX_TEMPS) { 1001 tcg_raise_tb_overflow(s); 1002 } 1003 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 1004 } 1005 1006 static TCGTemp *tcg_global_alloc(TCGContext *s) 1007 { 1008 TCGTemp *ts; 1009 1010 tcg_debug_assert(s->nb_globals == s->nb_temps); 1011 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 1012 s->nb_globals++; 1013 ts = tcg_temp_alloc(s); 1014 ts->kind = TEMP_GLOBAL; 1015 1016 return ts; 1017 } 1018 1019 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1020 TCGReg reg, const char *name) 1021 { 1022 TCGTemp *ts; 1023 1024 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 1025 tcg_abort(); 1026 } 1027 1028 ts = tcg_global_alloc(s); 1029 ts->base_type = type; 1030 ts->type = type; 1031 ts->kind = TEMP_FIXED; 1032 ts->reg = reg; 1033 ts->name = name; 1034 tcg_regset_set_reg(s->reserved_regs, reg); 1035 1036 return ts; 1037 } 1038 1039 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 1040 { 1041 s->frame_start = start; 1042 s->frame_end = start + size; 1043 s->frame_temp 1044 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 1045 } 1046 1047 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 1048 intptr_t offset, const char *name) 1049 { 1050 TCGContext *s = tcg_ctx; 1051 TCGTemp *base_ts = tcgv_ptr_temp(base); 1052 TCGTemp *ts = tcg_global_alloc(s); 1053 int indirect_reg = 0; 1054 1055 switch (base_ts->kind) { 1056 case TEMP_FIXED: 1057 break; 1058 case TEMP_GLOBAL: 1059 /* We do not support double-indirect registers. */ 1060 tcg_debug_assert(!base_ts->indirect_reg); 1061 base_ts->indirect_base = 1; 1062 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 1063 ? 2 : 1); 1064 indirect_reg = 1; 1065 break; 1066 default: 1067 g_assert_not_reached(); 1068 } 1069 1070 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1071 TCGTemp *ts2 = tcg_global_alloc(s); 1072 char buf[64]; 1073 1074 ts->base_type = TCG_TYPE_I64; 1075 ts->type = TCG_TYPE_I32; 1076 ts->indirect_reg = indirect_reg; 1077 ts->mem_allocated = 1; 1078 ts->mem_base = base_ts; 1079 ts->mem_offset = offset; 1080 pstrcpy(buf, sizeof(buf), name); 1081 pstrcat(buf, sizeof(buf), "_0"); 1082 ts->name = strdup(buf); 1083 1084 tcg_debug_assert(ts2 == ts + 1); 1085 ts2->base_type = TCG_TYPE_I64; 1086 ts2->type = TCG_TYPE_I32; 1087 ts2->indirect_reg = indirect_reg; 1088 ts2->mem_allocated = 1; 1089 ts2->mem_base = base_ts; 1090 ts2->mem_offset = offset + 4; 1091 ts2->temp_subindex = 1; 1092 pstrcpy(buf, sizeof(buf), name); 1093 pstrcat(buf, sizeof(buf), "_1"); 1094 ts2->name = strdup(buf); 1095 } else { 1096 ts->base_type = type; 1097 ts->type = type; 1098 ts->indirect_reg = indirect_reg; 1099 ts->mem_allocated = 1; 1100 ts->mem_base = base_ts; 1101 ts->mem_offset = offset; 1102 ts->name = name; 1103 } 1104 return ts; 1105 } 1106 1107 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) 1108 { 1109 TCGContext *s = tcg_ctx; 1110 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL; 1111 TCGTemp *ts; 1112 int idx, k; 1113 1114 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 1115 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 1116 if (idx < TCG_MAX_TEMPS) { 1117 /* There is already an available temp with the right type. */ 1118 clear_bit(idx, s->free_temps[k].l); 1119 1120 ts = &s->temps[idx]; 1121 ts->temp_allocated = 1; 1122 tcg_debug_assert(ts->base_type == type); 1123 tcg_debug_assert(ts->kind == kind); 1124 } else { 1125 ts = tcg_temp_alloc(s); 1126 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1127 TCGTemp *ts2 = tcg_temp_alloc(s); 1128 1129 ts->base_type = type; 1130 ts->type = TCG_TYPE_I32; 1131 ts->temp_allocated = 1; 1132 ts->kind = kind; 1133 1134 tcg_debug_assert(ts2 == ts + 1); 1135 ts2->base_type = TCG_TYPE_I64; 1136 ts2->type = TCG_TYPE_I32; 1137 ts2->temp_allocated = 1; 1138 ts2->temp_subindex = 1; 1139 ts2->kind = kind; 1140 } else { 1141 ts->base_type = type; 1142 ts->type = type; 1143 ts->temp_allocated = 1; 1144 ts->kind = kind; 1145 } 1146 } 1147 1148 #if defined(CONFIG_DEBUG_TCG) 1149 s->temps_in_use++; 1150 #endif 1151 return ts; 1152 } 1153 1154 TCGv_vec tcg_temp_new_vec(TCGType type) 1155 { 1156 TCGTemp *t; 1157 1158 #ifdef CONFIG_DEBUG_TCG 1159 switch (type) { 1160 case TCG_TYPE_V64: 1161 assert(TCG_TARGET_HAS_v64); 1162 break; 1163 case TCG_TYPE_V128: 1164 assert(TCG_TARGET_HAS_v128); 1165 break; 1166 case TCG_TYPE_V256: 1167 assert(TCG_TARGET_HAS_v256); 1168 break; 1169 default: 1170 g_assert_not_reached(); 1171 } 1172 #endif 1173 1174 t = tcg_temp_new_internal(type, 0); 1175 return temp_tcgv_vec(t); 1176 } 1177 1178 /* Create a new temp of the same type as an existing temp. */ 1179 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1180 { 1181 TCGTemp *t = tcgv_vec_temp(match); 1182 1183 tcg_debug_assert(t->temp_allocated != 0); 1184 1185 t = tcg_temp_new_internal(t->base_type, 0); 1186 return temp_tcgv_vec(t); 1187 } 1188 1189 void tcg_temp_free_internal(TCGTemp *ts) 1190 { 1191 TCGContext *s = tcg_ctx; 1192 int k, idx; 1193 1194 switch (ts->kind) { 1195 case TEMP_CONST: 1196 /* 1197 * In order to simplify users of tcg_constant_*, 1198 * silently ignore free. 1199 */ 1200 return; 1201 case TEMP_NORMAL: 1202 case TEMP_LOCAL: 1203 break; 1204 default: 1205 g_assert_not_reached(); 1206 } 1207 1208 #if defined(CONFIG_DEBUG_TCG) 1209 s->temps_in_use--; 1210 if (s->temps_in_use < 0) { 1211 fprintf(stderr, "More temporaries freed than allocated!\n"); 1212 } 1213 #endif 1214 1215 tcg_debug_assert(ts->temp_allocated != 0); 1216 ts->temp_allocated = 0; 1217 1218 idx = temp_idx(ts); 1219 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT); 1220 set_bit(idx, s->free_temps[k].l); 1221 } 1222 1223 TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1224 { 1225 TCGContext *s = tcg_ctx; 1226 GHashTable *h = s->const_table[type]; 1227 TCGTemp *ts; 1228 1229 if (h == NULL) { 1230 h = g_hash_table_new(g_int64_hash, g_int64_equal); 1231 s->const_table[type] = h; 1232 } 1233 1234 ts = g_hash_table_lookup(h, &val); 1235 if (ts == NULL) { 1236 int64_t *val_ptr; 1237 1238 ts = tcg_temp_alloc(s); 1239 1240 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1241 TCGTemp *ts2 = tcg_temp_alloc(s); 1242 1243 tcg_debug_assert(ts2 == ts + 1); 1244 1245 ts->base_type = TCG_TYPE_I64; 1246 ts->type = TCG_TYPE_I32; 1247 ts->kind = TEMP_CONST; 1248 ts->temp_allocated = 1; 1249 1250 ts2->base_type = TCG_TYPE_I64; 1251 ts2->type = TCG_TYPE_I32; 1252 ts2->kind = TEMP_CONST; 1253 ts2->temp_allocated = 1; 1254 ts2->temp_subindex = 1; 1255 1256 /* 1257 * Retain the full value of the 64-bit constant in the low 1258 * part, so that the hash table works. Actual uses will 1259 * truncate the value to the low part. 1260 */ 1261 ts[HOST_BIG_ENDIAN].val = val; 1262 ts[!HOST_BIG_ENDIAN].val = val >> 32; 1263 val_ptr = &ts[HOST_BIG_ENDIAN].val; 1264 } else { 1265 ts->base_type = type; 1266 ts->type = type; 1267 ts->kind = TEMP_CONST; 1268 ts->temp_allocated = 1; 1269 ts->val = val; 1270 val_ptr = &ts->val; 1271 } 1272 g_hash_table_insert(h, val_ptr, ts); 1273 } 1274 1275 return ts; 1276 } 1277 1278 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1279 { 1280 val = dup_const(vece, val); 1281 return temp_tcgv_vec(tcg_constant_internal(type, val)); 1282 } 1283 1284 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 1285 { 1286 TCGTemp *t = tcgv_vec_temp(match); 1287 1288 tcg_debug_assert(t->temp_allocated != 0); 1289 return tcg_constant_vec(t->base_type, vece, val); 1290 } 1291 1292 TCGv_i32 tcg_const_i32(int32_t val) 1293 { 1294 TCGv_i32 t0; 1295 t0 = tcg_temp_new_i32(); 1296 tcg_gen_movi_i32(t0, val); 1297 return t0; 1298 } 1299 1300 TCGv_i64 tcg_const_i64(int64_t val) 1301 { 1302 TCGv_i64 t0; 1303 t0 = tcg_temp_new_i64(); 1304 tcg_gen_movi_i64(t0, val); 1305 return t0; 1306 } 1307 1308 TCGv_i32 tcg_const_local_i32(int32_t val) 1309 { 1310 TCGv_i32 t0; 1311 t0 = tcg_temp_local_new_i32(); 1312 tcg_gen_movi_i32(t0, val); 1313 return t0; 1314 } 1315 1316 TCGv_i64 tcg_const_local_i64(int64_t val) 1317 { 1318 TCGv_i64 t0; 1319 t0 = tcg_temp_local_new_i64(); 1320 tcg_gen_movi_i64(t0, val); 1321 return t0; 1322 } 1323 1324 #if defined(CONFIG_DEBUG_TCG) 1325 void tcg_clear_temp_count(void) 1326 { 1327 TCGContext *s = tcg_ctx; 1328 s->temps_in_use = 0; 1329 } 1330 1331 int tcg_check_temp_count(void) 1332 { 1333 TCGContext *s = tcg_ctx; 1334 if (s->temps_in_use) { 1335 /* Clear the count so that we don't give another 1336 * warning immediately next time around. 1337 */ 1338 s->temps_in_use = 0; 1339 return 1; 1340 } 1341 return 0; 1342 } 1343 #endif 1344 1345 /* Return true if OP may appear in the opcode stream. 1346 Test the runtime variable that controls each opcode. */ 1347 bool tcg_op_supported(TCGOpcode op) 1348 { 1349 const bool have_vec 1350 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1351 1352 switch (op) { 1353 case INDEX_op_discard: 1354 case INDEX_op_set_label: 1355 case INDEX_op_call: 1356 case INDEX_op_br: 1357 case INDEX_op_mb: 1358 case INDEX_op_insn_start: 1359 case INDEX_op_exit_tb: 1360 case INDEX_op_goto_tb: 1361 case INDEX_op_goto_ptr: 1362 case INDEX_op_qemu_ld_i32: 1363 case INDEX_op_qemu_st_i32: 1364 case INDEX_op_qemu_ld_i64: 1365 case INDEX_op_qemu_st_i64: 1366 return true; 1367 1368 case INDEX_op_qemu_st8_i32: 1369 return TCG_TARGET_HAS_qemu_st8_i32; 1370 1371 case INDEX_op_mov_i32: 1372 case INDEX_op_setcond_i32: 1373 case INDEX_op_brcond_i32: 1374 case INDEX_op_ld8u_i32: 1375 case INDEX_op_ld8s_i32: 1376 case INDEX_op_ld16u_i32: 1377 case INDEX_op_ld16s_i32: 1378 case INDEX_op_ld_i32: 1379 case INDEX_op_st8_i32: 1380 case INDEX_op_st16_i32: 1381 case INDEX_op_st_i32: 1382 case INDEX_op_add_i32: 1383 case INDEX_op_sub_i32: 1384 case INDEX_op_mul_i32: 1385 case INDEX_op_and_i32: 1386 case INDEX_op_or_i32: 1387 case INDEX_op_xor_i32: 1388 case INDEX_op_shl_i32: 1389 case INDEX_op_shr_i32: 1390 case INDEX_op_sar_i32: 1391 return true; 1392 1393 case INDEX_op_movcond_i32: 1394 return TCG_TARGET_HAS_movcond_i32; 1395 case INDEX_op_div_i32: 1396 case INDEX_op_divu_i32: 1397 return TCG_TARGET_HAS_div_i32; 1398 case INDEX_op_rem_i32: 1399 case INDEX_op_remu_i32: 1400 return TCG_TARGET_HAS_rem_i32; 1401 case INDEX_op_div2_i32: 1402 case INDEX_op_divu2_i32: 1403 return TCG_TARGET_HAS_div2_i32; 1404 case INDEX_op_rotl_i32: 1405 case INDEX_op_rotr_i32: 1406 return TCG_TARGET_HAS_rot_i32; 1407 case INDEX_op_deposit_i32: 1408 return TCG_TARGET_HAS_deposit_i32; 1409 case INDEX_op_extract_i32: 1410 return TCG_TARGET_HAS_extract_i32; 1411 case INDEX_op_sextract_i32: 1412 return TCG_TARGET_HAS_sextract_i32; 1413 case INDEX_op_extract2_i32: 1414 return TCG_TARGET_HAS_extract2_i32; 1415 case INDEX_op_add2_i32: 1416 return TCG_TARGET_HAS_add2_i32; 1417 case INDEX_op_sub2_i32: 1418 return TCG_TARGET_HAS_sub2_i32; 1419 case INDEX_op_mulu2_i32: 1420 return TCG_TARGET_HAS_mulu2_i32; 1421 case INDEX_op_muls2_i32: 1422 return TCG_TARGET_HAS_muls2_i32; 1423 case INDEX_op_muluh_i32: 1424 return TCG_TARGET_HAS_muluh_i32; 1425 case INDEX_op_mulsh_i32: 1426 return TCG_TARGET_HAS_mulsh_i32; 1427 case INDEX_op_ext8s_i32: 1428 return TCG_TARGET_HAS_ext8s_i32; 1429 case INDEX_op_ext16s_i32: 1430 return TCG_TARGET_HAS_ext16s_i32; 1431 case INDEX_op_ext8u_i32: 1432 return TCG_TARGET_HAS_ext8u_i32; 1433 case INDEX_op_ext16u_i32: 1434 return TCG_TARGET_HAS_ext16u_i32; 1435 case INDEX_op_bswap16_i32: 1436 return TCG_TARGET_HAS_bswap16_i32; 1437 case INDEX_op_bswap32_i32: 1438 return TCG_TARGET_HAS_bswap32_i32; 1439 case INDEX_op_not_i32: 1440 return TCG_TARGET_HAS_not_i32; 1441 case INDEX_op_neg_i32: 1442 return TCG_TARGET_HAS_neg_i32; 1443 case INDEX_op_andc_i32: 1444 return TCG_TARGET_HAS_andc_i32; 1445 case INDEX_op_orc_i32: 1446 return TCG_TARGET_HAS_orc_i32; 1447 case INDEX_op_eqv_i32: 1448 return TCG_TARGET_HAS_eqv_i32; 1449 case INDEX_op_nand_i32: 1450 return TCG_TARGET_HAS_nand_i32; 1451 case INDEX_op_nor_i32: 1452 return TCG_TARGET_HAS_nor_i32; 1453 case INDEX_op_clz_i32: 1454 return TCG_TARGET_HAS_clz_i32; 1455 case INDEX_op_ctz_i32: 1456 return TCG_TARGET_HAS_ctz_i32; 1457 case INDEX_op_ctpop_i32: 1458 return TCG_TARGET_HAS_ctpop_i32; 1459 1460 case INDEX_op_brcond2_i32: 1461 case INDEX_op_setcond2_i32: 1462 return TCG_TARGET_REG_BITS == 32; 1463 1464 case INDEX_op_mov_i64: 1465 case INDEX_op_setcond_i64: 1466 case INDEX_op_brcond_i64: 1467 case INDEX_op_ld8u_i64: 1468 case INDEX_op_ld8s_i64: 1469 case INDEX_op_ld16u_i64: 1470 case INDEX_op_ld16s_i64: 1471 case INDEX_op_ld32u_i64: 1472 case INDEX_op_ld32s_i64: 1473 case INDEX_op_ld_i64: 1474 case INDEX_op_st8_i64: 1475 case INDEX_op_st16_i64: 1476 case INDEX_op_st32_i64: 1477 case INDEX_op_st_i64: 1478 case INDEX_op_add_i64: 1479 case INDEX_op_sub_i64: 1480 case INDEX_op_mul_i64: 1481 case INDEX_op_and_i64: 1482 case INDEX_op_or_i64: 1483 case INDEX_op_xor_i64: 1484 case INDEX_op_shl_i64: 1485 case INDEX_op_shr_i64: 1486 case INDEX_op_sar_i64: 1487 case INDEX_op_ext_i32_i64: 1488 case INDEX_op_extu_i32_i64: 1489 return TCG_TARGET_REG_BITS == 64; 1490 1491 case INDEX_op_movcond_i64: 1492 return TCG_TARGET_HAS_movcond_i64; 1493 case INDEX_op_div_i64: 1494 case INDEX_op_divu_i64: 1495 return TCG_TARGET_HAS_div_i64; 1496 case INDEX_op_rem_i64: 1497 case INDEX_op_remu_i64: 1498 return TCG_TARGET_HAS_rem_i64; 1499 case INDEX_op_div2_i64: 1500 case INDEX_op_divu2_i64: 1501 return TCG_TARGET_HAS_div2_i64; 1502 case INDEX_op_rotl_i64: 1503 case INDEX_op_rotr_i64: 1504 return TCG_TARGET_HAS_rot_i64; 1505 case INDEX_op_deposit_i64: 1506 return TCG_TARGET_HAS_deposit_i64; 1507 case INDEX_op_extract_i64: 1508 return TCG_TARGET_HAS_extract_i64; 1509 case INDEX_op_sextract_i64: 1510 return TCG_TARGET_HAS_sextract_i64; 1511 case INDEX_op_extract2_i64: 1512 return TCG_TARGET_HAS_extract2_i64; 1513 case INDEX_op_extrl_i64_i32: 1514 return TCG_TARGET_HAS_extrl_i64_i32; 1515 case INDEX_op_extrh_i64_i32: 1516 return TCG_TARGET_HAS_extrh_i64_i32; 1517 case INDEX_op_ext8s_i64: 1518 return TCG_TARGET_HAS_ext8s_i64; 1519 case INDEX_op_ext16s_i64: 1520 return TCG_TARGET_HAS_ext16s_i64; 1521 case INDEX_op_ext32s_i64: 1522 return TCG_TARGET_HAS_ext32s_i64; 1523 case INDEX_op_ext8u_i64: 1524 return TCG_TARGET_HAS_ext8u_i64; 1525 case INDEX_op_ext16u_i64: 1526 return TCG_TARGET_HAS_ext16u_i64; 1527 case INDEX_op_ext32u_i64: 1528 return TCG_TARGET_HAS_ext32u_i64; 1529 case INDEX_op_bswap16_i64: 1530 return TCG_TARGET_HAS_bswap16_i64; 1531 case INDEX_op_bswap32_i64: 1532 return TCG_TARGET_HAS_bswap32_i64; 1533 case INDEX_op_bswap64_i64: 1534 return TCG_TARGET_HAS_bswap64_i64; 1535 case INDEX_op_not_i64: 1536 return TCG_TARGET_HAS_not_i64; 1537 case INDEX_op_neg_i64: 1538 return TCG_TARGET_HAS_neg_i64; 1539 case INDEX_op_andc_i64: 1540 return TCG_TARGET_HAS_andc_i64; 1541 case INDEX_op_orc_i64: 1542 return TCG_TARGET_HAS_orc_i64; 1543 case INDEX_op_eqv_i64: 1544 return TCG_TARGET_HAS_eqv_i64; 1545 case INDEX_op_nand_i64: 1546 return TCG_TARGET_HAS_nand_i64; 1547 case INDEX_op_nor_i64: 1548 return TCG_TARGET_HAS_nor_i64; 1549 case INDEX_op_clz_i64: 1550 return TCG_TARGET_HAS_clz_i64; 1551 case INDEX_op_ctz_i64: 1552 return TCG_TARGET_HAS_ctz_i64; 1553 case INDEX_op_ctpop_i64: 1554 return TCG_TARGET_HAS_ctpop_i64; 1555 case INDEX_op_add2_i64: 1556 return TCG_TARGET_HAS_add2_i64; 1557 case INDEX_op_sub2_i64: 1558 return TCG_TARGET_HAS_sub2_i64; 1559 case INDEX_op_mulu2_i64: 1560 return TCG_TARGET_HAS_mulu2_i64; 1561 case INDEX_op_muls2_i64: 1562 return TCG_TARGET_HAS_muls2_i64; 1563 case INDEX_op_muluh_i64: 1564 return TCG_TARGET_HAS_muluh_i64; 1565 case INDEX_op_mulsh_i64: 1566 return TCG_TARGET_HAS_mulsh_i64; 1567 1568 case INDEX_op_mov_vec: 1569 case INDEX_op_dup_vec: 1570 case INDEX_op_dupm_vec: 1571 case INDEX_op_ld_vec: 1572 case INDEX_op_st_vec: 1573 case INDEX_op_add_vec: 1574 case INDEX_op_sub_vec: 1575 case INDEX_op_and_vec: 1576 case INDEX_op_or_vec: 1577 case INDEX_op_xor_vec: 1578 case INDEX_op_cmp_vec: 1579 return have_vec; 1580 case INDEX_op_dup2_vec: 1581 return have_vec && TCG_TARGET_REG_BITS == 32; 1582 case INDEX_op_not_vec: 1583 return have_vec && TCG_TARGET_HAS_not_vec; 1584 case INDEX_op_neg_vec: 1585 return have_vec && TCG_TARGET_HAS_neg_vec; 1586 case INDEX_op_abs_vec: 1587 return have_vec && TCG_TARGET_HAS_abs_vec; 1588 case INDEX_op_andc_vec: 1589 return have_vec && TCG_TARGET_HAS_andc_vec; 1590 case INDEX_op_orc_vec: 1591 return have_vec && TCG_TARGET_HAS_orc_vec; 1592 case INDEX_op_nand_vec: 1593 return have_vec && TCG_TARGET_HAS_nand_vec; 1594 case INDEX_op_nor_vec: 1595 return have_vec && TCG_TARGET_HAS_nor_vec; 1596 case INDEX_op_eqv_vec: 1597 return have_vec && TCG_TARGET_HAS_eqv_vec; 1598 case INDEX_op_mul_vec: 1599 return have_vec && TCG_TARGET_HAS_mul_vec; 1600 case INDEX_op_shli_vec: 1601 case INDEX_op_shri_vec: 1602 case INDEX_op_sari_vec: 1603 return have_vec && TCG_TARGET_HAS_shi_vec; 1604 case INDEX_op_shls_vec: 1605 case INDEX_op_shrs_vec: 1606 case INDEX_op_sars_vec: 1607 return have_vec && TCG_TARGET_HAS_shs_vec; 1608 case INDEX_op_shlv_vec: 1609 case INDEX_op_shrv_vec: 1610 case INDEX_op_sarv_vec: 1611 return have_vec && TCG_TARGET_HAS_shv_vec; 1612 case INDEX_op_rotli_vec: 1613 return have_vec && TCG_TARGET_HAS_roti_vec; 1614 case INDEX_op_rotls_vec: 1615 return have_vec && TCG_TARGET_HAS_rots_vec; 1616 case INDEX_op_rotlv_vec: 1617 case INDEX_op_rotrv_vec: 1618 return have_vec && TCG_TARGET_HAS_rotv_vec; 1619 case INDEX_op_ssadd_vec: 1620 case INDEX_op_usadd_vec: 1621 case INDEX_op_sssub_vec: 1622 case INDEX_op_ussub_vec: 1623 return have_vec && TCG_TARGET_HAS_sat_vec; 1624 case INDEX_op_smin_vec: 1625 case INDEX_op_umin_vec: 1626 case INDEX_op_smax_vec: 1627 case INDEX_op_umax_vec: 1628 return have_vec && TCG_TARGET_HAS_minmax_vec; 1629 case INDEX_op_bitsel_vec: 1630 return have_vec && TCG_TARGET_HAS_bitsel_vec; 1631 case INDEX_op_cmpsel_vec: 1632 return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1633 1634 default: 1635 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1636 return true; 1637 } 1638 } 1639 1640 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); 1641 1642 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1643 { 1644 const TCGHelperInfo *info; 1645 TCGv_i64 extend_free[MAX_CALL_IARGS]; 1646 int n_extend = 0; 1647 TCGOp *op; 1648 int i, n, pi = 0, total_args; 1649 1650 info = g_hash_table_lookup(helper_table, (gpointer)func); 1651 total_args = info->nr_out + info->nr_in + 2; 1652 op = tcg_op_alloc(INDEX_op_call, total_args); 1653 1654 #ifdef CONFIG_PLUGIN 1655 /* detect non-plugin helpers */ 1656 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) { 1657 tcg_ctx->plugin_insn->calls_helpers = true; 1658 } 1659 #endif 1660 1661 TCGOP_CALLO(op) = n = info->nr_out; 1662 switch (n) { 1663 case 0: 1664 tcg_debug_assert(ret == NULL); 1665 break; 1666 case 1: 1667 tcg_debug_assert(ret != NULL); 1668 op->args[pi++] = temp_arg(ret); 1669 break; 1670 case 2: 1671 tcg_debug_assert(ret != NULL); 1672 tcg_debug_assert(ret->base_type == ret->type + 1); 1673 tcg_debug_assert(ret->temp_subindex == 0); 1674 op->args[pi++] = temp_arg(ret); 1675 op->args[pi++] = temp_arg(ret + 1); 1676 break; 1677 default: 1678 g_assert_not_reached(); 1679 } 1680 1681 TCGOP_CALLI(op) = n = info->nr_in; 1682 for (i = 0; i < n; i++) { 1683 const TCGCallArgumentLoc *loc = &info->in[i]; 1684 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; 1685 1686 switch (loc->kind) { 1687 case TCG_CALL_ARG_NORMAL: 1688 op->args[pi++] = temp_arg(ts); 1689 break; 1690 1691 case TCG_CALL_ARG_EXTEND_U: 1692 case TCG_CALL_ARG_EXTEND_S: 1693 { 1694 TCGv_i64 temp = tcg_temp_new_i64(); 1695 TCGv_i32 orig = temp_tcgv_i32(ts); 1696 1697 if (loc->kind == TCG_CALL_ARG_EXTEND_S) { 1698 tcg_gen_ext_i32_i64(temp, orig); 1699 } else { 1700 tcg_gen_extu_i32_i64(temp, orig); 1701 } 1702 op->args[pi++] = tcgv_i64_arg(temp); 1703 extend_free[n_extend++] = temp; 1704 } 1705 break; 1706 1707 default: 1708 g_assert_not_reached(); 1709 } 1710 } 1711 op->args[pi++] = (uintptr_t)func; 1712 op->args[pi++] = (uintptr_t)info; 1713 tcg_debug_assert(pi == total_args); 1714 1715 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 1716 1717 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); 1718 for (i = 0; i < n_extend; ++i) { 1719 tcg_temp_free_i64(extend_free[i]); 1720 } 1721 } 1722 1723 static void tcg_reg_alloc_start(TCGContext *s) 1724 { 1725 int i, n; 1726 1727 for (i = 0, n = s->nb_temps; i < n; i++) { 1728 TCGTemp *ts = &s->temps[i]; 1729 TCGTempVal val = TEMP_VAL_MEM; 1730 1731 switch (ts->kind) { 1732 case TEMP_CONST: 1733 val = TEMP_VAL_CONST; 1734 break; 1735 case TEMP_FIXED: 1736 val = TEMP_VAL_REG; 1737 break; 1738 case TEMP_GLOBAL: 1739 break; 1740 case TEMP_NORMAL: 1741 case TEMP_EBB: 1742 val = TEMP_VAL_DEAD; 1743 /* fall through */ 1744 case TEMP_LOCAL: 1745 ts->mem_allocated = 0; 1746 break; 1747 default: 1748 g_assert_not_reached(); 1749 } 1750 ts->val_type = val; 1751 } 1752 1753 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1754 } 1755 1756 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1757 TCGTemp *ts) 1758 { 1759 int idx = temp_idx(ts); 1760 1761 switch (ts->kind) { 1762 case TEMP_FIXED: 1763 case TEMP_GLOBAL: 1764 pstrcpy(buf, buf_size, ts->name); 1765 break; 1766 case TEMP_LOCAL: 1767 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1768 break; 1769 case TEMP_EBB: 1770 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals); 1771 break; 1772 case TEMP_NORMAL: 1773 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1774 break; 1775 case TEMP_CONST: 1776 switch (ts->type) { 1777 case TCG_TYPE_I32: 1778 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1779 break; 1780 #if TCG_TARGET_REG_BITS > 32 1781 case TCG_TYPE_I64: 1782 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1783 break; 1784 #endif 1785 case TCG_TYPE_V64: 1786 case TCG_TYPE_V128: 1787 case TCG_TYPE_V256: 1788 snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1789 64 << (ts->type - TCG_TYPE_V64), ts->val); 1790 break; 1791 default: 1792 g_assert_not_reached(); 1793 } 1794 break; 1795 } 1796 return buf; 1797 } 1798 1799 static char *tcg_get_arg_str(TCGContext *s, char *buf, 1800 int buf_size, TCGArg arg) 1801 { 1802 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1803 } 1804 1805 static const char * const cond_name[] = 1806 { 1807 [TCG_COND_NEVER] = "never", 1808 [TCG_COND_ALWAYS] = "always", 1809 [TCG_COND_EQ] = "eq", 1810 [TCG_COND_NE] = "ne", 1811 [TCG_COND_LT] = "lt", 1812 [TCG_COND_GE] = "ge", 1813 [TCG_COND_LE] = "le", 1814 [TCG_COND_GT] = "gt", 1815 [TCG_COND_LTU] = "ltu", 1816 [TCG_COND_GEU] = "geu", 1817 [TCG_COND_LEU] = "leu", 1818 [TCG_COND_GTU] = "gtu" 1819 }; 1820 1821 static const char * const ldst_name[] = 1822 { 1823 [MO_UB] = "ub", 1824 [MO_SB] = "sb", 1825 [MO_LEUW] = "leuw", 1826 [MO_LESW] = "lesw", 1827 [MO_LEUL] = "leul", 1828 [MO_LESL] = "lesl", 1829 [MO_LEUQ] = "leq", 1830 [MO_BEUW] = "beuw", 1831 [MO_BESW] = "besw", 1832 [MO_BEUL] = "beul", 1833 [MO_BESL] = "besl", 1834 [MO_BEUQ] = "beq", 1835 }; 1836 1837 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1838 #ifdef TARGET_ALIGNED_ONLY 1839 [MO_UNALN >> MO_ASHIFT] = "un+", 1840 [MO_ALIGN >> MO_ASHIFT] = "", 1841 #else 1842 [MO_UNALN >> MO_ASHIFT] = "", 1843 [MO_ALIGN >> MO_ASHIFT] = "al+", 1844 #endif 1845 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1846 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1847 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1848 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1849 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1850 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1851 }; 1852 1853 static const char bswap_flag_name[][6] = { 1854 [TCG_BSWAP_IZ] = "iz", 1855 [TCG_BSWAP_OZ] = "oz", 1856 [TCG_BSWAP_OS] = "os", 1857 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1858 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1859 }; 1860 1861 static inline bool tcg_regset_single(TCGRegSet d) 1862 { 1863 return (d & (d - 1)) == 0; 1864 } 1865 1866 static inline TCGReg tcg_regset_first(TCGRegSet d) 1867 { 1868 if (TCG_TARGET_NB_REGS <= 32) { 1869 return ctz32(d); 1870 } else { 1871 return ctz64(d); 1872 } 1873 } 1874 1875 /* Return only the number of characters output -- no error return. */ 1876 #define ne_fprintf(...) \ 1877 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 1878 1879 static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 1880 { 1881 char buf[128]; 1882 TCGOp *op; 1883 1884 QTAILQ_FOREACH(op, &s->ops, link) { 1885 int i, k, nb_oargs, nb_iargs, nb_cargs; 1886 const TCGOpDef *def; 1887 TCGOpcode c; 1888 int col = 0; 1889 1890 c = op->opc; 1891 def = &tcg_op_defs[c]; 1892 1893 if (c == INDEX_op_insn_start) { 1894 nb_oargs = 0; 1895 col += ne_fprintf(f, "\n ----"); 1896 1897 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1898 target_ulong a; 1899 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1900 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 1901 #else 1902 a = op->args[i]; 1903 #endif 1904 col += ne_fprintf(f, " " TARGET_FMT_lx, a); 1905 } 1906 } else if (c == INDEX_op_call) { 1907 const TCGHelperInfo *info = tcg_call_info(op); 1908 void *func = tcg_call_func(op); 1909 1910 /* variable number of arguments */ 1911 nb_oargs = TCGOP_CALLO(op); 1912 nb_iargs = TCGOP_CALLI(op); 1913 nb_cargs = def->nb_cargs; 1914 1915 col += ne_fprintf(f, " %s ", def->name); 1916 1917 /* 1918 * Print the function name from TCGHelperInfo, if available. 1919 * Note that plugins have a template function for the info, 1920 * but the actual function pointer comes from the plugin. 1921 */ 1922 if (func == info->func) { 1923 col += ne_fprintf(f, "%s", info->name); 1924 } else { 1925 col += ne_fprintf(f, "plugin(%p)", func); 1926 } 1927 1928 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 1929 for (i = 0; i < nb_oargs; i++) { 1930 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 1931 op->args[i])); 1932 } 1933 for (i = 0; i < nb_iargs; i++) { 1934 TCGArg arg = op->args[nb_oargs + i]; 1935 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 1936 col += ne_fprintf(f, ",%s", t); 1937 } 1938 } else { 1939 col += ne_fprintf(f, " %s ", def->name); 1940 1941 nb_oargs = def->nb_oargs; 1942 nb_iargs = def->nb_iargs; 1943 nb_cargs = def->nb_cargs; 1944 1945 if (def->flags & TCG_OPF_VECTOR) { 1946 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op), 1947 8 << TCGOP_VECE(op)); 1948 } 1949 1950 k = 0; 1951 for (i = 0; i < nb_oargs; i++) { 1952 const char *sep = k ? "," : ""; 1953 col += ne_fprintf(f, "%s%s", sep, 1954 tcg_get_arg_str(s, buf, sizeof(buf), 1955 op->args[k++])); 1956 } 1957 for (i = 0; i < nb_iargs; i++) { 1958 const char *sep = k ? "," : ""; 1959 col += ne_fprintf(f, "%s%s", sep, 1960 tcg_get_arg_str(s, buf, sizeof(buf), 1961 op->args[k++])); 1962 } 1963 switch (c) { 1964 case INDEX_op_brcond_i32: 1965 case INDEX_op_setcond_i32: 1966 case INDEX_op_movcond_i32: 1967 case INDEX_op_brcond2_i32: 1968 case INDEX_op_setcond2_i32: 1969 case INDEX_op_brcond_i64: 1970 case INDEX_op_setcond_i64: 1971 case INDEX_op_movcond_i64: 1972 case INDEX_op_cmp_vec: 1973 case INDEX_op_cmpsel_vec: 1974 if (op->args[k] < ARRAY_SIZE(cond_name) 1975 && cond_name[op->args[k]]) { 1976 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 1977 } else { 1978 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 1979 } 1980 i = 1; 1981 break; 1982 case INDEX_op_qemu_ld_i32: 1983 case INDEX_op_qemu_st_i32: 1984 case INDEX_op_qemu_st8_i32: 1985 case INDEX_op_qemu_ld_i64: 1986 case INDEX_op_qemu_st_i64: 1987 { 1988 MemOpIdx oi = op->args[k++]; 1989 MemOp op = get_memop(oi); 1990 unsigned ix = get_mmuidx(oi); 1991 1992 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1993 col += ne_fprintf(f, ",$0x%x,%u", op, ix); 1994 } else { 1995 const char *s_al, *s_op; 1996 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1997 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1998 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix); 1999 } 2000 i = 1; 2001 } 2002 break; 2003 case INDEX_op_bswap16_i32: 2004 case INDEX_op_bswap16_i64: 2005 case INDEX_op_bswap32_i32: 2006 case INDEX_op_bswap32_i64: 2007 case INDEX_op_bswap64_i64: 2008 { 2009 TCGArg flags = op->args[k]; 2010 const char *name = NULL; 2011 2012 if (flags < ARRAY_SIZE(bswap_flag_name)) { 2013 name = bswap_flag_name[flags]; 2014 } 2015 if (name) { 2016 col += ne_fprintf(f, ",%s", name); 2017 } else { 2018 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 2019 } 2020 i = k = 1; 2021 } 2022 break; 2023 default: 2024 i = 0; 2025 break; 2026 } 2027 switch (c) { 2028 case INDEX_op_set_label: 2029 case INDEX_op_br: 2030 case INDEX_op_brcond_i32: 2031 case INDEX_op_brcond_i64: 2032 case INDEX_op_brcond2_i32: 2033 col += ne_fprintf(f, "%s$L%d", k ? "," : "", 2034 arg_label(op->args[k])->id); 2035 i++, k++; 2036 break; 2037 default: 2038 break; 2039 } 2040 for (; i < nb_cargs; i++, k++) { 2041 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 2042 op->args[k]); 2043 } 2044 } 2045 2046 if (have_prefs || op->life) { 2047 for (; col < 40; ++col) { 2048 putc(' ', f); 2049 } 2050 } 2051 2052 if (op->life) { 2053 unsigned life = op->life; 2054 2055 if (life & (SYNC_ARG * 3)) { 2056 ne_fprintf(f, " sync:"); 2057 for (i = 0; i < 2; ++i) { 2058 if (life & (SYNC_ARG << i)) { 2059 ne_fprintf(f, " %d", i); 2060 } 2061 } 2062 } 2063 life /= DEAD_ARG; 2064 if (life) { 2065 ne_fprintf(f, " dead:"); 2066 for (i = 0; life; ++i, life >>= 1) { 2067 if (life & 1) { 2068 ne_fprintf(f, " %d", i); 2069 } 2070 } 2071 } 2072 } 2073 2074 if (have_prefs) { 2075 for (i = 0; i < nb_oargs; ++i) { 2076 TCGRegSet set = output_pref(op, i); 2077 2078 if (i == 0) { 2079 ne_fprintf(f, " pref="); 2080 } else { 2081 ne_fprintf(f, ","); 2082 } 2083 if (set == 0) { 2084 ne_fprintf(f, "none"); 2085 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 2086 ne_fprintf(f, "all"); 2087 #ifdef CONFIG_DEBUG_TCG 2088 } else if (tcg_regset_single(set)) { 2089 TCGReg reg = tcg_regset_first(set); 2090 ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 2091 #endif 2092 } else if (TCG_TARGET_NB_REGS <= 32) { 2093 ne_fprintf(f, "0x%x", (uint32_t)set); 2094 } else { 2095 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 2096 } 2097 } 2098 } 2099 2100 putc('\n', f); 2101 } 2102 } 2103 2104 /* we give more priority to constraints with less registers */ 2105 static int get_constraint_priority(const TCGOpDef *def, int k) 2106 { 2107 const TCGArgConstraint *arg_ct = &def->args_ct[k]; 2108 int n = ctpop64(arg_ct->regs); 2109 2110 /* 2111 * Sort constraints of a single register first, which includes output 2112 * aliases (which must exactly match the input already allocated). 2113 */ 2114 if (n == 1 || arg_ct->oalias) { 2115 return INT_MAX; 2116 } 2117 2118 /* 2119 * Sort register pairs next, first then second immediately after. 2120 * Arbitrarily sort multiple pairs by the index of the first reg; 2121 * there shouldn't be many pairs. 2122 */ 2123 switch (arg_ct->pair) { 2124 case 1: 2125 case 3: 2126 return (k + 1) * 2; 2127 case 2: 2128 return (arg_ct->pair_index + 1) * 2 - 1; 2129 } 2130 2131 /* Finally, sort by decreasing register count. */ 2132 assert(n > 1); 2133 return -n; 2134 } 2135 2136 /* sort from highest priority to lowest */ 2137 static void sort_constraints(TCGOpDef *def, int start, int n) 2138 { 2139 int i, j; 2140 TCGArgConstraint *a = def->args_ct; 2141 2142 for (i = 0; i < n; i++) { 2143 a[start + i].sort_index = start + i; 2144 } 2145 if (n <= 1) { 2146 return; 2147 } 2148 for (i = 0; i < n - 1; i++) { 2149 for (j = i + 1; j < n; j++) { 2150 int p1 = get_constraint_priority(def, a[start + i].sort_index); 2151 int p2 = get_constraint_priority(def, a[start + j].sort_index); 2152 if (p1 < p2) { 2153 int tmp = a[start + i].sort_index; 2154 a[start + i].sort_index = a[start + j].sort_index; 2155 a[start + j].sort_index = tmp; 2156 } 2157 } 2158 } 2159 } 2160 2161 static void process_op_defs(TCGContext *s) 2162 { 2163 TCGOpcode op; 2164 2165 for (op = 0; op < NB_OPS; op++) { 2166 TCGOpDef *def = &tcg_op_defs[op]; 2167 const TCGTargetOpDef *tdefs; 2168 bool saw_alias_pair = false; 2169 int i, o, i2, o2, nb_args; 2170 2171 if (def->flags & TCG_OPF_NOT_PRESENT) { 2172 continue; 2173 } 2174 2175 nb_args = def->nb_iargs + def->nb_oargs; 2176 if (nb_args == 0) { 2177 continue; 2178 } 2179 2180 /* 2181 * Macro magic should make it impossible, but double-check that 2182 * the array index is in range. Since the signness of an enum 2183 * is implementation defined, force the result to unsigned. 2184 */ 2185 unsigned con_set = tcg_target_op_def(op); 2186 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 2187 tdefs = &constraint_sets[con_set]; 2188 2189 for (i = 0; i < nb_args; i++) { 2190 const char *ct_str = tdefs->args_ct_str[i]; 2191 bool input_p = i >= def->nb_oargs; 2192 2193 /* Incomplete TCGTargetOpDef entry. */ 2194 tcg_debug_assert(ct_str != NULL); 2195 2196 switch (*ct_str) { 2197 case '0' ... '9': 2198 o = *ct_str - '0'; 2199 tcg_debug_assert(input_p); 2200 tcg_debug_assert(o < def->nb_oargs); 2201 tcg_debug_assert(def->args_ct[o].regs != 0); 2202 tcg_debug_assert(!def->args_ct[o].oalias); 2203 def->args_ct[i] = def->args_ct[o]; 2204 /* The output sets oalias. */ 2205 def->args_ct[o].oalias = 1; 2206 def->args_ct[o].alias_index = i; 2207 /* The input sets ialias. */ 2208 def->args_ct[i].ialias = 1; 2209 def->args_ct[i].alias_index = o; 2210 if (def->args_ct[i].pair) { 2211 saw_alias_pair = true; 2212 } 2213 tcg_debug_assert(ct_str[1] == '\0'); 2214 continue; 2215 2216 case '&': 2217 tcg_debug_assert(!input_p); 2218 def->args_ct[i].newreg = true; 2219 ct_str++; 2220 break; 2221 2222 case 'p': /* plus */ 2223 /* Allocate to the register after the previous. */ 2224 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 2225 o = i - 1; 2226 tcg_debug_assert(!def->args_ct[o].pair); 2227 tcg_debug_assert(!def->args_ct[o].ct); 2228 def->args_ct[i] = (TCGArgConstraint){ 2229 .pair = 2, 2230 .pair_index = o, 2231 .regs = def->args_ct[o].regs << 1, 2232 }; 2233 def->args_ct[o].pair = 1; 2234 def->args_ct[o].pair_index = i; 2235 tcg_debug_assert(ct_str[1] == '\0'); 2236 continue; 2237 2238 case 'm': /* minus */ 2239 /* Allocate to the register before the previous. */ 2240 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 2241 o = i - 1; 2242 tcg_debug_assert(!def->args_ct[o].pair); 2243 tcg_debug_assert(!def->args_ct[o].ct); 2244 def->args_ct[i] = (TCGArgConstraint){ 2245 .pair = 1, 2246 .pair_index = o, 2247 .regs = def->args_ct[o].regs >> 1, 2248 }; 2249 def->args_ct[o].pair = 2; 2250 def->args_ct[o].pair_index = i; 2251 tcg_debug_assert(ct_str[1] == '\0'); 2252 continue; 2253 } 2254 2255 do { 2256 switch (*ct_str) { 2257 case 'i': 2258 def->args_ct[i].ct |= TCG_CT_CONST; 2259 break; 2260 2261 /* Include all of the target-specific constraints. */ 2262 2263 #undef CONST 2264 #define CONST(CASE, MASK) \ 2265 case CASE: def->args_ct[i].ct |= MASK; break; 2266 #define REGS(CASE, MASK) \ 2267 case CASE: def->args_ct[i].regs |= MASK; break; 2268 2269 #include "tcg-target-con-str.h" 2270 2271 #undef REGS 2272 #undef CONST 2273 default: 2274 case '0' ... '9': 2275 case '&': 2276 case 'p': 2277 case 'm': 2278 /* Typo in TCGTargetOpDef constraint. */ 2279 g_assert_not_reached(); 2280 } 2281 } while (*++ct_str != '\0'); 2282 } 2283 2284 /* TCGTargetOpDef entry with too much information? */ 2285 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2286 2287 /* 2288 * Fix up output pairs that are aliased with inputs. 2289 * When we created the alias, we copied pair from the output. 2290 * There are three cases: 2291 * (1a) Pairs of inputs alias pairs of outputs. 2292 * (1b) One input aliases the first of a pair of outputs. 2293 * (2) One input aliases the second of a pair of outputs. 2294 * 2295 * Case 1a is handled by making sure that the pair_index'es are 2296 * properly updated so that they appear the same as a pair of inputs. 2297 * 2298 * Case 1b is handled by setting the pair_index of the input to 2299 * itself, simply so it doesn't point to an unrelated argument. 2300 * Since we don't encounter the "second" during the input allocation 2301 * phase, nothing happens with the second half of the input pair. 2302 * 2303 * Case 2 is handled by setting the second input to pair=3, the 2304 * first output to pair=3, and the pair_index'es to match. 2305 */ 2306 if (saw_alias_pair) { 2307 for (i = def->nb_oargs; i < nb_args; i++) { 2308 /* 2309 * Since [0-9pm] must be alone in the constraint string, 2310 * the only way they can both be set is if the pair comes 2311 * from the output alias. 2312 */ 2313 if (!def->args_ct[i].ialias) { 2314 continue; 2315 } 2316 switch (def->args_ct[i].pair) { 2317 case 0: 2318 break; 2319 case 1: 2320 o = def->args_ct[i].alias_index; 2321 o2 = def->args_ct[o].pair_index; 2322 tcg_debug_assert(def->args_ct[o].pair == 1); 2323 tcg_debug_assert(def->args_ct[o2].pair == 2); 2324 if (def->args_ct[o2].oalias) { 2325 /* Case 1a */ 2326 i2 = def->args_ct[o2].alias_index; 2327 tcg_debug_assert(def->args_ct[i2].pair == 2); 2328 def->args_ct[i2].pair_index = i; 2329 def->args_ct[i].pair_index = i2; 2330 } else { 2331 /* Case 1b */ 2332 def->args_ct[i].pair_index = i; 2333 } 2334 break; 2335 case 2: 2336 o = def->args_ct[i].alias_index; 2337 o2 = def->args_ct[o].pair_index; 2338 tcg_debug_assert(def->args_ct[o].pair == 2); 2339 tcg_debug_assert(def->args_ct[o2].pair == 1); 2340 if (def->args_ct[o2].oalias) { 2341 /* Case 1a */ 2342 i2 = def->args_ct[o2].alias_index; 2343 tcg_debug_assert(def->args_ct[i2].pair == 1); 2344 def->args_ct[i2].pair_index = i; 2345 def->args_ct[i].pair_index = i2; 2346 } else { 2347 /* Case 2 */ 2348 def->args_ct[i].pair = 3; 2349 def->args_ct[o2].pair = 3; 2350 def->args_ct[i].pair_index = o2; 2351 def->args_ct[o2].pair_index = i; 2352 } 2353 break; 2354 default: 2355 g_assert_not_reached(); 2356 } 2357 } 2358 } 2359 2360 /* sort the constraints (XXX: this is just an heuristic) */ 2361 sort_constraints(def, 0, def->nb_oargs); 2362 sort_constraints(def, def->nb_oargs, def->nb_iargs); 2363 } 2364 } 2365 2366 void tcg_op_remove(TCGContext *s, TCGOp *op) 2367 { 2368 TCGLabel *label; 2369 2370 switch (op->opc) { 2371 case INDEX_op_br: 2372 label = arg_label(op->args[0]); 2373 label->refs--; 2374 break; 2375 case INDEX_op_brcond_i32: 2376 case INDEX_op_brcond_i64: 2377 label = arg_label(op->args[3]); 2378 label->refs--; 2379 break; 2380 case INDEX_op_brcond2_i32: 2381 label = arg_label(op->args[5]); 2382 label->refs--; 2383 break; 2384 default: 2385 break; 2386 } 2387 2388 QTAILQ_REMOVE(&s->ops, op, link); 2389 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2390 s->nb_ops--; 2391 2392 #ifdef CONFIG_PROFILER 2393 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 2394 #endif 2395 } 2396 2397 void tcg_remove_ops_after(TCGOp *op) 2398 { 2399 TCGContext *s = tcg_ctx; 2400 2401 while (true) { 2402 TCGOp *last = tcg_last_op(); 2403 if (last == op) { 2404 return; 2405 } 2406 tcg_op_remove(s, last); 2407 } 2408 } 2409 2410 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) 2411 { 2412 TCGContext *s = tcg_ctx; 2413 TCGOp *op = NULL; 2414 2415 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { 2416 QTAILQ_FOREACH(op, &s->free_ops, link) { 2417 if (nargs <= op->nargs) { 2418 QTAILQ_REMOVE(&s->free_ops, op, link); 2419 nargs = op->nargs; 2420 goto found; 2421 } 2422 } 2423 } 2424 2425 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ 2426 nargs = MAX(4, nargs); 2427 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); 2428 2429 found: 2430 memset(op, 0, offsetof(TCGOp, link)); 2431 op->opc = opc; 2432 op->nargs = nargs; 2433 2434 /* Check for bitfield overflow. */ 2435 tcg_debug_assert(op->nargs == nargs); 2436 2437 s->nb_ops++; 2438 return op; 2439 } 2440 2441 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) 2442 { 2443 TCGOp *op = tcg_op_alloc(opc, nargs); 2444 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2445 return op; 2446 } 2447 2448 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 2449 TCGOpcode opc, unsigned nargs) 2450 { 2451 TCGOp *new_op = tcg_op_alloc(opc, nargs); 2452 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 2453 return new_op; 2454 } 2455 2456 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 2457 TCGOpcode opc, unsigned nargs) 2458 { 2459 TCGOp *new_op = tcg_op_alloc(opc, nargs); 2460 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 2461 return new_op; 2462 } 2463 2464 /* Reachable analysis : remove unreachable code. */ 2465 static void reachable_code_pass(TCGContext *s) 2466 { 2467 TCGOp *op, *op_next; 2468 bool dead = false; 2469 2470 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2471 bool remove = dead; 2472 TCGLabel *label; 2473 2474 switch (op->opc) { 2475 case INDEX_op_set_label: 2476 label = arg_label(op->args[0]); 2477 if (label->refs == 0) { 2478 /* 2479 * While there is an occasional backward branch, virtually 2480 * all branches generated by the translators are forward. 2481 * Which means that generally we will have already removed 2482 * all references to the label that will be, and there is 2483 * little to be gained by iterating. 2484 */ 2485 remove = true; 2486 } else { 2487 /* Once we see a label, insns become live again. */ 2488 dead = false; 2489 remove = false; 2490 2491 /* 2492 * Optimization can fold conditional branches to unconditional. 2493 * If we find a label with one reference which is preceded by 2494 * an unconditional branch to it, remove both. This needed to 2495 * wait until the dead code in between them was removed. 2496 */ 2497 if (label->refs == 1) { 2498 TCGOp *op_prev = QTAILQ_PREV(op, link); 2499 if (op_prev->opc == INDEX_op_br && 2500 label == arg_label(op_prev->args[0])) { 2501 tcg_op_remove(s, op_prev); 2502 remove = true; 2503 } 2504 } 2505 } 2506 break; 2507 2508 case INDEX_op_br: 2509 case INDEX_op_exit_tb: 2510 case INDEX_op_goto_ptr: 2511 /* Unconditional branches; everything following is dead. */ 2512 dead = true; 2513 break; 2514 2515 case INDEX_op_call: 2516 /* Notice noreturn helper calls, raising exceptions. */ 2517 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2518 dead = true; 2519 } 2520 break; 2521 2522 case INDEX_op_insn_start: 2523 /* Never remove -- we need to keep these for unwind. */ 2524 remove = false; 2525 break; 2526 2527 default: 2528 break; 2529 } 2530 2531 if (remove) { 2532 tcg_op_remove(s, op); 2533 } 2534 } 2535 } 2536 2537 #define TS_DEAD 1 2538 #define TS_MEM 2 2539 2540 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 2541 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 2542 2543 /* For liveness_pass_1, the register preferences for a given temp. */ 2544 static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 2545 { 2546 return ts->state_ptr; 2547 } 2548 2549 /* For liveness_pass_1, reset the preferences for a given temp to the 2550 * maximal regset for its type. 2551 */ 2552 static inline void la_reset_pref(TCGTemp *ts) 2553 { 2554 *la_temp_pref(ts) 2555 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 2556 } 2557 2558 /* liveness analysis: end of function: all temps are dead, and globals 2559 should be in memory. */ 2560 static void la_func_end(TCGContext *s, int ng, int nt) 2561 { 2562 int i; 2563 2564 for (i = 0; i < ng; ++i) { 2565 s->temps[i].state = TS_DEAD | TS_MEM; 2566 la_reset_pref(&s->temps[i]); 2567 } 2568 for (i = ng; i < nt; ++i) { 2569 s->temps[i].state = TS_DEAD; 2570 la_reset_pref(&s->temps[i]); 2571 } 2572 } 2573 2574 /* liveness analysis: end of basic block: all temps are dead, globals 2575 and local temps should be in memory. */ 2576 static void la_bb_end(TCGContext *s, int ng, int nt) 2577 { 2578 int i; 2579 2580 for (i = 0; i < nt; ++i) { 2581 TCGTemp *ts = &s->temps[i]; 2582 int state; 2583 2584 switch (ts->kind) { 2585 case TEMP_FIXED: 2586 case TEMP_GLOBAL: 2587 case TEMP_LOCAL: 2588 state = TS_DEAD | TS_MEM; 2589 break; 2590 case TEMP_NORMAL: 2591 case TEMP_EBB: 2592 case TEMP_CONST: 2593 state = TS_DEAD; 2594 break; 2595 default: 2596 g_assert_not_reached(); 2597 } 2598 ts->state = state; 2599 la_reset_pref(ts); 2600 } 2601 } 2602 2603 /* liveness analysis: sync globals back to memory. */ 2604 static void la_global_sync(TCGContext *s, int ng) 2605 { 2606 int i; 2607 2608 for (i = 0; i < ng; ++i) { 2609 int state = s->temps[i].state; 2610 s->temps[i].state = state | TS_MEM; 2611 if (state == TS_DEAD) { 2612 /* If the global was previously dead, reset prefs. */ 2613 la_reset_pref(&s->temps[i]); 2614 } 2615 } 2616 } 2617 2618 /* 2619 * liveness analysis: conditional branch: all temps are dead unless 2620 * explicitly live-across-conditional-branch, globals and local temps 2621 * should be synced. 2622 */ 2623 static void la_bb_sync(TCGContext *s, int ng, int nt) 2624 { 2625 la_global_sync(s, ng); 2626 2627 for (int i = ng; i < nt; ++i) { 2628 TCGTemp *ts = &s->temps[i]; 2629 int state; 2630 2631 switch (ts->kind) { 2632 case TEMP_LOCAL: 2633 state = ts->state; 2634 ts->state = state | TS_MEM; 2635 if (state != TS_DEAD) { 2636 continue; 2637 } 2638 break; 2639 case TEMP_NORMAL: 2640 s->temps[i].state = TS_DEAD; 2641 break; 2642 case TEMP_EBB: 2643 case TEMP_CONST: 2644 continue; 2645 default: 2646 g_assert_not_reached(); 2647 } 2648 la_reset_pref(&s->temps[i]); 2649 } 2650 } 2651 2652 /* liveness analysis: sync globals back to memory and kill. */ 2653 static void la_global_kill(TCGContext *s, int ng) 2654 { 2655 int i; 2656 2657 for (i = 0; i < ng; i++) { 2658 s->temps[i].state = TS_DEAD | TS_MEM; 2659 la_reset_pref(&s->temps[i]); 2660 } 2661 } 2662 2663 /* liveness analysis: note live globals crossing calls. */ 2664 static void la_cross_call(TCGContext *s, int nt) 2665 { 2666 TCGRegSet mask = ~tcg_target_call_clobber_regs; 2667 int i; 2668 2669 for (i = 0; i < nt; i++) { 2670 TCGTemp *ts = &s->temps[i]; 2671 if (!(ts->state & TS_DEAD)) { 2672 TCGRegSet *pset = la_temp_pref(ts); 2673 TCGRegSet set = *pset; 2674 2675 set &= mask; 2676 /* If the combination is not possible, restart. */ 2677 if (set == 0) { 2678 set = tcg_target_available_regs[ts->type] & mask; 2679 } 2680 *pset = set; 2681 } 2682 } 2683 } 2684 2685 /* Liveness analysis : update the opc_arg_life array to tell if a 2686 given input arguments is dead. Instructions updating dead 2687 temporaries are removed. */ 2688 static void liveness_pass_1(TCGContext *s) 2689 { 2690 int nb_globals = s->nb_globals; 2691 int nb_temps = s->nb_temps; 2692 TCGOp *op, *op_prev; 2693 TCGRegSet *prefs; 2694 int i; 2695 2696 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 2697 for (i = 0; i < nb_temps; ++i) { 2698 s->temps[i].state_ptr = prefs + i; 2699 } 2700 2701 /* ??? Should be redundant with the exit_tb that ends the TB. */ 2702 la_func_end(s, nb_globals, nb_temps); 2703 2704 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 2705 int nb_iargs, nb_oargs; 2706 TCGOpcode opc_new, opc_new2; 2707 bool have_opc_new2; 2708 TCGLifeData arg_life = 0; 2709 TCGTemp *ts; 2710 TCGOpcode opc = op->opc; 2711 const TCGOpDef *def = &tcg_op_defs[opc]; 2712 2713 switch (opc) { 2714 case INDEX_op_call: 2715 { 2716 const TCGHelperInfo *info = tcg_call_info(op); 2717 int call_flags = tcg_call_flags(op); 2718 2719 nb_oargs = TCGOP_CALLO(op); 2720 nb_iargs = TCGOP_CALLI(op); 2721 2722 /* pure functions can be removed if their result is unused */ 2723 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 2724 for (i = 0; i < nb_oargs; i++) { 2725 ts = arg_temp(op->args[i]); 2726 if (ts->state != TS_DEAD) { 2727 goto do_not_remove_call; 2728 } 2729 } 2730 goto do_remove; 2731 } 2732 do_not_remove_call: 2733 2734 /* Output args are dead. */ 2735 for (i = 0; i < nb_oargs; i++) { 2736 ts = arg_temp(op->args[i]); 2737 if (ts->state & TS_DEAD) { 2738 arg_life |= DEAD_ARG << i; 2739 } 2740 if (ts->state & TS_MEM) { 2741 arg_life |= SYNC_ARG << i; 2742 } 2743 ts->state = TS_DEAD; 2744 la_reset_pref(ts); 2745 } 2746 2747 /* Not used -- it will be tcg_target_call_oarg_reg(). */ 2748 memset(op->output_pref, 0, sizeof(op->output_pref)); 2749 2750 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 2751 TCG_CALL_NO_READ_GLOBALS))) { 2752 la_global_kill(s, nb_globals); 2753 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 2754 la_global_sync(s, nb_globals); 2755 } 2756 2757 /* Record arguments that die in this helper. */ 2758 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2759 ts = arg_temp(op->args[i]); 2760 if (ts->state & TS_DEAD) { 2761 arg_life |= DEAD_ARG << i; 2762 } 2763 } 2764 2765 /* For all live registers, remove call-clobbered prefs. */ 2766 la_cross_call(s, nb_temps); 2767 2768 /* 2769 * Input arguments are live for preceding opcodes. 2770 * 2771 * For those arguments that die, and will be allocated in 2772 * registers, clear the register set for that arg, to be 2773 * filled in below. For args that will be on the stack, 2774 * reset to any available reg. Process arguments in reverse 2775 * order so that if a temp is used more than once, the stack 2776 * reset to max happens before the register reset to 0. 2777 */ 2778 for (i = nb_iargs - 1; i >= 0; i--) { 2779 const TCGCallArgumentLoc *loc = &info->in[i]; 2780 ts = arg_temp(op->args[nb_oargs + i]); 2781 2782 if (ts->state & TS_DEAD) { 2783 switch (loc->kind) { 2784 case TCG_CALL_ARG_NORMAL: 2785 case TCG_CALL_ARG_EXTEND_U: 2786 case TCG_CALL_ARG_EXTEND_S: 2787 if (REG_P(loc)) { 2788 *la_temp_pref(ts) = 0; 2789 break; 2790 } 2791 /* fall through */ 2792 default: 2793 *la_temp_pref(ts) = 2794 tcg_target_available_regs[ts->type]; 2795 break; 2796 } 2797 ts->state &= ~TS_DEAD; 2798 } 2799 } 2800 2801 /* 2802 * For each input argument, add its input register to prefs. 2803 * If a temp is used once, this produces a single set bit; 2804 * if a temp is used multiple times, this produces a set. 2805 */ 2806 for (i = 0; i < nb_iargs; i++) { 2807 const TCGCallArgumentLoc *loc = &info->in[i]; 2808 ts = arg_temp(op->args[nb_oargs + i]); 2809 2810 switch (loc->kind) { 2811 case TCG_CALL_ARG_NORMAL: 2812 case TCG_CALL_ARG_EXTEND_U: 2813 case TCG_CALL_ARG_EXTEND_S: 2814 if (REG_P(loc)) { 2815 tcg_regset_set_reg(*la_temp_pref(ts), 2816 tcg_target_call_iarg_regs[loc->arg_slot]); 2817 } 2818 break; 2819 default: 2820 break; 2821 } 2822 } 2823 } 2824 break; 2825 case INDEX_op_insn_start: 2826 break; 2827 case INDEX_op_discard: 2828 /* mark the temporary as dead */ 2829 ts = arg_temp(op->args[0]); 2830 ts->state = TS_DEAD; 2831 la_reset_pref(ts); 2832 break; 2833 2834 case INDEX_op_add2_i32: 2835 opc_new = INDEX_op_add_i32; 2836 goto do_addsub2; 2837 case INDEX_op_sub2_i32: 2838 opc_new = INDEX_op_sub_i32; 2839 goto do_addsub2; 2840 case INDEX_op_add2_i64: 2841 opc_new = INDEX_op_add_i64; 2842 goto do_addsub2; 2843 case INDEX_op_sub2_i64: 2844 opc_new = INDEX_op_sub_i64; 2845 do_addsub2: 2846 nb_iargs = 4; 2847 nb_oargs = 2; 2848 /* Test if the high part of the operation is dead, but not 2849 the low part. The result can be optimized to a simple 2850 add or sub. This happens often for x86_64 guest when the 2851 cpu mode is set to 32 bit. */ 2852 if (arg_temp(op->args[1])->state == TS_DEAD) { 2853 if (arg_temp(op->args[0])->state == TS_DEAD) { 2854 goto do_remove; 2855 } 2856 /* Replace the opcode and adjust the args in place, 2857 leaving 3 unused args at the end. */ 2858 op->opc = opc = opc_new; 2859 op->args[1] = op->args[2]; 2860 op->args[2] = op->args[4]; 2861 /* Fall through and mark the single-word operation live. */ 2862 nb_iargs = 2; 2863 nb_oargs = 1; 2864 } 2865 goto do_not_remove; 2866 2867 case INDEX_op_mulu2_i32: 2868 opc_new = INDEX_op_mul_i32; 2869 opc_new2 = INDEX_op_muluh_i32; 2870 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 2871 goto do_mul2; 2872 case INDEX_op_muls2_i32: 2873 opc_new = INDEX_op_mul_i32; 2874 opc_new2 = INDEX_op_mulsh_i32; 2875 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 2876 goto do_mul2; 2877 case INDEX_op_mulu2_i64: 2878 opc_new = INDEX_op_mul_i64; 2879 opc_new2 = INDEX_op_muluh_i64; 2880 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 2881 goto do_mul2; 2882 case INDEX_op_muls2_i64: 2883 opc_new = INDEX_op_mul_i64; 2884 opc_new2 = INDEX_op_mulsh_i64; 2885 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 2886 goto do_mul2; 2887 do_mul2: 2888 nb_iargs = 2; 2889 nb_oargs = 2; 2890 if (arg_temp(op->args[1])->state == TS_DEAD) { 2891 if (arg_temp(op->args[0])->state == TS_DEAD) { 2892 /* Both parts of the operation are dead. */ 2893 goto do_remove; 2894 } 2895 /* The high part of the operation is dead; generate the low. */ 2896 op->opc = opc = opc_new; 2897 op->args[1] = op->args[2]; 2898 op->args[2] = op->args[3]; 2899 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 2900 /* The low part of the operation is dead; generate the high. */ 2901 op->opc = opc = opc_new2; 2902 op->args[0] = op->args[1]; 2903 op->args[1] = op->args[2]; 2904 op->args[2] = op->args[3]; 2905 } else { 2906 goto do_not_remove; 2907 } 2908 /* Mark the single-word operation live. */ 2909 nb_oargs = 1; 2910 goto do_not_remove; 2911 2912 default: 2913 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 2914 nb_iargs = def->nb_iargs; 2915 nb_oargs = def->nb_oargs; 2916 2917 /* Test if the operation can be removed because all 2918 its outputs are dead. We assume that nb_oargs == 0 2919 implies side effects */ 2920 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 2921 for (i = 0; i < nb_oargs; i++) { 2922 if (arg_temp(op->args[i])->state != TS_DEAD) { 2923 goto do_not_remove; 2924 } 2925 } 2926 goto do_remove; 2927 } 2928 goto do_not_remove; 2929 2930 do_remove: 2931 tcg_op_remove(s, op); 2932 break; 2933 2934 do_not_remove: 2935 for (i = 0; i < nb_oargs; i++) { 2936 ts = arg_temp(op->args[i]); 2937 2938 /* Remember the preference of the uses that followed. */ 2939 if (i < ARRAY_SIZE(op->output_pref)) { 2940 op->output_pref[i] = *la_temp_pref(ts); 2941 } 2942 2943 /* Output args are dead. */ 2944 if (ts->state & TS_DEAD) { 2945 arg_life |= DEAD_ARG << i; 2946 } 2947 if (ts->state & TS_MEM) { 2948 arg_life |= SYNC_ARG << i; 2949 } 2950 ts->state = TS_DEAD; 2951 la_reset_pref(ts); 2952 } 2953 2954 /* If end of basic block, update. */ 2955 if (def->flags & TCG_OPF_BB_EXIT) { 2956 la_func_end(s, nb_globals, nb_temps); 2957 } else if (def->flags & TCG_OPF_COND_BRANCH) { 2958 la_bb_sync(s, nb_globals, nb_temps); 2959 } else if (def->flags & TCG_OPF_BB_END) { 2960 la_bb_end(s, nb_globals, nb_temps); 2961 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2962 la_global_sync(s, nb_globals); 2963 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2964 la_cross_call(s, nb_temps); 2965 } 2966 } 2967 2968 /* Record arguments that die in this opcode. */ 2969 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2970 ts = arg_temp(op->args[i]); 2971 if (ts->state & TS_DEAD) { 2972 arg_life |= DEAD_ARG << i; 2973 } 2974 } 2975 2976 /* Input arguments are live for preceding opcodes. */ 2977 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2978 ts = arg_temp(op->args[i]); 2979 if (ts->state & TS_DEAD) { 2980 /* For operands that were dead, initially allow 2981 all regs for the type. */ 2982 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 2983 ts->state &= ~TS_DEAD; 2984 } 2985 } 2986 2987 /* Incorporate constraints for this operand. */ 2988 switch (opc) { 2989 case INDEX_op_mov_i32: 2990 case INDEX_op_mov_i64: 2991 /* Note that these are TCG_OPF_NOT_PRESENT and do not 2992 have proper constraints. That said, special case 2993 moves to propagate preferences backward. */ 2994 if (IS_DEAD_ARG(1)) { 2995 *la_temp_pref(arg_temp(op->args[0])) 2996 = *la_temp_pref(arg_temp(op->args[1])); 2997 } 2998 break; 2999 3000 default: 3001 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3002 const TCGArgConstraint *ct = &def->args_ct[i]; 3003 TCGRegSet set, *pset; 3004 3005 ts = arg_temp(op->args[i]); 3006 pset = la_temp_pref(ts); 3007 set = *pset; 3008 3009 set &= ct->regs; 3010 if (ct->ialias) { 3011 set &= output_pref(op, ct->alias_index); 3012 } 3013 /* If the combination is not possible, restart. */ 3014 if (set == 0) { 3015 set = ct->regs; 3016 } 3017 *pset = set; 3018 } 3019 break; 3020 } 3021 break; 3022 } 3023 op->life = arg_life; 3024 } 3025 } 3026 3027 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 3028 static bool liveness_pass_2(TCGContext *s) 3029 { 3030 int nb_globals = s->nb_globals; 3031 int nb_temps, i; 3032 bool changes = false; 3033 TCGOp *op, *op_next; 3034 3035 /* Create a temporary for each indirect global. */ 3036 for (i = 0; i < nb_globals; ++i) { 3037 TCGTemp *its = &s->temps[i]; 3038 if (its->indirect_reg) { 3039 TCGTemp *dts = tcg_temp_alloc(s); 3040 dts->type = its->type; 3041 dts->base_type = its->base_type; 3042 dts->kind = TEMP_EBB; 3043 its->state_ptr = dts; 3044 } else { 3045 its->state_ptr = NULL; 3046 } 3047 /* All globals begin dead. */ 3048 its->state = TS_DEAD; 3049 } 3050 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 3051 TCGTemp *its = &s->temps[i]; 3052 its->state_ptr = NULL; 3053 its->state = TS_DEAD; 3054 } 3055 3056 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 3057 TCGOpcode opc = op->opc; 3058 const TCGOpDef *def = &tcg_op_defs[opc]; 3059 TCGLifeData arg_life = op->life; 3060 int nb_iargs, nb_oargs, call_flags; 3061 TCGTemp *arg_ts, *dir_ts; 3062 3063 if (opc == INDEX_op_call) { 3064 nb_oargs = TCGOP_CALLO(op); 3065 nb_iargs = TCGOP_CALLI(op); 3066 call_flags = tcg_call_flags(op); 3067 } else { 3068 nb_iargs = def->nb_iargs; 3069 nb_oargs = def->nb_oargs; 3070 3071 /* Set flags similar to how calls require. */ 3072 if (def->flags & TCG_OPF_COND_BRANCH) { 3073 /* Like reading globals: sync_globals */ 3074 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 3075 } else if (def->flags & TCG_OPF_BB_END) { 3076 /* Like writing globals: save_globals */ 3077 call_flags = 0; 3078 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3079 /* Like reading globals: sync_globals */ 3080 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 3081 } else { 3082 /* No effect on globals. */ 3083 call_flags = (TCG_CALL_NO_READ_GLOBALS | 3084 TCG_CALL_NO_WRITE_GLOBALS); 3085 } 3086 } 3087 3088 /* Make sure that input arguments are available. */ 3089 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3090 arg_ts = arg_temp(op->args[i]); 3091 dir_ts = arg_ts->state_ptr; 3092 if (dir_ts && arg_ts->state == TS_DEAD) { 3093 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 3094 ? INDEX_op_ld_i32 3095 : INDEX_op_ld_i64); 3096 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 3097 3098 lop->args[0] = temp_arg(dir_ts); 3099 lop->args[1] = temp_arg(arg_ts->mem_base); 3100 lop->args[2] = arg_ts->mem_offset; 3101 3102 /* Loaded, but synced with memory. */ 3103 arg_ts->state = TS_MEM; 3104 } 3105 } 3106 3107 /* Perform input replacement, and mark inputs that became dead. 3108 No action is required except keeping temp_state up to date 3109 so that we reload when needed. */ 3110 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3111 arg_ts = arg_temp(op->args[i]); 3112 dir_ts = arg_ts->state_ptr; 3113 if (dir_ts) { 3114 op->args[i] = temp_arg(dir_ts); 3115 changes = true; 3116 if (IS_DEAD_ARG(i)) { 3117 arg_ts->state = TS_DEAD; 3118 } 3119 } 3120 } 3121 3122 /* Liveness analysis should ensure that the following are 3123 all correct, for call sites and basic block end points. */ 3124 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 3125 /* Nothing to do */ 3126 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 3127 for (i = 0; i < nb_globals; ++i) { 3128 /* Liveness should see that globals are synced back, 3129 that is, either TS_DEAD or TS_MEM. */ 3130 arg_ts = &s->temps[i]; 3131 tcg_debug_assert(arg_ts->state_ptr == 0 3132 || arg_ts->state != 0); 3133 } 3134 } else { 3135 for (i = 0; i < nb_globals; ++i) { 3136 /* Liveness should see that globals are saved back, 3137 that is, TS_DEAD, waiting to be reloaded. */ 3138 arg_ts = &s->temps[i]; 3139 tcg_debug_assert(arg_ts->state_ptr == 0 3140 || arg_ts->state == TS_DEAD); 3141 } 3142 } 3143 3144 /* Outputs become available. */ 3145 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 3146 arg_ts = arg_temp(op->args[0]); 3147 dir_ts = arg_ts->state_ptr; 3148 if (dir_ts) { 3149 op->args[0] = temp_arg(dir_ts); 3150 changes = true; 3151 3152 /* The output is now live and modified. */ 3153 arg_ts->state = 0; 3154 3155 if (NEED_SYNC_ARG(0)) { 3156 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 3157 ? INDEX_op_st_i32 3158 : INDEX_op_st_i64); 3159 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 3160 TCGTemp *out_ts = dir_ts; 3161 3162 if (IS_DEAD_ARG(0)) { 3163 out_ts = arg_temp(op->args[1]); 3164 arg_ts->state = TS_DEAD; 3165 tcg_op_remove(s, op); 3166 } else { 3167 arg_ts->state = TS_MEM; 3168 } 3169 3170 sop->args[0] = temp_arg(out_ts); 3171 sop->args[1] = temp_arg(arg_ts->mem_base); 3172 sop->args[2] = arg_ts->mem_offset; 3173 } else { 3174 tcg_debug_assert(!IS_DEAD_ARG(0)); 3175 } 3176 } 3177 } else { 3178 for (i = 0; i < nb_oargs; i++) { 3179 arg_ts = arg_temp(op->args[i]); 3180 dir_ts = arg_ts->state_ptr; 3181 if (!dir_ts) { 3182 continue; 3183 } 3184 op->args[i] = temp_arg(dir_ts); 3185 changes = true; 3186 3187 /* The output is now live and modified. */ 3188 arg_ts->state = 0; 3189 3190 /* Sync outputs upon their last write. */ 3191 if (NEED_SYNC_ARG(i)) { 3192 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 3193 ? INDEX_op_st_i32 3194 : INDEX_op_st_i64); 3195 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 3196 3197 sop->args[0] = temp_arg(dir_ts); 3198 sop->args[1] = temp_arg(arg_ts->mem_base); 3199 sop->args[2] = arg_ts->mem_offset; 3200 3201 arg_ts->state = TS_MEM; 3202 } 3203 /* Drop outputs that are dead. */ 3204 if (IS_DEAD_ARG(i)) { 3205 arg_ts->state = TS_DEAD; 3206 } 3207 } 3208 } 3209 } 3210 3211 return changes; 3212 } 3213 3214 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3215 { 3216 int size = tcg_type_size(ts->type); 3217 int align; 3218 intptr_t off; 3219 3220 switch (ts->type) { 3221 case TCG_TYPE_I32: 3222 align = 4; 3223 break; 3224 case TCG_TYPE_I64: 3225 case TCG_TYPE_V64: 3226 align = 8; 3227 break; 3228 case TCG_TYPE_V128: 3229 case TCG_TYPE_V256: 3230 /* Note that we do not require aligned storage for V256. */ 3231 align = 16; 3232 break; 3233 default: 3234 g_assert_not_reached(); 3235 } 3236 3237 /* 3238 * Assume the stack is sufficiently aligned. 3239 * This affects e.g. ARM NEON, where we have 8 byte stack alignment 3240 * and do not require 16 byte vector alignment. This seems slightly 3241 * easier than fully parameterizing the above switch statement. 3242 */ 3243 align = MIN(TCG_TARGET_STACK_ALIGN, align); 3244 off = ROUND_UP(s->current_frame_offset, align); 3245 3246 /* If we've exhausted the stack frame, restart with a smaller TB. */ 3247 if (off + size > s->frame_end) { 3248 tcg_raise_tb_overflow(s); 3249 } 3250 s->current_frame_offset = off + size; 3251 3252 ts->mem_offset = off; 3253 #if defined(__sparc__) 3254 ts->mem_offset += TCG_TARGET_STACK_BIAS; 3255 #endif 3256 ts->mem_base = s->frame_temp; 3257 ts->mem_allocated = 1; 3258 } 3259 3260 /* Assign @reg to @ts, and update reg_to_temp[]. */ 3261 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 3262 { 3263 if (ts->val_type == TEMP_VAL_REG) { 3264 TCGReg old = ts->reg; 3265 tcg_debug_assert(s->reg_to_temp[old] == ts); 3266 if (old == reg) { 3267 return; 3268 } 3269 s->reg_to_temp[old] = NULL; 3270 } 3271 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3272 s->reg_to_temp[reg] = ts; 3273 ts->val_type = TEMP_VAL_REG; 3274 ts->reg = reg; 3275 } 3276 3277 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 3278 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 3279 { 3280 tcg_debug_assert(type != TEMP_VAL_REG); 3281 if (ts->val_type == TEMP_VAL_REG) { 3282 TCGReg reg = ts->reg; 3283 tcg_debug_assert(s->reg_to_temp[reg] == ts); 3284 s->reg_to_temp[reg] = NULL; 3285 } 3286 ts->val_type = type; 3287 } 3288 3289 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3290 3291 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 3292 mark it free; otherwise mark it dead. */ 3293 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3294 { 3295 TCGTempVal new_type; 3296 3297 switch (ts->kind) { 3298 case TEMP_FIXED: 3299 return; 3300 case TEMP_GLOBAL: 3301 case TEMP_LOCAL: 3302 new_type = TEMP_VAL_MEM; 3303 break; 3304 case TEMP_NORMAL: 3305 case TEMP_EBB: 3306 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3307 break; 3308 case TEMP_CONST: 3309 new_type = TEMP_VAL_CONST; 3310 break; 3311 default: 3312 g_assert_not_reached(); 3313 } 3314 set_temp_val_nonreg(s, ts, new_type); 3315 } 3316 3317 /* Mark a temporary as dead. */ 3318 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 3319 { 3320 temp_free_or_dead(s, ts, 1); 3321 } 3322 3323 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 3324 registers needs to be allocated to store a constant. If 'free_or_dead' 3325 is non-zero, subsequently release the temporary; if it is positive, the 3326 temp is dead; if it is negative, the temp is free. */ 3327 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 3328 TCGRegSet preferred_regs, int free_or_dead) 3329 { 3330 if (!temp_readonly(ts) && !ts->mem_coherent) { 3331 if (!ts->mem_allocated) { 3332 temp_allocate_frame(s, ts); 3333 } 3334 switch (ts->val_type) { 3335 case TEMP_VAL_CONST: 3336 /* If we're going to free the temp immediately, then we won't 3337 require it later in a register, so attempt to store the 3338 constant to memory directly. */ 3339 if (free_or_dead 3340 && tcg_out_sti(s, ts->type, ts->val, 3341 ts->mem_base->reg, ts->mem_offset)) { 3342 break; 3343 } 3344 temp_load(s, ts, tcg_target_available_regs[ts->type], 3345 allocated_regs, preferred_regs); 3346 /* fallthrough */ 3347 3348 case TEMP_VAL_REG: 3349 tcg_out_st(s, ts->type, ts->reg, 3350 ts->mem_base->reg, ts->mem_offset); 3351 break; 3352 3353 case TEMP_VAL_MEM: 3354 break; 3355 3356 case TEMP_VAL_DEAD: 3357 default: 3358 tcg_abort(); 3359 } 3360 ts->mem_coherent = 1; 3361 } 3362 if (free_or_dead) { 3363 temp_free_or_dead(s, ts, free_or_dead); 3364 } 3365 } 3366 3367 /* free register 'reg' by spilling the corresponding temporary if necessary */ 3368 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 3369 { 3370 TCGTemp *ts = s->reg_to_temp[reg]; 3371 if (ts != NULL) { 3372 temp_sync(s, ts, allocated_regs, 0, -1); 3373 } 3374 } 3375 3376 /** 3377 * tcg_reg_alloc: 3378 * @required_regs: Set of registers in which we must allocate. 3379 * @allocated_regs: Set of registers which must be avoided. 3380 * @preferred_regs: Set of registers we should prefer. 3381 * @rev: True if we search the registers in "indirect" order. 3382 * 3383 * The allocated register must be in @required_regs & ~@allocated_regs, 3384 * but if we can put it in @preferred_regs we may save a move later. 3385 */ 3386 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3387 TCGRegSet allocated_regs, 3388 TCGRegSet preferred_regs, bool rev) 3389 { 3390 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3391 TCGRegSet reg_ct[2]; 3392 const int *order; 3393 3394 reg_ct[1] = required_regs & ~allocated_regs; 3395 tcg_debug_assert(reg_ct[1] != 0); 3396 reg_ct[0] = reg_ct[1] & preferred_regs; 3397 3398 /* Skip the preferred_regs option if it cannot be satisfied, 3399 or if the preference made no difference. */ 3400 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3401 3402 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3403 3404 /* Try free registers, preferences first. */ 3405 for (j = f; j < 2; j++) { 3406 TCGRegSet set = reg_ct[j]; 3407 3408 if (tcg_regset_single(set)) { 3409 /* One register in the set. */ 3410 TCGReg reg = tcg_regset_first(set); 3411 if (s->reg_to_temp[reg] == NULL) { 3412 return reg; 3413 } 3414 } else { 3415 for (i = 0; i < n; i++) { 3416 TCGReg reg = order[i]; 3417 if (s->reg_to_temp[reg] == NULL && 3418 tcg_regset_test_reg(set, reg)) { 3419 return reg; 3420 } 3421 } 3422 } 3423 } 3424 3425 /* We must spill something. */ 3426 for (j = f; j < 2; j++) { 3427 TCGRegSet set = reg_ct[j]; 3428 3429 if (tcg_regset_single(set)) { 3430 /* One register in the set. */ 3431 TCGReg reg = tcg_regset_first(set); 3432 tcg_reg_free(s, reg, allocated_regs); 3433 return reg; 3434 } else { 3435 for (i = 0; i < n; i++) { 3436 TCGReg reg = order[i]; 3437 if (tcg_regset_test_reg(set, reg)) { 3438 tcg_reg_free(s, reg, allocated_regs); 3439 return reg; 3440 } 3441 } 3442 } 3443 } 3444 3445 tcg_abort(); 3446 } 3447 3448 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 3449 TCGRegSet allocated_regs, 3450 TCGRegSet preferred_regs, bool rev) 3451 { 3452 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3453 TCGRegSet reg_ct[2]; 3454 const int *order; 3455 3456 /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 3457 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 3458 tcg_debug_assert(reg_ct[1] != 0); 3459 reg_ct[0] = reg_ct[1] & preferred_regs; 3460 3461 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3462 3463 /* 3464 * Skip the preferred_regs option if it cannot be satisfied, 3465 * or if the preference made no difference. 3466 */ 3467 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3468 3469 /* 3470 * Minimize the number of flushes by looking for 2 free registers first, 3471 * then a single flush, then two flushes. 3472 */ 3473 for (fmin = 2; fmin >= 0; fmin--) { 3474 for (j = k; j < 2; j++) { 3475 TCGRegSet set = reg_ct[j]; 3476 3477 for (i = 0; i < n; i++) { 3478 TCGReg reg = order[i]; 3479 3480 if (tcg_regset_test_reg(set, reg)) { 3481 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 3482 if (f >= fmin) { 3483 tcg_reg_free(s, reg, allocated_regs); 3484 tcg_reg_free(s, reg + 1, allocated_regs); 3485 return reg; 3486 } 3487 } 3488 } 3489 } 3490 } 3491 tcg_abort(); 3492 } 3493 3494 /* Make sure the temporary is in a register. If needed, allocate the register 3495 from DESIRED while avoiding ALLOCATED. */ 3496 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3497 TCGRegSet allocated_regs, TCGRegSet preferred_regs) 3498 { 3499 TCGReg reg; 3500 3501 switch (ts->val_type) { 3502 case TEMP_VAL_REG: 3503 return; 3504 case TEMP_VAL_CONST: 3505 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3506 preferred_regs, ts->indirect_base); 3507 if (ts->type <= TCG_TYPE_I64) { 3508 tcg_out_movi(s, ts->type, reg, ts->val); 3509 } else { 3510 uint64_t val = ts->val; 3511 MemOp vece = MO_64; 3512 3513 /* 3514 * Find the minimal vector element that matches the constant. 3515 * The targets will, in general, have to do this search anyway, 3516 * do this generically. 3517 */ 3518 if (val == dup_const(MO_8, val)) { 3519 vece = MO_8; 3520 } else if (val == dup_const(MO_16, val)) { 3521 vece = MO_16; 3522 } else if (val == dup_const(MO_32, val)) { 3523 vece = MO_32; 3524 } 3525 3526 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 3527 } 3528 ts->mem_coherent = 0; 3529 break; 3530 case TEMP_VAL_MEM: 3531 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3532 preferred_regs, ts->indirect_base); 3533 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 3534 ts->mem_coherent = 1; 3535 break; 3536 case TEMP_VAL_DEAD: 3537 default: 3538 tcg_abort(); 3539 } 3540 set_temp_val_reg(s, ts, reg); 3541 } 3542 3543 /* Save a temporary to memory. 'allocated_regs' is used in case a 3544 temporary registers needs to be allocated to store a constant. */ 3545 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 3546 { 3547 /* The liveness analysis already ensures that globals are back 3548 in memory. Keep an tcg_debug_assert for safety. */ 3549 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 3550 } 3551 3552 /* save globals to their canonical location and assume they can be 3553 modified be the following code. 'allocated_regs' is used in case a 3554 temporary registers needs to be allocated to store a constant. */ 3555 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3556 { 3557 int i, n; 3558 3559 for (i = 0, n = s->nb_globals; i < n; i++) { 3560 temp_save(s, &s->temps[i], allocated_regs); 3561 } 3562 } 3563 3564 /* sync globals to their canonical location and assume they can be 3565 read by the following code. 'allocated_regs' is used in case a 3566 temporary registers needs to be allocated to store a constant. */ 3567 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 3568 { 3569 int i, n; 3570 3571 for (i = 0, n = s->nb_globals; i < n; i++) { 3572 TCGTemp *ts = &s->temps[i]; 3573 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3574 || ts->kind == TEMP_FIXED 3575 || ts->mem_coherent); 3576 } 3577 } 3578 3579 /* at the end of a basic block, we assume all temporaries are dead and 3580 all globals are stored at their canonical location. */ 3581 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3582 { 3583 int i; 3584 3585 for (i = s->nb_globals; i < s->nb_temps; i++) { 3586 TCGTemp *ts = &s->temps[i]; 3587 3588 switch (ts->kind) { 3589 case TEMP_LOCAL: 3590 temp_save(s, ts, allocated_regs); 3591 break; 3592 case TEMP_NORMAL: 3593 case TEMP_EBB: 3594 /* The liveness analysis already ensures that temps are dead. 3595 Keep an tcg_debug_assert for safety. */ 3596 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3597 break; 3598 case TEMP_CONST: 3599 /* Similarly, we should have freed any allocated register. */ 3600 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3601 break; 3602 default: 3603 g_assert_not_reached(); 3604 } 3605 } 3606 3607 save_globals(s, allocated_regs); 3608 } 3609 3610 /* 3611 * At a conditional branch, we assume all temporaries are dead unless 3612 * explicitly live-across-conditional-branch; all globals and local 3613 * temps are synced to their location. 3614 */ 3615 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3616 { 3617 sync_globals(s, allocated_regs); 3618 3619 for (int i = s->nb_globals; i < s->nb_temps; i++) { 3620 TCGTemp *ts = &s->temps[i]; 3621 /* 3622 * The liveness analysis already ensures that temps are dead. 3623 * Keep tcg_debug_asserts for safety. 3624 */ 3625 switch (ts->kind) { 3626 case TEMP_LOCAL: 3627 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3628 break; 3629 case TEMP_NORMAL: 3630 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3631 break; 3632 case TEMP_EBB: 3633 case TEMP_CONST: 3634 break; 3635 default: 3636 g_assert_not_reached(); 3637 } 3638 } 3639 } 3640 3641 /* 3642 * Specialized code generation for INDEX_op_mov_* with a constant. 3643 */ 3644 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3645 tcg_target_ulong val, TCGLifeData arg_life, 3646 TCGRegSet preferred_regs) 3647 { 3648 /* ENV should not be modified. */ 3649 tcg_debug_assert(!temp_readonly(ots)); 3650 3651 /* The movi is not explicitly generated here. */ 3652 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 3653 ots->val = val; 3654 ots->mem_coherent = 0; 3655 if (NEED_SYNC_ARG(0)) { 3656 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 3657 } else if (IS_DEAD_ARG(0)) { 3658 temp_dead(s, ots); 3659 } 3660 } 3661 3662 /* 3663 * Specialized code generation for INDEX_op_mov_*. 3664 */ 3665 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 3666 { 3667 const TCGLifeData arg_life = op->life; 3668 TCGRegSet allocated_regs, preferred_regs; 3669 TCGTemp *ts, *ots; 3670 TCGType otype, itype; 3671 TCGReg oreg, ireg; 3672 3673 allocated_regs = s->reserved_regs; 3674 preferred_regs = output_pref(op, 0); 3675 ots = arg_temp(op->args[0]); 3676 ts = arg_temp(op->args[1]); 3677 3678 /* ENV should not be modified. */ 3679 tcg_debug_assert(!temp_readonly(ots)); 3680 3681 /* Note that otype != itype for no-op truncation. */ 3682 otype = ots->type; 3683 itype = ts->type; 3684 3685 if (ts->val_type == TEMP_VAL_CONST) { 3686 /* propagate constant or generate sti */ 3687 tcg_target_ulong val = ts->val; 3688 if (IS_DEAD_ARG(1)) { 3689 temp_dead(s, ts); 3690 } 3691 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 3692 return; 3693 } 3694 3695 /* If the source value is in memory we're going to be forced 3696 to have it in a register in order to perform the copy. Copy 3697 the SOURCE value into its own register first, that way we 3698 don't have to reload SOURCE the next time it is used. */ 3699 if (ts->val_type == TEMP_VAL_MEM) { 3700 temp_load(s, ts, tcg_target_available_regs[itype], 3701 allocated_regs, preferred_regs); 3702 } 3703 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 3704 ireg = ts->reg; 3705 3706 if (IS_DEAD_ARG(0)) { 3707 /* mov to a non-saved dead register makes no sense (even with 3708 liveness analysis disabled). */ 3709 tcg_debug_assert(NEED_SYNC_ARG(0)); 3710 if (!ots->mem_allocated) { 3711 temp_allocate_frame(s, ots); 3712 } 3713 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 3714 if (IS_DEAD_ARG(1)) { 3715 temp_dead(s, ts); 3716 } 3717 temp_dead(s, ots); 3718 return; 3719 } 3720 3721 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 3722 /* 3723 * The mov can be suppressed. Kill input first, so that it 3724 * is unlinked from reg_to_temp, then set the output to the 3725 * reg that we saved from the input. 3726 */ 3727 temp_dead(s, ts); 3728 oreg = ireg; 3729 } else { 3730 if (ots->val_type == TEMP_VAL_REG) { 3731 oreg = ots->reg; 3732 } else { 3733 /* Make sure to not spill the input register during allocation. */ 3734 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 3735 allocated_regs | ((TCGRegSet)1 << ireg), 3736 preferred_regs, ots->indirect_base); 3737 } 3738 if (!tcg_out_mov(s, otype, oreg, ireg)) { 3739 /* 3740 * Cross register class move not supported. 3741 * Store the source register into the destination slot 3742 * and leave the destination temp as TEMP_VAL_MEM. 3743 */ 3744 assert(!temp_readonly(ots)); 3745 if (!ts->mem_allocated) { 3746 temp_allocate_frame(s, ots); 3747 } 3748 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 3749 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 3750 ots->mem_coherent = 1; 3751 return; 3752 } 3753 } 3754 set_temp_val_reg(s, ots, oreg); 3755 ots->mem_coherent = 0; 3756 3757 if (NEED_SYNC_ARG(0)) { 3758 temp_sync(s, ots, allocated_regs, 0, 0); 3759 } 3760 } 3761 3762 /* 3763 * Specialized code generation for INDEX_op_dup_vec. 3764 */ 3765 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 3766 { 3767 const TCGLifeData arg_life = op->life; 3768 TCGRegSet dup_out_regs, dup_in_regs; 3769 TCGTemp *its, *ots; 3770 TCGType itype, vtype; 3771 unsigned vece; 3772 int lowpart_ofs; 3773 bool ok; 3774 3775 ots = arg_temp(op->args[0]); 3776 its = arg_temp(op->args[1]); 3777 3778 /* ENV should not be modified. */ 3779 tcg_debug_assert(!temp_readonly(ots)); 3780 3781 itype = its->type; 3782 vece = TCGOP_VECE(op); 3783 vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3784 3785 if (its->val_type == TEMP_VAL_CONST) { 3786 /* Propagate constant via movi -> dupi. */ 3787 tcg_target_ulong val = its->val; 3788 if (IS_DEAD_ARG(1)) { 3789 temp_dead(s, its); 3790 } 3791 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); 3792 return; 3793 } 3794 3795 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 3796 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 3797 3798 /* Allocate the output register now. */ 3799 if (ots->val_type != TEMP_VAL_REG) { 3800 TCGRegSet allocated_regs = s->reserved_regs; 3801 TCGReg oreg; 3802 3803 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 3804 /* Make sure to not spill the input register. */ 3805 tcg_regset_set_reg(allocated_regs, its->reg); 3806 } 3807 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3808 output_pref(op, 0), ots->indirect_base); 3809 set_temp_val_reg(s, ots, oreg); 3810 } 3811 3812 switch (its->val_type) { 3813 case TEMP_VAL_REG: 3814 /* 3815 * The dup constriaints must be broad, covering all possible VECE. 3816 * However, tcg_op_dup_vec() gets to see the VECE and we allow it 3817 * to fail, indicating that extra moves are required for that case. 3818 */ 3819 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 3820 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 3821 goto done; 3822 } 3823 /* Try again from memory or a vector input register. */ 3824 } 3825 if (!its->mem_coherent) { 3826 /* 3827 * The input register is not synced, and so an extra store 3828 * would be required to use memory. Attempt an integer-vector 3829 * register move first. We do not have a TCGRegSet for this. 3830 */ 3831 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 3832 break; 3833 } 3834 /* Sync the temp back to its slot and load from there. */ 3835 temp_sync(s, its, s->reserved_regs, 0, 0); 3836 } 3837 /* fall through */ 3838 3839 case TEMP_VAL_MEM: 3840 lowpart_ofs = 0; 3841 if (HOST_BIG_ENDIAN) { 3842 lowpart_ofs = tcg_type_size(itype) - (1 << vece); 3843 } 3844 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 3845 its->mem_offset + lowpart_ofs)) { 3846 goto done; 3847 } 3848 /* Load the input into the destination vector register. */ 3849 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 3850 break; 3851 3852 default: 3853 g_assert_not_reached(); 3854 } 3855 3856 /* We now have a vector input register, so dup must succeed. */ 3857 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 3858 tcg_debug_assert(ok); 3859 3860 done: 3861 ots->mem_coherent = 0; 3862 if (IS_DEAD_ARG(1)) { 3863 temp_dead(s, its); 3864 } 3865 if (NEED_SYNC_ARG(0)) { 3866 temp_sync(s, ots, s->reserved_regs, 0, 0); 3867 } 3868 if (IS_DEAD_ARG(0)) { 3869 temp_dead(s, ots); 3870 } 3871 } 3872 3873 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 3874 { 3875 const TCGLifeData arg_life = op->life; 3876 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 3877 TCGRegSet i_allocated_regs; 3878 TCGRegSet o_allocated_regs; 3879 int i, k, nb_iargs, nb_oargs; 3880 TCGReg reg; 3881 TCGArg arg; 3882 const TCGArgConstraint *arg_ct; 3883 TCGTemp *ts; 3884 TCGArg new_args[TCG_MAX_OP_ARGS]; 3885 int const_args[TCG_MAX_OP_ARGS]; 3886 3887 nb_oargs = def->nb_oargs; 3888 nb_iargs = def->nb_iargs; 3889 3890 /* copy constants */ 3891 memcpy(new_args + nb_oargs + nb_iargs, 3892 op->args + nb_oargs + nb_iargs, 3893 sizeof(TCGArg) * def->nb_cargs); 3894 3895 i_allocated_regs = s->reserved_regs; 3896 o_allocated_regs = s->reserved_regs; 3897 3898 /* satisfy input constraints */ 3899 for (k = 0; k < nb_iargs; k++) { 3900 TCGRegSet i_preferred_regs, i_required_regs; 3901 bool allocate_new_reg, copyto_new_reg; 3902 TCGTemp *ts2; 3903 int i1, i2; 3904 3905 i = def->args_ct[nb_oargs + k].sort_index; 3906 arg = op->args[i]; 3907 arg_ct = &def->args_ct[i]; 3908 ts = arg_temp(arg); 3909 3910 if (ts->val_type == TEMP_VAL_CONST 3911 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 3912 /* constant is OK for instruction */ 3913 const_args[i] = 1; 3914 new_args[i] = ts->val; 3915 continue; 3916 } 3917 3918 reg = ts->reg; 3919 i_preferred_regs = 0; 3920 i_required_regs = arg_ct->regs; 3921 allocate_new_reg = false; 3922 copyto_new_reg = false; 3923 3924 switch (arg_ct->pair) { 3925 case 0: /* not paired */ 3926 if (arg_ct->ialias) { 3927 i_preferred_regs = output_pref(op, arg_ct->alias_index); 3928 3929 /* 3930 * If the input is readonly, then it cannot also be an 3931 * output and aliased to itself. If the input is not 3932 * dead after the instruction, we must allocate a new 3933 * register and move it. 3934 */ 3935 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 3936 allocate_new_reg = true; 3937 } else if (ts->val_type == TEMP_VAL_REG) { 3938 /* 3939 * Check if the current register has already been 3940 * allocated for another input. 3941 */ 3942 allocate_new_reg = 3943 tcg_regset_test_reg(i_allocated_regs, reg); 3944 } 3945 } 3946 if (!allocate_new_reg) { 3947 temp_load(s, ts, i_required_regs, i_allocated_regs, 3948 i_preferred_regs); 3949 reg = ts->reg; 3950 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 3951 } 3952 if (allocate_new_reg) { 3953 /* 3954 * Allocate a new register matching the constraint 3955 * and move the temporary register into it. 3956 */ 3957 temp_load(s, ts, tcg_target_available_regs[ts->type], 3958 i_allocated_regs, 0); 3959 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 3960 i_preferred_regs, ts->indirect_base); 3961 copyto_new_reg = true; 3962 } 3963 break; 3964 3965 case 1: 3966 /* First of an input pair; if i1 == i2, the second is an output. */ 3967 i1 = i; 3968 i2 = arg_ct->pair_index; 3969 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 3970 3971 /* 3972 * It is easier to default to allocating a new pair 3973 * and to identify a few cases where it's not required. 3974 */ 3975 if (arg_ct->ialias) { 3976 i_preferred_regs = output_pref(op, arg_ct->alias_index); 3977 if (IS_DEAD_ARG(i1) && 3978 IS_DEAD_ARG(i2) && 3979 !temp_readonly(ts) && 3980 ts->val_type == TEMP_VAL_REG && 3981 ts->reg < TCG_TARGET_NB_REGS - 1 && 3982 tcg_regset_test_reg(i_required_regs, reg) && 3983 !tcg_regset_test_reg(i_allocated_regs, reg) && 3984 !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 3985 (ts2 3986 ? ts2->val_type == TEMP_VAL_REG && 3987 ts2->reg == reg + 1 && 3988 !temp_readonly(ts2) 3989 : s->reg_to_temp[reg + 1] == NULL)) { 3990 break; 3991 } 3992 } else { 3993 /* Without aliasing, the pair must also be an input. */ 3994 tcg_debug_assert(ts2); 3995 if (ts->val_type == TEMP_VAL_REG && 3996 ts2->val_type == TEMP_VAL_REG && 3997 ts2->reg == reg + 1 && 3998 tcg_regset_test_reg(i_required_regs, reg)) { 3999 break; 4000 } 4001 } 4002 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 4003 0, ts->indirect_base); 4004 goto do_pair; 4005 4006 case 2: /* pair second */ 4007 reg = new_args[arg_ct->pair_index] + 1; 4008 goto do_pair; 4009 4010 case 3: /* ialias with second output, no first input */ 4011 tcg_debug_assert(arg_ct->ialias); 4012 i_preferred_regs = output_pref(op, arg_ct->alias_index); 4013 4014 if (IS_DEAD_ARG(i) && 4015 !temp_readonly(ts) && 4016 ts->val_type == TEMP_VAL_REG && 4017 reg > 0 && 4018 s->reg_to_temp[reg - 1] == NULL && 4019 tcg_regset_test_reg(i_required_regs, reg) && 4020 !tcg_regset_test_reg(i_allocated_regs, reg) && 4021 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 4022 tcg_regset_set_reg(i_allocated_regs, reg - 1); 4023 break; 4024 } 4025 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 4026 i_allocated_regs, 0, 4027 ts->indirect_base); 4028 tcg_regset_set_reg(i_allocated_regs, reg); 4029 reg += 1; 4030 goto do_pair; 4031 4032 do_pair: 4033 /* 4034 * If an aliased input is not dead after the instruction, 4035 * we must allocate a new register and move it. 4036 */ 4037 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 4038 TCGRegSet t_allocated_regs = i_allocated_regs; 4039 4040 /* 4041 * Because of the alias, and the continued life, make sure 4042 * that the temp is somewhere *other* than the reg pair, 4043 * and we get a copy in reg. 4044 */ 4045 tcg_regset_set_reg(t_allocated_regs, reg); 4046 tcg_regset_set_reg(t_allocated_regs, reg + 1); 4047 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 4048 /* If ts was already in reg, copy it somewhere else. */ 4049 TCGReg nr; 4050 bool ok; 4051 4052 tcg_debug_assert(ts->kind != TEMP_FIXED); 4053 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 4054 t_allocated_regs, 0, ts->indirect_base); 4055 ok = tcg_out_mov(s, ts->type, nr, reg); 4056 tcg_debug_assert(ok); 4057 4058 set_temp_val_reg(s, ts, nr); 4059 } else { 4060 temp_load(s, ts, tcg_target_available_regs[ts->type], 4061 t_allocated_regs, 0); 4062 copyto_new_reg = true; 4063 } 4064 } else { 4065 /* Preferably allocate to reg, otherwise copy. */ 4066 i_required_regs = (TCGRegSet)1 << reg; 4067 temp_load(s, ts, i_required_regs, i_allocated_regs, 4068 i_preferred_regs); 4069 copyto_new_reg = ts->reg != reg; 4070 } 4071 break; 4072 4073 default: 4074 g_assert_not_reached(); 4075 } 4076 4077 if (copyto_new_reg) { 4078 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4079 /* 4080 * Cross register class move not supported. Sync the 4081 * temp back to its slot and load from there. 4082 */ 4083 temp_sync(s, ts, i_allocated_regs, 0, 0); 4084 tcg_out_ld(s, ts->type, reg, 4085 ts->mem_base->reg, ts->mem_offset); 4086 } 4087 } 4088 new_args[i] = reg; 4089 const_args[i] = 0; 4090 tcg_regset_set_reg(i_allocated_regs, reg); 4091 } 4092 4093 /* mark dead temporaries and free the associated registers */ 4094 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4095 if (IS_DEAD_ARG(i)) { 4096 temp_dead(s, arg_temp(op->args[i])); 4097 } 4098 } 4099 4100 if (def->flags & TCG_OPF_COND_BRANCH) { 4101 tcg_reg_alloc_cbranch(s, i_allocated_regs); 4102 } else if (def->flags & TCG_OPF_BB_END) { 4103 tcg_reg_alloc_bb_end(s, i_allocated_regs); 4104 } else { 4105 if (def->flags & TCG_OPF_CALL_CLOBBER) { 4106 /* XXX: permit generic clobber register list ? */ 4107 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4108 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4109 tcg_reg_free(s, i, i_allocated_regs); 4110 } 4111 } 4112 } 4113 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 4114 /* sync globals if the op has side effects and might trigger 4115 an exception. */ 4116 sync_globals(s, i_allocated_regs); 4117 } 4118 4119 /* satisfy the output constraints */ 4120 for(k = 0; k < nb_oargs; k++) { 4121 i = def->args_ct[k].sort_index; 4122 arg = op->args[i]; 4123 arg_ct = &def->args_ct[i]; 4124 ts = arg_temp(arg); 4125 4126 /* ENV should not be modified. */ 4127 tcg_debug_assert(!temp_readonly(ts)); 4128 4129 switch (arg_ct->pair) { 4130 case 0: /* not paired */ 4131 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 4132 reg = new_args[arg_ct->alias_index]; 4133 } else if (arg_ct->newreg) { 4134 reg = tcg_reg_alloc(s, arg_ct->regs, 4135 i_allocated_regs | o_allocated_regs, 4136 output_pref(op, k), ts->indirect_base); 4137 } else { 4138 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 4139 output_pref(op, k), ts->indirect_base); 4140 } 4141 break; 4142 4143 case 1: /* first of pair */ 4144 tcg_debug_assert(!arg_ct->newreg); 4145 if (arg_ct->oalias) { 4146 reg = new_args[arg_ct->alias_index]; 4147 break; 4148 } 4149 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 4150 output_pref(op, k), ts->indirect_base); 4151 break; 4152 4153 case 2: /* second of pair */ 4154 tcg_debug_assert(!arg_ct->newreg); 4155 if (arg_ct->oalias) { 4156 reg = new_args[arg_ct->alias_index]; 4157 } else { 4158 reg = new_args[arg_ct->pair_index] + 1; 4159 } 4160 break; 4161 4162 case 3: /* first of pair, aliasing with a second input */ 4163 tcg_debug_assert(!arg_ct->newreg); 4164 reg = new_args[arg_ct->pair_index] - 1; 4165 break; 4166 4167 default: 4168 g_assert_not_reached(); 4169 } 4170 tcg_regset_set_reg(o_allocated_regs, reg); 4171 set_temp_val_reg(s, ts, reg); 4172 ts->mem_coherent = 0; 4173 new_args[i] = reg; 4174 } 4175 } 4176 4177 /* emit instruction */ 4178 if (def->flags & TCG_OPF_VECTOR) { 4179 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 4180 new_args, const_args); 4181 } else { 4182 tcg_out_op(s, op->opc, new_args, const_args); 4183 } 4184 4185 /* move the outputs in the correct register if needed */ 4186 for(i = 0; i < nb_oargs; i++) { 4187 ts = arg_temp(op->args[i]); 4188 4189 /* ENV should not be modified. */ 4190 tcg_debug_assert(!temp_readonly(ts)); 4191 4192 if (NEED_SYNC_ARG(i)) { 4193 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 4194 } else if (IS_DEAD_ARG(i)) { 4195 temp_dead(s, ts); 4196 } 4197 } 4198 } 4199 4200 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 4201 { 4202 const TCGLifeData arg_life = op->life; 4203 TCGTemp *ots, *itsl, *itsh; 4204 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4205 4206 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 4207 tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 4208 tcg_debug_assert(TCGOP_VECE(op) == MO_64); 4209 4210 ots = arg_temp(op->args[0]); 4211 itsl = arg_temp(op->args[1]); 4212 itsh = arg_temp(op->args[2]); 4213 4214 /* ENV should not be modified. */ 4215 tcg_debug_assert(!temp_readonly(ots)); 4216 4217 /* Allocate the output register now. */ 4218 if (ots->val_type != TEMP_VAL_REG) { 4219 TCGRegSet allocated_regs = s->reserved_regs; 4220 TCGRegSet dup_out_regs = 4221 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 4222 TCGReg oreg; 4223 4224 /* Make sure to not spill the input registers. */ 4225 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 4226 tcg_regset_set_reg(allocated_regs, itsl->reg); 4227 } 4228 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 4229 tcg_regset_set_reg(allocated_regs, itsh->reg); 4230 } 4231 4232 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 4233 output_pref(op, 0), ots->indirect_base); 4234 set_temp_val_reg(s, ots, oreg); 4235 } 4236 4237 /* Promote dup2 of immediates to dupi_vec. */ 4238 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 4239 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 4240 MemOp vece = MO_64; 4241 4242 if (val == dup_const(MO_8, val)) { 4243 vece = MO_8; 4244 } else if (val == dup_const(MO_16, val)) { 4245 vece = MO_16; 4246 } else if (val == dup_const(MO_32, val)) { 4247 vece = MO_32; 4248 } 4249 4250 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 4251 goto done; 4252 } 4253 4254 /* If the two inputs form one 64-bit value, try dupm_vec. */ 4255 if (itsl->temp_subindex == HOST_BIG_ENDIAN && 4256 itsh->temp_subindex == !HOST_BIG_ENDIAN && 4257 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { 4258 TCGTemp *its = itsl - HOST_BIG_ENDIAN; 4259 4260 temp_sync(s, its + 0, s->reserved_regs, 0, 0); 4261 temp_sync(s, its + 1, s->reserved_regs, 0, 0); 4262 4263 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 4264 its->mem_base->reg, its->mem_offset)) { 4265 goto done; 4266 } 4267 } 4268 4269 /* Fall back to generic expansion. */ 4270 return false; 4271 4272 done: 4273 ots->mem_coherent = 0; 4274 if (IS_DEAD_ARG(1)) { 4275 temp_dead(s, itsl); 4276 } 4277 if (IS_DEAD_ARG(2)) { 4278 temp_dead(s, itsh); 4279 } 4280 if (NEED_SYNC_ARG(0)) { 4281 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 4282 } else if (IS_DEAD_ARG(0)) { 4283 temp_dead(s, ots); 4284 } 4285 return true; 4286 } 4287 4288 static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, 4289 TCGRegSet allocated_regs) 4290 { 4291 if (ts->val_type == TEMP_VAL_REG) { 4292 if (ts->reg != reg) { 4293 tcg_reg_free(s, reg, allocated_regs); 4294 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4295 /* 4296 * Cross register class move not supported. Sync the 4297 * temp back to its slot and load from there. 4298 */ 4299 temp_sync(s, ts, allocated_regs, 0, 0); 4300 tcg_out_ld(s, ts->type, reg, 4301 ts->mem_base->reg, ts->mem_offset); 4302 } 4303 } 4304 } else { 4305 TCGRegSet arg_set = 0; 4306 4307 tcg_reg_free(s, reg, allocated_regs); 4308 tcg_regset_set_reg(arg_set, reg); 4309 temp_load(s, ts, arg_set, allocated_regs, 0); 4310 } 4311 } 4312 4313 static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, 4314 TCGRegSet allocated_regs) 4315 { 4316 /* 4317 * When the destination is on the stack, load up the temp and store. 4318 * If there are many call-saved registers, the temp might live to 4319 * see another use; otherwise it'll be discarded. 4320 */ 4321 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); 4322 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, 4323 TCG_TARGET_CALL_STACK_OFFSET + 4324 stk_slot * sizeof(tcg_target_long)); 4325 } 4326 4327 static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, 4328 TCGTemp *ts, TCGRegSet *allocated_regs) 4329 { 4330 if (REG_P(l)) { 4331 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; 4332 load_arg_reg(s, reg, ts, *allocated_regs); 4333 tcg_regset_set_reg(*allocated_regs, reg); 4334 } else { 4335 load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs), 4336 ts, *allocated_regs); 4337 } 4338 } 4339 4340 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 4341 { 4342 const int nb_oargs = TCGOP_CALLO(op); 4343 const int nb_iargs = TCGOP_CALLI(op); 4344 const TCGLifeData arg_life = op->life; 4345 const TCGHelperInfo *info = tcg_call_info(op); 4346 TCGRegSet allocated_regs = s->reserved_regs; 4347 int i; 4348 4349 /* 4350 * Move inputs into place in reverse order, 4351 * so that we place stacked arguments first. 4352 */ 4353 for (i = nb_iargs - 1; i >= 0; --i) { 4354 const TCGCallArgumentLoc *loc = &info->in[i]; 4355 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); 4356 4357 switch (loc->kind) { 4358 case TCG_CALL_ARG_NORMAL: 4359 case TCG_CALL_ARG_EXTEND_U: 4360 case TCG_CALL_ARG_EXTEND_S: 4361 load_arg_normal(s, loc, ts, &allocated_regs); 4362 break; 4363 default: 4364 g_assert_not_reached(); 4365 } 4366 } 4367 4368 /* Mark dead temporaries and free the associated registers. */ 4369 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4370 if (IS_DEAD_ARG(i)) { 4371 temp_dead(s, arg_temp(op->args[i])); 4372 } 4373 } 4374 4375 /* Clobber call registers. */ 4376 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4377 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4378 tcg_reg_free(s, i, allocated_regs); 4379 } 4380 } 4381 4382 /* 4383 * Save globals if they might be written by the helper, 4384 * sync them if they might be read. 4385 */ 4386 if (info->flags & TCG_CALL_NO_READ_GLOBALS) { 4387 /* Nothing to do */ 4388 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { 4389 sync_globals(s, allocated_regs); 4390 } else { 4391 save_globals(s, allocated_regs); 4392 } 4393 4394 tcg_out_call(s, tcg_call_func(op), info); 4395 4396 /* Assign output registers and emit moves if needed. */ 4397 switch (info->out_kind) { 4398 case TCG_CALL_RET_NORMAL: 4399 for (i = 0; i < nb_oargs; i++) { 4400 TCGTemp *ts = arg_temp(op->args[i]); 4401 TCGReg reg = tcg_target_call_oarg_regs[i]; 4402 4403 /* ENV should not be modified. */ 4404 tcg_debug_assert(!temp_readonly(ts)); 4405 4406 set_temp_val_reg(s, ts, reg); 4407 ts->mem_coherent = 0; 4408 } 4409 break; 4410 default: 4411 g_assert_not_reached(); 4412 } 4413 4414 /* Flush or discard output registers as needed. */ 4415 for (i = 0; i < nb_oargs; i++) { 4416 TCGTemp *ts = arg_temp(op->args[i]); 4417 if (NEED_SYNC_ARG(i)) { 4418 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); 4419 } else if (IS_DEAD_ARG(i)) { 4420 temp_dead(s, ts); 4421 } 4422 } 4423 } 4424 4425 #ifdef CONFIG_PROFILER 4426 4427 /* avoid copy/paste errors */ 4428 #define PROF_ADD(to, from, field) \ 4429 do { \ 4430 (to)->field += qatomic_read(&((from)->field)); \ 4431 } while (0) 4432 4433 #define PROF_MAX(to, from, field) \ 4434 do { \ 4435 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4436 if (val__ > (to)->field) { \ 4437 (to)->field = val__; \ 4438 } \ 4439 } while (0) 4440 4441 /* Pass in a zero'ed @prof */ 4442 static inline 4443 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4444 { 4445 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4446 unsigned int i; 4447 4448 for (i = 0; i < n_ctxs; i++) { 4449 TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4450 const TCGProfile *orig = &s->prof; 4451 4452 if (counters) { 4453 PROF_ADD(prof, orig, cpu_exec_time); 4454 PROF_ADD(prof, orig, tb_count1); 4455 PROF_ADD(prof, orig, tb_count); 4456 PROF_ADD(prof, orig, op_count); 4457 PROF_MAX(prof, orig, op_count_max); 4458 PROF_ADD(prof, orig, temp_count); 4459 PROF_MAX(prof, orig, temp_count_max); 4460 PROF_ADD(prof, orig, del_op_count); 4461 PROF_ADD(prof, orig, code_in_len); 4462 PROF_ADD(prof, orig, code_out_len); 4463 PROF_ADD(prof, orig, search_out_len); 4464 PROF_ADD(prof, orig, interm_time); 4465 PROF_ADD(prof, orig, code_time); 4466 PROF_ADD(prof, orig, la_time); 4467 PROF_ADD(prof, orig, opt_time); 4468 PROF_ADD(prof, orig, restore_count); 4469 PROF_ADD(prof, orig, restore_time); 4470 } 4471 if (table) { 4472 int i; 4473 4474 for (i = 0; i < NB_OPS; i++) { 4475 PROF_ADD(prof, orig, table_op_count[i]); 4476 } 4477 } 4478 } 4479 } 4480 4481 #undef PROF_ADD 4482 #undef PROF_MAX 4483 4484 static void tcg_profile_snapshot_counters(TCGProfile *prof) 4485 { 4486 tcg_profile_snapshot(prof, true, false); 4487 } 4488 4489 static void tcg_profile_snapshot_table(TCGProfile *prof) 4490 { 4491 tcg_profile_snapshot(prof, false, true); 4492 } 4493 4494 void tcg_dump_op_count(GString *buf) 4495 { 4496 TCGProfile prof = {}; 4497 int i; 4498 4499 tcg_profile_snapshot_table(&prof); 4500 for (i = 0; i < NB_OPS; i++) { 4501 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name, 4502 prof.table_op_count[i]); 4503 } 4504 } 4505 4506 int64_t tcg_cpu_exec_time(void) 4507 { 4508 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4509 unsigned int i; 4510 int64_t ret = 0; 4511 4512 for (i = 0; i < n_ctxs; i++) { 4513 const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4514 const TCGProfile *prof = &s->prof; 4515 4516 ret += qatomic_read(&prof->cpu_exec_time); 4517 } 4518 return ret; 4519 } 4520 #else 4521 void tcg_dump_op_count(GString *buf) 4522 { 4523 g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 4524 } 4525 4526 int64_t tcg_cpu_exec_time(void) 4527 { 4528 error_report("%s: TCG profiler not compiled", __func__); 4529 exit(EXIT_FAILURE); 4530 } 4531 #endif 4532 4533 4534 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) 4535 { 4536 #ifdef CONFIG_PROFILER 4537 TCGProfile *prof = &s->prof; 4538 #endif 4539 int i, num_insns; 4540 TCGOp *op; 4541 4542 #ifdef CONFIG_PROFILER 4543 { 4544 int n = 0; 4545 4546 QTAILQ_FOREACH(op, &s->ops, link) { 4547 n++; 4548 } 4549 qatomic_set(&prof->op_count, prof->op_count + n); 4550 if (n > prof->op_count_max) { 4551 qatomic_set(&prof->op_count_max, n); 4552 } 4553 4554 n = s->nb_temps; 4555 qatomic_set(&prof->temp_count, prof->temp_count + n); 4556 if (n > prof->temp_count_max) { 4557 qatomic_set(&prof->temp_count_max, n); 4558 } 4559 } 4560 #endif 4561 4562 #ifdef DEBUG_DISAS 4563 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4564 && qemu_log_in_addr_range(pc_start))) { 4565 FILE *logfile = qemu_log_trylock(); 4566 if (logfile) { 4567 fprintf(logfile, "OP:\n"); 4568 tcg_dump_ops(s, logfile, false); 4569 fprintf(logfile, "\n"); 4570 qemu_log_unlock(logfile); 4571 } 4572 } 4573 #endif 4574 4575 #ifdef CONFIG_DEBUG_TCG 4576 /* Ensure all labels referenced have been emitted. */ 4577 { 4578 TCGLabel *l; 4579 bool error = false; 4580 4581 QSIMPLEQ_FOREACH(l, &s->labels, next) { 4582 if (unlikely(!l->present) && l->refs) { 4583 qemu_log_mask(CPU_LOG_TB_OP, 4584 "$L%d referenced but not present.\n", l->id); 4585 error = true; 4586 } 4587 } 4588 assert(!error); 4589 } 4590 #endif 4591 4592 #ifdef CONFIG_PROFILER 4593 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 4594 #endif 4595 4596 #ifdef USE_TCG_OPTIMIZATIONS 4597 tcg_optimize(s); 4598 #endif 4599 4600 #ifdef CONFIG_PROFILER 4601 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 4602 qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 4603 #endif 4604 4605 reachable_code_pass(s); 4606 liveness_pass_1(s); 4607 4608 if (s->nb_indirects > 0) { 4609 #ifdef DEBUG_DISAS 4610 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 4611 && qemu_log_in_addr_range(pc_start))) { 4612 FILE *logfile = qemu_log_trylock(); 4613 if (logfile) { 4614 fprintf(logfile, "OP before indirect lowering:\n"); 4615 tcg_dump_ops(s, logfile, false); 4616 fprintf(logfile, "\n"); 4617 qemu_log_unlock(logfile); 4618 } 4619 } 4620 #endif 4621 /* Replace indirect temps with direct temps. */ 4622 if (liveness_pass_2(s)) { 4623 /* If changes were made, re-run liveness. */ 4624 liveness_pass_1(s); 4625 } 4626 } 4627 4628 #ifdef CONFIG_PROFILER 4629 qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 4630 #endif 4631 4632 #ifdef DEBUG_DISAS 4633 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 4634 && qemu_log_in_addr_range(pc_start))) { 4635 FILE *logfile = qemu_log_trylock(); 4636 if (logfile) { 4637 fprintf(logfile, "OP after optimization and liveness analysis:\n"); 4638 tcg_dump_ops(s, logfile, true); 4639 fprintf(logfile, "\n"); 4640 qemu_log_unlock(logfile); 4641 } 4642 } 4643 #endif 4644 4645 /* Initialize goto_tb jump offsets. */ 4646 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 4647 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 4648 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 4649 if (TCG_TARGET_HAS_direct_jump) { 4650 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 4651 tcg_ctx->tb_jmp_target_addr = NULL; 4652 } else { 4653 tcg_ctx->tb_jmp_insn_offset = NULL; 4654 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 4655 } 4656 4657 tcg_reg_alloc_start(s); 4658 4659 /* 4660 * Reset the buffer pointers when restarting after overflow. 4661 * TODO: Move this into translate-all.c with the rest of the 4662 * buffer management. Having only this done here is confusing. 4663 */ 4664 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 4665 s->code_ptr = s->code_buf; 4666 4667 #ifdef TCG_TARGET_NEED_LDST_LABELS 4668 QSIMPLEQ_INIT(&s->ldst_labels); 4669 #endif 4670 #ifdef TCG_TARGET_NEED_POOL_LABELS 4671 s->pool_labels = NULL; 4672 #endif 4673 4674 num_insns = -1; 4675 QTAILQ_FOREACH(op, &s->ops, link) { 4676 TCGOpcode opc = op->opc; 4677 4678 #ifdef CONFIG_PROFILER 4679 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 4680 #endif 4681 4682 switch (opc) { 4683 case INDEX_op_mov_i32: 4684 case INDEX_op_mov_i64: 4685 case INDEX_op_mov_vec: 4686 tcg_reg_alloc_mov(s, op); 4687 break; 4688 case INDEX_op_dup_vec: 4689 tcg_reg_alloc_dup(s, op); 4690 break; 4691 case INDEX_op_insn_start: 4692 if (num_insns >= 0) { 4693 size_t off = tcg_current_code_size(s); 4694 s->gen_insn_end_off[num_insns] = off; 4695 /* Assert that we do not overflow our stored offset. */ 4696 assert(s->gen_insn_end_off[num_insns] == off); 4697 } 4698 num_insns++; 4699 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 4700 target_ulong a; 4701 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 4702 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 4703 #else 4704 a = op->args[i]; 4705 #endif 4706 s->gen_insn_data[num_insns][i] = a; 4707 } 4708 break; 4709 case INDEX_op_discard: 4710 temp_dead(s, arg_temp(op->args[0])); 4711 break; 4712 case INDEX_op_set_label: 4713 tcg_reg_alloc_bb_end(s, s->reserved_regs); 4714 tcg_out_label(s, arg_label(op->args[0])); 4715 break; 4716 case INDEX_op_call: 4717 tcg_reg_alloc_call(s, op); 4718 break; 4719 case INDEX_op_dup2_vec: 4720 if (tcg_reg_alloc_dup2(s, op)) { 4721 break; 4722 } 4723 /* fall through */ 4724 default: 4725 /* Sanity check that we've not introduced any unhandled opcodes. */ 4726 tcg_debug_assert(tcg_op_supported(opc)); 4727 /* Note: in order to speed up the code, it would be much 4728 faster to have specialized register allocator functions for 4729 some common argument patterns */ 4730 tcg_reg_alloc_op(s, op); 4731 break; 4732 } 4733 /* Test for (pending) buffer overflow. The assumption is that any 4734 one operation beginning below the high water mark cannot overrun 4735 the buffer completely. Thus we can test for overflow after 4736 generating code without having to check during generation. */ 4737 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 4738 return -1; 4739 } 4740 /* Test for TB overflow, as seen by gen_insn_end_off. */ 4741 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 4742 return -2; 4743 } 4744 } 4745 tcg_debug_assert(num_insns >= 0); 4746 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 4747 4748 /* Generate TB finalization at the end of block */ 4749 #ifdef TCG_TARGET_NEED_LDST_LABELS 4750 i = tcg_out_ldst_finalize(s); 4751 if (i < 0) { 4752 return i; 4753 } 4754 #endif 4755 #ifdef TCG_TARGET_NEED_POOL_LABELS 4756 i = tcg_out_pool_finalize(s); 4757 if (i < 0) { 4758 return i; 4759 } 4760 #endif 4761 if (!tcg_resolve_relocs(s)) { 4762 return -2; 4763 } 4764 4765 #ifndef CONFIG_TCG_INTERPRETER 4766 /* flush instruction cache */ 4767 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 4768 (uintptr_t)s->code_buf, 4769 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 4770 #endif 4771 4772 return tcg_current_code_size(s); 4773 } 4774 4775 #ifdef CONFIG_PROFILER 4776 void tcg_dump_info(GString *buf) 4777 { 4778 TCGProfile prof = {}; 4779 const TCGProfile *s; 4780 int64_t tb_count; 4781 int64_t tb_div_count; 4782 int64_t tot; 4783 4784 tcg_profile_snapshot_counters(&prof); 4785 s = &prof; 4786 tb_count = s->tb_count; 4787 tb_div_count = tb_count ? tb_count : 1; 4788 tot = s->interm_time + s->code_time; 4789 4790 g_string_append_printf(buf, "JIT cycles %" PRId64 4791 " (%0.3f s at 2.4 GHz)\n", 4792 tot, tot / 2.4e9); 4793 g_string_append_printf(buf, "translated TBs %" PRId64 4794 " (aborted=%" PRId64 " %0.1f%%)\n", 4795 tb_count, s->tb_count1 - tb_count, 4796 (double)(s->tb_count1 - s->tb_count) 4797 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 4798 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n", 4799 (double)s->op_count / tb_div_count, s->op_count_max); 4800 g_string_append_printf(buf, "deleted ops/TB %0.2f\n", 4801 (double)s->del_op_count / tb_div_count); 4802 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n", 4803 (double)s->temp_count / tb_div_count, 4804 s->temp_count_max); 4805 g_string_append_printf(buf, "avg host code/TB %0.1f\n", 4806 (double)s->code_out_len / tb_div_count); 4807 g_string_append_printf(buf, "avg search data/TB %0.1f\n", 4808 (double)s->search_out_len / tb_div_count); 4809 4810 g_string_append_printf(buf, "cycles/op %0.1f\n", 4811 s->op_count ? (double)tot / s->op_count : 0); 4812 g_string_append_printf(buf, "cycles/in byte %0.1f\n", 4813 s->code_in_len ? (double)tot / s->code_in_len : 0); 4814 g_string_append_printf(buf, "cycles/out byte %0.1f\n", 4815 s->code_out_len ? (double)tot / s->code_out_len : 0); 4816 g_string_append_printf(buf, "cycles/search byte %0.1f\n", 4817 s->search_out_len ? 4818 (double)tot / s->search_out_len : 0); 4819 if (tot == 0) { 4820 tot = 1; 4821 } 4822 g_string_append_printf(buf, " gen_interm time %0.1f%%\n", 4823 (double)s->interm_time / tot * 100.0); 4824 g_string_append_printf(buf, " gen_code time %0.1f%%\n", 4825 (double)s->code_time / tot * 100.0); 4826 g_string_append_printf(buf, "optim./code time %0.1f%%\n", 4827 (double)s->opt_time / (s->code_time ? 4828 s->code_time : 1) 4829 * 100.0); 4830 g_string_append_printf(buf, "liveness/code time %0.1f%%\n", 4831 (double)s->la_time / (s->code_time ? 4832 s->code_time : 1) * 100.0); 4833 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n", 4834 s->restore_count); 4835 g_string_append_printf(buf, " avg cycles %0.1f\n", 4836 s->restore_count ? 4837 (double)s->restore_time / s->restore_count : 0); 4838 } 4839 #else 4840 void tcg_dump_info(GString *buf) 4841 { 4842 g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 4843 } 4844 #endif 4845 4846 #ifdef ELF_HOST_MACHINE 4847 /* In order to use this feature, the backend needs to do three things: 4848 4849 (1) Define ELF_HOST_MACHINE to indicate both what value to 4850 put into the ELF image and to indicate support for the feature. 4851 4852 (2) Define tcg_register_jit. This should create a buffer containing 4853 the contents of a .debug_frame section that describes the post- 4854 prologue unwind info for the tcg machine. 4855 4856 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 4857 */ 4858 4859 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 4860 typedef enum { 4861 JIT_NOACTION = 0, 4862 JIT_REGISTER_FN, 4863 JIT_UNREGISTER_FN 4864 } jit_actions_t; 4865 4866 struct jit_code_entry { 4867 struct jit_code_entry *next_entry; 4868 struct jit_code_entry *prev_entry; 4869 const void *symfile_addr; 4870 uint64_t symfile_size; 4871 }; 4872 4873 struct jit_descriptor { 4874 uint32_t version; 4875 uint32_t action_flag; 4876 struct jit_code_entry *relevant_entry; 4877 struct jit_code_entry *first_entry; 4878 }; 4879 4880 void __jit_debug_register_code(void) __attribute__((noinline)); 4881 void __jit_debug_register_code(void) 4882 { 4883 asm(""); 4884 } 4885 4886 /* Must statically initialize the version, because GDB may check 4887 the version before we can set it. */ 4888 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 4889 4890 /* End GDB interface. */ 4891 4892 static int find_string(const char *strtab, const char *str) 4893 { 4894 const char *p = strtab + 1; 4895 4896 while (1) { 4897 if (strcmp(p, str) == 0) { 4898 return p - strtab; 4899 } 4900 p += strlen(p) + 1; 4901 } 4902 } 4903 4904 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 4905 const void *debug_frame, 4906 size_t debug_frame_size) 4907 { 4908 struct __attribute__((packed)) DebugInfo { 4909 uint32_t len; 4910 uint16_t version; 4911 uint32_t abbrev; 4912 uint8_t ptr_size; 4913 uint8_t cu_die; 4914 uint16_t cu_lang; 4915 uintptr_t cu_low_pc; 4916 uintptr_t cu_high_pc; 4917 uint8_t fn_die; 4918 char fn_name[16]; 4919 uintptr_t fn_low_pc; 4920 uintptr_t fn_high_pc; 4921 uint8_t cu_eoc; 4922 }; 4923 4924 struct ElfImage { 4925 ElfW(Ehdr) ehdr; 4926 ElfW(Phdr) phdr; 4927 ElfW(Shdr) shdr[7]; 4928 ElfW(Sym) sym[2]; 4929 struct DebugInfo di; 4930 uint8_t da[24]; 4931 char str[80]; 4932 }; 4933 4934 struct ElfImage *img; 4935 4936 static const struct ElfImage img_template = { 4937 .ehdr = { 4938 .e_ident[EI_MAG0] = ELFMAG0, 4939 .e_ident[EI_MAG1] = ELFMAG1, 4940 .e_ident[EI_MAG2] = ELFMAG2, 4941 .e_ident[EI_MAG3] = ELFMAG3, 4942 .e_ident[EI_CLASS] = ELF_CLASS, 4943 .e_ident[EI_DATA] = ELF_DATA, 4944 .e_ident[EI_VERSION] = EV_CURRENT, 4945 .e_type = ET_EXEC, 4946 .e_machine = ELF_HOST_MACHINE, 4947 .e_version = EV_CURRENT, 4948 .e_phoff = offsetof(struct ElfImage, phdr), 4949 .e_shoff = offsetof(struct ElfImage, shdr), 4950 .e_ehsize = sizeof(ElfW(Shdr)), 4951 .e_phentsize = sizeof(ElfW(Phdr)), 4952 .e_phnum = 1, 4953 .e_shentsize = sizeof(ElfW(Shdr)), 4954 .e_shnum = ARRAY_SIZE(img->shdr), 4955 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 4956 #ifdef ELF_HOST_FLAGS 4957 .e_flags = ELF_HOST_FLAGS, 4958 #endif 4959 #ifdef ELF_OSABI 4960 .e_ident[EI_OSABI] = ELF_OSABI, 4961 #endif 4962 }, 4963 .phdr = { 4964 .p_type = PT_LOAD, 4965 .p_flags = PF_X, 4966 }, 4967 .shdr = { 4968 [0] = { .sh_type = SHT_NULL }, 4969 /* Trick: The contents of code_gen_buffer are not present in 4970 this fake ELF file; that got allocated elsewhere. Therefore 4971 we mark .text as SHT_NOBITS (similar to .bss) so that readers 4972 will not look for contents. We can record any address. */ 4973 [1] = { /* .text */ 4974 .sh_type = SHT_NOBITS, 4975 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 4976 }, 4977 [2] = { /* .debug_info */ 4978 .sh_type = SHT_PROGBITS, 4979 .sh_offset = offsetof(struct ElfImage, di), 4980 .sh_size = sizeof(struct DebugInfo), 4981 }, 4982 [3] = { /* .debug_abbrev */ 4983 .sh_type = SHT_PROGBITS, 4984 .sh_offset = offsetof(struct ElfImage, da), 4985 .sh_size = sizeof(img->da), 4986 }, 4987 [4] = { /* .debug_frame */ 4988 .sh_type = SHT_PROGBITS, 4989 .sh_offset = sizeof(struct ElfImage), 4990 }, 4991 [5] = { /* .symtab */ 4992 .sh_type = SHT_SYMTAB, 4993 .sh_offset = offsetof(struct ElfImage, sym), 4994 .sh_size = sizeof(img->sym), 4995 .sh_info = 1, 4996 .sh_link = ARRAY_SIZE(img->shdr) - 1, 4997 .sh_entsize = sizeof(ElfW(Sym)), 4998 }, 4999 [6] = { /* .strtab */ 5000 .sh_type = SHT_STRTAB, 5001 .sh_offset = offsetof(struct ElfImage, str), 5002 .sh_size = sizeof(img->str), 5003 } 5004 }, 5005 .sym = { 5006 [1] = { /* code_gen_buffer */ 5007 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 5008 .st_shndx = 1, 5009 } 5010 }, 5011 .di = { 5012 .len = sizeof(struct DebugInfo) - 4, 5013 .version = 2, 5014 .ptr_size = sizeof(void *), 5015 .cu_die = 1, 5016 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 5017 .fn_die = 2, 5018 .fn_name = "code_gen_buffer" 5019 }, 5020 .da = { 5021 1, /* abbrev number (the cu) */ 5022 0x11, 1, /* DW_TAG_compile_unit, has children */ 5023 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 5024 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 5025 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 5026 0, 0, /* end of abbrev */ 5027 2, /* abbrev number (the fn) */ 5028 0x2e, 0, /* DW_TAG_subprogram, no children */ 5029 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 5030 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 5031 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 5032 0, 0, /* end of abbrev */ 5033 0 /* no more abbrev */ 5034 }, 5035 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 5036 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 5037 }; 5038 5039 /* We only need a single jit entry; statically allocate it. */ 5040 static struct jit_code_entry one_entry; 5041 5042 uintptr_t buf = (uintptr_t)buf_ptr; 5043 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 5044 DebugFrameHeader *dfh; 5045 5046 img = g_malloc(img_size); 5047 *img = img_template; 5048 5049 img->phdr.p_vaddr = buf; 5050 img->phdr.p_paddr = buf; 5051 img->phdr.p_memsz = buf_size; 5052 5053 img->shdr[1].sh_name = find_string(img->str, ".text"); 5054 img->shdr[1].sh_addr = buf; 5055 img->shdr[1].sh_size = buf_size; 5056 5057 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 5058 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 5059 5060 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 5061 img->shdr[4].sh_size = debug_frame_size; 5062 5063 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 5064 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 5065 5066 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 5067 img->sym[1].st_value = buf; 5068 img->sym[1].st_size = buf_size; 5069 5070 img->di.cu_low_pc = buf; 5071 img->di.cu_high_pc = buf + buf_size; 5072 img->di.fn_low_pc = buf; 5073 img->di.fn_high_pc = buf + buf_size; 5074 5075 dfh = (DebugFrameHeader *)(img + 1); 5076 memcpy(dfh, debug_frame, debug_frame_size); 5077 dfh->fde.func_start = buf; 5078 dfh->fde.func_len = buf_size; 5079 5080 #ifdef DEBUG_JIT 5081 /* Enable this block to be able to debug the ELF image file creation. 5082 One can use readelf, objdump, or other inspection utilities. */ 5083 { 5084 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 5085 FILE *f = fopen(jit, "w+b"); 5086 if (f) { 5087 if (fwrite(img, img_size, 1, f) != img_size) { 5088 /* Avoid stupid unused return value warning for fwrite. */ 5089 } 5090 fclose(f); 5091 } 5092 } 5093 #endif 5094 5095 one_entry.symfile_addr = img; 5096 one_entry.symfile_size = img_size; 5097 5098 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 5099 __jit_debug_descriptor.relevant_entry = &one_entry; 5100 __jit_debug_descriptor.first_entry = &one_entry; 5101 __jit_debug_register_code(); 5102 } 5103 #else 5104 /* No support for the feature. Provide the entry point expected by exec.c, 5105 and implement the internal function we declared earlier. */ 5106 5107 static void tcg_register_jit_int(const void *buf, size_t size, 5108 const void *debug_frame, 5109 size_t debug_frame_size) 5110 { 5111 } 5112 5113 void tcg_register_jit(const void *buf, size_t buf_size) 5114 { 5115 } 5116 #endif /* ELF_HOST_MACHINE */ 5117 5118 #if !TCG_TARGET_MAYBE_vec 5119 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 5120 { 5121 g_assert_not_reached(); 5122 } 5123 #endif 5124