1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/error-report.h" 34 #include "qemu/cutils.h" 35 #include "qemu/host-utils.h" 36 #include "qemu/qemu-print.h" 37 #include "qemu/cacheflush.h" 38 #include "qemu/cacheinfo.h" 39 #include "qemu/timer.h" 40 41 /* Note: the long term plan is to reduce the dependencies on the QEMU 42 CPU definitions. Currently they are used for qemu_ld/st 43 instructions */ 44 #define NO_CPU_IO_DEFS 45 46 #include "exec/exec-all.h" 47 #include "tcg/tcg-op.h" 48 49 #if UINTPTR_MAX == UINT32_MAX 50 # define ELF_CLASS ELFCLASS32 51 #else 52 # define ELF_CLASS ELFCLASS64 53 #endif 54 #if HOST_BIG_ENDIAN 55 # define ELF_DATA ELFDATA2MSB 56 #else 57 # define ELF_DATA ELFDATA2LSB 58 #endif 59 60 #include "elf.h" 61 #include "exec/log.h" 62 #include "tcg/tcg-ldst.h" 63 #include "tcg/tcg-temp-internal.h" 64 #include "tcg-internal.h" 65 #include "accel/tcg/perf.h" 66 67 /* Forward declarations for functions declared in tcg-target.c.inc and 68 used here. */ 69 static void tcg_target_init(TCGContext *s); 70 static void tcg_target_qemu_prologue(TCGContext *s); 71 static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 72 intptr_t value, intptr_t addend); 73 74 /* The CIE and FDE header definitions will be common to all hosts. */ 75 typedef struct { 76 uint32_t len __attribute__((aligned((sizeof(void *))))); 77 uint32_t id; 78 uint8_t version; 79 char augmentation[1]; 80 uint8_t code_align; 81 uint8_t data_align; 82 uint8_t return_column; 83 } DebugFrameCIE; 84 85 typedef struct QEMU_PACKED { 86 uint32_t len __attribute__((aligned((sizeof(void *))))); 87 uint32_t cie_offset; 88 uintptr_t func_start; 89 uintptr_t func_len; 90 } DebugFrameFDEHeader; 91 92 typedef struct QEMU_PACKED { 93 DebugFrameCIE cie; 94 DebugFrameFDEHeader fde; 95 } DebugFrameHeader; 96 97 static void tcg_register_jit_int(const void *buf, size_t size, 98 const void *debug_frame, 99 size_t debug_frame_size) 100 __attribute__((unused)); 101 102 /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 103 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 104 intptr_t arg2); 105 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 106 static void tcg_out_movi(TCGContext *s, TCGType type, 107 TCGReg ret, tcg_target_long arg); 108 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); 109 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); 110 static void tcg_out_goto_tb(TCGContext *s, int which); 111 static void tcg_out_op(TCGContext *s, TCGOpcode opc, 112 const TCGArg args[TCG_MAX_OP_ARGS], 113 const int const_args[TCG_MAX_OP_ARGS]); 114 #if TCG_TARGET_MAYBE_vec 115 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 116 TCGReg dst, TCGReg src); 117 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 118 TCGReg dst, TCGReg base, intptr_t offset); 119 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 120 TCGReg dst, int64_t arg); 121 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 122 unsigned vecl, unsigned vece, 123 const TCGArg args[TCG_MAX_OP_ARGS], 124 const int const_args[TCG_MAX_OP_ARGS]); 125 #else 126 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 127 TCGReg dst, TCGReg src) 128 { 129 g_assert_not_reached(); 130 } 131 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 132 TCGReg dst, TCGReg base, intptr_t offset) 133 { 134 g_assert_not_reached(); 135 } 136 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 137 TCGReg dst, int64_t arg) 138 { 139 g_assert_not_reached(); 140 } 141 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 142 unsigned vecl, unsigned vece, 143 const TCGArg args[TCG_MAX_OP_ARGS], 144 const int const_args[TCG_MAX_OP_ARGS]) 145 { 146 g_assert_not_reached(); 147 } 148 #endif 149 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 150 intptr_t arg2); 151 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 152 TCGReg base, intptr_t ofs); 153 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 154 const TCGHelperInfo *info); 155 static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); 156 static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 157 #ifdef TCG_TARGET_NEED_LDST_LABELS 158 static int tcg_out_ldst_finalize(TCGContext *s); 159 #endif 160 161 TCGContext tcg_init_ctx; 162 __thread TCGContext *tcg_ctx; 163 164 TCGContext **tcg_ctxs; 165 unsigned int tcg_cur_ctxs; 166 unsigned int tcg_max_ctxs; 167 TCGv_env cpu_env = 0; 168 const void *tcg_code_gen_epilogue; 169 uintptr_t tcg_splitwx_diff; 170 171 #ifndef CONFIG_TCG_INTERPRETER 172 tcg_prologue_fn *tcg_qemu_tb_exec; 173 #endif 174 175 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 176 static TCGRegSet tcg_target_call_clobber_regs; 177 178 #if TCG_TARGET_INSN_UNIT_SIZE == 1 179 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 180 { 181 *s->code_ptr++ = v; 182 } 183 184 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 185 uint8_t v) 186 { 187 *p = v; 188 } 189 #endif 190 191 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 192 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 193 { 194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 195 *s->code_ptr++ = v; 196 } else { 197 tcg_insn_unit *p = s->code_ptr; 198 memcpy(p, &v, sizeof(v)); 199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 200 } 201 } 202 203 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 204 uint16_t v) 205 { 206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 207 *p = v; 208 } else { 209 memcpy(p, &v, sizeof(v)); 210 } 211 } 212 #endif 213 214 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 215 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 216 { 217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 218 *s->code_ptr++ = v; 219 } else { 220 tcg_insn_unit *p = s->code_ptr; 221 memcpy(p, &v, sizeof(v)); 222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 223 } 224 } 225 226 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 227 uint32_t v) 228 { 229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 230 *p = v; 231 } else { 232 memcpy(p, &v, sizeof(v)); 233 } 234 } 235 #endif 236 237 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 238 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 239 { 240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 241 *s->code_ptr++ = v; 242 } else { 243 tcg_insn_unit *p = s->code_ptr; 244 memcpy(p, &v, sizeof(v)); 245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 246 } 247 } 248 249 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 250 uint64_t v) 251 { 252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 253 *p = v; 254 } else { 255 memcpy(p, &v, sizeof(v)); 256 } 257 } 258 #endif 259 260 /* label relocation processing */ 261 262 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 263 TCGLabel *l, intptr_t addend) 264 { 265 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 266 267 r->type = type; 268 r->ptr = code_ptr; 269 r->addend = addend; 270 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 271 } 272 273 static void tcg_out_label(TCGContext *s, TCGLabel *l) 274 { 275 tcg_debug_assert(!l->has_value); 276 l->has_value = 1; 277 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 278 } 279 280 TCGLabel *gen_new_label(void) 281 { 282 TCGContext *s = tcg_ctx; 283 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 284 285 memset(l, 0, sizeof(TCGLabel)); 286 l->id = s->nb_labels++; 287 QSIMPLEQ_INIT(&l->branches); 288 QSIMPLEQ_INIT(&l->relocs); 289 290 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 291 292 return l; 293 } 294 295 static bool tcg_resolve_relocs(TCGContext *s) 296 { 297 TCGLabel *l; 298 299 QSIMPLEQ_FOREACH(l, &s->labels, next) { 300 TCGRelocation *r; 301 uintptr_t value = l->u.value; 302 303 QSIMPLEQ_FOREACH(r, &l->relocs, next) { 304 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 305 return false; 306 } 307 } 308 } 309 return true; 310 } 311 312 static void set_jmp_reset_offset(TCGContext *s, int which) 313 { 314 /* 315 * We will check for overflow at the end of the opcode loop in 316 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 317 */ 318 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); 319 } 320 321 static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) 322 { 323 /* 324 * We will check for overflow at the end of the opcode loop in 325 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 326 */ 327 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); 328 } 329 330 static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) 331 { 332 /* 333 * Return the read-execute version of the pointer, for the benefit 334 * of any pc-relative addressing mode. 335 */ 336 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); 337 } 338 339 /* Signal overflow, starting over with fewer guest insns. */ 340 static G_NORETURN 341 void tcg_raise_tb_overflow(TCGContext *s) 342 { 343 siglongjmp(s->jmp_trans, -2); 344 } 345 346 #define C_PFX1(P, A) P##A 347 #define C_PFX2(P, A, B) P##A##_##B 348 #define C_PFX3(P, A, B, C) P##A##_##B##_##C 349 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 350 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 351 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 352 353 /* Define an enumeration for the various combinations. */ 354 355 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 356 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 357 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 358 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 359 360 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 361 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 362 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 363 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 364 365 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 366 367 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 368 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 369 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 370 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 371 372 typedef enum { 373 #include "tcg-target-con-set.h" 374 } TCGConstraintSetIndex; 375 376 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 377 378 #undef C_O0_I1 379 #undef C_O0_I2 380 #undef C_O0_I3 381 #undef C_O0_I4 382 #undef C_O1_I1 383 #undef C_O1_I2 384 #undef C_O1_I3 385 #undef C_O1_I4 386 #undef C_N1_I2 387 #undef C_O2_I1 388 #undef C_O2_I2 389 #undef C_O2_I3 390 #undef C_O2_I4 391 392 /* Put all of the constraint sets into an array, indexed by the enum. */ 393 394 #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 395 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 396 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 397 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 398 399 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 400 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 401 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 402 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 403 404 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 405 406 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 407 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 408 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 409 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 410 411 static const TCGTargetOpDef constraint_sets[] = { 412 #include "tcg-target-con-set.h" 413 }; 414 415 416 #undef C_O0_I1 417 #undef C_O0_I2 418 #undef C_O0_I3 419 #undef C_O0_I4 420 #undef C_O1_I1 421 #undef C_O1_I2 422 #undef C_O1_I3 423 #undef C_O1_I4 424 #undef C_N1_I2 425 #undef C_O2_I1 426 #undef C_O2_I2 427 #undef C_O2_I3 428 #undef C_O2_I4 429 430 /* Expand the enumerator to be returned from tcg_target_op_def(). */ 431 432 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 433 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 434 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 435 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 436 437 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 438 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 439 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 440 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 441 442 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 443 444 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 445 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 446 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 447 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 448 449 #include "tcg-target.c.inc" 450 451 static void alloc_tcg_plugin_context(TCGContext *s) 452 { 453 #ifdef CONFIG_PLUGIN 454 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 455 s->plugin_tb->insns = 456 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 457 #endif 458 } 459 460 /* 461 * All TCG threads except the parent (i.e. the one that called tcg_context_init 462 * and registered the target's TCG globals) must register with this function 463 * before initiating translation. 464 * 465 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 466 * of tcg_region_init() for the reasoning behind this. 467 * 468 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 469 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 470 * is not used anymore for translation once this function is called. 471 * 472 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 473 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 474 */ 475 #ifdef CONFIG_USER_ONLY 476 void tcg_register_thread(void) 477 { 478 tcg_ctx = &tcg_init_ctx; 479 } 480 #else 481 void tcg_register_thread(void) 482 { 483 TCGContext *s = g_malloc(sizeof(*s)); 484 unsigned int i, n; 485 486 *s = tcg_init_ctx; 487 488 /* Relink mem_base. */ 489 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 490 if (tcg_init_ctx.temps[i].mem_base) { 491 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 492 tcg_debug_assert(b >= 0 && b < n); 493 s->temps[i].mem_base = &s->temps[b]; 494 } 495 } 496 497 /* Claim an entry in tcg_ctxs */ 498 n = qatomic_fetch_inc(&tcg_cur_ctxs); 499 g_assert(n < tcg_max_ctxs); 500 qatomic_set(&tcg_ctxs[n], s); 501 502 if (n > 0) { 503 alloc_tcg_plugin_context(s); 504 tcg_region_initial_alloc(s); 505 } 506 507 tcg_ctx = s; 508 } 509 #endif /* !CONFIG_USER_ONLY */ 510 511 /* pool based memory allocation */ 512 void *tcg_malloc_internal(TCGContext *s, int size) 513 { 514 TCGPool *p; 515 int pool_size; 516 517 if (size > TCG_POOL_CHUNK_SIZE) { 518 /* big malloc: insert a new pool (XXX: could optimize) */ 519 p = g_malloc(sizeof(TCGPool) + size); 520 p->size = size; 521 p->next = s->pool_first_large; 522 s->pool_first_large = p; 523 return p->data; 524 } else { 525 p = s->pool_current; 526 if (!p) { 527 p = s->pool_first; 528 if (!p) 529 goto new_pool; 530 } else { 531 if (!p->next) { 532 new_pool: 533 pool_size = TCG_POOL_CHUNK_SIZE; 534 p = g_malloc(sizeof(TCGPool) + pool_size); 535 p->size = pool_size; 536 p->next = NULL; 537 if (s->pool_current) { 538 s->pool_current->next = p; 539 } else { 540 s->pool_first = p; 541 } 542 } else { 543 p = p->next; 544 } 545 } 546 } 547 s->pool_current = p; 548 s->pool_cur = p->data + size; 549 s->pool_end = p->data + p->size; 550 return p->data; 551 } 552 553 void tcg_pool_reset(TCGContext *s) 554 { 555 TCGPool *p, *t; 556 for (p = s->pool_first_large; p; p = t) { 557 t = p->next; 558 g_free(p); 559 } 560 s->pool_first_large = NULL; 561 s->pool_cur = s->pool_end = NULL; 562 s->pool_current = NULL; 563 } 564 565 #include "exec/helper-proto.h" 566 567 static TCGHelperInfo all_helpers[] = { 568 #include "exec/helper-tcg.h" 569 }; 570 static GHashTable *helper_table; 571 572 #ifdef CONFIG_TCG_INTERPRETER 573 static ffi_type *typecode_to_ffi(int argmask) 574 { 575 /* 576 * libffi does not support __int128_t, so we have forced Int128 577 * to use the structure definition instead of the builtin type. 578 */ 579 static ffi_type *ffi_type_i128_elements[3] = { 580 &ffi_type_uint64, 581 &ffi_type_uint64, 582 NULL 583 }; 584 static ffi_type ffi_type_i128 = { 585 .size = 16, 586 .alignment = __alignof__(Int128), 587 .type = FFI_TYPE_STRUCT, 588 .elements = ffi_type_i128_elements, 589 }; 590 591 switch (argmask) { 592 case dh_typecode_void: 593 return &ffi_type_void; 594 case dh_typecode_i32: 595 return &ffi_type_uint32; 596 case dh_typecode_s32: 597 return &ffi_type_sint32; 598 case dh_typecode_i64: 599 return &ffi_type_uint64; 600 case dh_typecode_s64: 601 return &ffi_type_sint64; 602 case dh_typecode_ptr: 603 return &ffi_type_pointer; 604 case dh_typecode_i128: 605 return &ffi_type_i128; 606 } 607 g_assert_not_reached(); 608 } 609 610 static void init_ffi_layouts(void) 611 { 612 /* g_direct_hash/equal for direct comparisons on uint32_t. */ 613 GHashTable *ffi_table = g_hash_table_new(NULL, NULL); 614 615 for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 616 TCGHelperInfo *info = &all_helpers[i]; 617 unsigned typemask = info->typemask; 618 gpointer hash = (gpointer)(uintptr_t)typemask; 619 struct { 620 ffi_cif cif; 621 ffi_type *args[]; 622 } *ca; 623 ffi_status status; 624 int nargs; 625 ffi_cif *cif; 626 627 cif = g_hash_table_lookup(ffi_table, hash); 628 if (cif) { 629 info->cif = cif; 630 continue; 631 } 632 633 /* Ignoring the return type, find the last non-zero field. */ 634 nargs = 32 - clz32(typemask >> 3); 635 nargs = DIV_ROUND_UP(nargs, 3); 636 assert(nargs <= MAX_CALL_IARGS); 637 638 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 639 ca->cif.rtype = typecode_to_ffi(typemask & 7); 640 ca->cif.nargs = nargs; 641 642 if (nargs != 0) { 643 ca->cif.arg_types = ca->args; 644 for (int j = 0; j < nargs; ++j) { 645 int typecode = extract32(typemask, (j + 1) * 3, 3); 646 ca->args[j] = typecode_to_ffi(typecode); 647 } 648 } 649 650 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 651 ca->cif.rtype, ca->cif.arg_types); 652 assert(status == FFI_OK); 653 654 cif = &ca->cif; 655 info->cif = cif; 656 g_hash_table_insert(ffi_table, hash, (gpointer)cif); 657 } 658 659 g_hash_table_destroy(ffi_table); 660 } 661 #endif /* CONFIG_TCG_INTERPRETER */ 662 663 typedef struct TCGCumulativeArgs { 664 int arg_idx; /* tcg_gen_callN args[] */ 665 int info_in_idx; /* TCGHelperInfo in[] */ 666 int arg_slot; /* regs+stack slot */ 667 int ref_slot; /* stack slots for references */ 668 } TCGCumulativeArgs; 669 670 static void layout_arg_even(TCGCumulativeArgs *cum) 671 { 672 cum->arg_slot += cum->arg_slot & 1; 673 } 674 675 static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, 676 TCGCallArgumentKind kind) 677 { 678 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 679 680 *loc = (TCGCallArgumentLoc){ 681 .kind = kind, 682 .arg_idx = cum->arg_idx, 683 .arg_slot = cum->arg_slot, 684 }; 685 cum->info_in_idx++; 686 cum->arg_slot++; 687 } 688 689 static void layout_arg_normal_n(TCGCumulativeArgs *cum, 690 TCGHelperInfo *info, int n) 691 { 692 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 693 694 for (int i = 0; i < n; ++i) { 695 /* Layout all using the same arg_idx, adjusting the subindex. */ 696 loc[i] = (TCGCallArgumentLoc){ 697 .kind = TCG_CALL_ARG_NORMAL, 698 .arg_idx = cum->arg_idx, 699 .tmp_subindex = i, 700 .arg_slot = cum->arg_slot + i, 701 }; 702 } 703 cum->info_in_idx += n; 704 cum->arg_slot += n; 705 } 706 707 static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info) 708 { 709 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 710 int n = 128 / TCG_TARGET_REG_BITS; 711 712 /* The first subindex carries the pointer. */ 713 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF); 714 715 /* 716 * The callee is allowed to clobber memory associated with 717 * structure pass by-reference. Therefore we must make copies. 718 * Allocate space from "ref_slot", which will be adjusted to 719 * follow the parameters on the stack. 720 */ 721 loc[0].ref_slot = cum->ref_slot; 722 723 /* 724 * Subsequent words also go into the reference slot, but 725 * do not accumulate into the regular arguments. 726 */ 727 for (int i = 1; i < n; ++i) { 728 loc[i] = (TCGCallArgumentLoc){ 729 .kind = TCG_CALL_ARG_BY_REF_N, 730 .arg_idx = cum->arg_idx, 731 .tmp_subindex = i, 732 .ref_slot = cum->ref_slot + i, 733 }; 734 } 735 cum->info_in_idx += n; 736 cum->ref_slot += n; 737 } 738 739 static void init_call_layout(TCGHelperInfo *info) 740 { 741 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); 742 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 743 unsigned typemask = info->typemask; 744 unsigned typecode; 745 TCGCumulativeArgs cum = { }; 746 747 /* 748 * Parse and place any function return value. 749 */ 750 typecode = typemask & 7; 751 switch (typecode) { 752 case dh_typecode_void: 753 info->nr_out = 0; 754 break; 755 case dh_typecode_i32: 756 case dh_typecode_s32: 757 case dh_typecode_ptr: 758 info->nr_out = 1; 759 info->out_kind = TCG_CALL_RET_NORMAL; 760 break; 761 case dh_typecode_i64: 762 case dh_typecode_s64: 763 info->nr_out = 64 / TCG_TARGET_REG_BITS; 764 info->out_kind = TCG_CALL_RET_NORMAL; 765 /* Query the last register now to trigger any assert early. */ 766 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 767 break; 768 case dh_typecode_i128: 769 info->nr_out = 128 / TCG_TARGET_REG_BITS; 770 info->out_kind = TCG_TARGET_CALL_RET_I128; 771 switch (TCG_TARGET_CALL_RET_I128) { 772 case TCG_CALL_RET_NORMAL: 773 /* Query the last register now to trigger any assert early. */ 774 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 775 break; 776 case TCG_CALL_RET_BY_VEC: 777 /* Query the single register now to trigger any assert early. */ 778 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0); 779 break; 780 case TCG_CALL_RET_BY_REF: 781 /* 782 * Allocate the first argument to the output. 783 * We don't need to store this anywhere, just make it 784 * unavailable for use in the input loop below. 785 */ 786 cum.arg_slot = 1; 787 break; 788 default: 789 qemu_build_not_reached(); 790 } 791 break; 792 default: 793 g_assert_not_reached(); 794 } 795 796 /* 797 * Parse and place function arguments. 798 */ 799 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { 800 TCGCallArgumentKind kind; 801 TCGType type; 802 803 typecode = typemask & 7; 804 switch (typecode) { 805 case dh_typecode_i32: 806 case dh_typecode_s32: 807 type = TCG_TYPE_I32; 808 break; 809 case dh_typecode_i64: 810 case dh_typecode_s64: 811 type = TCG_TYPE_I64; 812 break; 813 case dh_typecode_ptr: 814 type = TCG_TYPE_PTR; 815 break; 816 case dh_typecode_i128: 817 type = TCG_TYPE_I128; 818 break; 819 default: 820 g_assert_not_reached(); 821 } 822 823 switch (type) { 824 case TCG_TYPE_I32: 825 switch (TCG_TARGET_CALL_ARG_I32) { 826 case TCG_CALL_ARG_EVEN: 827 layout_arg_even(&cum); 828 /* fall through */ 829 case TCG_CALL_ARG_NORMAL: 830 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 831 break; 832 case TCG_CALL_ARG_EXTEND: 833 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); 834 layout_arg_1(&cum, info, kind); 835 break; 836 default: 837 qemu_build_not_reached(); 838 } 839 break; 840 841 case TCG_TYPE_I64: 842 switch (TCG_TARGET_CALL_ARG_I64) { 843 case TCG_CALL_ARG_EVEN: 844 layout_arg_even(&cum); 845 /* fall through */ 846 case TCG_CALL_ARG_NORMAL: 847 if (TCG_TARGET_REG_BITS == 32) { 848 layout_arg_normal_n(&cum, info, 2); 849 } else { 850 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 851 } 852 break; 853 default: 854 qemu_build_not_reached(); 855 } 856 break; 857 858 case TCG_TYPE_I128: 859 switch (TCG_TARGET_CALL_ARG_I128) { 860 case TCG_CALL_ARG_EVEN: 861 layout_arg_even(&cum); 862 /* fall through */ 863 case TCG_CALL_ARG_NORMAL: 864 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS); 865 break; 866 case TCG_CALL_ARG_BY_REF: 867 layout_arg_by_ref(&cum, info); 868 break; 869 default: 870 qemu_build_not_reached(); 871 } 872 break; 873 874 default: 875 g_assert_not_reached(); 876 } 877 } 878 info->nr_in = cum.info_in_idx; 879 880 /* Validate that we didn't overrun the input array. */ 881 assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); 882 /* Validate the backend has enough argument space. */ 883 assert(cum.arg_slot <= max_reg_slots + max_stk_slots); 884 885 /* 886 * Relocate the "ref_slot" area to the end of the parameters. 887 * Minimizing this stack offset helps code size for x86, 888 * which has a signed 8-bit offset encoding. 889 */ 890 if (cum.ref_slot != 0) { 891 int ref_base = 0; 892 893 if (cum.arg_slot > max_reg_slots) { 894 int align = __alignof(Int128) / sizeof(tcg_target_long); 895 896 ref_base = cum.arg_slot - max_reg_slots; 897 if (align > 1) { 898 ref_base = ROUND_UP(ref_base, align); 899 } 900 } 901 assert(ref_base + cum.ref_slot <= max_stk_slots); 902 903 if (ref_base != 0) { 904 for (int i = cum.info_in_idx - 1; i >= 0; --i) { 905 TCGCallArgumentLoc *loc = &info->in[i]; 906 switch (loc->kind) { 907 case TCG_CALL_ARG_BY_REF: 908 case TCG_CALL_ARG_BY_REF_N: 909 loc->ref_slot += ref_base; 910 break; 911 default: 912 break; 913 } 914 } 915 } 916 } 917 } 918 919 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 920 static void process_op_defs(TCGContext *s); 921 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 922 TCGReg reg, const char *name); 923 924 static void tcg_context_init(unsigned max_cpus) 925 { 926 TCGContext *s = &tcg_init_ctx; 927 int op, total_args, n, i; 928 TCGOpDef *def; 929 TCGArgConstraint *args_ct; 930 TCGTemp *ts; 931 932 memset(s, 0, sizeof(*s)); 933 s->nb_globals = 0; 934 935 /* Count total number of arguments and allocate the corresponding 936 space */ 937 total_args = 0; 938 for(op = 0; op < NB_OPS; op++) { 939 def = &tcg_op_defs[op]; 940 n = def->nb_iargs + def->nb_oargs; 941 total_args += n; 942 } 943 944 args_ct = g_new0(TCGArgConstraint, total_args); 945 946 for(op = 0; op < NB_OPS; op++) { 947 def = &tcg_op_defs[op]; 948 def->args_ct = args_ct; 949 n = def->nb_iargs + def->nb_oargs; 950 args_ct += n; 951 } 952 953 /* Register helpers. */ 954 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 955 helper_table = g_hash_table_new(NULL, NULL); 956 957 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 958 init_call_layout(&all_helpers[i]); 959 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 960 (gpointer)&all_helpers[i]); 961 } 962 963 #ifdef CONFIG_TCG_INTERPRETER 964 init_ffi_layouts(); 965 #endif 966 967 tcg_target_init(s); 968 process_op_defs(s); 969 970 /* Reverse the order of the saved registers, assuming they're all at 971 the start of tcg_target_reg_alloc_order. */ 972 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 973 int r = tcg_target_reg_alloc_order[n]; 974 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 975 break; 976 } 977 } 978 for (i = 0; i < n; ++i) { 979 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 980 } 981 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 982 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 983 } 984 985 alloc_tcg_plugin_context(s); 986 987 tcg_ctx = s; 988 /* 989 * In user-mode we simply share the init context among threads, since we 990 * use a single region. See the documentation tcg_region_init() for the 991 * reasoning behind this. 992 * In softmmu we will have at most max_cpus TCG threads. 993 */ 994 #ifdef CONFIG_USER_ONLY 995 tcg_ctxs = &tcg_ctx; 996 tcg_cur_ctxs = 1; 997 tcg_max_ctxs = 1; 998 #else 999 tcg_max_ctxs = max_cpus; 1000 tcg_ctxs = g_new0(TCGContext *, max_cpus); 1001 #endif 1002 1003 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 1004 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 1005 cpu_env = temp_tcgv_ptr(ts); 1006 } 1007 1008 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 1009 { 1010 tcg_context_init(max_cpus); 1011 tcg_region_init(tb_size, splitwx, max_cpus); 1012 } 1013 1014 /* 1015 * Allocate TBs right before their corresponding translated code, making 1016 * sure that TBs and code are on different cache lines. 1017 */ 1018 TranslationBlock *tcg_tb_alloc(TCGContext *s) 1019 { 1020 uintptr_t align = qemu_icache_linesize; 1021 TranslationBlock *tb; 1022 void *next; 1023 1024 retry: 1025 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 1026 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 1027 1028 if (unlikely(next > s->code_gen_highwater)) { 1029 if (tcg_region_alloc(s)) { 1030 return NULL; 1031 } 1032 goto retry; 1033 } 1034 qatomic_set(&s->code_gen_ptr, next); 1035 s->data_gen_ptr = NULL; 1036 return tb; 1037 } 1038 1039 void tcg_prologue_init(TCGContext *s) 1040 { 1041 size_t prologue_size; 1042 1043 s->code_ptr = s->code_gen_ptr; 1044 s->code_buf = s->code_gen_ptr; 1045 s->data_gen_ptr = NULL; 1046 1047 #ifndef CONFIG_TCG_INTERPRETER 1048 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 1049 #endif 1050 1051 #ifdef TCG_TARGET_NEED_POOL_LABELS 1052 s->pool_labels = NULL; 1053 #endif 1054 1055 qemu_thread_jit_write(); 1056 /* Generate the prologue. */ 1057 tcg_target_qemu_prologue(s); 1058 1059 #ifdef TCG_TARGET_NEED_POOL_LABELS 1060 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 1061 { 1062 int result = tcg_out_pool_finalize(s); 1063 tcg_debug_assert(result == 0); 1064 } 1065 #endif 1066 1067 prologue_size = tcg_current_code_size(s); 1068 perf_report_prologue(s->code_gen_ptr, prologue_size); 1069 1070 #ifndef CONFIG_TCG_INTERPRETER 1071 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 1072 (uintptr_t)s->code_buf, prologue_size); 1073 #endif 1074 1075 #ifdef DEBUG_DISAS 1076 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 1077 FILE *logfile = qemu_log_trylock(); 1078 if (logfile) { 1079 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 1080 if (s->data_gen_ptr) { 1081 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 1082 size_t data_size = prologue_size - code_size; 1083 size_t i; 1084 1085 disas(logfile, s->code_gen_ptr, code_size); 1086 1087 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1088 if (sizeof(tcg_target_ulong) == 8) { 1089 fprintf(logfile, 1090 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1091 (uintptr_t)s->data_gen_ptr + i, 1092 *(uint64_t *)(s->data_gen_ptr + i)); 1093 } else { 1094 fprintf(logfile, 1095 "0x%08" PRIxPTR ": .long 0x%08x\n", 1096 (uintptr_t)s->data_gen_ptr + i, 1097 *(uint32_t *)(s->data_gen_ptr + i)); 1098 } 1099 } 1100 } else { 1101 disas(logfile, s->code_gen_ptr, prologue_size); 1102 } 1103 fprintf(logfile, "\n"); 1104 qemu_log_unlock(logfile); 1105 } 1106 } 1107 #endif 1108 1109 #ifndef CONFIG_TCG_INTERPRETER 1110 /* 1111 * Assert that goto_ptr is implemented completely, setting an epilogue. 1112 * For tci, we use NULL as the signal to return from the interpreter, 1113 * so skip this check. 1114 */ 1115 tcg_debug_assert(tcg_code_gen_epilogue != NULL); 1116 #endif 1117 1118 tcg_region_prologue_set(s); 1119 } 1120 1121 void tcg_func_start(TCGContext *s) 1122 { 1123 tcg_pool_reset(s); 1124 s->nb_temps = s->nb_globals; 1125 1126 /* No temps have been previously allocated for size or locality. */ 1127 memset(s->free_temps, 0, sizeof(s->free_temps)); 1128 1129 /* No constant temps have been previously allocated. */ 1130 for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 1131 if (s->const_table[i]) { 1132 g_hash_table_remove_all(s->const_table[i]); 1133 } 1134 } 1135 1136 s->nb_ops = 0; 1137 s->nb_labels = 0; 1138 s->current_frame_offset = s->frame_start; 1139 1140 #ifdef CONFIG_DEBUG_TCG 1141 s->goto_tb_issue_mask = 0; 1142 #endif 1143 1144 QTAILQ_INIT(&s->ops); 1145 QTAILQ_INIT(&s->free_ops); 1146 QSIMPLEQ_INIT(&s->labels); 1147 } 1148 1149 static TCGTemp *tcg_temp_alloc(TCGContext *s) 1150 { 1151 int n = s->nb_temps++; 1152 1153 if (n >= TCG_MAX_TEMPS) { 1154 tcg_raise_tb_overflow(s); 1155 } 1156 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 1157 } 1158 1159 static TCGTemp *tcg_global_alloc(TCGContext *s) 1160 { 1161 TCGTemp *ts; 1162 1163 tcg_debug_assert(s->nb_globals == s->nb_temps); 1164 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 1165 s->nb_globals++; 1166 ts = tcg_temp_alloc(s); 1167 ts->kind = TEMP_GLOBAL; 1168 1169 return ts; 1170 } 1171 1172 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1173 TCGReg reg, const char *name) 1174 { 1175 TCGTemp *ts; 1176 1177 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 1178 tcg_abort(); 1179 } 1180 1181 ts = tcg_global_alloc(s); 1182 ts->base_type = type; 1183 ts->type = type; 1184 ts->kind = TEMP_FIXED; 1185 ts->reg = reg; 1186 ts->name = name; 1187 tcg_regset_set_reg(s->reserved_regs, reg); 1188 1189 return ts; 1190 } 1191 1192 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 1193 { 1194 s->frame_start = start; 1195 s->frame_end = start + size; 1196 s->frame_temp 1197 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 1198 } 1199 1200 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 1201 intptr_t offset, const char *name) 1202 { 1203 TCGContext *s = tcg_ctx; 1204 TCGTemp *base_ts = tcgv_ptr_temp(base); 1205 TCGTemp *ts = tcg_global_alloc(s); 1206 int indirect_reg = 0; 1207 1208 switch (base_ts->kind) { 1209 case TEMP_FIXED: 1210 break; 1211 case TEMP_GLOBAL: 1212 /* We do not support double-indirect registers. */ 1213 tcg_debug_assert(!base_ts->indirect_reg); 1214 base_ts->indirect_base = 1; 1215 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 1216 ? 2 : 1); 1217 indirect_reg = 1; 1218 break; 1219 default: 1220 g_assert_not_reached(); 1221 } 1222 1223 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1224 TCGTemp *ts2 = tcg_global_alloc(s); 1225 char buf[64]; 1226 1227 ts->base_type = TCG_TYPE_I64; 1228 ts->type = TCG_TYPE_I32; 1229 ts->indirect_reg = indirect_reg; 1230 ts->mem_allocated = 1; 1231 ts->mem_base = base_ts; 1232 ts->mem_offset = offset; 1233 pstrcpy(buf, sizeof(buf), name); 1234 pstrcat(buf, sizeof(buf), "_0"); 1235 ts->name = strdup(buf); 1236 1237 tcg_debug_assert(ts2 == ts + 1); 1238 ts2->base_type = TCG_TYPE_I64; 1239 ts2->type = TCG_TYPE_I32; 1240 ts2->indirect_reg = indirect_reg; 1241 ts2->mem_allocated = 1; 1242 ts2->mem_base = base_ts; 1243 ts2->mem_offset = offset + 4; 1244 ts2->temp_subindex = 1; 1245 pstrcpy(buf, sizeof(buf), name); 1246 pstrcat(buf, sizeof(buf), "_1"); 1247 ts2->name = strdup(buf); 1248 } else { 1249 ts->base_type = type; 1250 ts->type = type; 1251 ts->indirect_reg = indirect_reg; 1252 ts->mem_allocated = 1; 1253 ts->mem_base = base_ts; 1254 ts->mem_offset = offset; 1255 ts->name = name; 1256 } 1257 return ts; 1258 } 1259 1260 TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) 1261 { 1262 TCGContext *s = tcg_ctx; 1263 TCGTemp *ts; 1264 int n; 1265 1266 if (kind == TEMP_EBB) { 1267 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); 1268 1269 if (idx < TCG_MAX_TEMPS) { 1270 /* There is already an available temp with the right type. */ 1271 clear_bit(idx, s->free_temps[type].l); 1272 1273 ts = &s->temps[idx]; 1274 ts->temp_allocated = 1; 1275 tcg_debug_assert(ts->base_type == type); 1276 tcg_debug_assert(ts->kind == kind); 1277 return ts; 1278 } 1279 } else { 1280 tcg_debug_assert(kind == TEMP_TB); 1281 } 1282 1283 switch (type) { 1284 case TCG_TYPE_I32: 1285 case TCG_TYPE_V64: 1286 case TCG_TYPE_V128: 1287 case TCG_TYPE_V256: 1288 n = 1; 1289 break; 1290 case TCG_TYPE_I64: 1291 n = 64 / TCG_TARGET_REG_BITS; 1292 break; 1293 case TCG_TYPE_I128: 1294 n = 128 / TCG_TARGET_REG_BITS; 1295 break; 1296 default: 1297 g_assert_not_reached(); 1298 } 1299 1300 ts = tcg_temp_alloc(s); 1301 ts->base_type = type; 1302 ts->temp_allocated = 1; 1303 ts->kind = kind; 1304 1305 if (n == 1) { 1306 ts->type = type; 1307 } else { 1308 ts->type = TCG_TYPE_REG; 1309 1310 for (int i = 1; i < n; ++i) { 1311 TCGTemp *ts2 = tcg_temp_alloc(s); 1312 1313 tcg_debug_assert(ts2 == ts + i); 1314 ts2->base_type = type; 1315 ts2->type = TCG_TYPE_REG; 1316 ts2->temp_allocated = 1; 1317 ts2->temp_subindex = i; 1318 ts2->kind = kind; 1319 } 1320 } 1321 return ts; 1322 } 1323 1324 TCGv_vec tcg_temp_new_vec(TCGType type) 1325 { 1326 TCGTemp *t; 1327 1328 #ifdef CONFIG_DEBUG_TCG 1329 switch (type) { 1330 case TCG_TYPE_V64: 1331 assert(TCG_TARGET_HAS_v64); 1332 break; 1333 case TCG_TYPE_V128: 1334 assert(TCG_TARGET_HAS_v128); 1335 break; 1336 case TCG_TYPE_V256: 1337 assert(TCG_TARGET_HAS_v256); 1338 break; 1339 default: 1340 g_assert_not_reached(); 1341 } 1342 #endif 1343 1344 t = tcg_temp_new_internal(type, TEMP_EBB); 1345 return temp_tcgv_vec(t); 1346 } 1347 1348 /* Create a new temp of the same type as an existing temp. */ 1349 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1350 { 1351 TCGTemp *t = tcgv_vec_temp(match); 1352 1353 tcg_debug_assert(t->temp_allocated != 0); 1354 1355 t = tcg_temp_new_internal(t->base_type, TEMP_EBB); 1356 return temp_tcgv_vec(t); 1357 } 1358 1359 void tcg_temp_free_internal(TCGTemp *ts) 1360 { 1361 TCGContext *s = tcg_ctx; 1362 1363 switch (ts->kind) { 1364 case TEMP_CONST: 1365 case TEMP_TB: 1366 /* Silently ignore free. */ 1367 break; 1368 case TEMP_EBB: 1369 tcg_debug_assert(ts->temp_allocated != 0); 1370 ts->temp_allocated = 0; 1371 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); 1372 break; 1373 default: 1374 /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */ 1375 g_assert_not_reached(); 1376 } 1377 } 1378 1379 TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1380 { 1381 TCGContext *s = tcg_ctx; 1382 GHashTable *h = s->const_table[type]; 1383 TCGTemp *ts; 1384 1385 if (h == NULL) { 1386 h = g_hash_table_new(g_int64_hash, g_int64_equal); 1387 s->const_table[type] = h; 1388 } 1389 1390 ts = g_hash_table_lookup(h, &val); 1391 if (ts == NULL) { 1392 int64_t *val_ptr; 1393 1394 ts = tcg_temp_alloc(s); 1395 1396 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1397 TCGTemp *ts2 = tcg_temp_alloc(s); 1398 1399 tcg_debug_assert(ts2 == ts + 1); 1400 1401 ts->base_type = TCG_TYPE_I64; 1402 ts->type = TCG_TYPE_I32; 1403 ts->kind = TEMP_CONST; 1404 ts->temp_allocated = 1; 1405 1406 ts2->base_type = TCG_TYPE_I64; 1407 ts2->type = TCG_TYPE_I32; 1408 ts2->kind = TEMP_CONST; 1409 ts2->temp_allocated = 1; 1410 ts2->temp_subindex = 1; 1411 1412 /* 1413 * Retain the full value of the 64-bit constant in the low 1414 * part, so that the hash table works. Actual uses will 1415 * truncate the value to the low part. 1416 */ 1417 ts[HOST_BIG_ENDIAN].val = val; 1418 ts[!HOST_BIG_ENDIAN].val = val >> 32; 1419 val_ptr = &ts[HOST_BIG_ENDIAN].val; 1420 } else { 1421 ts->base_type = type; 1422 ts->type = type; 1423 ts->kind = TEMP_CONST; 1424 ts->temp_allocated = 1; 1425 ts->val = val; 1426 val_ptr = &ts->val; 1427 } 1428 g_hash_table_insert(h, val_ptr, ts); 1429 } 1430 1431 return ts; 1432 } 1433 1434 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1435 { 1436 val = dup_const(vece, val); 1437 return temp_tcgv_vec(tcg_constant_internal(type, val)); 1438 } 1439 1440 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 1441 { 1442 TCGTemp *t = tcgv_vec_temp(match); 1443 1444 tcg_debug_assert(t->temp_allocated != 0); 1445 return tcg_constant_vec(t->base_type, vece, val); 1446 } 1447 1448 TCGv_i32 tcg_const_i32(int32_t val) 1449 { 1450 TCGv_i32 t0; 1451 t0 = tcg_temp_new_i32(); 1452 tcg_gen_movi_i32(t0, val); 1453 return t0; 1454 } 1455 1456 TCGv_i64 tcg_const_i64(int64_t val) 1457 { 1458 TCGv_i64 t0; 1459 t0 = tcg_temp_new_i64(); 1460 tcg_gen_movi_i64(t0, val); 1461 return t0; 1462 } 1463 1464 /* Return true if OP may appear in the opcode stream. 1465 Test the runtime variable that controls each opcode. */ 1466 bool tcg_op_supported(TCGOpcode op) 1467 { 1468 const bool have_vec 1469 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1470 1471 switch (op) { 1472 case INDEX_op_discard: 1473 case INDEX_op_set_label: 1474 case INDEX_op_call: 1475 case INDEX_op_br: 1476 case INDEX_op_mb: 1477 case INDEX_op_insn_start: 1478 case INDEX_op_exit_tb: 1479 case INDEX_op_goto_tb: 1480 case INDEX_op_goto_ptr: 1481 case INDEX_op_qemu_ld_i32: 1482 case INDEX_op_qemu_st_i32: 1483 case INDEX_op_qemu_ld_i64: 1484 case INDEX_op_qemu_st_i64: 1485 return true; 1486 1487 case INDEX_op_qemu_st8_i32: 1488 return TCG_TARGET_HAS_qemu_st8_i32; 1489 1490 case INDEX_op_mov_i32: 1491 case INDEX_op_setcond_i32: 1492 case INDEX_op_brcond_i32: 1493 case INDEX_op_ld8u_i32: 1494 case INDEX_op_ld8s_i32: 1495 case INDEX_op_ld16u_i32: 1496 case INDEX_op_ld16s_i32: 1497 case INDEX_op_ld_i32: 1498 case INDEX_op_st8_i32: 1499 case INDEX_op_st16_i32: 1500 case INDEX_op_st_i32: 1501 case INDEX_op_add_i32: 1502 case INDEX_op_sub_i32: 1503 case INDEX_op_mul_i32: 1504 case INDEX_op_and_i32: 1505 case INDEX_op_or_i32: 1506 case INDEX_op_xor_i32: 1507 case INDEX_op_shl_i32: 1508 case INDEX_op_shr_i32: 1509 case INDEX_op_sar_i32: 1510 return true; 1511 1512 case INDEX_op_movcond_i32: 1513 return TCG_TARGET_HAS_movcond_i32; 1514 case INDEX_op_div_i32: 1515 case INDEX_op_divu_i32: 1516 return TCG_TARGET_HAS_div_i32; 1517 case INDEX_op_rem_i32: 1518 case INDEX_op_remu_i32: 1519 return TCG_TARGET_HAS_rem_i32; 1520 case INDEX_op_div2_i32: 1521 case INDEX_op_divu2_i32: 1522 return TCG_TARGET_HAS_div2_i32; 1523 case INDEX_op_rotl_i32: 1524 case INDEX_op_rotr_i32: 1525 return TCG_TARGET_HAS_rot_i32; 1526 case INDEX_op_deposit_i32: 1527 return TCG_TARGET_HAS_deposit_i32; 1528 case INDEX_op_extract_i32: 1529 return TCG_TARGET_HAS_extract_i32; 1530 case INDEX_op_sextract_i32: 1531 return TCG_TARGET_HAS_sextract_i32; 1532 case INDEX_op_extract2_i32: 1533 return TCG_TARGET_HAS_extract2_i32; 1534 case INDEX_op_add2_i32: 1535 return TCG_TARGET_HAS_add2_i32; 1536 case INDEX_op_sub2_i32: 1537 return TCG_TARGET_HAS_sub2_i32; 1538 case INDEX_op_mulu2_i32: 1539 return TCG_TARGET_HAS_mulu2_i32; 1540 case INDEX_op_muls2_i32: 1541 return TCG_TARGET_HAS_muls2_i32; 1542 case INDEX_op_muluh_i32: 1543 return TCG_TARGET_HAS_muluh_i32; 1544 case INDEX_op_mulsh_i32: 1545 return TCG_TARGET_HAS_mulsh_i32; 1546 case INDEX_op_ext8s_i32: 1547 return TCG_TARGET_HAS_ext8s_i32; 1548 case INDEX_op_ext16s_i32: 1549 return TCG_TARGET_HAS_ext16s_i32; 1550 case INDEX_op_ext8u_i32: 1551 return TCG_TARGET_HAS_ext8u_i32; 1552 case INDEX_op_ext16u_i32: 1553 return TCG_TARGET_HAS_ext16u_i32; 1554 case INDEX_op_bswap16_i32: 1555 return TCG_TARGET_HAS_bswap16_i32; 1556 case INDEX_op_bswap32_i32: 1557 return TCG_TARGET_HAS_bswap32_i32; 1558 case INDEX_op_not_i32: 1559 return TCG_TARGET_HAS_not_i32; 1560 case INDEX_op_neg_i32: 1561 return TCG_TARGET_HAS_neg_i32; 1562 case INDEX_op_andc_i32: 1563 return TCG_TARGET_HAS_andc_i32; 1564 case INDEX_op_orc_i32: 1565 return TCG_TARGET_HAS_orc_i32; 1566 case INDEX_op_eqv_i32: 1567 return TCG_TARGET_HAS_eqv_i32; 1568 case INDEX_op_nand_i32: 1569 return TCG_TARGET_HAS_nand_i32; 1570 case INDEX_op_nor_i32: 1571 return TCG_TARGET_HAS_nor_i32; 1572 case INDEX_op_clz_i32: 1573 return TCG_TARGET_HAS_clz_i32; 1574 case INDEX_op_ctz_i32: 1575 return TCG_TARGET_HAS_ctz_i32; 1576 case INDEX_op_ctpop_i32: 1577 return TCG_TARGET_HAS_ctpop_i32; 1578 1579 case INDEX_op_brcond2_i32: 1580 case INDEX_op_setcond2_i32: 1581 return TCG_TARGET_REG_BITS == 32; 1582 1583 case INDEX_op_mov_i64: 1584 case INDEX_op_setcond_i64: 1585 case INDEX_op_brcond_i64: 1586 case INDEX_op_ld8u_i64: 1587 case INDEX_op_ld8s_i64: 1588 case INDEX_op_ld16u_i64: 1589 case INDEX_op_ld16s_i64: 1590 case INDEX_op_ld32u_i64: 1591 case INDEX_op_ld32s_i64: 1592 case INDEX_op_ld_i64: 1593 case INDEX_op_st8_i64: 1594 case INDEX_op_st16_i64: 1595 case INDEX_op_st32_i64: 1596 case INDEX_op_st_i64: 1597 case INDEX_op_add_i64: 1598 case INDEX_op_sub_i64: 1599 case INDEX_op_mul_i64: 1600 case INDEX_op_and_i64: 1601 case INDEX_op_or_i64: 1602 case INDEX_op_xor_i64: 1603 case INDEX_op_shl_i64: 1604 case INDEX_op_shr_i64: 1605 case INDEX_op_sar_i64: 1606 case INDEX_op_ext_i32_i64: 1607 case INDEX_op_extu_i32_i64: 1608 return TCG_TARGET_REG_BITS == 64; 1609 1610 case INDEX_op_movcond_i64: 1611 return TCG_TARGET_HAS_movcond_i64; 1612 case INDEX_op_div_i64: 1613 case INDEX_op_divu_i64: 1614 return TCG_TARGET_HAS_div_i64; 1615 case INDEX_op_rem_i64: 1616 case INDEX_op_remu_i64: 1617 return TCG_TARGET_HAS_rem_i64; 1618 case INDEX_op_div2_i64: 1619 case INDEX_op_divu2_i64: 1620 return TCG_TARGET_HAS_div2_i64; 1621 case INDEX_op_rotl_i64: 1622 case INDEX_op_rotr_i64: 1623 return TCG_TARGET_HAS_rot_i64; 1624 case INDEX_op_deposit_i64: 1625 return TCG_TARGET_HAS_deposit_i64; 1626 case INDEX_op_extract_i64: 1627 return TCG_TARGET_HAS_extract_i64; 1628 case INDEX_op_sextract_i64: 1629 return TCG_TARGET_HAS_sextract_i64; 1630 case INDEX_op_extract2_i64: 1631 return TCG_TARGET_HAS_extract2_i64; 1632 case INDEX_op_extrl_i64_i32: 1633 return TCG_TARGET_HAS_extrl_i64_i32; 1634 case INDEX_op_extrh_i64_i32: 1635 return TCG_TARGET_HAS_extrh_i64_i32; 1636 case INDEX_op_ext8s_i64: 1637 return TCG_TARGET_HAS_ext8s_i64; 1638 case INDEX_op_ext16s_i64: 1639 return TCG_TARGET_HAS_ext16s_i64; 1640 case INDEX_op_ext32s_i64: 1641 return TCG_TARGET_HAS_ext32s_i64; 1642 case INDEX_op_ext8u_i64: 1643 return TCG_TARGET_HAS_ext8u_i64; 1644 case INDEX_op_ext16u_i64: 1645 return TCG_TARGET_HAS_ext16u_i64; 1646 case INDEX_op_ext32u_i64: 1647 return TCG_TARGET_HAS_ext32u_i64; 1648 case INDEX_op_bswap16_i64: 1649 return TCG_TARGET_HAS_bswap16_i64; 1650 case INDEX_op_bswap32_i64: 1651 return TCG_TARGET_HAS_bswap32_i64; 1652 case INDEX_op_bswap64_i64: 1653 return TCG_TARGET_HAS_bswap64_i64; 1654 case INDEX_op_not_i64: 1655 return TCG_TARGET_HAS_not_i64; 1656 case INDEX_op_neg_i64: 1657 return TCG_TARGET_HAS_neg_i64; 1658 case INDEX_op_andc_i64: 1659 return TCG_TARGET_HAS_andc_i64; 1660 case INDEX_op_orc_i64: 1661 return TCG_TARGET_HAS_orc_i64; 1662 case INDEX_op_eqv_i64: 1663 return TCG_TARGET_HAS_eqv_i64; 1664 case INDEX_op_nand_i64: 1665 return TCG_TARGET_HAS_nand_i64; 1666 case INDEX_op_nor_i64: 1667 return TCG_TARGET_HAS_nor_i64; 1668 case INDEX_op_clz_i64: 1669 return TCG_TARGET_HAS_clz_i64; 1670 case INDEX_op_ctz_i64: 1671 return TCG_TARGET_HAS_ctz_i64; 1672 case INDEX_op_ctpop_i64: 1673 return TCG_TARGET_HAS_ctpop_i64; 1674 case INDEX_op_add2_i64: 1675 return TCG_TARGET_HAS_add2_i64; 1676 case INDEX_op_sub2_i64: 1677 return TCG_TARGET_HAS_sub2_i64; 1678 case INDEX_op_mulu2_i64: 1679 return TCG_TARGET_HAS_mulu2_i64; 1680 case INDEX_op_muls2_i64: 1681 return TCG_TARGET_HAS_muls2_i64; 1682 case INDEX_op_muluh_i64: 1683 return TCG_TARGET_HAS_muluh_i64; 1684 case INDEX_op_mulsh_i64: 1685 return TCG_TARGET_HAS_mulsh_i64; 1686 1687 case INDEX_op_mov_vec: 1688 case INDEX_op_dup_vec: 1689 case INDEX_op_dupm_vec: 1690 case INDEX_op_ld_vec: 1691 case INDEX_op_st_vec: 1692 case INDEX_op_add_vec: 1693 case INDEX_op_sub_vec: 1694 case INDEX_op_and_vec: 1695 case INDEX_op_or_vec: 1696 case INDEX_op_xor_vec: 1697 case INDEX_op_cmp_vec: 1698 return have_vec; 1699 case INDEX_op_dup2_vec: 1700 return have_vec && TCG_TARGET_REG_BITS == 32; 1701 case INDEX_op_not_vec: 1702 return have_vec && TCG_TARGET_HAS_not_vec; 1703 case INDEX_op_neg_vec: 1704 return have_vec && TCG_TARGET_HAS_neg_vec; 1705 case INDEX_op_abs_vec: 1706 return have_vec && TCG_TARGET_HAS_abs_vec; 1707 case INDEX_op_andc_vec: 1708 return have_vec && TCG_TARGET_HAS_andc_vec; 1709 case INDEX_op_orc_vec: 1710 return have_vec && TCG_TARGET_HAS_orc_vec; 1711 case INDEX_op_nand_vec: 1712 return have_vec && TCG_TARGET_HAS_nand_vec; 1713 case INDEX_op_nor_vec: 1714 return have_vec && TCG_TARGET_HAS_nor_vec; 1715 case INDEX_op_eqv_vec: 1716 return have_vec && TCG_TARGET_HAS_eqv_vec; 1717 case INDEX_op_mul_vec: 1718 return have_vec && TCG_TARGET_HAS_mul_vec; 1719 case INDEX_op_shli_vec: 1720 case INDEX_op_shri_vec: 1721 case INDEX_op_sari_vec: 1722 return have_vec && TCG_TARGET_HAS_shi_vec; 1723 case INDEX_op_shls_vec: 1724 case INDEX_op_shrs_vec: 1725 case INDEX_op_sars_vec: 1726 return have_vec && TCG_TARGET_HAS_shs_vec; 1727 case INDEX_op_shlv_vec: 1728 case INDEX_op_shrv_vec: 1729 case INDEX_op_sarv_vec: 1730 return have_vec && TCG_TARGET_HAS_shv_vec; 1731 case INDEX_op_rotli_vec: 1732 return have_vec && TCG_TARGET_HAS_roti_vec; 1733 case INDEX_op_rotls_vec: 1734 return have_vec && TCG_TARGET_HAS_rots_vec; 1735 case INDEX_op_rotlv_vec: 1736 case INDEX_op_rotrv_vec: 1737 return have_vec && TCG_TARGET_HAS_rotv_vec; 1738 case INDEX_op_ssadd_vec: 1739 case INDEX_op_usadd_vec: 1740 case INDEX_op_sssub_vec: 1741 case INDEX_op_ussub_vec: 1742 return have_vec && TCG_TARGET_HAS_sat_vec; 1743 case INDEX_op_smin_vec: 1744 case INDEX_op_umin_vec: 1745 case INDEX_op_smax_vec: 1746 case INDEX_op_umax_vec: 1747 return have_vec && TCG_TARGET_HAS_minmax_vec; 1748 case INDEX_op_bitsel_vec: 1749 return have_vec && TCG_TARGET_HAS_bitsel_vec; 1750 case INDEX_op_cmpsel_vec: 1751 return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1752 1753 default: 1754 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1755 return true; 1756 } 1757 } 1758 1759 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); 1760 1761 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1762 { 1763 const TCGHelperInfo *info; 1764 TCGv_i64 extend_free[MAX_CALL_IARGS]; 1765 int n_extend = 0; 1766 TCGOp *op; 1767 int i, n, pi = 0, total_args; 1768 1769 info = g_hash_table_lookup(helper_table, (gpointer)func); 1770 total_args = info->nr_out + info->nr_in + 2; 1771 op = tcg_op_alloc(INDEX_op_call, total_args); 1772 1773 #ifdef CONFIG_PLUGIN 1774 /* Flag helpers that may affect guest state */ 1775 if (tcg_ctx->plugin_insn && 1776 !(info->flags & TCG_CALL_PLUGIN) && 1777 !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { 1778 tcg_ctx->plugin_insn->calls_helpers = true; 1779 } 1780 #endif 1781 1782 TCGOP_CALLO(op) = n = info->nr_out; 1783 switch (n) { 1784 case 0: 1785 tcg_debug_assert(ret == NULL); 1786 break; 1787 case 1: 1788 tcg_debug_assert(ret != NULL); 1789 op->args[pi++] = temp_arg(ret); 1790 break; 1791 case 2: 1792 case 4: 1793 tcg_debug_assert(ret != NULL); 1794 tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); 1795 tcg_debug_assert(ret->temp_subindex == 0); 1796 for (i = 0; i < n; ++i) { 1797 op->args[pi++] = temp_arg(ret + i); 1798 } 1799 break; 1800 default: 1801 g_assert_not_reached(); 1802 } 1803 1804 TCGOP_CALLI(op) = n = info->nr_in; 1805 for (i = 0; i < n; i++) { 1806 const TCGCallArgumentLoc *loc = &info->in[i]; 1807 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; 1808 1809 switch (loc->kind) { 1810 case TCG_CALL_ARG_NORMAL: 1811 case TCG_CALL_ARG_BY_REF: 1812 case TCG_CALL_ARG_BY_REF_N: 1813 op->args[pi++] = temp_arg(ts); 1814 break; 1815 1816 case TCG_CALL_ARG_EXTEND_U: 1817 case TCG_CALL_ARG_EXTEND_S: 1818 { 1819 TCGv_i64 temp = tcg_temp_ebb_new_i64(); 1820 TCGv_i32 orig = temp_tcgv_i32(ts); 1821 1822 if (loc->kind == TCG_CALL_ARG_EXTEND_S) { 1823 tcg_gen_ext_i32_i64(temp, orig); 1824 } else { 1825 tcg_gen_extu_i32_i64(temp, orig); 1826 } 1827 op->args[pi++] = tcgv_i64_arg(temp); 1828 extend_free[n_extend++] = temp; 1829 } 1830 break; 1831 1832 default: 1833 g_assert_not_reached(); 1834 } 1835 } 1836 op->args[pi++] = (uintptr_t)func; 1837 op->args[pi++] = (uintptr_t)info; 1838 tcg_debug_assert(pi == total_args); 1839 1840 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 1841 1842 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); 1843 for (i = 0; i < n_extend; ++i) { 1844 tcg_temp_free_i64(extend_free[i]); 1845 } 1846 } 1847 1848 static void tcg_reg_alloc_start(TCGContext *s) 1849 { 1850 int i, n; 1851 1852 for (i = 0, n = s->nb_temps; i < n; i++) { 1853 TCGTemp *ts = &s->temps[i]; 1854 TCGTempVal val = TEMP_VAL_MEM; 1855 1856 switch (ts->kind) { 1857 case TEMP_CONST: 1858 val = TEMP_VAL_CONST; 1859 break; 1860 case TEMP_FIXED: 1861 val = TEMP_VAL_REG; 1862 break; 1863 case TEMP_GLOBAL: 1864 break; 1865 case TEMP_EBB: 1866 val = TEMP_VAL_DEAD; 1867 /* fall through */ 1868 case TEMP_TB: 1869 ts->mem_allocated = 0; 1870 break; 1871 default: 1872 g_assert_not_reached(); 1873 } 1874 ts->val_type = val; 1875 } 1876 1877 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1878 } 1879 1880 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1881 TCGTemp *ts) 1882 { 1883 int idx = temp_idx(ts); 1884 1885 switch (ts->kind) { 1886 case TEMP_FIXED: 1887 case TEMP_GLOBAL: 1888 pstrcpy(buf, buf_size, ts->name); 1889 break; 1890 case TEMP_TB: 1891 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1892 break; 1893 case TEMP_EBB: 1894 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1895 break; 1896 case TEMP_CONST: 1897 switch (ts->type) { 1898 case TCG_TYPE_I32: 1899 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1900 break; 1901 #if TCG_TARGET_REG_BITS > 32 1902 case TCG_TYPE_I64: 1903 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1904 break; 1905 #endif 1906 case TCG_TYPE_V64: 1907 case TCG_TYPE_V128: 1908 case TCG_TYPE_V256: 1909 snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1910 64 << (ts->type - TCG_TYPE_V64), ts->val); 1911 break; 1912 default: 1913 g_assert_not_reached(); 1914 } 1915 break; 1916 } 1917 return buf; 1918 } 1919 1920 static char *tcg_get_arg_str(TCGContext *s, char *buf, 1921 int buf_size, TCGArg arg) 1922 { 1923 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1924 } 1925 1926 static const char * const cond_name[] = 1927 { 1928 [TCG_COND_NEVER] = "never", 1929 [TCG_COND_ALWAYS] = "always", 1930 [TCG_COND_EQ] = "eq", 1931 [TCG_COND_NE] = "ne", 1932 [TCG_COND_LT] = "lt", 1933 [TCG_COND_GE] = "ge", 1934 [TCG_COND_LE] = "le", 1935 [TCG_COND_GT] = "gt", 1936 [TCG_COND_LTU] = "ltu", 1937 [TCG_COND_GEU] = "geu", 1938 [TCG_COND_LEU] = "leu", 1939 [TCG_COND_GTU] = "gtu" 1940 }; 1941 1942 static const char * const ldst_name[] = 1943 { 1944 [MO_UB] = "ub", 1945 [MO_SB] = "sb", 1946 [MO_LEUW] = "leuw", 1947 [MO_LESW] = "lesw", 1948 [MO_LEUL] = "leul", 1949 [MO_LESL] = "lesl", 1950 [MO_LEUQ] = "leq", 1951 [MO_BEUW] = "beuw", 1952 [MO_BESW] = "besw", 1953 [MO_BEUL] = "beul", 1954 [MO_BESL] = "besl", 1955 [MO_BEUQ] = "beq", 1956 }; 1957 1958 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1959 #ifdef TARGET_ALIGNED_ONLY 1960 [MO_UNALN >> MO_ASHIFT] = "un+", 1961 [MO_ALIGN >> MO_ASHIFT] = "", 1962 #else 1963 [MO_UNALN >> MO_ASHIFT] = "", 1964 [MO_ALIGN >> MO_ASHIFT] = "al+", 1965 #endif 1966 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1967 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1968 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1969 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1970 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1971 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1972 }; 1973 1974 static const char bswap_flag_name[][6] = { 1975 [TCG_BSWAP_IZ] = "iz", 1976 [TCG_BSWAP_OZ] = "oz", 1977 [TCG_BSWAP_OS] = "os", 1978 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1979 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1980 }; 1981 1982 static inline bool tcg_regset_single(TCGRegSet d) 1983 { 1984 return (d & (d - 1)) == 0; 1985 } 1986 1987 static inline TCGReg tcg_regset_first(TCGRegSet d) 1988 { 1989 if (TCG_TARGET_NB_REGS <= 32) { 1990 return ctz32(d); 1991 } else { 1992 return ctz64(d); 1993 } 1994 } 1995 1996 /* Return only the number of characters output -- no error return. */ 1997 #define ne_fprintf(...) \ 1998 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 1999 2000 static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 2001 { 2002 char buf[128]; 2003 TCGOp *op; 2004 2005 QTAILQ_FOREACH(op, &s->ops, link) { 2006 int i, k, nb_oargs, nb_iargs, nb_cargs; 2007 const TCGOpDef *def; 2008 TCGOpcode c; 2009 int col = 0; 2010 2011 c = op->opc; 2012 def = &tcg_op_defs[c]; 2013 2014 if (c == INDEX_op_insn_start) { 2015 nb_oargs = 0; 2016 col += ne_fprintf(f, "\n ----"); 2017 2018 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 2019 target_ulong a; 2020 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 2021 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 2022 #else 2023 a = op->args[i]; 2024 #endif 2025 col += ne_fprintf(f, " " TARGET_FMT_lx, a); 2026 } 2027 } else if (c == INDEX_op_call) { 2028 const TCGHelperInfo *info = tcg_call_info(op); 2029 void *func = tcg_call_func(op); 2030 2031 /* variable number of arguments */ 2032 nb_oargs = TCGOP_CALLO(op); 2033 nb_iargs = TCGOP_CALLI(op); 2034 nb_cargs = def->nb_cargs; 2035 2036 col += ne_fprintf(f, " %s ", def->name); 2037 2038 /* 2039 * Print the function name from TCGHelperInfo, if available. 2040 * Note that plugins have a template function for the info, 2041 * but the actual function pointer comes from the plugin. 2042 */ 2043 if (func == info->func) { 2044 col += ne_fprintf(f, "%s", info->name); 2045 } else { 2046 col += ne_fprintf(f, "plugin(%p)", func); 2047 } 2048 2049 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 2050 for (i = 0; i < nb_oargs; i++) { 2051 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 2052 op->args[i])); 2053 } 2054 for (i = 0; i < nb_iargs; i++) { 2055 TCGArg arg = op->args[nb_oargs + i]; 2056 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 2057 col += ne_fprintf(f, ",%s", t); 2058 } 2059 } else { 2060 col += ne_fprintf(f, " %s ", def->name); 2061 2062 nb_oargs = def->nb_oargs; 2063 nb_iargs = def->nb_iargs; 2064 nb_cargs = def->nb_cargs; 2065 2066 if (def->flags & TCG_OPF_VECTOR) { 2067 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op), 2068 8 << TCGOP_VECE(op)); 2069 } 2070 2071 k = 0; 2072 for (i = 0; i < nb_oargs; i++) { 2073 const char *sep = k ? "," : ""; 2074 col += ne_fprintf(f, "%s%s", sep, 2075 tcg_get_arg_str(s, buf, sizeof(buf), 2076 op->args[k++])); 2077 } 2078 for (i = 0; i < nb_iargs; i++) { 2079 const char *sep = k ? "," : ""; 2080 col += ne_fprintf(f, "%s%s", sep, 2081 tcg_get_arg_str(s, buf, sizeof(buf), 2082 op->args[k++])); 2083 } 2084 switch (c) { 2085 case INDEX_op_brcond_i32: 2086 case INDEX_op_setcond_i32: 2087 case INDEX_op_movcond_i32: 2088 case INDEX_op_brcond2_i32: 2089 case INDEX_op_setcond2_i32: 2090 case INDEX_op_brcond_i64: 2091 case INDEX_op_setcond_i64: 2092 case INDEX_op_movcond_i64: 2093 case INDEX_op_cmp_vec: 2094 case INDEX_op_cmpsel_vec: 2095 if (op->args[k] < ARRAY_SIZE(cond_name) 2096 && cond_name[op->args[k]]) { 2097 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 2098 } else { 2099 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 2100 } 2101 i = 1; 2102 break; 2103 case INDEX_op_qemu_ld_i32: 2104 case INDEX_op_qemu_st_i32: 2105 case INDEX_op_qemu_st8_i32: 2106 case INDEX_op_qemu_ld_i64: 2107 case INDEX_op_qemu_st_i64: 2108 { 2109 MemOpIdx oi = op->args[k++]; 2110 MemOp op = get_memop(oi); 2111 unsigned ix = get_mmuidx(oi); 2112 2113 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 2114 col += ne_fprintf(f, ",$0x%x,%u", op, ix); 2115 } else { 2116 const char *s_al, *s_op; 2117 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 2118 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 2119 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix); 2120 } 2121 i = 1; 2122 } 2123 break; 2124 case INDEX_op_bswap16_i32: 2125 case INDEX_op_bswap16_i64: 2126 case INDEX_op_bswap32_i32: 2127 case INDEX_op_bswap32_i64: 2128 case INDEX_op_bswap64_i64: 2129 { 2130 TCGArg flags = op->args[k]; 2131 const char *name = NULL; 2132 2133 if (flags < ARRAY_SIZE(bswap_flag_name)) { 2134 name = bswap_flag_name[flags]; 2135 } 2136 if (name) { 2137 col += ne_fprintf(f, ",%s", name); 2138 } else { 2139 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 2140 } 2141 i = k = 1; 2142 } 2143 break; 2144 default: 2145 i = 0; 2146 break; 2147 } 2148 switch (c) { 2149 case INDEX_op_set_label: 2150 case INDEX_op_br: 2151 case INDEX_op_brcond_i32: 2152 case INDEX_op_brcond_i64: 2153 case INDEX_op_brcond2_i32: 2154 col += ne_fprintf(f, "%s$L%d", k ? "," : "", 2155 arg_label(op->args[k])->id); 2156 i++, k++; 2157 break; 2158 case INDEX_op_mb: 2159 { 2160 TCGBar membar = op->args[k]; 2161 const char *b_op, *m_op; 2162 2163 switch (membar & TCG_BAR_SC) { 2164 case 0: 2165 b_op = "none"; 2166 break; 2167 case TCG_BAR_LDAQ: 2168 b_op = "acq"; 2169 break; 2170 case TCG_BAR_STRL: 2171 b_op = "rel"; 2172 break; 2173 case TCG_BAR_SC: 2174 b_op = "seq"; 2175 break; 2176 default: 2177 g_assert_not_reached(); 2178 } 2179 2180 switch (membar & TCG_MO_ALL) { 2181 case 0: 2182 m_op = "none"; 2183 break; 2184 case TCG_MO_LD_LD: 2185 m_op = "rr"; 2186 break; 2187 case TCG_MO_LD_ST: 2188 m_op = "rw"; 2189 break; 2190 case TCG_MO_ST_LD: 2191 m_op = "wr"; 2192 break; 2193 case TCG_MO_ST_ST: 2194 m_op = "ww"; 2195 break; 2196 case TCG_MO_LD_LD | TCG_MO_LD_ST: 2197 m_op = "rr+rw"; 2198 break; 2199 case TCG_MO_LD_LD | TCG_MO_ST_LD: 2200 m_op = "rr+wr"; 2201 break; 2202 case TCG_MO_LD_LD | TCG_MO_ST_ST: 2203 m_op = "rr+ww"; 2204 break; 2205 case TCG_MO_LD_ST | TCG_MO_ST_LD: 2206 m_op = "rw+wr"; 2207 break; 2208 case TCG_MO_LD_ST | TCG_MO_ST_ST: 2209 m_op = "rw+ww"; 2210 break; 2211 case TCG_MO_ST_LD | TCG_MO_ST_ST: 2212 m_op = "wr+ww"; 2213 break; 2214 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD: 2215 m_op = "rr+rw+wr"; 2216 break; 2217 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST: 2218 m_op = "rr+rw+ww"; 2219 break; 2220 case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST: 2221 m_op = "rr+wr+ww"; 2222 break; 2223 case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST: 2224 m_op = "rw+wr+ww"; 2225 break; 2226 case TCG_MO_ALL: 2227 m_op = "all"; 2228 break; 2229 default: 2230 g_assert_not_reached(); 2231 } 2232 2233 col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op); 2234 i++, k++; 2235 } 2236 break; 2237 default: 2238 break; 2239 } 2240 for (; i < nb_cargs; i++, k++) { 2241 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 2242 op->args[k]); 2243 } 2244 } 2245 2246 if (have_prefs || op->life) { 2247 for (; col < 40; ++col) { 2248 putc(' ', f); 2249 } 2250 } 2251 2252 if (op->life) { 2253 unsigned life = op->life; 2254 2255 if (life & (SYNC_ARG * 3)) { 2256 ne_fprintf(f, " sync:"); 2257 for (i = 0; i < 2; ++i) { 2258 if (life & (SYNC_ARG << i)) { 2259 ne_fprintf(f, " %d", i); 2260 } 2261 } 2262 } 2263 life /= DEAD_ARG; 2264 if (life) { 2265 ne_fprintf(f, " dead:"); 2266 for (i = 0; life; ++i, life >>= 1) { 2267 if (life & 1) { 2268 ne_fprintf(f, " %d", i); 2269 } 2270 } 2271 } 2272 } 2273 2274 if (have_prefs) { 2275 for (i = 0; i < nb_oargs; ++i) { 2276 TCGRegSet set = output_pref(op, i); 2277 2278 if (i == 0) { 2279 ne_fprintf(f, " pref="); 2280 } else { 2281 ne_fprintf(f, ","); 2282 } 2283 if (set == 0) { 2284 ne_fprintf(f, "none"); 2285 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 2286 ne_fprintf(f, "all"); 2287 #ifdef CONFIG_DEBUG_TCG 2288 } else if (tcg_regset_single(set)) { 2289 TCGReg reg = tcg_regset_first(set); 2290 ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 2291 #endif 2292 } else if (TCG_TARGET_NB_REGS <= 32) { 2293 ne_fprintf(f, "0x%x", (uint32_t)set); 2294 } else { 2295 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 2296 } 2297 } 2298 } 2299 2300 putc('\n', f); 2301 } 2302 } 2303 2304 /* we give more priority to constraints with less registers */ 2305 static int get_constraint_priority(const TCGOpDef *def, int k) 2306 { 2307 const TCGArgConstraint *arg_ct = &def->args_ct[k]; 2308 int n = ctpop64(arg_ct->regs); 2309 2310 /* 2311 * Sort constraints of a single register first, which includes output 2312 * aliases (which must exactly match the input already allocated). 2313 */ 2314 if (n == 1 || arg_ct->oalias) { 2315 return INT_MAX; 2316 } 2317 2318 /* 2319 * Sort register pairs next, first then second immediately after. 2320 * Arbitrarily sort multiple pairs by the index of the first reg; 2321 * there shouldn't be many pairs. 2322 */ 2323 switch (arg_ct->pair) { 2324 case 1: 2325 case 3: 2326 return (k + 1) * 2; 2327 case 2: 2328 return (arg_ct->pair_index + 1) * 2 - 1; 2329 } 2330 2331 /* Finally, sort by decreasing register count. */ 2332 assert(n > 1); 2333 return -n; 2334 } 2335 2336 /* sort from highest priority to lowest */ 2337 static void sort_constraints(TCGOpDef *def, int start, int n) 2338 { 2339 int i, j; 2340 TCGArgConstraint *a = def->args_ct; 2341 2342 for (i = 0; i < n; i++) { 2343 a[start + i].sort_index = start + i; 2344 } 2345 if (n <= 1) { 2346 return; 2347 } 2348 for (i = 0; i < n - 1; i++) { 2349 for (j = i + 1; j < n; j++) { 2350 int p1 = get_constraint_priority(def, a[start + i].sort_index); 2351 int p2 = get_constraint_priority(def, a[start + j].sort_index); 2352 if (p1 < p2) { 2353 int tmp = a[start + i].sort_index; 2354 a[start + i].sort_index = a[start + j].sort_index; 2355 a[start + j].sort_index = tmp; 2356 } 2357 } 2358 } 2359 } 2360 2361 static void process_op_defs(TCGContext *s) 2362 { 2363 TCGOpcode op; 2364 2365 for (op = 0; op < NB_OPS; op++) { 2366 TCGOpDef *def = &tcg_op_defs[op]; 2367 const TCGTargetOpDef *tdefs; 2368 bool saw_alias_pair = false; 2369 int i, o, i2, o2, nb_args; 2370 2371 if (def->flags & TCG_OPF_NOT_PRESENT) { 2372 continue; 2373 } 2374 2375 nb_args = def->nb_iargs + def->nb_oargs; 2376 if (nb_args == 0) { 2377 continue; 2378 } 2379 2380 /* 2381 * Macro magic should make it impossible, but double-check that 2382 * the array index is in range. Since the signness of an enum 2383 * is implementation defined, force the result to unsigned. 2384 */ 2385 unsigned con_set = tcg_target_op_def(op); 2386 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 2387 tdefs = &constraint_sets[con_set]; 2388 2389 for (i = 0; i < nb_args; i++) { 2390 const char *ct_str = tdefs->args_ct_str[i]; 2391 bool input_p = i >= def->nb_oargs; 2392 2393 /* Incomplete TCGTargetOpDef entry. */ 2394 tcg_debug_assert(ct_str != NULL); 2395 2396 switch (*ct_str) { 2397 case '0' ... '9': 2398 o = *ct_str - '0'; 2399 tcg_debug_assert(input_p); 2400 tcg_debug_assert(o < def->nb_oargs); 2401 tcg_debug_assert(def->args_ct[o].regs != 0); 2402 tcg_debug_assert(!def->args_ct[o].oalias); 2403 def->args_ct[i] = def->args_ct[o]; 2404 /* The output sets oalias. */ 2405 def->args_ct[o].oalias = 1; 2406 def->args_ct[o].alias_index = i; 2407 /* The input sets ialias. */ 2408 def->args_ct[i].ialias = 1; 2409 def->args_ct[i].alias_index = o; 2410 if (def->args_ct[i].pair) { 2411 saw_alias_pair = true; 2412 } 2413 tcg_debug_assert(ct_str[1] == '\0'); 2414 continue; 2415 2416 case '&': 2417 tcg_debug_assert(!input_p); 2418 def->args_ct[i].newreg = true; 2419 ct_str++; 2420 break; 2421 2422 case 'p': /* plus */ 2423 /* Allocate to the register after the previous. */ 2424 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 2425 o = i - 1; 2426 tcg_debug_assert(!def->args_ct[o].pair); 2427 tcg_debug_assert(!def->args_ct[o].ct); 2428 def->args_ct[i] = (TCGArgConstraint){ 2429 .pair = 2, 2430 .pair_index = o, 2431 .regs = def->args_ct[o].regs << 1, 2432 }; 2433 def->args_ct[o].pair = 1; 2434 def->args_ct[o].pair_index = i; 2435 tcg_debug_assert(ct_str[1] == '\0'); 2436 continue; 2437 2438 case 'm': /* minus */ 2439 /* Allocate to the register before the previous. */ 2440 tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); 2441 o = i - 1; 2442 tcg_debug_assert(!def->args_ct[o].pair); 2443 tcg_debug_assert(!def->args_ct[o].ct); 2444 def->args_ct[i] = (TCGArgConstraint){ 2445 .pair = 1, 2446 .pair_index = o, 2447 .regs = def->args_ct[o].regs >> 1, 2448 }; 2449 def->args_ct[o].pair = 2; 2450 def->args_ct[o].pair_index = i; 2451 tcg_debug_assert(ct_str[1] == '\0'); 2452 continue; 2453 } 2454 2455 do { 2456 switch (*ct_str) { 2457 case 'i': 2458 def->args_ct[i].ct |= TCG_CT_CONST; 2459 break; 2460 2461 /* Include all of the target-specific constraints. */ 2462 2463 #undef CONST 2464 #define CONST(CASE, MASK) \ 2465 case CASE: def->args_ct[i].ct |= MASK; break; 2466 #define REGS(CASE, MASK) \ 2467 case CASE: def->args_ct[i].regs |= MASK; break; 2468 2469 #include "tcg-target-con-str.h" 2470 2471 #undef REGS 2472 #undef CONST 2473 default: 2474 case '0' ... '9': 2475 case '&': 2476 case 'p': 2477 case 'm': 2478 /* Typo in TCGTargetOpDef constraint. */ 2479 g_assert_not_reached(); 2480 } 2481 } while (*++ct_str != '\0'); 2482 } 2483 2484 /* TCGTargetOpDef entry with too much information? */ 2485 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2486 2487 /* 2488 * Fix up output pairs that are aliased with inputs. 2489 * When we created the alias, we copied pair from the output. 2490 * There are three cases: 2491 * (1a) Pairs of inputs alias pairs of outputs. 2492 * (1b) One input aliases the first of a pair of outputs. 2493 * (2) One input aliases the second of a pair of outputs. 2494 * 2495 * Case 1a is handled by making sure that the pair_index'es are 2496 * properly updated so that they appear the same as a pair of inputs. 2497 * 2498 * Case 1b is handled by setting the pair_index of the input to 2499 * itself, simply so it doesn't point to an unrelated argument. 2500 * Since we don't encounter the "second" during the input allocation 2501 * phase, nothing happens with the second half of the input pair. 2502 * 2503 * Case 2 is handled by setting the second input to pair=3, the 2504 * first output to pair=3, and the pair_index'es to match. 2505 */ 2506 if (saw_alias_pair) { 2507 for (i = def->nb_oargs; i < nb_args; i++) { 2508 /* 2509 * Since [0-9pm] must be alone in the constraint string, 2510 * the only way they can both be set is if the pair comes 2511 * from the output alias. 2512 */ 2513 if (!def->args_ct[i].ialias) { 2514 continue; 2515 } 2516 switch (def->args_ct[i].pair) { 2517 case 0: 2518 break; 2519 case 1: 2520 o = def->args_ct[i].alias_index; 2521 o2 = def->args_ct[o].pair_index; 2522 tcg_debug_assert(def->args_ct[o].pair == 1); 2523 tcg_debug_assert(def->args_ct[o2].pair == 2); 2524 if (def->args_ct[o2].oalias) { 2525 /* Case 1a */ 2526 i2 = def->args_ct[o2].alias_index; 2527 tcg_debug_assert(def->args_ct[i2].pair == 2); 2528 def->args_ct[i2].pair_index = i; 2529 def->args_ct[i].pair_index = i2; 2530 } else { 2531 /* Case 1b */ 2532 def->args_ct[i].pair_index = i; 2533 } 2534 break; 2535 case 2: 2536 o = def->args_ct[i].alias_index; 2537 o2 = def->args_ct[o].pair_index; 2538 tcg_debug_assert(def->args_ct[o].pair == 2); 2539 tcg_debug_assert(def->args_ct[o2].pair == 1); 2540 if (def->args_ct[o2].oalias) { 2541 /* Case 1a */ 2542 i2 = def->args_ct[o2].alias_index; 2543 tcg_debug_assert(def->args_ct[i2].pair == 1); 2544 def->args_ct[i2].pair_index = i; 2545 def->args_ct[i].pair_index = i2; 2546 } else { 2547 /* Case 2 */ 2548 def->args_ct[i].pair = 3; 2549 def->args_ct[o2].pair = 3; 2550 def->args_ct[i].pair_index = o2; 2551 def->args_ct[o2].pair_index = i; 2552 } 2553 break; 2554 default: 2555 g_assert_not_reached(); 2556 } 2557 } 2558 } 2559 2560 /* sort the constraints (XXX: this is just an heuristic) */ 2561 sort_constraints(def, 0, def->nb_oargs); 2562 sort_constraints(def, def->nb_oargs, def->nb_iargs); 2563 } 2564 } 2565 2566 static void remove_label_use(TCGOp *op, int idx) 2567 { 2568 TCGLabel *label = arg_label(op->args[idx]); 2569 TCGLabelUse *use; 2570 2571 QSIMPLEQ_FOREACH(use, &label->branches, next) { 2572 if (use->op == op) { 2573 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); 2574 return; 2575 } 2576 } 2577 g_assert_not_reached(); 2578 } 2579 2580 void tcg_op_remove(TCGContext *s, TCGOp *op) 2581 { 2582 switch (op->opc) { 2583 case INDEX_op_br: 2584 remove_label_use(op, 0); 2585 break; 2586 case INDEX_op_brcond_i32: 2587 case INDEX_op_brcond_i64: 2588 remove_label_use(op, 3); 2589 break; 2590 case INDEX_op_brcond2_i32: 2591 remove_label_use(op, 5); 2592 break; 2593 default: 2594 break; 2595 } 2596 2597 QTAILQ_REMOVE(&s->ops, op, link); 2598 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2599 s->nb_ops--; 2600 2601 #ifdef CONFIG_PROFILER 2602 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 2603 #endif 2604 } 2605 2606 void tcg_remove_ops_after(TCGOp *op) 2607 { 2608 TCGContext *s = tcg_ctx; 2609 2610 while (true) { 2611 TCGOp *last = tcg_last_op(); 2612 if (last == op) { 2613 return; 2614 } 2615 tcg_op_remove(s, last); 2616 } 2617 } 2618 2619 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) 2620 { 2621 TCGContext *s = tcg_ctx; 2622 TCGOp *op = NULL; 2623 2624 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { 2625 QTAILQ_FOREACH(op, &s->free_ops, link) { 2626 if (nargs <= op->nargs) { 2627 QTAILQ_REMOVE(&s->free_ops, op, link); 2628 nargs = op->nargs; 2629 goto found; 2630 } 2631 } 2632 } 2633 2634 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ 2635 nargs = MAX(4, nargs); 2636 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); 2637 2638 found: 2639 memset(op, 0, offsetof(TCGOp, link)); 2640 op->opc = opc; 2641 op->nargs = nargs; 2642 2643 /* Check for bitfield overflow. */ 2644 tcg_debug_assert(op->nargs == nargs); 2645 2646 s->nb_ops++; 2647 return op; 2648 } 2649 2650 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) 2651 { 2652 TCGOp *op = tcg_op_alloc(opc, nargs); 2653 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2654 return op; 2655 } 2656 2657 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 2658 TCGOpcode opc, unsigned nargs) 2659 { 2660 TCGOp *new_op = tcg_op_alloc(opc, nargs); 2661 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 2662 return new_op; 2663 } 2664 2665 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 2666 TCGOpcode opc, unsigned nargs) 2667 { 2668 TCGOp *new_op = tcg_op_alloc(opc, nargs); 2669 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 2670 return new_op; 2671 } 2672 2673 static void move_label_uses(TCGLabel *to, TCGLabel *from) 2674 { 2675 TCGLabelUse *u; 2676 2677 QSIMPLEQ_FOREACH(u, &from->branches, next) { 2678 TCGOp *op = u->op; 2679 switch (op->opc) { 2680 case INDEX_op_br: 2681 op->args[0] = label_arg(to); 2682 break; 2683 case INDEX_op_brcond_i32: 2684 case INDEX_op_brcond_i64: 2685 op->args[3] = label_arg(to); 2686 break; 2687 case INDEX_op_brcond2_i32: 2688 op->args[5] = label_arg(to); 2689 break; 2690 default: 2691 g_assert_not_reached(); 2692 } 2693 } 2694 2695 QSIMPLEQ_CONCAT(&to->branches, &from->branches); 2696 } 2697 2698 /* Reachable analysis : remove unreachable code. */ 2699 static void __attribute__((noinline)) 2700 reachable_code_pass(TCGContext *s) 2701 { 2702 TCGOp *op, *op_next, *op_prev; 2703 bool dead = false; 2704 2705 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2706 bool remove = dead; 2707 TCGLabel *label; 2708 2709 switch (op->opc) { 2710 case INDEX_op_set_label: 2711 label = arg_label(op->args[0]); 2712 2713 /* 2714 * Note that the first op in the TB is always a load, 2715 * so there is always something before a label. 2716 */ 2717 op_prev = QTAILQ_PREV(op, link); 2718 2719 /* 2720 * If we find two sequential labels, move all branches to 2721 * reference the second label and remove the first label. 2722 * Do this before branch to next optimization, so that the 2723 * middle label is out of the way. 2724 */ 2725 if (op_prev->opc == INDEX_op_set_label) { 2726 move_label_uses(label, arg_label(op_prev->args[0])); 2727 tcg_op_remove(s, op_prev); 2728 op_prev = QTAILQ_PREV(op, link); 2729 } 2730 2731 /* 2732 * Optimization can fold conditional branches to unconditional. 2733 * If we find a label which is preceded by an unconditional 2734 * branch to next, remove the branch. We couldn't do this when 2735 * processing the branch because any dead code between the branch 2736 * and label had not yet been removed. 2737 */ 2738 if (op_prev->opc == INDEX_op_br && 2739 label == arg_label(op_prev->args[0])) { 2740 tcg_op_remove(s, op_prev); 2741 /* Fall through means insns become live again. */ 2742 dead = false; 2743 } 2744 2745 if (QSIMPLEQ_EMPTY(&label->branches)) { 2746 /* 2747 * While there is an occasional backward branch, virtually 2748 * all branches generated by the translators are forward. 2749 * Which means that generally we will have already removed 2750 * all references to the label that will be, and there is 2751 * little to be gained by iterating. 2752 */ 2753 remove = true; 2754 } else { 2755 /* Once we see a label, insns become live again. */ 2756 dead = false; 2757 remove = false; 2758 } 2759 break; 2760 2761 case INDEX_op_br: 2762 case INDEX_op_exit_tb: 2763 case INDEX_op_goto_ptr: 2764 /* Unconditional branches; everything following is dead. */ 2765 dead = true; 2766 break; 2767 2768 case INDEX_op_call: 2769 /* Notice noreturn helper calls, raising exceptions. */ 2770 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2771 dead = true; 2772 } 2773 break; 2774 2775 case INDEX_op_insn_start: 2776 /* Never remove -- we need to keep these for unwind. */ 2777 remove = false; 2778 break; 2779 2780 default: 2781 break; 2782 } 2783 2784 if (remove) { 2785 tcg_op_remove(s, op); 2786 } 2787 } 2788 } 2789 2790 #define TS_DEAD 1 2791 #define TS_MEM 2 2792 2793 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 2794 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 2795 2796 /* For liveness_pass_1, the register preferences for a given temp. */ 2797 static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 2798 { 2799 return ts->state_ptr; 2800 } 2801 2802 /* For liveness_pass_1, reset the preferences for a given temp to the 2803 * maximal regset for its type. 2804 */ 2805 static inline void la_reset_pref(TCGTemp *ts) 2806 { 2807 *la_temp_pref(ts) 2808 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 2809 } 2810 2811 /* liveness analysis: end of function: all temps are dead, and globals 2812 should be in memory. */ 2813 static void la_func_end(TCGContext *s, int ng, int nt) 2814 { 2815 int i; 2816 2817 for (i = 0; i < ng; ++i) { 2818 s->temps[i].state = TS_DEAD | TS_MEM; 2819 la_reset_pref(&s->temps[i]); 2820 } 2821 for (i = ng; i < nt; ++i) { 2822 s->temps[i].state = TS_DEAD; 2823 la_reset_pref(&s->temps[i]); 2824 } 2825 } 2826 2827 /* liveness analysis: end of basic block: all temps are dead, globals 2828 and local temps should be in memory. */ 2829 static void la_bb_end(TCGContext *s, int ng, int nt) 2830 { 2831 int i; 2832 2833 for (i = 0; i < nt; ++i) { 2834 TCGTemp *ts = &s->temps[i]; 2835 int state; 2836 2837 switch (ts->kind) { 2838 case TEMP_FIXED: 2839 case TEMP_GLOBAL: 2840 case TEMP_TB: 2841 state = TS_DEAD | TS_MEM; 2842 break; 2843 case TEMP_EBB: 2844 case TEMP_CONST: 2845 state = TS_DEAD; 2846 break; 2847 default: 2848 g_assert_not_reached(); 2849 } 2850 ts->state = state; 2851 la_reset_pref(ts); 2852 } 2853 } 2854 2855 /* liveness analysis: sync globals back to memory. */ 2856 static void la_global_sync(TCGContext *s, int ng) 2857 { 2858 int i; 2859 2860 for (i = 0; i < ng; ++i) { 2861 int state = s->temps[i].state; 2862 s->temps[i].state = state | TS_MEM; 2863 if (state == TS_DEAD) { 2864 /* If the global was previously dead, reset prefs. */ 2865 la_reset_pref(&s->temps[i]); 2866 } 2867 } 2868 } 2869 2870 /* 2871 * liveness analysis: conditional branch: all temps are dead unless 2872 * explicitly live-across-conditional-branch, globals and local temps 2873 * should be synced. 2874 */ 2875 static void la_bb_sync(TCGContext *s, int ng, int nt) 2876 { 2877 la_global_sync(s, ng); 2878 2879 for (int i = ng; i < nt; ++i) { 2880 TCGTemp *ts = &s->temps[i]; 2881 int state; 2882 2883 switch (ts->kind) { 2884 case TEMP_TB: 2885 state = ts->state; 2886 ts->state = state | TS_MEM; 2887 if (state != TS_DEAD) { 2888 continue; 2889 } 2890 break; 2891 case TEMP_EBB: 2892 case TEMP_CONST: 2893 continue; 2894 default: 2895 g_assert_not_reached(); 2896 } 2897 la_reset_pref(&s->temps[i]); 2898 } 2899 } 2900 2901 /* liveness analysis: sync globals back to memory and kill. */ 2902 static void la_global_kill(TCGContext *s, int ng) 2903 { 2904 int i; 2905 2906 for (i = 0; i < ng; i++) { 2907 s->temps[i].state = TS_DEAD | TS_MEM; 2908 la_reset_pref(&s->temps[i]); 2909 } 2910 } 2911 2912 /* liveness analysis: note live globals crossing calls. */ 2913 static void la_cross_call(TCGContext *s, int nt) 2914 { 2915 TCGRegSet mask = ~tcg_target_call_clobber_regs; 2916 int i; 2917 2918 for (i = 0; i < nt; i++) { 2919 TCGTemp *ts = &s->temps[i]; 2920 if (!(ts->state & TS_DEAD)) { 2921 TCGRegSet *pset = la_temp_pref(ts); 2922 TCGRegSet set = *pset; 2923 2924 set &= mask; 2925 /* If the combination is not possible, restart. */ 2926 if (set == 0) { 2927 set = tcg_target_available_regs[ts->type] & mask; 2928 } 2929 *pset = set; 2930 } 2931 } 2932 } 2933 2934 /* 2935 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce 2936 * to TEMP_EBB, if possible. 2937 */ 2938 static void __attribute__((noinline)) 2939 liveness_pass_0(TCGContext *s) 2940 { 2941 void * const multiple_ebb = (void *)(uintptr_t)-1; 2942 int nb_temps = s->nb_temps; 2943 TCGOp *op, *ebb; 2944 2945 for (int i = s->nb_globals; i < nb_temps; ++i) { 2946 s->temps[i].state_ptr = NULL; 2947 } 2948 2949 /* 2950 * Represent each EBB by the op at which it begins. In the case of 2951 * the first EBB, this is the first op, otherwise it is a label. 2952 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use 2953 * within a single EBB, else MULTIPLE_EBB. 2954 */ 2955 ebb = QTAILQ_FIRST(&s->ops); 2956 QTAILQ_FOREACH(op, &s->ops, link) { 2957 const TCGOpDef *def; 2958 int nb_oargs, nb_iargs; 2959 2960 switch (op->opc) { 2961 case INDEX_op_set_label: 2962 ebb = op; 2963 continue; 2964 case INDEX_op_discard: 2965 continue; 2966 case INDEX_op_call: 2967 nb_oargs = TCGOP_CALLO(op); 2968 nb_iargs = TCGOP_CALLI(op); 2969 break; 2970 default: 2971 def = &tcg_op_defs[op->opc]; 2972 nb_oargs = def->nb_oargs; 2973 nb_iargs = def->nb_iargs; 2974 break; 2975 } 2976 2977 for (int i = 0; i < nb_oargs + nb_iargs; ++i) { 2978 TCGTemp *ts = arg_temp(op->args[i]); 2979 2980 if (ts->kind != TEMP_TB) { 2981 continue; 2982 } 2983 if (ts->state_ptr == NULL) { 2984 ts->state_ptr = ebb; 2985 } else if (ts->state_ptr != ebb) { 2986 ts->state_ptr = multiple_ebb; 2987 } 2988 } 2989 } 2990 2991 /* 2992 * For TEMP_TB that turned out not to be used beyond one EBB, 2993 * reduce the liveness to TEMP_EBB. 2994 */ 2995 for (int i = s->nb_globals; i < nb_temps; ++i) { 2996 TCGTemp *ts = &s->temps[i]; 2997 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { 2998 ts->kind = TEMP_EBB; 2999 } 3000 } 3001 } 3002 3003 /* Liveness analysis : update the opc_arg_life array to tell if a 3004 given input arguments is dead. Instructions updating dead 3005 temporaries are removed. */ 3006 static void __attribute__((noinline)) 3007 liveness_pass_1(TCGContext *s) 3008 { 3009 int nb_globals = s->nb_globals; 3010 int nb_temps = s->nb_temps; 3011 TCGOp *op, *op_prev; 3012 TCGRegSet *prefs; 3013 int i; 3014 3015 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 3016 for (i = 0; i < nb_temps; ++i) { 3017 s->temps[i].state_ptr = prefs + i; 3018 } 3019 3020 /* ??? Should be redundant with the exit_tb that ends the TB. */ 3021 la_func_end(s, nb_globals, nb_temps); 3022 3023 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 3024 int nb_iargs, nb_oargs; 3025 TCGOpcode opc_new, opc_new2; 3026 bool have_opc_new2; 3027 TCGLifeData arg_life = 0; 3028 TCGTemp *ts; 3029 TCGOpcode opc = op->opc; 3030 const TCGOpDef *def = &tcg_op_defs[opc]; 3031 3032 switch (opc) { 3033 case INDEX_op_call: 3034 { 3035 const TCGHelperInfo *info = tcg_call_info(op); 3036 int call_flags = tcg_call_flags(op); 3037 3038 nb_oargs = TCGOP_CALLO(op); 3039 nb_iargs = TCGOP_CALLI(op); 3040 3041 /* pure functions can be removed if their result is unused */ 3042 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 3043 for (i = 0; i < nb_oargs; i++) { 3044 ts = arg_temp(op->args[i]); 3045 if (ts->state != TS_DEAD) { 3046 goto do_not_remove_call; 3047 } 3048 } 3049 goto do_remove; 3050 } 3051 do_not_remove_call: 3052 3053 /* Output args are dead. */ 3054 for (i = 0; i < nb_oargs; i++) { 3055 ts = arg_temp(op->args[i]); 3056 if (ts->state & TS_DEAD) { 3057 arg_life |= DEAD_ARG << i; 3058 } 3059 if (ts->state & TS_MEM) { 3060 arg_life |= SYNC_ARG << i; 3061 } 3062 ts->state = TS_DEAD; 3063 la_reset_pref(ts); 3064 } 3065 3066 /* Not used -- it will be tcg_target_call_oarg_reg(). */ 3067 memset(op->output_pref, 0, sizeof(op->output_pref)); 3068 3069 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 3070 TCG_CALL_NO_READ_GLOBALS))) { 3071 la_global_kill(s, nb_globals); 3072 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 3073 la_global_sync(s, nb_globals); 3074 } 3075 3076 /* Record arguments that die in this helper. */ 3077 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3078 ts = arg_temp(op->args[i]); 3079 if (ts->state & TS_DEAD) { 3080 arg_life |= DEAD_ARG << i; 3081 } 3082 } 3083 3084 /* For all live registers, remove call-clobbered prefs. */ 3085 la_cross_call(s, nb_temps); 3086 3087 /* 3088 * Input arguments are live for preceding opcodes. 3089 * 3090 * For those arguments that die, and will be allocated in 3091 * registers, clear the register set for that arg, to be 3092 * filled in below. For args that will be on the stack, 3093 * reset to any available reg. Process arguments in reverse 3094 * order so that if a temp is used more than once, the stack 3095 * reset to max happens before the register reset to 0. 3096 */ 3097 for (i = nb_iargs - 1; i >= 0; i--) { 3098 const TCGCallArgumentLoc *loc = &info->in[i]; 3099 ts = arg_temp(op->args[nb_oargs + i]); 3100 3101 if (ts->state & TS_DEAD) { 3102 switch (loc->kind) { 3103 case TCG_CALL_ARG_NORMAL: 3104 case TCG_CALL_ARG_EXTEND_U: 3105 case TCG_CALL_ARG_EXTEND_S: 3106 if (REG_P(loc)) { 3107 *la_temp_pref(ts) = 0; 3108 break; 3109 } 3110 /* fall through */ 3111 default: 3112 *la_temp_pref(ts) = 3113 tcg_target_available_regs[ts->type]; 3114 break; 3115 } 3116 ts->state &= ~TS_DEAD; 3117 } 3118 } 3119 3120 /* 3121 * For each input argument, add its input register to prefs. 3122 * If a temp is used once, this produces a single set bit; 3123 * if a temp is used multiple times, this produces a set. 3124 */ 3125 for (i = 0; i < nb_iargs; i++) { 3126 const TCGCallArgumentLoc *loc = &info->in[i]; 3127 ts = arg_temp(op->args[nb_oargs + i]); 3128 3129 switch (loc->kind) { 3130 case TCG_CALL_ARG_NORMAL: 3131 case TCG_CALL_ARG_EXTEND_U: 3132 case TCG_CALL_ARG_EXTEND_S: 3133 if (REG_P(loc)) { 3134 tcg_regset_set_reg(*la_temp_pref(ts), 3135 tcg_target_call_iarg_regs[loc->arg_slot]); 3136 } 3137 break; 3138 default: 3139 break; 3140 } 3141 } 3142 } 3143 break; 3144 case INDEX_op_insn_start: 3145 break; 3146 case INDEX_op_discard: 3147 /* mark the temporary as dead */ 3148 ts = arg_temp(op->args[0]); 3149 ts->state = TS_DEAD; 3150 la_reset_pref(ts); 3151 break; 3152 3153 case INDEX_op_add2_i32: 3154 opc_new = INDEX_op_add_i32; 3155 goto do_addsub2; 3156 case INDEX_op_sub2_i32: 3157 opc_new = INDEX_op_sub_i32; 3158 goto do_addsub2; 3159 case INDEX_op_add2_i64: 3160 opc_new = INDEX_op_add_i64; 3161 goto do_addsub2; 3162 case INDEX_op_sub2_i64: 3163 opc_new = INDEX_op_sub_i64; 3164 do_addsub2: 3165 nb_iargs = 4; 3166 nb_oargs = 2; 3167 /* Test if the high part of the operation is dead, but not 3168 the low part. The result can be optimized to a simple 3169 add or sub. This happens often for x86_64 guest when the 3170 cpu mode is set to 32 bit. */ 3171 if (arg_temp(op->args[1])->state == TS_DEAD) { 3172 if (arg_temp(op->args[0])->state == TS_DEAD) { 3173 goto do_remove; 3174 } 3175 /* Replace the opcode and adjust the args in place, 3176 leaving 3 unused args at the end. */ 3177 op->opc = opc = opc_new; 3178 op->args[1] = op->args[2]; 3179 op->args[2] = op->args[4]; 3180 /* Fall through and mark the single-word operation live. */ 3181 nb_iargs = 2; 3182 nb_oargs = 1; 3183 } 3184 goto do_not_remove; 3185 3186 case INDEX_op_mulu2_i32: 3187 opc_new = INDEX_op_mul_i32; 3188 opc_new2 = INDEX_op_muluh_i32; 3189 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 3190 goto do_mul2; 3191 case INDEX_op_muls2_i32: 3192 opc_new = INDEX_op_mul_i32; 3193 opc_new2 = INDEX_op_mulsh_i32; 3194 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 3195 goto do_mul2; 3196 case INDEX_op_mulu2_i64: 3197 opc_new = INDEX_op_mul_i64; 3198 opc_new2 = INDEX_op_muluh_i64; 3199 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 3200 goto do_mul2; 3201 case INDEX_op_muls2_i64: 3202 opc_new = INDEX_op_mul_i64; 3203 opc_new2 = INDEX_op_mulsh_i64; 3204 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 3205 goto do_mul2; 3206 do_mul2: 3207 nb_iargs = 2; 3208 nb_oargs = 2; 3209 if (arg_temp(op->args[1])->state == TS_DEAD) { 3210 if (arg_temp(op->args[0])->state == TS_DEAD) { 3211 /* Both parts of the operation are dead. */ 3212 goto do_remove; 3213 } 3214 /* The high part of the operation is dead; generate the low. */ 3215 op->opc = opc = opc_new; 3216 op->args[1] = op->args[2]; 3217 op->args[2] = op->args[3]; 3218 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 3219 /* The low part of the operation is dead; generate the high. */ 3220 op->opc = opc = opc_new2; 3221 op->args[0] = op->args[1]; 3222 op->args[1] = op->args[2]; 3223 op->args[2] = op->args[3]; 3224 } else { 3225 goto do_not_remove; 3226 } 3227 /* Mark the single-word operation live. */ 3228 nb_oargs = 1; 3229 goto do_not_remove; 3230 3231 default: 3232 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 3233 nb_iargs = def->nb_iargs; 3234 nb_oargs = def->nb_oargs; 3235 3236 /* Test if the operation can be removed because all 3237 its outputs are dead. We assume that nb_oargs == 0 3238 implies side effects */ 3239 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 3240 for (i = 0; i < nb_oargs; i++) { 3241 if (arg_temp(op->args[i])->state != TS_DEAD) { 3242 goto do_not_remove; 3243 } 3244 } 3245 goto do_remove; 3246 } 3247 goto do_not_remove; 3248 3249 do_remove: 3250 tcg_op_remove(s, op); 3251 break; 3252 3253 do_not_remove: 3254 for (i = 0; i < nb_oargs; i++) { 3255 ts = arg_temp(op->args[i]); 3256 3257 /* Remember the preference of the uses that followed. */ 3258 if (i < ARRAY_SIZE(op->output_pref)) { 3259 op->output_pref[i] = *la_temp_pref(ts); 3260 } 3261 3262 /* Output args are dead. */ 3263 if (ts->state & TS_DEAD) { 3264 arg_life |= DEAD_ARG << i; 3265 } 3266 if (ts->state & TS_MEM) { 3267 arg_life |= SYNC_ARG << i; 3268 } 3269 ts->state = TS_DEAD; 3270 la_reset_pref(ts); 3271 } 3272 3273 /* If end of basic block, update. */ 3274 if (def->flags & TCG_OPF_BB_EXIT) { 3275 la_func_end(s, nb_globals, nb_temps); 3276 } else if (def->flags & TCG_OPF_COND_BRANCH) { 3277 la_bb_sync(s, nb_globals, nb_temps); 3278 } else if (def->flags & TCG_OPF_BB_END) { 3279 la_bb_end(s, nb_globals, nb_temps); 3280 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3281 la_global_sync(s, nb_globals); 3282 if (def->flags & TCG_OPF_CALL_CLOBBER) { 3283 la_cross_call(s, nb_temps); 3284 } 3285 } 3286 3287 /* Record arguments that die in this opcode. */ 3288 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3289 ts = arg_temp(op->args[i]); 3290 if (ts->state & TS_DEAD) { 3291 arg_life |= DEAD_ARG << i; 3292 } 3293 } 3294 3295 /* Input arguments are live for preceding opcodes. */ 3296 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3297 ts = arg_temp(op->args[i]); 3298 if (ts->state & TS_DEAD) { 3299 /* For operands that were dead, initially allow 3300 all regs for the type. */ 3301 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 3302 ts->state &= ~TS_DEAD; 3303 } 3304 } 3305 3306 /* Incorporate constraints for this operand. */ 3307 switch (opc) { 3308 case INDEX_op_mov_i32: 3309 case INDEX_op_mov_i64: 3310 /* Note that these are TCG_OPF_NOT_PRESENT and do not 3311 have proper constraints. That said, special case 3312 moves to propagate preferences backward. */ 3313 if (IS_DEAD_ARG(1)) { 3314 *la_temp_pref(arg_temp(op->args[0])) 3315 = *la_temp_pref(arg_temp(op->args[1])); 3316 } 3317 break; 3318 3319 default: 3320 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3321 const TCGArgConstraint *ct = &def->args_ct[i]; 3322 TCGRegSet set, *pset; 3323 3324 ts = arg_temp(op->args[i]); 3325 pset = la_temp_pref(ts); 3326 set = *pset; 3327 3328 set &= ct->regs; 3329 if (ct->ialias) { 3330 set &= output_pref(op, ct->alias_index); 3331 } 3332 /* If the combination is not possible, restart. */ 3333 if (set == 0) { 3334 set = ct->regs; 3335 } 3336 *pset = set; 3337 } 3338 break; 3339 } 3340 break; 3341 } 3342 op->life = arg_life; 3343 } 3344 } 3345 3346 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 3347 static bool __attribute__((noinline)) 3348 liveness_pass_2(TCGContext *s) 3349 { 3350 int nb_globals = s->nb_globals; 3351 int nb_temps, i; 3352 bool changes = false; 3353 TCGOp *op, *op_next; 3354 3355 /* Create a temporary for each indirect global. */ 3356 for (i = 0; i < nb_globals; ++i) { 3357 TCGTemp *its = &s->temps[i]; 3358 if (its->indirect_reg) { 3359 TCGTemp *dts = tcg_temp_alloc(s); 3360 dts->type = its->type; 3361 dts->base_type = its->base_type; 3362 dts->temp_subindex = its->temp_subindex; 3363 dts->kind = TEMP_EBB; 3364 its->state_ptr = dts; 3365 } else { 3366 its->state_ptr = NULL; 3367 } 3368 /* All globals begin dead. */ 3369 its->state = TS_DEAD; 3370 } 3371 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 3372 TCGTemp *its = &s->temps[i]; 3373 its->state_ptr = NULL; 3374 its->state = TS_DEAD; 3375 } 3376 3377 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 3378 TCGOpcode opc = op->opc; 3379 const TCGOpDef *def = &tcg_op_defs[opc]; 3380 TCGLifeData arg_life = op->life; 3381 int nb_iargs, nb_oargs, call_flags; 3382 TCGTemp *arg_ts, *dir_ts; 3383 3384 if (opc == INDEX_op_call) { 3385 nb_oargs = TCGOP_CALLO(op); 3386 nb_iargs = TCGOP_CALLI(op); 3387 call_flags = tcg_call_flags(op); 3388 } else { 3389 nb_iargs = def->nb_iargs; 3390 nb_oargs = def->nb_oargs; 3391 3392 /* Set flags similar to how calls require. */ 3393 if (def->flags & TCG_OPF_COND_BRANCH) { 3394 /* Like reading globals: sync_globals */ 3395 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 3396 } else if (def->flags & TCG_OPF_BB_END) { 3397 /* Like writing globals: save_globals */ 3398 call_flags = 0; 3399 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3400 /* Like reading globals: sync_globals */ 3401 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 3402 } else { 3403 /* No effect on globals. */ 3404 call_flags = (TCG_CALL_NO_READ_GLOBALS | 3405 TCG_CALL_NO_WRITE_GLOBALS); 3406 } 3407 } 3408 3409 /* Make sure that input arguments are available. */ 3410 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3411 arg_ts = arg_temp(op->args[i]); 3412 dir_ts = arg_ts->state_ptr; 3413 if (dir_ts && arg_ts->state == TS_DEAD) { 3414 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 3415 ? INDEX_op_ld_i32 3416 : INDEX_op_ld_i64); 3417 TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); 3418 3419 lop->args[0] = temp_arg(dir_ts); 3420 lop->args[1] = temp_arg(arg_ts->mem_base); 3421 lop->args[2] = arg_ts->mem_offset; 3422 3423 /* Loaded, but synced with memory. */ 3424 arg_ts->state = TS_MEM; 3425 } 3426 } 3427 3428 /* Perform input replacement, and mark inputs that became dead. 3429 No action is required except keeping temp_state up to date 3430 so that we reload when needed. */ 3431 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3432 arg_ts = arg_temp(op->args[i]); 3433 dir_ts = arg_ts->state_ptr; 3434 if (dir_ts) { 3435 op->args[i] = temp_arg(dir_ts); 3436 changes = true; 3437 if (IS_DEAD_ARG(i)) { 3438 arg_ts->state = TS_DEAD; 3439 } 3440 } 3441 } 3442 3443 /* Liveness analysis should ensure that the following are 3444 all correct, for call sites and basic block end points. */ 3445 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 3446 /* Nothing to do */ 3447 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 3448 for (i = 0; i < nb_globals; ++i) { 3449 /* Liveness should see that globals are synced back, 3450 that is, either TS_DEAD or TS_MEM. */ 3451 arg_ts = &s->temps[i]; 3452 tcg_debug_assert(arg_ts->state_ptr == 0 3453 || arg_ts->state != 0); 3454 } 3455 } else { 3456 for (i = 0; i < nb_globals; ++i) { 3457 /* Liveness should see that globals are saved back, 3458 that is, TS_DEAD, waiting to be reloaded. */ 3459 arg_ts = &s->temps[i]; 3460 tcg_debug_assert(arg_ts->state_ptr == 0 3461 || arg_ts->state == TS_DEAD); 3462 } 3463 } 3464 3465 /* Outputs become available. */ 3466 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 3467 arg_ts = arg_temp(op->args[0]); 3468 dir_ts = arg_ts->state_ptr; 3469 if (dir_ts) { 3470 op->args[0] = temp_arg(dir_ts); 3471 changes = true; 3472 3473 /* The output is now live and modified. */ 3474 arg_ts->state = 0; 3475 3476 if (NEED_SYNC_ARG(0)) { 3477 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 3478 ? INDEX_op_st_i32 3479 : INDEX_op_st_i64); 3480 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 3481 TCGTemp *out_ts = dir_ts; 3482 3483 if (IS_DEAD_ARG(0)) { 3484 out_ts = arg_temp(op->args[1]); 3485 arg_ts->state = TS_DEAD; 3486 tcg_op_remove(s, op); 3487 } else { 3488 arg_ts->state = TS_MEM; 3489 } 3490 3491 sop->args[0] = temp_arg(out_ts); 3492 sop->args[1] = temp_arg(arg_ts->mem_base); 3493 sop->args[2] = arg_ts->mem_offset; 3494 } else { 3495 tcg_debug_assert(!IS_DEAD_ARG(0)); 3496 } 3497 } 3498 } else { 3499 for (i = 0; i < nb_oargs; i++) { 3500 arg_ts = arg_temp(op->args[i]); 3501 dir_ts = arg_ts->state_ptr; 3502 if (!dir_ts) { 3503 continue; 3504 } 3505 op->args[i] = temp_arg(dir_ts); 3506 changes = true; 3507 3508 /* The output is now live and modified. */ 3509 arg_ts->state = 0; 3510 3511 /* Sync outputs upon their last write. */ 3512 if (NEED_SYNC_ARG(i)) { 3513 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 3514 ? INDEX_op_st_i32 3515 : INDEX_op_st_i64); 3516 TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); 3517 3518 sop->args[0] = temp_arg(dir_ts); 3519 sop->args[1] = temp_arg(arg_ts->mem_base); 3520 sop->args[2] = arg_ts->mem_offset; 3521 3522 arg_ts->state = TS_MEM; 3523 } 3524 /* Drop outputs that are dead. */ 3525 if (IS_DEAD_ARG(i)) { 3526 arg_ts->state = TS_DEAD; 3527 } 3528 } 3529 } 3530 } 3531 3532 return changes; 3533 } 3534 3535 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3536 { 3537 intptr_t off; 3538 int size, align; 3539 3540 /* When allocating an object, look at the full type. */ 3541 size = tcg_type_size(ts->base_type); 3542 switch (ts->base_type) { 3543 case TCG_TYPE_I32: 3544 align = 4; 3545 break; 3546 case TCG_TYPE_I64: 3547 case TCG_TYPE_V64: 3548 align = 8; 3549 break; 3550 case TCG_TYPE_I128: 3551 case TCG_TYPE_V128: 3552 case TCG_TYPE_V256: 3553 /* 3554 * Note that we do not require aligned storage for V256, 3555 * and that we provide alignment for I128 to match V128, 3556 * even if that's above what the host ABI requires. 3557 */ 3558 align = 16; 3559 break; 3560 default: 3561 g_assert_not_reached(); 3562 } 3563 3564 /* 3565 * Assume the stack is sufficiently aligned. 3566 * This affects e.g. ARM NEON, where we have 8 byte stack alignment 3567 * and do not require 16 byte vector alignment. This seems slightly 3568 * easier than fully parameterizing the above switch statement. 3569 */ 3570 align = MIN(TCG_TARGET_STACK_ALIGN, align); 3571 off = ROUND_UP(s->current_frame_offset, align); 3572 3573 /* If we've exhausted the stack frame, restart with a smaller TB. */ 3574 if (off + size > s->frame_end) { 3575 tcg_raise_tb_overflow(s); 3576 } 3577 s->current_frame_offset = off + size; 3578 #if defined(__sparc__) 3579 off += TCG_TARGET_STACK_BIAS; 3580 #endif 3581 3582 /* If the object was subdivided, assign memory to all the parts. */ 3583 if (ts->base_type != ts->type) { 3584 int part_size = tcg_type_size(ts->type); 3585 int part_count = size / part_size; 3586 3587 /* 3588 * Each part is allocated sequentially in tcg_temp_new_internal. 3589 * Jump back to the first part by subtracting the current index. 3590 */ 3591 ts -= ts->temp_subindex; 3592 for (int i = 0; i < part_count; ++i) { 3593 ts[i].mem_offset = off + i * part_size; 3594 ts[i].mem_base = s->frame_temp; 3595 ts[i].mem_allocated = 1; 3596 } 3597 } else { 3598 ts->mem_offset = off; 3599 ts->mem_base = s->frame_temp; 3600 ts->mem_allocated = 1; 3601 } 3602 } 3603 3604 /* Assign @reg to @ts, and update reg_to_temp[]. */ 3605 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 3606 { 3607 if (ts->val_type == TEMP_VAL_REG) { 3608 TCGReg old = ts->reg; 3609 tcg_debug_assert(s->reg_to_temp[old] == ts); 3610 if (old == reg) { 3611 return; 3612 } 3613 s->reg_to_temp[old] = NULL; 3614 } 3615 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 3616 s->reg_to_temp[reg] = ts; 3617 ts->val_type = TEMP_VAL_REG; 3618 ts->reg = reg; 3619 } 3620 3621 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 3622 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 3623 { 3624 tcg_debug_assert(type != TEMP_VAL_REG); 3625 if (ts->val_type == TEMP_VAL_REG) { 3626 TCGReg reg = ts->reg; 3627 tcg_debug_assert(s->reg_to_temp[reg] == ts); 3628 s->reg_to_temp[reg] = NULL; 3629 } 3630 ts->val_type = type; 3631 } 3632 3633 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3634 3635 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 3636 mark it free; otherwise mark it dead. */ 3637 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3638 { 3639 TCGTempVal new_type; 3640 3641 switch (ts->kind) { 3642 case TEMP_FIXED: 3643 return; 3644 case TEMP_GLOBAL: 3645 case TEMP_TB: 3646 new_type = TEMP_VAL_MEM; 3647 break; 3648 case TEMP_EBB: 3649 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3650 break; 3651 case TEMP_CONST: 3652 new_type = TEMP_VAL_CONST; 3653 break; 3654 default: 3655 g_assert_not_reached(); 3656 } 3657 set_temp_val_nonreg(s, ts, new_type); 3658 } 3659 3660 /* Mark a temporary as dead. */ 3661 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 3662 { 3663 temp_free_or_dead(s, ts, 1); 3664 } 3665 3666 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 3667 registers needs to be allocated to store a constant. If 'free_or_dead' 3668 is non-zero, subsequently release the temporary; if it is positive, the 3669 temp is dead; if it is negative, the temp is free. */ 3670 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 3671 TCGRegSet preferred_regs, int free_or_dead) 3672 { 3673 if (!temp_readonly(ts) && !ts->mem_coherent) { 3674 if (!ts->mem_allocated) { 3675 temp_allocate_frame(s, ts); 3676 } 3677 switch (ts->val_type) { 3678 case TEMP_VAL_CONST: 3679 /* If we're going to free the temp immediately, then we won't 3680 require it later in a register, so attempt to store the 3681 constant to memory directly. */ 3682 if (free_or_dead 3683 && tcg_out_sti(s, ts->type, ts->val, 3684 ts->mem_base->reg, ts->mem_offset)) { 3685 break; 3686 } 3687 temp_load(s, ts, tcg_target_available_regs[ts->type], 3688 allocated_regs, preferred_regs); 3689 /* fallthrough */ 3690 3691 case TEMP_VAL_REG: 3692 tcg_out_st(s, ts->type, ts->reg, 3693 ts->mem_base->reg, ts->mem_offset); 3694 break; 3695 3696 case TEMP_VAL_MEM: 3697 break; 3698 3699 case TEMP_VAL_DEAD: 3700 default: 3701 tcg_abort(); 3702 } 3703 ts->mem_coherent = 1; 3704 } 3705 if (free_or_dead) { 3706 temp_free_or_dead(s, ts, free_or_dead); 3707 } 3708 } 3709 3710 /* free register 'reg' by spilling the corresponding temporary if necessary */ 3711 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 3712 { 3713 TCGTemp *ts = s->reg_to_temp[reg]; 3714 if (ts != NULL) { 3715 temp_sync(s, ts, allocated_regs, 0, -1); 3716 } 3717 } 3718 3719 /** 3720 * tcg_reg_alloc: 3721 * @required_regs: Set of registers in which we must allocate. 3722 * @allocated_regs: Set of registers which must be avoided. 3723 * @preferred_regs: Set of registers we should prefer. 3724 * @rev: True if we search the registers in "indirect" order. 3725 * 3726 * The allocated register must be in @required_regs & ~@allocated_regs, 3727 * but if we can put it in @preferred_regs we may save a move later. 3728 */ 3729 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3730 TCGRegSet allocated_regs, 3731 TCGRegSet preferred_regs, bool rev) 3732 { 3733 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3734 TCGRegSet reg_ct[2]; 3735 const int *order; 3736 3737 reg_ct[1] = required_regs & ~allocated_regs; 3738 tcg_debug_assert(reg_ct[1] != 0); 3739 reg_ct[0] = reg_ct[1] & preferred_regs; 3740 3741 /* Skip the preferred_regs option if it cannot be satisfied, 3742 or if the preference made no difference. */ 3743 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3744 3745 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3746 3747 /* Try free registers, preferences first. */ 3748 for (j = f; j < 2; j++) { 3749 TCGRegSet set = reg_ct[j]; 3750 3751 if (tcg_regset_single(set)) { 3752 /* One register in the set. */ 3753 TCGReg reg = tcg_regset_first(set); 3754 if (s->reg_to_temp[reg] == NULL) { 3755 return reg; 3756 } 3757 } else { 3758 for (i = 0; i < n; i++) { 3759 TCGReg reg = order[i]; 3760 if (s->reg_to_temp[reg] == NULL && 3761 tcg_regset_test_reg(set, reg)) { 3762 return reg; 3763 } 3764 } 3765 } 3766 } 3767 3768 /* We must spill something. */ 3769 for (j = f; j < 2; j++) { 3770 TCGRegSet set = reg_ct[j]; 3771 3772 if (tcg_regset_single(set)) { 3773 /* One register in the set. */ 3774 TCGReg reg = tcg_regset_first(set); 3775 tcg_reg_free(s, reg, allocated_regs); 3776 return reg; 3777 } else { 3778 for (i = 0; i < n; i++) { 3779 TCGReg reg = order[i]; 3780 if (tcg_regset_test_reg(set, reg)) { 3781 tcg_reg_free(s, reg, allocated_regs); 3782 return reg; 3783 } 3784 } 3785 } 3786 } 3787 3788 tcg_abort(); 3789 } 3790 3791 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 3792 TCGRegSet allocated_regs, 3793 TCGRegSet preferred_regs, bool rev) 3794 { 3795 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3796 TCGRegSet reg_ct[2]; 3797 const int *order; 3798 3799 /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 3800 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 3801 tcg_debug_assert(reg_ct[1] != 0); 3802 reg_ct[0] = reg_ct[1] & preferred_regs; 3803 3804 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3805 3806 /* 3807 * Skip the preferred_regs option if it cannot be satisfied, 3808 * or if the preference made no difference. 3809 */ 3810 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3811 3812 /* 3813 * Minimize the number of flushes by looking for 2 free registers first, 3814 * then a single flush, then two flushes. 3815 */ 3816 for (fmin = 2; fmin >= 0; fmin--) { 3817 for (j = k; j < 2; j++) { 3818 TCGRegSet set = reg_ct[j]; 3819 3820 for (i = 0; i < n; i++) { 3821 TCGReg reg = order[i]; 3822 3823 if (tcg_regset_test_reg(set, reg)) { 3824 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 3825 if (f >= fmin) { 3826 tcg_reg_free(s, reg, allocated_regs); 3827 tcg_reg_free(s, reg + 1, allocated_regs); 3828 return reg; 3829 } 3830 } 3831 } 3832 } 3833 } 3834 tcg_abort(); 3835 } 3836 3837 /* Make sure the temporary is in a register. If needed, allocate the register 3838 from DESIRED while avoiding ALLOCATED. */ 3839 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3840 TCGRegSet allocated_regs, TCGRegSet preferred_regs) 3841 { 3842 TCGReg reg; 3843 3844 switch (ts->val_type) { 3845 case TEMP_VAL_REG: 3846 return; 3847 case TEMP_VAL_CONST: 3848 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3849 preferred_regs, ts->indirect_base); 3850 if (ts->type <= TCG_TYPE_I64) { 3851 tcg_out_movi(s, ts->type, reg, ts->val); 3852 } else { 3853 uint64_t val = ts->val; 3854 MemOp vece = MO_64; 3855 3856 /* 3857 * Find the minimal vector element that matches the constant. 3858 * The targets will, in general, have to do this search anyway, 3859 * do this generically. 3860 */ 3861 if (val == dup_const(MO_8, val)) { 3862 vece = MO_8; 3863 } else if (val == dup_const(MO_16, val)) { 3864 vece = MO_16; 3865 } else if (val == dup_const(MO_32, val)) { 3866 vece = MO_32; 3867 } 3868 3869 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 3870 } 3871 ts->mem_coherent = 0; 3872 break; 3873 case TEMP_VAL_MEM: 3874 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3875 preferred_regs, ts->indirect_base); 3876 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 3877 ts->mem_coherent = 1; 3878 break; 3879 case TEMP_VAL_DEAD: 3880 default: 3881 tcg_abort(); 3882 } 3883 set_temp_val_reg(s, ts, reg); 3884 } 3885 3886 /* Save a temporary to memory. 'allocated_regs' is used in case a 3887 temporary registers needs to be allocated to store a constant. */ 3888 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 3889 { 3890 /* The liveness analysis already ensures that globals are back 3891 in memory. Keep an tcg_debug_assert for safety. */ 3892 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 3893 } 3894 3895 /* save globals to their canonical location and assume they can be 3896 modified be the following code. 'allocated_regs' is used in case a 3897 temporary registers needs to be allocated to store a constant. */ 3898 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3899 { 3900 int i, n; 3901 3902 for (i = 0, n = s->nb_globals; i < n; i++) { 3903 temp_save(s, &s->temps[i], allocated_regs); 3904 } 3905 } 3906 3907 /* sync globals to their canonical location and assume they can be 3908 read by the following code. 'allocated_regs' is used in case a 3909 temporary registers needs to be allocated to store a constant. */ 3910 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 3911 { 3912 int i, n; 3913 3914 for (i = 0, n = s->nb_globals; i < n; i++) { 3915 TCGTemp *ts = &s->temps[i]; 3916 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3917 || ts->kind == TEMP_FIXED 3918 || ts->mem_coherent); 3919 } 3920 } 3921 3922 /* at the end of a basic block, we assume all temporaries are dead and 3923 all globals are stored at their canonical location. */ 3924 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3925 { 3926 int i; 3927 3928 for (i = s->nb_globals; i < s->nb_temps; i++) { 3929 TCGTemp *ts = &s->temps[i]; 3930 3931 switch (ts->kind) { 3932 case TEMP_TB: 3933 temp_save(s, ts, allocated_regs); 3934 break; 3935 case TEMP_EBB: 3936 /* The liveness analysis already ensures that temps are dead. 3937 Keep an tcg_debug_assert for safety. */ 3938 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3939 break; 3940 case TEMP_CONST: 3941 /* Similarly, we should have freed any allocated register. */ 3942 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3943 break; 3944 default: 3945 g_assert_not_reached(); 3946 } 3947 } 3948 3949 save_globals(s, allocated_regs); 3950 } 3951 3952 /* 3953 * At a conditional branch, we assume all temporaries are dead unless 3954 * explicitly live-across-conditional-branch; all globals and local 3955 * temps are synced to their location. 3956 */ 3957 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3958 { 3959 sync_globals(s, allocated_regs); 3960 3961 for (int i = s->nb_globals; i < s->nb_temps; i++) { 3962 TCGTemp *ts = &s->temps[i]; 3963 /* 3964 * The liveness analysis already ensures that temps are dead. 3965 * Keep tcg_debug_asserts for safety. 3966 */ 3967 switch (ts->kind) { 3968 case TEMP_TB: 3969 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3970 break; 3971 case TEMP_EBB: 3972 case TEMP_CONST: 3973 break; 3974 default: 3975 g_assert_not_reached(); 3976 } 3977 } 3978 } 3979 3980 /* 3981 * Specialized code generation for INDEX_op_mov_* with a constant. 3982 */ 3983 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3984 tcg_target_ulong val, TCGLifeData arg_life, 3985 TCGRegSet preferred_regs) 3986 { 3987 /* ENV should not be modified. */ 3988 tcg_debug_assert(!temp_readonly(ots)); 3989 3990 /* The movi is not explicitly generated here. */ 3991 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 3992 ots->val = val; 3993 ots->mem_coherent = 0; 3994 if (NEED_SYNC_ARG(0)) { 3995 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 3996 } else if (IS_DEAD_ARG(0)) { 3997 temp_dead(s, ots); 3998 } 3999 } 4000 4001 /* 4002 * Specialized code generation for INDEX_op_mov_*. 4003 */ 4004 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 4005 { 4006 const TCGLifeData arg_life = op->life; 4007 TCGRegSet allocated_regs, preferred_regs; 4008 TCGTemp *ts, *ots; 4009 TCGType otype, itype; 4010 TCGReg oreg, ireg; 4011 4012 allocated_regs = s->reserved_regs; 4013 preferred_regs = output_pref(op, 0); 4014 ots = arg_temp(op->args[0]); 4015 ts = arg_temp(op->args[1]); 4016 4017 /* ENV should not be modified. */ 4018 tcg_debug_assert(!temp_readonly(ots)); 4019 4020 /* Note that otype != itype for no-op truncation. */ 4021 otype = ots->type; 4022 itype = ts->type; 4023 4024 if (ts->val_type == TEMP_VAL_CONST) { 4025 /* propagate constant or generate sti */ 4026 tcg_target_ulong val = ts->val; 4027 if (IS_DEAD_ARG(1)) { 4028 temp_dead(s, ts); 4029 } 4030 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 4031 return; 4032 } 4033 4034 /* If the source value is in memory we're going to be forced 4035 to have it in a register in order to perform the copy. Copy 4036 the SOURCE value into its own register first, that way we 4037 don't have to reload SOURCE the next time it is used. */ 4038 if (ts->val_type == TEMP_VAL_MEM) { 4039 temp_load(s, ts, tcg_target_available_regs[itype], 4040 allocated_regs, preferred_regs); 4041 } 4042 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 4043 ireg = ts->reg; 4044 4045 if (IS_DEAD_ARG(0)) { 4046 /* mov to a non-saved dead register makes no sense (even with 4047 liveness analysis disabled). */ 4048 tcg_debug_assert(NEED_SYNC_ARG(0)); 4049 if (!ots->mem_allocated) { 4050 temp_allocate_frame(s, ots); 4051 } 4052 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 4053 if (IS_DEAD_ARG(1)) { 4054 temp_dead(s, ts); 4055 } 4056 temp_dead(s, ots); 4057 return; 4058 } 4059 4060 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 4061 /* 4062 * The mov can be suppressed. Kill input first, so that it 4063 * is unlinked from reg_to_temp, then set the output to the 4064 * reg that we saved from the input. 4065 */ 4066 temp_dead(s, ts); 4067 oreg = ireg; 4068 } else { 4069 if (ots->val_type == TEMP_VAL_REG) { 4070 oreg = ots->reg; 4071 } else { 4072 /* Make sure to not spill the input register during allocation. */ 4073 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 4074 allocated_regs | ((TCGRegSet)1 << ireg), 4075 preferred_regs, ots->indirect_base); 4076 } 4077 if (!tcg_out_mov(s, otype, oreg, ireg)) { 4078 /* 4079 * Cross register class move not supported. 4080 * Store the source register into the destination slot 4081 * and leave the destination temp as TEMP_VAL_MEM. 4082 */ 4083 assert(!temp_readonly(ots)); 4084 if (!ts->mem_allocated) { 4085 temp_allocate_frame(s, ots); 4086 } 4087 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 4088 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 4089 ots->mem_coherent = 1; 4090 return; 4091 } 4092 } 4093 set_temp_val_reg(s, ots, oreg); 4094 ots->mem_coherent = 0; 4095 4096 if (NEED_SYNC_ARG(0)) { 4097 temp_sync(s, ots, allocated_regs, 0, 0); 4098 } 4099 } 4100 4101 /* 4102 * Specialized code generation for INDEX_op_dup_vec. 4103 */ 4104 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 4105 { 4106 const TCGLifeData arg_life = op->life; 4107 TCGRegSet dup_out_regs, dup_in_regs; 4108 TCGTemp *its, *ots; 4109 TCGType itype, vtype; 4110 unsigned vece; 4111 int lowpart_ofs; 4112 bool ok; 4113 4114 ots = arg_temp(op->args[0]); 4115 its = arg_temp(op->args[1]); 4116 4117 /* ENV should not be modified. */ 4118 tcg_debug_assert(!temp_readonly(ots)); 4119 4120 itype = its->type; 4121 vece = TCGOP_VECE(op); 4122 vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4123 4124 if (its->val_type == TEMP_VAL_CONST) { 4125 /* Propagate constant via movi -> dupi. */ 4126 tcg_target_ulong val = its->val; 4127 if (IS_DEAD_ARG(1)) { 4128 temp_dead(s, its); 4129 } 4130 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); 4131 return; 4132 } 4133 4134 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 4135 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 4136 4137 /* Allocate the output register now. */ 4138 if (ots->val_type != TEMP_VAL_REG) { 4139 TCGRegSet allocated_regs = s->reserved_regs; 4140 TCGReg oreg; 4141 4142 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 4143 /* Make sure to not spill the input register. */ 4144 tcg_regset_set_reg(allocated_regs, its->reg); 4145 } 4146 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 4147 output_pref(op, 0), ots->indirect_base); 4148 set_temp_val_reg(s, ots, oreg); 4149 } 4150 4151 switch (its->val_type) { 4152 case TEMP_VAL_REG: 4153 /* 4154 * The dup constriaints must be broad, covering all possible VECE. 4155 * However, tcg_op_dup_vec() gets to see the VECE and we allow it 4156 * to fail, indicating that extra moves are required for that case. 4157 */ 4158 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 4159 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 4160 goto done; 4161 } 4162 /* Try again from memory or a vector input register. */ 4163 } 4164 if (!its->mem_coherent) { 4165 /* 4166 * The input register is not synced, and so an extra store 4167 * would be required to use memory. Attempt an integer-vector 4168 * register move first. We do not have a TCGRegSet for this. 4169 */ 4170 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 4171 break; 4172 } 4173 /* Sync the temp back to its slot and load from there. */ 4174 temp_sync(s, its, s->reserved_regs, 0, 0); 4175 } 4176 /* fall through */ 4177 4178 case TEMP_VAL_MEM: 4179 lowpart_ofs = 0; 4180 if (HOST_BIG_ENDIAN) { 4181 lowpart_ofs = tcg_type_size(itype) - (1 << vece); 4182 } 4183 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 4184 its->mem_offset + lowpart_ofs)) { 4185 goto done; 4186 } 4187 /* Load the input into the destination vector register. */ 4188 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 4189 break; 4190 4191 default: 4192 g_assert_not_reached(); 4193 } 4194 4195 /* We now have a vector input register, so dup must succeed. */ 4196 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 4197 tcg_debug_assert(ok); 4198 4199 done: 4200 ots->mem_coherent = 0; 4201 if (IS_DEAD_ARG(1)) { 4202 temp_dead(s, its); 4203 } 4204 if (NEED_SYNC_ARG(0)) { 4205 temp_sync(s, ots, s->reserved_regs, 0, 0); 4206 } 4207 if (IS_DEAD_ARG(0)) { 4208 temp_dead(s, ots); 4209 } 4210 } 4211 4212 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 4213 { 4214 const TCGLifeData arg_life = op->life; 4215 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 4216 TCGRegSet i_allocated_regs; 4217 TCGRegSet o_allocated_regs; 4218 int i, k, nb_iargs, nb_oargs; 4219 TCGReg reg; 4220 TCGArg arg; 4221 const TCGArgConstraint *arg_ct; 4222 TCGTemp *ts; 4223 TCGArg new_args[TCG_MAX_OP_ARGS]; 4224 int const_args[TCG_MAX_OP_ARGS]; 4225 4226 nb_oargs = def->nb_oargs; 4227 nb_iargs = def->nb_iargs; 4228 4229 /* copy constants */ 4230 memcpy(new_args + nb_oargs + nb_iargs, 4231 op->args + nb_oargs + nb_iargs, 4232 sizeof(TCGArg) * def->nb_cargs); 4233 4234 i_allocated_regs = s->reserved_regs; 4235 o_allocated_regs = s->reserved_regs; 4236 4237 /* satisfy input constraints */ 4238 for (k = 0; k < nb_iargs; k++) { 4239 TCGRegSet i_preferred_regs, i_required_regs; 4240 bool allocate_new_reg, copyto_new_reg; 4241 TCGTemp *ts2; 4242 int i1, i2; 4243 4244 i = def->args_ct[nb_oargs + k].sort_index; 4245 arg = op->args[i]; 4246 arg_ct = &def->args_ct[i]; 4247 ts = arg_temp(arg); 4248 4249 if (ts->val_type == TEMP_VAL_CONST 4250 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 4251 /* constant is OK for instruction */ 4252 const_args[i] = 1; 4253 new_args[i] = ts->val; 4254 continue; 4255 } 4256 4257 reg = ts->reg; 4258 i_preferred_regs = 0; 4259 i_required_regs = arg_ct->regs; 4260 allocate_new_reg = false; 4261 copyto_new_reg = false; 4262 4263 switch (arg_ct->pair) { 4264 case 0: /* not paired */ 4265 if (arg_ct->ialias) { 4266 i_preferred_regs = output_pref(op, arg_ct->alias_index); 4267 4268 /* 4269 * If the input is readonly, then it cannot also be an 4270 * output and aliased to itself. If the input is not 4271 * dead after the instruction, we must allocate a new 4272 * register and move it. 4273 */ 4274 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 4275 allocate_new_reg = true; 4276 } else if (ts->val_type == TEMP_VAL_REG) { 4277 /* 4278 * Check if the current register has already been 4279 * allocated for another input. 4280 */ 4281 allocate_new_reg = 4282 tcg_regset_test_reg(i_allocated_regs, reg); 4283 } 4284 } 4285 if (!allocate_new_reg) { 4286 temp_load(s, ts, i_required_regs, i_allocated_regs, 4287 i_preferred_regs); 4288 reg = ts->reg; 4289 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 4290 } 4291 if (allocate_new_reg) { 4292 /* 4293 * Allocate a new register matching the constraint 4294 * and move the temporary register into it. 4295 */ 4296 temp_load(s, ts, tcg_target_available_regs[ts->type], 4297 i_allocated_regs, 0); 4298 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 4299 i_preferred_regs, ts->indirect_base); 4300 copyto_new_reg = true; 4301 } 4302 break; 4303 4304 case 1: 4305 /* First of an input pair; if i1 == i2, the second is an output. */ 4306 i1 = i; 4307 i2 = arg_ct->pair_index; 4308 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 4309 4310 /* 4311 * It is easier to default to allocating a new pair 4312 * and to identify a few cases where it's not required. 4313 */ 4314 if (arg_ct->ialias) { 4315 i_preferred_regs = output_pref(op, arg_ct->alias_index); 4316 if (IS_DEAD_ARG(i1) && 4317 IS_DEAD_ARG(i2) && 4318 !temp_readonly(ts) && 4319 ts->val_type == TEMP_VAL_REG && 4320 ts->reg < TCG_TARGET_NB_REGS - 1 && 4321 tcg_regset_test_reg(i_required_regs, reg) && 4322 !tcg_regset_test_reg(i_allocated_regs, reg) && 4323 !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 4324 (ts2 4325 ? ts2->val_type == TEMP_VAL_REG && 4326 ts2->reg == reg + 1 && 4327 !temp_readonly(ts2) 4328 : s->reg_to_temp[reg + 1] == NULL)) { 4329 break; 4330 } 4331 } else { 4332 /* Without aliasing, the pair must also be an input. */ 4333 tcg_debug_assert(ts2); 4334 if (ts->val_type == TEMP_VAL_REG && 4335 ts2->val_type == TEMP_VAL_REG && 4336 ts2->reg == reg + 1 && 4337 tcg_regset_test_reg(i_required_regs, reg)) { 4338 break; 4339 } 4340 } 4341 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 4342 0, ts->indirect_base); 4343 goto do_pair; 4344 4345 case 2: /* pair second */ 4346 reg = new_args[arg_ct->pair_index] + 1; 4347 goto do_pair; 4348 4349 case 3: /* ialias with second output, no first input */ 4350 tcg_debug_assert(arg_ct->ialias); 4351 i_preferred_regs = output_pref(op, arg_ct->alias_index); 4352 4353 if (IS_DEAD_ARG(i) && 4354 !temp_readonly(ts) && 4355 ts->val_type == TEMP_VAL_REG && 4356 reg > 0 && 4357 s->reg_to_temp[reg - 1] == NULL && 4358 tcg_regset_test_reg(i_required_regs, reg) && 4359 !tcg_regset_test_reg(i_allocated_regs, reg) && 4360 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 4361 tcg_regset_set_reg(i_allocated_regs, reg - 1); 4362 break; 4363 } 4364 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 4365 i_allocated_regs, 0, 4366 ts->indirect_base); 4367 tcg_regset_set_reg(i_allocated_regs, reg); 4368 reg += 1; 4369 goto do_pair; 4370 4371 do_pair: 4372 /* 4373 * If an aliased input is not dead after the instruction, 4374 * we must allocate a new register and move it. 4375 */ 4376 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 4377 TCGRegSet t_allocated_regs = i_allocated_regs; 4378 4379 /* 4380 * Because of the alias, and the continued life, make sure 4381 * that the temp is somewhere *other* than the reg pair, 4382 * and we get a copy in reg. 4383 */ 4384 tcg_regset_set_reg(t_allocated_regs, reg); 4385 tcg_regset_set_reg(t_allocated_regs, reg + 1); 4386 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 4387 /* If ts was already in reg, copy it somewhere else. */ 4388 TCGReg nr; 4389 bool ok; 4390 4391 tcg_debug_assert(ts->kind != TEMP_FIXED); 4392 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 4393 t_allocated_regs, 0, ts->indirect_base); 4394 ok = tcg_out_mov(s, ts->type, nr, reg); 4395 tcg_debug_assert(ok); 4396 4397 set_temp_val_reg(s, ts, nr); 4398 } else { 4399 temp_load(s, ts, tcg_target_available_regs[ts->type], 4400 t_allocated_regs, 0); 4401 copyto_new_reg = true; 4402 } 4403 } else { 4404 /* Preferably allocate to reg, otherwise copy. */ 4405 i_required_regs = (TCGRegSet)1 << reg; 4406 temp_load(s, ts, i_required_regs, i_allocated_regs, 4407 i_preferred_regs); 4408 copyto_new_reg = ts->reg != reg; 4409 } 4410 break; 4411 4412 default: 4413 g_assert_not_reached(); 4414 } 4415 4416 if (copyto_new_reg) { 4417 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4418 /* 4419 * Cross register class move not supported. Sync the 4420 * temp back to its slot and load from there. 4421 */ 4422 temp_sync(s, ts, i_allocated_regs, 0, 0); 4423 tcg_out_ld(s, ts->type, reg, 4424 ts->mem_base->reg, ts->mem_offset); 4425 } 4426 } 4427 new_args[i] = reg; 4428 const_args[i] = 0; 4429 tcg_regset_set_reg(i_allocated_regs, reg); 4430 } 4431 4432 /* mark dead temporaries and free the associated registers */ 4433 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4434 if (IS_DEAD_ARG(i)) { 4435 temp_dead(s, arg_temp(op->args[i])); 4436 } 4437 } 4438 4439 if (def->flags & TCG_OPF_COND_BRANCH) { 4440 tcg_reg_alloc_cbranch(s, i_allocated_regs); 4441 } else if (def->flags & TCG_OPF_BB_END) { 4442 tcg_reg_alloc_bb_end(s, i_allocated_regs); 4443 } else { 4444 if (def->flags & TCG_OPF_CALL_CLOBBER) { 4445 /* XXX: permit generic clobber register list ? */ 4446 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4447 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4448 tcg_reg_free(s, i, i_allocated_regs); 4449 } 4450 } 4451 } 4452 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 4453 /* sync globals if the op has side effects and might trigger 4454 an exception. */ 4455 sync_globals(s, i_allocated_regs); 4456 } 4457 4458 /* satisfy the output constraints */ 4459 for(k = 0; k < nb_oargs; k++) { 4460 i = def->args_ct[k].sort_index; 4461 arg = op->args[i]; 4462 arg_ct = &def->args_ct[i]; 4463 ts = arg_temp(arg); 4464 4465 /* ENV should not be modified. */ 4466 tcg_debug_assert(!temp_readonly(ts)); 4467 4468 switch (arg_ct->pair) { 4469 case 0: /* not paired */ 4470 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 4471 reg = new_args[arg_ct->alias_index]; 4472 } else if (arg_ct->newreg) { 4473 reg = tcg_reg_alloc(s, arg_ct->regs, 4474 i_allocated_regs | o_allocated_regs, 4475 output_pref(op, k), ts->indirect_base); 4476 } else { 4477 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 4478 output_pref(op, k), ts->indirect_base); 4479 } 4480 break; 4481 4482 case 1: /* first of pair */ 4483 tcg_debug_assert(!arg_ct->newreg); 4484 if (arg_ct->oalias) { 4485 reg = new_args[arg_ct->alias_index]; 4486 break; 4487 } 4488 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 4489 output_pref(op, k), ts->indirect_base); 4490 break; 4491 4492 case 2: /* second of pair */ 4493 tcg_debug_assert(!arg_ct->newreg); 4494 if (arg_ct->oalias) { 4495 reg = new_args[arg_ct->alias_index]; 4496 } else { 4497 reg = new_args[arg_ct->pair_index] + 1; 4498 } 4499 break; 4500 4501 case 3: /* first of pair, aliasing with a second input */ 4502 tcg_debug_assert(!arg_ct->newreg); 4503 reg = new_args[arg_ct->pair_index] - 1; 4504 break; 4505 4506 default: 4507 g_assert_not_reached(); 4508 } 4509 tcg_regset_set_reg(o_allocated_regs, reg); 4510 set_temp_val_reg(s, ts, reg); 4511 ts->mem_coherent = 0; 4512 new_args[i] = reg; 4513 } 4514 } 4515 4516 /* emit instruction */ 4517 if (def->flags & TCG_OPF_VECTOR) { 4518 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 4519 new_args, const_args); 4520 } else { 4521 tcg_out_op(s, op->opc, new_args, const_args); 4522 } 4523 4524 /* move the outputs in the correct register if needed */ 4525 for(i = 0; i < nb_oargs; i++) { 4526 ts = arg_temp(op->args[i]); 4527 4528 /* ENV should not be modified. */ 4529 tcg_debug_assert(!temp_readonly(ts)); 4530 4531 if (NEED_SYNC_ARG(i)) { 4532 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 4533 } else if (IS_DEAD_ARG(i)) { 4534 temp_dead(s, ts); 4535 } 4536 } 4537 } 4538 4539 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 4540 { 4541 const TCGLifeData arg_life = op->life; 4542 TCGTemp *ots, *itsl, *itsh; 4543 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 4544 4545 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 4546 tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 4547 tcg_debug_assert(TCGOP_VECE(op) == MO_64); 4548 4549 ots = arg_temp(op->args[0]); 4550 itsl = arg_temp(op->args[1]); 4551 itsh = arg_temp(op->args[2]); 4552 4553 /* ENV should not be modified. */ 4554 tcg_debug_assert(!temp_readonly(ots)); 4555 4556 /* Allocate the output register now. */ 4557 if (ots->val_type != TEMP_VAL_REG) { 4558 TCGRegSet allocated_regs = s->reserved_regs; 4559 TCGRegSet dup_out_regs = 4560 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 4561 TCGReg oreg; 4562 4563 /* Make sure to not spill the input registers. */ 4564 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 4565 tcg_regset_set_reg(allocated_regs, itsl->reg); 4566 } 4567 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 4568 tcg_regset_set_reg(allocated_regs, itsh->reg); 4569 } 4570 4571 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 4572 output_pref(op, 0), ots->indirect_base); 4573 set_temp_val_reg(s, ots, oreg); 4574 } 4575 4576 /* Promote dup2 of immediates to dupi_vec. */ 4577 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 4578 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 4579 MemOp vece = MO_64; 4580 4581 if (val == dup_const(MO_8, val)) { 4582 vece = MO_8; 4583 } else if (val == dup_const(MO_16, val)) { 4584 vece = MO_16; 4585 } else if (val == dup_const(MO_32, val)) { 4586 vece = MO_32; 4587 } 4588 4589 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 4590 goto done; 4591 } 4592 4593 /* If the two inputs form one 64-bit value, try dupm_vec. */ 4594 if (itsl->temp_subindex == HOST_BIG_ENDIAN && 4595 itsh->temp_subindex == !HOST_BIG_ENDIAN && 4596 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { 4597 TCGTemp *its = itsl - HOST_BIG_ENDIAN; 4598 4599 temp_sync(s, its + 0, s->reserved_regs, 0, 0); 4600 temp_sync(s, its + 1, s->reserved_regs, 0, 0); 4601 4602 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 4603 its->mem_base->reg, its->mem_offset)) { 4604 goto done; 4605 } 4606 } 4607 4608 /* Fall back to generic expansion. */ 4609 return false; 4610 4611 done: 4612 ots->mem_coherent = 0; 4613 if (IS_DEAD_ARG(1)) { 4614 temp_dead(s, itsl); 4615 } 4616 if (IS_DEAD_ARG(2)) { 4617 temp_dead(s, itsh); 4618 } 4619 if (NEED_SYNC_ARG(0)) { 4620 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 4621 } else if (IS_DEAD_ARG(0)) { 4622 temp_dead(s, ots); 4623 } 4624 return true; 4625 } 4626 4627 static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, 4628 TCGRegSet allocated_regs) 4629 { 4630 if (ts->val_type == TEMP_VAL_REG) { 4631 if (ts->reg != reg) { 4632 tcg_reg_free(s, reg, allocated_regs); 4633 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 4634 /* 4635 * Cross register class move not supported. Sync the 4636 * temp back to its slot and load from there. 4637 */ 4638 temp_sync(s, ts, allocated_regs, 0, 0); 4639 tcg_out_ld(s, ts->type, reg, 4640 ts->mem_base->reg, ts->mem_offset); 4641 } 4642 } 4643 } else { 4644 TCGRegSet arg_set = 0; 4645 4646 tcg_reg_free(s, reg, allocated_regs); 4647 tcg_regset_set_reg(arg_set, reg); 4648 temp_load(s, ts, arg_set, allocated_regs, 0); 4649 } 4650 } 4651 4652 static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, 4653 TCGRegSet allocated_regs) 4654 { 4655 /* 4656 * When the destination is on the stack, load up the temp and store. 4657 * If there are many call-saved registers, the temp might live to 4658 * see another use; otherwise it'll be discarded. 4659 */ 4660 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); 4661 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, 4662 TCG_TARGET_CALL_STACK_OFFSET + 4663 stk_slot * sizeof(tcg_target_long)); 4664 } 4665 4666 static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, 4667 TCGTemp *ts, TCGRegSet *allocated_regs) 4668 { 4669 if (REG_P(l)) { 4670 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; 4671 load_arg_reg(s, reg, ts, *allocated_regs); 4672 tcg_regset_set_reg(*allocated_regs, reg); 4673 } else { 4674 load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs), 4675 ts, *allocated_regs); 4676 } 4677 } 4678 4679 static void load_arg_ref(TCGContext *s, int arg_slot, TCGReg ref_base, 4680 intptr_t ref_off, TCGRegSet *allocated_regs) 4681 { 4682 TCGReg reg; 4683 int stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); 4684 4685 if (stk_slot < 0) { 4686 reg = tcg_target_call_iarg_regs[arg_slot]; 4687 tcg_reg_free(s, reg, *allocated_regs); 4688 tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4689 tcg_regset_set_reg(*allocated_regs, reg); 4690 } else { 4691 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR], 4692 *allocated_regs, 0, false); 4693 tcg_out_addi_ptr(s, reg, ref_base, ref_off); 4694 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, 4695 TCG_TARGET_CALL_STACK_OFFSET 4696 + stk_slot * sizeof(tcg_target_long)); 4697 } 4698 } 4699 4700 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 4701 { 4702 const int nb_oargs = TCGOP_CALLO(op); 4703 const int nb_iargs = TCGOP_CALLI(op); 4704 const TCGLifeData arg_life = op->life; 4705 const TCGHelperInfo *info = tcg_call_info(op); 4706 TCGRegSet allocated_regs = s->reserved_regs; 4707 int i; 4708 4709 /* 4710 * Move inputs into place in reverse order, 4711 * so that we place stacked arguments first. 4712 */ 4713 for (i = nb_iargs - 1; i >= 0; --i) { 4714 const TCGCallArgumentLoc *loc = &info->in[i]; 4715 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); 4716 4717 switch (loc->kind) { 4718 case TCG_CALL_ARG_NORMAL: 4719 case TCG_CALL_ARG_EXTEND_U: 4720 case TCG_CALL_ARG_EXTEND_S: 4721 load_arg_normal(s, loc, ts, &allocated_regs); 4722 break; 4723 case TCG_CALL_ARG_BY_REF: 4724 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4725 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, 4726 TCG_TARGET_CALL_STACK_OFFSET 4727 + loc->ref_slot * sizeof(tcg_target_long), 4728 &allocated_regs); 4729 break; 4730 case TCG_CALL_ARG_BY_REF_N: 4731 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 4732 break; 4733 default: 4734 g_assert_not_reached(); 4735 } 4736 } 4737 4738 /* Mark dead temporaries and free the associated registers. */ 4739 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4740 if (IS_DEAD_ARG(i)) { 4741 temp_dead(s, arg_temp(op->args[i])); 4742 } 4743 } 4744 4745 /* Clobber call registers. */ 4746 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 4747 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 4748 tcg_reg_free(s, i, allocated_regs); 4749 } 4750 } 4751 4752 /* 4753 * Save globals if they might be written by the helper, 4754 * sync them if they might be read. 4755 */ 4756 if (info->flags & TCG_CALL_NO_READ_GLOBALS) { 4757 /* Nothing to do */ 4758 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { 4759 sync_globals(s, allocated_regs); 4760 } else { 4761 save_globals(s, allocated_regs); 4762 } 4763 4764 /* 4765 * If the ABI passes a pointer to the returned struct as the first 4766 * argument, load that now. Pass a pointer to the output home slot. 4767 */ 4768 if (info->out_kind == TCG_CALL_RET_BY_REF) { 4769 TCGTemp *ts = arg_temp(op->args[0]); 4770 4771 if (!ts->mem_allocated) { 4772 temp_allocate_frame(s, ts); 4773 } 4774 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); 4775 } 4776 4777 tcg_out_call(s, tcg_call_func(op), info); 4778 4779 /* Assign output registers and emit moves if needed. */ 4780 switch (info->out_kind) { 4781 case TCG_CALL_RET_NORMAL: 4782 for (i = 0; i < nb_oargs; i++) { 4783 TCGTemp *ts = arg_temp(op->args[i]); 4784 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i); 4785 4786 /* ENV should not be modified. */ 4787 tcg_debug_assert(!temp_readonly(ts)); 4788 4789 set_temp_val_reg(s, ts, reg); 4790 ts->mem_coherent = 0; 4791 } 4792 break; 4793 4794 case TCG_CALL_RET_BY_VEC: 4795 { 4796 TCGTemp *ts = arg_temp(op->args[0]); 4797 4798 tcg_debug_assert(ts->base_type == TCG_TYPE_I128); 4799 tcg_debug_assert(ts->temp_subindex == 0); 4800 if (!ts->mem_allocated) { 4801 temp_allocate_frame(s, ts); 4802 } 4803 tcg_out_st(s, TCG_TYPE_V128, 4804 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 4805 ts->mem_base->reg, ts->mem_offset); 4806 } 4807 /* fall through to mark all parts in memory */ 4808 4809 case TCG_CALL_RET_BY_REF: 4810 /* The callee has performed a write through the reference. */ 4811 for (i = 0; i < nb_oargs; i++) { 4812 TCGTemp *ts = arg_temp(op->args[i]); 4813 ts->val_type = TEMP_VAL_MEM; 4814 } 4815 break; 4816 4817 default: 4818 g_assert_not_reached(); 4819 } 4820 4821 /* Flush or discard output registers as needed. */ 4822 for (i = 0; i < nb_oargs; i++) { 4823 TCGTemp *ts = arg_temp(op->args[i]); 4824 if (NEED_SYNC_ARG(i)) { 4825 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); 4826 } else if (IS_DEAD_ARG(i)) { 4827 temp_dead(s, ts); 4828 } 4829 } 4830 } 4831 4832 #ifdef CONFIG_PROFILER 4833 4834 /* avoid copy/paste errors */ 4835 #define PROF_ADD(to, from, field) \ 4836 do { \ 4837 (to)->field += qatomic_read(&((from)->field)); \ 4838 } while (0) 4839 4840 #define PROF_MAX(to, from, field) \ 4841 do { \ 4842 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4843 if (val__ > (to)->field) { \ 4844 (to)->field = val__; \ 4845 } \ 4846 } while (0) 4847 4848 /* Pass in a zero'ed @prof */ 4849 static inline 4850 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4851 { 4852 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4853 unsigned int i; 4854 4855 for (i = 0; i < n_ctxs; i++) { 4856 TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4857 const TCGProfile *orig = &s->prof; 4858 4859 if (counters) { 4860 PROF_ADD(prof, orig, cpu_exec_time); 4861 PROF_ADD(prof, orig, tb_count1); 4862 PROF_ADD(prof, orig, tb_count); 4863 PROF_ADD(prof, orig, op_count); 4864 PROF_MAX(prof, orig, op_count_max); 4865 PROF_ADD(prof, orig, temp_count); 4866 PROF_MAX(prof, orig, temp_count_max); 4867 PROF_ADD(prof, orig, del_op_count); 4868 PROF_ADD(prof, orig, code_in_len); 4869 PROF_ADD(prof, orig, code_out_len); 4870 PROF_ADD(prof, orig, search_out_len); 4871 PROF_ADD(prof, orig, interm_time); 4872 PROF_ADD(prof, orig, code_time); 4873 PROF_ADD(prof, orig, la_time); 4874 PROF_ADD(prof, orig, opt_time); 4875 PROF_ADD(prof, orig, restore_count); 4876 PROF_ADD(prof, orig, restore_time); 4877 } 4878 if (table) { 4879 int i; 4880 4881 for (i = 0; i < NB_OPS; i++) { 4882 PROF_ADD(prof, orig, table_op_count[i]); 4883 } 4884 } 4885 } 4886 } 4887 4888 #undef PROF_ADD 4889 #undef PROF_MAX 4890 4891 static void tcg_profile_snapshot_counters(TCGProfile *prof) 4892 { 4893 tcg_profile_snapshot(prof, true, false); 4894 } 4895 4896 static void tcg_profile_snapshot_table(TCGProfile *prof) 4897 { 4898 tcg_profile_snapshot(prof, false, true); 4899 } 4900 4901 void tcg_dump_op_count(GString *buf) 4902 { 4903 TCGProfile prof = {}; 4904 int i; 4905 4906 tcg_profile_snapshot_table(&prof); 4907 for (i = 0; i < NB_OPS; i++) { 4908 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name, 4909 prof.table_op_count[i]); 4910 } 4911 } 4912 4913 int64_t tcg_cpu_exec_time(void) 4914 { 4915 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4916 unsigned int i; 4917 int64_t ret = 0; 4918 4919 for (i = 0; i < n_ctxs; i++) { 4920 const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4921 const TCGProfile *prof = &s->prof; 4922 4923 ret += qatomic_read(&prof->cpu_exec_time); 4924 } 4925 return ret; 4926 } 4927 #else 4928 void tcg_dump_op_count(GString *buf) 4929 { 4930 g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 4931 } 4932 4933 int64_t tcg_cpu_exec_time(void) 4934 { 4935 error_report("%s: TCG profiler not compiled", __func__); 4936 exit(EXIT_FAILURE); 4937 } 4938 #endif 4939 4940 4941 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) 4942 { 4943 #ifdef CONFIG_PROFILER 4944 TCGProfile *prof = &s->prof; 4945 #endif 4946 int i, num_insns; 4947 TCGOp *op; 4948 4949 #ifdef CONFIG_PROFILER 4950 { 4951 int n = 0; 4952 4953 QTAILQ_FOREACH(op, &s->ops, link) { 4954 n++; 4955 } 4956 qatomic_set(&prof->op_count, prof->op_count + n); 4957 if (n > prof->op_count_max) { 4958 qatomic_set(&prof->op_count_max, n); 4959 } 4960 4961 n = s->nb_temps; 4962 qatomic_set(&prof->temp_count, prof->temp_count + n); 4963 if (n > prof->temp_count_max) { 4964 qatomic_set(&prof->temp_count_max, n); 4965 } 4966 } 4967 #endif 4968 4969 #ifdef DEBUG_DISAS 4970 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4971 && qemu_log_in_addr_range(pc_start))) { 4972 FILE *logfile = qemu_log_trylock(); 4973 if (logfile) { 4974 fprintf(logfile, "OP:\n"); 4975 tcg_dump_ops(s, logfile, false); 4976 fprintf(logfile, "\n"); 4977 qemu_log_unlock(logfile); 4978 } 4979 } 4980 #endif 4981 4982 #ifdef CONFIG_DEBUG_TCG 4983 /* Ensure all labels referenced have been emitted. */ 4984 { 4985 TCGLabel *l; 4986 bool error = false; 4987 4988 QSIMPLEQ_FOREACH(l, &s->labels, next) { 4989 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { 4990 qemu_log_mask(CPU_LOG_TB_OP, 4991 "$L%d referenced but not present.\n", l->id); 4992 error = true; 4993 } 4994 } 4995 assert(!error); 4996 } 4997 #endif 4998 4999 #ifdef CONFIG_PROFILER 5000 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 5001 #endif 5002 5003 #ifdef USE_TCG_OPTIMIZATIONS 5004 tcg_optimize(s); 5005 #endif 5006 5007 #ifdef CONFIG_PROFILER 5008 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 5009 qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 5010 #endif 5011 5012 reachable_code_pass(s); 5013 liveness_pass_0(s); 5014 liveness_pass_1(s); 5015 5016 if (s->nb_indirects > 0) { 5017 #ifdef DEBUG_DISAS 5018 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 5019 && qemu_log_in_addr_range(pc_start))) { 5020 FILE *logfile = qemu_log_trylock(); 5021 if (logfile) { 5022 fprintf(logfile, "OP before indirect lowering:\n"); 5023 tcg_dump_ops(s, logfile, false); 5024 fprintf(logfile, "\n"); 5025 qemu_log_unlock(logfile); 5026 } 5027 } 5028 #endif 5029 /* Replace indirect temps with direct temps. */ 5030 if (liveness_pass_2(s)) { 5031 /* If changes were made, re-run liveness. */ 5032 liveness_pass_1(s); 5033 } 5034 } 5035 5036 #ifdef CONFIG_PROFILER 5037 qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 5038 #endif 5039 5040 #ifdef DEBUG_DISAS 5041 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 5042 && qemu_log_in_addr_range(pc_start))) { 5043 FILE *logfile = qemu_log_trylock(); 5044 if (logfile) { 5045 fprintf(logfile, "OP after optimization and liveness analysis:\n"); 5046 tcg_dump_ops(s, logfile, true); 5047 fprintf(logfile, "\n"); 5048 qemu_log_unlock(logfile); 5049 } 5050 } 5051 #endif 5052 5053 /* Initialize goto_tb jump offsets. */ 5054 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; 5055 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; 5056 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; 5057 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; 5058 5059 tcg_reg_alloc_start(s); 5060 5061 /* 5062 * Reset the buffer pointers when restarting after overflow. 5063 * TODO: Move this into translate-all.c with the rest of the 5064 * buffer management. Having only this done here is confusing. 5065 */ 5066 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 5067 s->code_ptr = s->code_buf; 5068 5069 #ifdef TCG_TARGET_NEED_LDST_LABELS 5070 QSIMPLEQ_INIT(&s->ldst_labels); 5071 #endif 5072 #ifdef TCG_TARGET_NEED_POOL_LABELS 5073 s->pool_labels = NULL; 5074 #endif 5075 5076 num_insns = -1; 5077 QTAILQ_FOREACH(op, &s->ops, link) { 5078 TCGOpcode opc = op->opc; 5079 5080 #ifdef CONFIG_PROFILER 5081 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 5082 #endif 5083 5084 switch (opc) { 5085 case INDEX_op_mov_i32: 5086 case INDEX_op_mov_i64: 5087 case INDEX_op_mov_vec: 5088 tcg_reg_alloc_mov(s, op); 5089 break; 5090 case INDEX_op_dup_vec: 5091 tcg_reg_alloc_dup(s, op); 5092 break; 5093 case INDEX_op_insn_start: 5094 if (num_insns >= 0) { 5095 size_t off = tcg_current_code_size(s); 5096 s->gen_insn_end_off[num_insns] = off; 5097 /* Assert that we do not overflow our stored offset. */ 5098 assert(s->gen_insn_end_off[num_insns] == off); 5099 } 5100 num_insns++; 5101 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 5102 target_ulong a; 5103 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 5104 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 5105 #else 5106 a = op->args[i]; 5107 #endif 5108 s->gen_insn_data[num_insns][i] = a; 5109 } 5110 break; 5111 case INDEX_op_discard: 5112 temp_dead(s, arg_temp(op->args[0])); 5113 break; 5114 case INDEX_op_set_label: 5115 tcg_reg_alloc_bb_end(s, s->reserved_regs); 5116 tcg_out_label(s, arg_label(op->args[0])); 5117 break; 5118 case INDEX_op_call: 5119 tcg_reg_alloc_call(s, op); 5120 break; 5121 case INDEX_op_exit_tb: 5122 tcg_out_exit_tb(s, op->args[0]); 5123 break; 5124 case INDEX_op_goto_tb: 5125 tcg_out_goto_tb(s, op->args[0]); 5126 break; 5127 case INDEX_op_dup2_vec: 5128 if (tcg_reg_alloc_dup2(s, op)) { 5129 break; 5130 } 5131 /* fall through */ 5132 default: 5133 /* Sanity check that we've not introduced any unhandled opcodes. */ 5134 tcg_debug_assert(tcg_op_supported(opc)); 5135 /* Note: in order to speed up the code, it would be much 5136 faster to have specialized register allocator functions for 5137 some common argument patterns */ 5138 tcg_reg_alloc_op(s, op); 5139 break; 5140 } 5141 /* Test for (pending) buffer overflow. The assumption is that any 5142 one operation beginning below the high water mark cannot overrun 5143 the buffer completely. Thus we can test for overflow after 5144 generating code without having to check during generation. */ 5145 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 5146 return -1; 5147 } 5148 /* Test for TB overflow, as seen by gen_insn_end_off. */ 5149 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 5150 return -2; 5151 } 5152 } 5153 tcg_debug_assert(num_insns >= 0); 5154 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 5155 5156 /* Generate TB finalization at the end of block */ 5157 #ifdef TCG_TARGET_NEED_LDST_LABELS 5158 i = tcg_out_ldst_finalize(s); 5159 if (i < 0) { 5160 return i; 5161 } 5162 #endif 5163 #ifdef TCG_TARGET_NEED_POOL_LABELS 5164 i = tcg_out_pool_finalize(s); 5165 if (i < 0) { 5166 return i; 5167 } 5168 #endif 5169 if (!tcg_resolve_relocs(s)) { 5170 return -2; 5171 } 5172 5173 #ifndef CONFIG_TCG_INTERPRETER 5174 /* flush instruction cache */ 5175 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 5176 (uintptr_t)s->code_buf, 5177 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 5178 #endif 5179 5180 return tcg_current_code_size(s); 5181 } 5182 5183 #ifdef CONFIG_PROFILER 5184 void tcg_dump_info(GString *buf) 5185 { 5186 TCGProfile prof = {}; 5187 const TCGProfile *s; 5188 int64_t tb_count; 5189 int64_t tb_div_count; 5190 int64_t tot; 5191 5192 tcg_profile_snapshot_counters(&prof); 5193 s = &prof; 5194 tb_count = s->tb_count; 5195 tb_div_count = tb_count ? tb_count : 1; 5196 tot = s->interm_time + s->code_time; 5197 5198 g_string_append_printf(buf, "JIT cycles %" PRId64 5199 " (%0.3f s at 2.4 GHz)\n", 5200 tot, tot / 2.4e9); 5201 g_string_append_printf(buf, "translated TBs %" PRId64 5202 " (aborted=%" PRId64 " %0.1f%%)\n", 5203 tb_count, s->tb_count1 - tb_count, 5204 (double)(s->tb_count1 - s->tb_count) 5205 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 5206 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n", 5207 (double)s->op_count / tb_div_count, s->op_count_max); 5208 g_string_append_printf(buf, "deleted ops/TB %0.2f\n", 5209 (double)s->del_op_count / tb_div_count); 5210 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n", 5211 (double)s->temp_count / tb_div_count, 5212 s->temp_count_max); 5213 g_string_append_printf(buf, "avg host code/TB %0.1f\n", 5214 (double)s->code_out_len / tb_div_count); 5215 g_string_append_printf(buf, "avg search data/TB %0.1f\n", 5216 (double)s->search_out_len / tb_div_count); 5217 5218 g_string_append_printf(buf, "cycles/op %0.1f\n", 5219 s->op_count ? (double)tot / s->op_count : 0); 5220 g_string_append_printf(buf, "cycles/in byte %0.1f\n", 5221 s->code_in_len ? (double)tot / s->code_in_len : 0); 5222 g_string_append_printf(buf, "cycles/out byte %0.1f\n", 5223 s->code_out_len ? (double)tot / s->code_out_len : 0); 5224 g_string_append_printf(buf, "cycles/search byte %0.1f\n", 5225 s->search_out_len ? 5226 (double)tot / s->search_out_len : 0); 5227 if (tot == 0) { 5228 tot = 1; 5229 } 5230 g_string_append_printf(buf, " gen_interm time %0.1f%%\n", 5231 (double)s->interm_time / tot * 100.0); 5232 g_string_append_printf(buf, " gen_code time %0.1f%%\n", 5233 (double)s->code_time / tot * 100.0); 5234 g_string_append_printf(buf, "optim./code time %0.1f%%\n", 5235 (double)s->opt_time / (s->code_time ? 5236 s->code_time : 1) 5237 * 100.0); 5238 g_string_append_printf(buf, "liveness/code time %0.1f%%\n", 5239 (double)s->la_time / (s->code_time ? 5240 s->code_time : 1) * 100.0); 5241 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n", 5242 s->restore_count); 5243 g_string_append_printf(buf, " avg cycles %0.1f\n", 5244 s->restore_count ? 5245 (double)s->restore_time / s->restore_count : 0); 5246 } 5247 #else 5248 void tcg_dump_info(GString *buf) 5249 { 5250 g_string_append_printf(buf, "[TCG profiler not compiled]\n"); 5251 } 5252 #endif 5253 5254 #ifdef ELF_HOST_MACHINE 5255 /* In order to use this feature, the backend needs to do three things: 5256 5257 (1) Define ELF_HOST_MACHINE to indicate both what value to 5258 put into the ELF image and to indicate support for the feature. 5259 5260 (2) Define tcg_register_jit. This should create a buffer containing 5261 the contents of a .debug_frame section that describes the post- 5262 prologue unwind info for the tcg machine. 5263 5264 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 5265 */ 5266 5267 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 5268 typedef enum { 5269 JIT_NOACTION = 0, 5270 JIT_REGISTER_FN, 5271 JIT_UNREGISTER_FN 5272 } jit_actions_t; 5273 5274 struct jit_code_entry { 5275 struct jit_code_entry *next_entry; 5276 struct jit_code_entry *prev_entry; 5277 const void *symfile_addr; 5278 uint64_t symfile_size; 5279 }; 5280 5281 struct jit_descriptor { 5282 uint32_t version; 5283 uint32_t action_flag; 5284 struct jit_code_entry *relevant_entry; 5285 struct jit_code_entry *first_entry; 5286 }; 5287 5288 void __jit_debug_register_code(void) __attribute__((noinline)); 5289 void __jit_debug_register_code(void) 5290 { 5291 asm(""); 5292 } 5293 5294 /* Must statically initialize the version, because GDB may check 5295 the version before we can set it. */ 5296 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 5297 5298 /* End GDB interface. */ 5299 5300 static int find_string(const char *strtab, const char *str) 5301 { 5302 const char *p = strtab + 1; 5303 5304 while (1) { 5305 if (strcmp(p, str) == 0) { 5306 return p - strtab; 5307 } 5308 p += strlen(p) + 1; 5309 } 5310 } 5311 5312 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 5313 const void *debug_frame, 5314 size_t debug_frame_size) 5315 { 5316 struct __attribute__((packed)) DebugInfo { 5317 uint32_t len; 5318 uint16_t version; 5319 uint32_t abbrev; 5320 uint8_t ptr_size; 5321 uint8_t cu_die; 5322 uint16_t cu_lang; 5323 uintptr_t cu_low_pc; 5324 uintptr_t cu_high_pc; 5325 uint8_t fn_die; 5326 char fn_name[16]; 5327 uintptr_t fn_low_pc; 5328 uintptr_t fn_high_pc; 5329 uint8_t cu_eoc; 5330 }; 5331 5332 struct ElfImage { 5333 ElfW(Ehdr) ehdr; 5334 ElfW(Phdr) phdr; 5335 ElfW(Shdr) shdr[7]; 5336 ElfW(Sym) sym[2]; 5337 struct DebugInfo di; 5338 uint8_t da[24]; 5339 char str[80]; 5340 }; 5341 5342 struct ElfImage *img; 5343 5344 static const struct ElfImage img_template = { 5345 .ehdr = { 5346 .e_ident[EI_MAG0] = ELFMAG0, 5347 .e_ident[EI_MAG1] = ELFMAG1, 5348 .e_ident[EI_MAG2] = ELFMAG2, 5349 .e_ident[EI_MAG3] = ELFMAG3, 5350 .e_ident[EI_CLASS] = ELF_CLASS, 5351 .e_ident[EI_DATA] = ELF_DATA, 5352 .e_ident[EI_VERSION] = EV_CURRENT, 5353 .e_type = ET_EXEC, 5354 .e_machine = ELF_HOST_MACHINE, 5355 .e_version = EV_CURRENT, 5356 .e_phoff = offsetof(struct ElfImage, phdr), 5357 .e_shoff = offsetof(struct ElfImage, shdr), 5358 .e_ehsize = sizeof(ElfW(Shdr)), 5359 .e_phentsize = sizeof(ElfW(Phdr)), 5360 .e_phnum = 1, 5361 .e_shentsize = sizeof(ElfW(Shdr)), 5362 .e_shnum = ARRAY_SIZE(img->shdr), 5363 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 5364 #ifdef ELF_HOST_FLAGS 5365 .e_flags = ELF_HOST_FLAGS, 5366 #endif 5367 #ifdef ELF_OSABI 5368 .e_ident[EI_OSABI] = ELF_OSABI, 5369 #endif 5370 }, 5371 .phdr = { 5372 .p_type = PT_LOAD, 5373 .p_flags = PF_X, 5374 }, 5375 .shdr = { 5376 [0] = { .sh_type = SHT_NULL }, 5377 /* Trick: The contents of code_gen_buffer are not present in 5378 this fake ELF file; that got allocated elsewhere. Therefore 5379 we mark .text as SHT_NOBITS (similar to .bss) so that readers 5380 will not look for contents. We can record any address. */ 5381 [1] = { /* .text */ 5382 .sh_type = SHT_NOBITS, 5383 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 5384 }, 5385 [2] = { /* .debug_info */ 5386 .sh_type = SHT_PROGBITS, 5387 .sh_offset = offsetof(struct ElfImage, di), 5388 .sh_size = sizeof(struct DebugInfo), 5389 }, 5390 [3] = { /* .debug_abbrev */ 5391 .sh_type = SHT_PROGBITS, 5392 .sh_offset = offsetof(struct ElfImage, da), 5393 .sh_size = sizeof(img->da), 5394 }, 5395 [4] = { /* .debug_frame */ 5396 .sh_type = SHT_PROGBITS, 5397 .sh_offset = sizeof(struct ElfImage), 5398 }, 5399 [5] = { /* .symtab */ 5400 .sh_type = SHT_SYMTAB, 5401 .sh_offset = offsetof(struct ElfImage, sym), 5402 .sh_size = sizeof(img->sym), 5403 .sh_info = 1, 5404 .sh_link = ARRAY_SIZE(img->shdr) - 1, 5405 .sh_entsize = sizeof(ElfW(Sym)), 5406 }, 5407 [6] = { /* .strtab */ 5408 .sh_type = SHT_STRTAB, 5409 .sh_offset = offsetof(struct ElfImage, str), 5410 .sh_size = sizeof(img->str), 5411 } 5412 }, 5413 .sym = { 5414 [1] = { /* code_gen_buffer */ 5415 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 5416 .st_shndx = 1, 5417 } 5418 }, 5419 .di = { 5420 .len = sizeof(struct DebugInfo) - 4, 5421 .version = 2, 5422 .ptr_size = sizeof(void *), 5423 .cu_die = 1, 5424 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 5425 .fn_die = 2, 5426 .fn_name = "code_gen_buffer" 5427 }, 5428 .da = { 5429 1, /* abbrev number (the cu) */ 5430 0x11, 1, /* DW_TAG_compile_unit, has children */ 5431 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 5432 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 5433 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 5434 0, 0, /* end of abbrev */ 5435 2, /* abbrev number (the fn) */ 5436 0x2e, 0, /* DW_TAG_subprogram, no children */ 5437 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 5438 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 5439 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 5440 0, 0, /* end of abbrev */ 5441 0 /* no more abbrev */ 5442 }, 5443 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 5444 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 5445 }; 5446 5447 /* We only need a single jit entry; statically allocate it. */ 5448 static struct jit_code_entry one_entry; 5449 5450 uintptr_t buf = (uintptr_t)buf_ptr; 5451 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 5452 DebugFrameHeader *dfh; 5453 5454 img = g_malloc(img_size); 5455 *img = img_template; 5456 5457 img->phdr.p_vaddr = buf; 5458 img->phdr.p_paddr = buf; 5459 img->phdr.p_memsz = buf_size; 5460 5461 img->shdr[1].sh_name = find_string(img->str, ".text"); 5462 img->shdr[1].sh_addr = buf; 5463 img->shdr[1].sh_size = buf_size; 5464 5465 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 5466 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 5467 5468 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 5469 img->shdr[4].sh_size = debug_frame_size; 5470 5471 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 5472 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 5473 5474 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 5475 img->sym[1].st_value = buf; 5476 img->sym[1].st_size = buf_size; 5477 5478 img->di.cu_low_pc = buf; 5479 img->di.cu_high_pc = buf + buf_size; 5480 img->di.fn_low_pc = buf; 5481 img->di.fn_high_pc = buf + buf_size; 5482 5483 dfh = (DebugFrameHeader *)(img + 1); 5484 memcpy(dfh, debug_frame, debug_frame_size); 5485 dfh->fde.func_start = buf; 5486 dfh->fde.func_len = buf_size; 5487 5488 #ifdef DEBUG_JIT 5489 /* Enable this block to be able to debug the ELF image file creation. 5490 One can use readelf, objdump, or other inspection utilities. */ 5491 { 5492 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 5493 FILE *f = fopen(jit, "w+b"); 5494 if (f) { 5495 if (fwrite(img, img_size, 1, f) != img_size) { 5496 /* Avoid stupid unused return value warning for fwrite. */ 5497 } 5498 fclose(f); 5499 } 5500 } 5501 #endif 5502 5503 one_entry.symfile_addr = img; 5504 one_entry.symfile_size = img_size; 5505 5506 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 5507 __jit_debug_descriptor.relevant_entry = &one_entry; 5508 __jit_debug_descriptor.first_entry = &one_entry; 5509 __jit_debug_register_code(); 5510 } 5511 #else 5512 /* No support for the feature. Provide the entry point expected by exec.c, 5513 and implement the internal function we declared earlier. */ 5514 5515 static void tcg_register_jit_int(const void *buf, size_t size, 5516 const void *debug_frame, 5517 size_t debug_frame_size) 5518 { 5519 } 5520 5521 void tcg_register_jit(const void *buf, size_t buf_size) 5522 { 5523 } 5524 #endif /* ELF_HOST_MACHINE */ 5525 5526 #if !TCG_TARGET_MAYBE_vec 5527 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 5528 { 5529 g_assert_not_reached(); 5530 } 5531 #endif 5532