1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 22 #include "trace.h" 23 #include "disas/disas.h" 24 #include "tcg/tcg.h" 25 #include "exec/mmap-lock.h" 26 #include "tb-internal.h" 27 #include "exec/tb-flush.h" 28 #include "qemu/cacheinfo.h" 29 #include "qemu/target-info.h" 30 #include "exec/log.h" 31 #include "exec/icount.h" 32 #include "accel/tcg/cpu-ops.h" 33 #include "tb-jmp-cache.h" 34 #include "tb-hash.h" 35 #include "tb-context.h" 36 #include "tb-internal.h" 37 #include "internal-common.h" 38 #include "tcg/perf.h" 39 #include "tcg/insn-start-words.h" 40 41 TBContext tb_ctx; 42 43 /* 44 * Encode VAL as a signed leb128 sequence at P. 45 * Return P incremented past the encoded value. 46 */ 47 static uint8_t *encode_sleb128(uint8_t *p, int64_t val) 48 { 49 int more, byte; 50 51 do { 52 byte = val & 0x7f; 53 val >>= 7; 54 more = !((val == 0 && (byte & 0x40) == 0) 55 || (val == -1 && (byte & 0x40) != 0)); 56 if (more) { 57 byte |= 0x80; 58 } 59 *p++ = byte; 60 } while (more); 61 62 return p; 63 } 64 65 /* 66 * Decode a signed leb128 sequence at *PP; increment *PP past the 67 * decoded value. Return the decoded value. 68 */ 69 static int64_t decode_sleb128(const uint8_t **pp) 70 { 71 const uint8_t *p = *pp; 72 int64_t val = 0; 73 int byte, shift = 0; 74 75 do { 76 byte = *p++; 77 val |= (int64_t)(byte & 0x7f) << shift; 78 shift += 7; 79 } while (byte & 0x80); 80 if (shift < 64 && (byte & 0x40)) { 81 val |= -(int64_t)1 << shift; 82 } 83 84 *pp = p; 85 return val; 86 } 87 88 /* Encode the data collected about the instructions while compiling TB. 89 Place the data at BLOCK, and return the number of bytes consumed. 90 91 The logical table consists of INSN_START_WORDS uint64_t's, 92 which come from the target's insn_start data, followed by a uintptr_t 93 which comes from the host pc of the end of the code implementing the insn. 94 95 Each line of the table is encoded as sleb128 deltas from the previous 96 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 97 That is, the first column is seeded with the guest pc, the last column 98 with the host pc, and the middle columns with zeros. */ 99 100 static int encode_search(TranslationBlock *tb, uint8_t *block) 101 { 102 uint8_t *highwater = tcg_ctx->code_gen_highwater; 103 uint64_t *insn_data = tcg_ctx->gen_insn_data; 104 uint16_t *insn_end_off = tcg_ctx->gen_insn_end_off; 105 uint8_t *p = block; 106 int i, j, n; 107 108 for (i = 0, n = tb->icount; i < n; ++i) { 109 uint64_t prev, curr; 110 111 for (j = 0; j < INSN_START_WORDS; ++j) { 112 if (i == 0) { 113 prev = (!(tb_cflags(tb) & CF_PCREL) && j == 0 ? tb->pc : 0); 114 } else { 115 prev = insn_data[(i - 1) * INSN_START_WORDS + j]; 116 } 117 curr = insn_data[i * INSN_START_WORDS + j]; 118 p = encode_sleb128(p, curr - prev); 119 } 120 prev = (i == 0 ? 0 : insn_end_off[i - 1]); 121 curr = insn_end_off[i]; 122 p = encode_sleb128(p, curr - prev); 123 124 /* Test for (pending) buffer overflow. The assumption is that any 125 one row beginning below the high water mark cannot overrun 126 the buffer completely. Thus we can test for overflow after 127 encoding a row without having to check during encoding. */ 128 if (unlikely(p > highwater)) { 129 return -1; 130 } 131 } 132 133 return p - block; 134 } 135 136 static int cpu_unwind_data_from_tb(TranslationBlock *tb, uintptr_t host_pc, 137 uint64_t *data) 138 { 139 uintptr_t iter_pc = (uintptr_t)tb->tc.ptr; 140 const uint8_t *p = tb->tc.ptr + tb->tc.size; 141 int i, j, num_insns = tb->icount; 142 143 host_pc -= GETPC_ADJ; 144 145 if (host_pc < iter_pc) { 146 return -1; 147 } 148 149 memset(data, 0, sizeof(uint64_t) * INSN_START_WORDS); 150 if (!(tb_cflags(tb) & CF_PCREL)) { 151 data[0] = tb->pc; 152 } 153 154 /* 155 * Reconstruct the stored insn data while looking for the point 156 * at which the end of the insn exceeds host_pc. 157 */ 158 for (i = 0; i < num_insns; ++i) { 159 for (j = 0; j < INSN_START_WORDS; ++j) { 160 data[j] += decode_sleb128(&p); 161 } 162 iter_pc += decode_sleb128(&p); 163 if (iter_pc > host_pc) { 164 return num_insns - i; 165 } 166 } 167 return -1; 168 } 169 170 /* 171 * The cpu state corresponding to 'host_pc' is restored in 172 * preparation for exiting the TB. 173 */ 174 void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 175 uintptr_t host_pc) 176 { 177 uint64_t data[INSN_START_WORDS]; 178 int insns_left = cpu_unwind_data_from_tb(tb, host_pc, data); 179 180 if (insns_left < 0) { 181 return; 182 } 183 184 if (tb_cflags(tb) & CF_USE_ICOUNT) { 185 assert(icount_enabled()); 186 /* 187 * Reset the cycle counter to the start of the block and 188 * shift if to the number of actually executed instructions. 189 */ 190 cpu->neg.icount_decr.u16.low += insns_left; 191 } 192 193 cpu->cc->tcg_ops->restore_state_to_opc(cpu, tb, data); 194 } 195 196 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc) 197 { 198 /* 199 * The host_pc has to be in the rx region of the code buffer. 200 * If it is not we will not be able to resolve it here. 201 * The two cases where host_pc will not be correct are: 202 * 203 * - fault during translation (instruction fetch) 204 * - fault from helper (not using GETPC() macro) 205 * 206 * Either way we need return early as we can't resolve it here. 207 */ 208 if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { 209 TranslationBlock *tb = tcg_tb_lookup(host_pc); 210 if (tb) { 211 cpu_restore_state_from_tb(cpu, tb, host_pc); 212 return true; 213 } 214 } 215 return false; 216 } 217 218 bool cpu_unwind_state_data(CPUState *cpu, uintptr_t host_pc, uint64_t *data) 219 { 220 if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { 221 TranslationBlock *tb = tcg_tb_lookup(host_pc); 222 if (tb) { 223 return cpu_unwind_data_from_tb(tb, host_pc, data) >= 0; 224 } 225 } 226 return false; 227 } 228 229 void page_init(void) 230 { 231 page_table_config_init(); 232 } 233 234 /* 235 * Isolate the portion of code gen which can setjmp/longjmp. 236 * Return the size of the generated code, or negative on error. 237 */ 238 static int setjmp_gen_code(CPUArchState *env, TranslationBlock *tb, 239 vaddr pc, void *host_pc, 240 int *max_insns, int64_t *ti) 241 { 242 int ret = sigsetjmp(tcg_ctx->jmp_trans, 0); 243 if (unlikely(ret != 0)) { 244 return ret; 245 } 246 247 tcg_func_start(tcg_ctx); 248 249 CPUState *cs = env_cpu(env); 250 tcg_ctx->cpu = cs; 251 cs->cc->tcg_ops->translate_code(cs, tb, max_insns, pc, host_pc); 252 253 assert(tb->size != 0); 254 tcg_ctx->cpu = NULL; 255 *max_insns = tb->icount; 256 257 return tcg_gen_code(tcg_ctx, tb, pc); 258 } 259 260 /* Called with mmap_lock held for user mode emulation. */ 261 TranslationBlock *tb_gen_code(CPUState *cpu, TCGTBCPUState s) 262 { 263 CPUArchState *env = cpu_env(cpu); 264 TranslationBlock *tb, *existing_tb; 265 tb_page_addr_t phys_pc, phys_p2; 266 tcg_insn_unit *gen_code_buf; 267 int gen_code_size, search_size, max_insns; 268 int64_t ti; 269 void *host_pc; 270 271 assert_memory_lock(); 272 qemu_thread_jit_write(); 273 274 phys_pc = get_page_addr_code_hostp(env, s.pc, &host_pc); 275 276 if (phys_pc == -1) { 277 /* Generate a one-shot TB with 1 insn in it */ 278 s.cflags = (s.cflags & ~CF_COUNT_MASK) | 1; 279 } 280 281 max_insns = s.cflags & CF_COUNT_MASK; 282 if (max_insns == 0) { 283 max_insns = TCG_MAX_INSNS; 284 } 285 QEMU_BUILD_BUG_ON(CF_COUNT_MASK + 1 != TCG_MAX_INSNS); 286 287 buffer_overflow: 288 assert_no_pages_locked(); 289 tb = tcg_tb_alloc(tcg_ctx); 290 if (unlikely(!tb)) { 291 /* flush must be done */ 292 if (cpu_in_serial_context(cpu)) { 293 trace_tb_gen_code_buffer_overflow("tcg_tb_alloc"); 294 tb_flush__exclusive_or_serial(); 295 goto buffer_overflow; 296 } 297 queue_tb_flush(cpu); 298 mmap_unlock(); 299 /* Make the execution loop process the flush as soon as possible. */ 300 cpu->exception_index = EXCP_INTERRUPT; 301 cpu_loop_exit(cpu); 302 } 303 304 gen_code_buf = tcg_ctx->code_gen_ptr; 305 tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); 306 if (!(s.cflags & CF_PCREL)) { 307 tb->pc = s.pc; 308 } 309 tb->cs_base = s.cs_base; 310 tb->flags = s.flags; 311 tb->cflags = s.cflags; 312 tb_set_page_addr0(tb, phys_pc); 313 tb_set_page_addr1(tb, -1); 314 if (phys_pc != -1) { 315 tb_lock_page0(phys_pc); 316 } 317 318 tcg_ctx->gen_tb = tb; 319 tcg_ctx->addr_type = target_long_bits() == 32 ? TCG_TYPE_I32 : TCG_TYPE_I64; 320 tcg_ctx->guest_mo = cpu->cc->tcg_ops->guest_default_memory_order; 321 322 restart_translate: 323 trace_translate_block(tb, s.pc, tb->tc.ptr); 324 325 gen_code_size = setjmp_gen_code(env, tb, s.pc, host_pc, &max_insns, &ti); 326 if (unlikely(gen_code_size < 0)) { 327 switch (gen_code_size) { 328 case -1: 329 trace_tb_gen_code_buffer_overflow("setjmp_gen_code"); 330 /* 331 * Overflow of code_gen_buffer, or the current slice of it. 332 * 333 * TODO: We don't need to re-do tcg_ops->translate_code, nor 334 * should we re-do the tcg optimization currently hidden 335 * inside tcg_gen_code. All that should be required is to 336 * flush the TBs, allocate a new TB, re-initialize it per 337 * above, and re-do the actual code generation. 338 */ 339 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 340 "Restarting code generation for " 341 "code_gen_buffer overflow\n"); 342 tb_unlock_pages(tb); 343 tcg_ctx->gen_tb = NULL; 344 goto buffer_overflow; 345 346 case -2: 347 /* 348 * The code generated for the TranslationBlock is too large. 349 * The maximum size allowed by the unwind info is 64k. 350 * There may be stricter constraints from relocations 351 * in the tcg backend. 352 * 353 * Try again with half as many insns as we attempted this time. 354 * If a single insn overflows, there's a bug somewhere... 355 */ 356 assert(max_insns > 1); 357 max_insns /= 2; 358 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 359 "Restarting code generation with " 360 "smaller translation block (max %d insns)\n", 361 max_insns); 362 363 /* 364 * The half-sized TB may not cross pages. 365 * TODO: Fix all targets that cross pages except with 366 * the first insn, at which point this can't be reached. 367 */ 368 phys_p2 = tb_page_addr1(tb); 369 if (unlikely(phys_p2 != -1)) { 370 tb_unlock_page1(phys_pc, phys_p2); 371 tb_set_page_addr1(tb, -1); 372 } 373 goto restart_translate; 374 375 case -3: 376 /* 377 * We had a page lock ordering problem. In order to avoid 378 * deadlock we had to drop the lock on page0, which means 379 * that everything we translated so far is compromised. 380 * Restart with locks held on both pages. 381 */ 382 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 383 "Restarting code generation with re-locked pages"); 384 goto restart_translate; 385 386 default: 387 g_assert_not_reached(); 388 } 389 } 390 tcg_ctx->gen_tb = NULL; 391 392 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 393 if (unlikely(search_size < 0)) { 394 trace_tb_gen_code_buffer_overflow("encode_search"); 395 tb_unlock_pages(tb); 396 goto buffer_overflow; 397 } 398 tb->tc.size = gen_code_size; 399 400 /* 401 * For CF_PCREL, attribute all executions of the generated code 402 * to its first mapping. 403 */ 404 perf_report_code(s.pc, tb, tcg_splitwx_to_rx(gen_code_buf)); 405 406 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 407 qemu_log_in_addr_range(s.pc)) { 408 FILE *logfile = qemu_log_trylock(); 409 if (logfile) { 410 int code_size, data_size; 411 const tcg_target_ulong *rx_data_gen_ptr; 412 size_t chunk_start; 413 int insn = 0; 414 415 if (tcg_ctx->data_gen_ptr) { 416 rx_data_gen_ptr = tcg_splitwx_to_rx(tcg_ctx->data_gen_ptr); 417 code_size = (const void *)rx_data_gen_ptr - tb->tc.ptr; 418 data_size = gen_code_size - code_size; 419 } else { 420 rx_data_gen_ptr = 0; 421 code_size = gen_code_size; 422 data_size = 0; 423 } 424 425 /* Dump header and the first instruction */ 426 fprintf(logfile, "OUT: [size=%d]\n", gen_code_size); 427 fprintf(logfile, 428 " -- guest addr 0x%016" PRIx64 " + tb prologue\n", 429 tcg_ctx->gen_insn_data[insn * INSN_START_WORDS]); 430 chunk_start = tcg_ctx->gen_insn_end_off[insn]; 431 disas(logfile, tb->tc.ptr, chunk_start); 432 433 /* 434 * Dump each instruction chunk, wrapping up empty chunks into 435 * the next instruction. The whole array is offset so the 436 * first entry is the beginning of the 2nd instruction. 437 */ 438 while (insn < tb->icount) { 439 size_t chunk_end = tcg_ctx->gen_insn_end_off[insn]; 440 if (chunk_end > chunk_start) { 441 fprintf(logfile, " -- guest addr 0x%016" PRIx64 "\n", 442 tcg_ctx->gen_insn_data[insn * INSN_START_WORDS]); 443 disas(logfile, tb->tc.ptr + chunk_start, 444 chunk_end - chunk_start); 445 chunk_start = chunk_end; 446 } 447 insn++; 448 } 449 450 if (chunk_start < code_size) { 451 fprintf(logfile, " -- tb slow paths + alignment\n"); 452 disas(logfile, tb->tc.ptr + chunk_start, 453 code_size - chunk_start); 454 } 455 456 /* Finally dump any data we may have after the block */ 457 if (data_size) { 458 int i; 459 fprintf(logfile, " data: [size=%d]\n", data_size); 460 for (i = 0; i < data_size / sizeof(tcg_target_ulong); i++) { 461 if (sizeof(tcg_target_ulong) == 8) { 462 fprintf(logfile, 463 "0x%08" PRIxPTR ": .quad 0x%016" TCG_PRIlx "\n", 464 (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]); 465 } else if (sizeof(tcg_target_ulong) == 4) { 466 fprintf(logfile, 467 "0x%08" PRIxPTR ": .long 0x%08" TCG_PRIlx "\n", 468 (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]); 469 } else { 470 qemu_build_not_reached(); 471 } 472 } 473 } 474 fprintf(logfile, "\n"); 475 qemu_log_unlock(logfile); 476 } 477 } 478 479 qatomic_set(&tcg_ctx->code_gen_ptr, (void *) 480 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 481 CODE_GEN_ALIGN)); 482 483 /* init jump list */ 484 qemu_spin_init(&tb->jmp_lock); 485 tb->jmp_list_head = (uintptr_t)NULL; 486 tb->jmp_list_next[0] = (uintptr_t)NULL; 487 tb->jmp_list_next[1] = (uintptr_t)NULL; 488 tb->jmp_dest[0] = (uintptr_t)NULL; 489 tb->jmp_dest[1] = (uintptr_t)NULL; 490 491 /* init original jump addresses which have been set during tcg_gen_code() */ 492 if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) { 493 tb_reset_jump(tb, 0); 494 } 495 if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) { 496 tb_reset_jump(tb, 1); 497 } 498 499 /* 500 * Insert TB into the corresponding region tree before publishing it 501 * through QHT. Otherwise rewinding happened in the TB might fail to 502 * lookup itself using host PC. 503 */ 504 tcg_tb_insert(tb); 505 506 /* 507 * If the TB is not associated with a physical RAM page then it must be 508 * a temporary one-insn TB. 509 * 510 * Such TBs must be added to region trees in order to make sure that 511 * restore_state_to_opc() - which on some architectures is not limited to 512 * rewinding, but also affects exception handling! - is called when such a 513 * TB causes an exception. 514 * 515 * At the same time, temporary one-insn TBs must be executed at most once, 516 * because subsequent reads from, e.g., I/O memory may return different 517 * values. So return early before attempting to link to other TBs or add 518 * to the QHT. 519 */ 520 if (tb_page_addr0(tb) == -1) { 521 assert_no_pages_locked(); 522 return tb; 523 } 524 525 /* 526 * No explicit memory barrier is required -- tb_link_page() makes the 527 * TB visible in a consistent state. 528 */ 529 existing_tb = tb_link_page(tb); 530 assert_no_pages_locked(); 531 532 /* if the TB already exists, discard what we just translated */ 533 if (unlikely(existing_tb != tb)) { 534 uintptr_t orig_aligned = (uintptr_t)gen_code_buf; 535 536 orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize); 537 qatomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned); 538 tcg_tb_remove(tb); 539 return existing_tb; 540 } 541 return tb; 542 } 543 544 /* user-mode: call with mmap_lock held */ 545 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr) 546 { 547 TranslationBlock *tb; 548 549 assert_memory_lock(); 550 551 tb = tcg_tb_lookup(retaddr); 552 if (tb) { 553 /* We can use retranslation to find the PC. */ 554 cpu_restore_state_from_tb(cpu, tb, retaddr); 555 tb_phys_invalidate(tb, -1); 556 } else { 557 /* The exception probably happened in a helper. The CPU state should 558 have been saved before calling it. Fetch the PC from there. */ 559 CPUArchState *env = cpu_env(cpu); 560 TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu); 561 tb_page_addr_t addr = get_page_addr_code(env, s.pc); 562 563 if (addr != -1) { 564 tb_invalidate_phys_range(cpu, addr, addr); 565 } 566 } 567 } 568 569 #ifndef CONFIG_USER_ONLY 570 /* 571 * In deterministic execution mode, instructions doing device I/Os 572 * must be at the end of the TB. 573 * 574 * Called by softmmu_template.h, with iothread mutex not held. 575 */ 576 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 577 { 578 TranslationBlock *tb; 579 CPUClass *cc; 580 uint32_t n; 581 582 tb = tcg_tb_lookup(retaddr); 583 if (!tb) { 584 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 585 (void *)retaddr); 586 } 587 cpu_restore_state_from_tb(cpu, tb, retaddr); 588 589 /* 590 * Some guests must re-execute the branch when re-executing a delay 591 * slot instruction. When this is the case, adjust icount and N 592 * to account for the re-execution of the branch. 593 */ 594 n = 1; 595 cc = cpu->cc; 596 if (cc->tcg_ops->io_recompile_replay_branch && 597 cc->tcg_ops->io_recompile_replay_branch(cpu, tb)) { 598 cpu->neg.icount_decr.u16.low++; 599 n = 2; 600 } 601 602 /* 603 * Exit the loop and potentially generate a new TB executing the 604 * just the I/O insns. We also limit instrumentation to memory 605 * operations only (which execute after completion) so we don't 606 * double instrument the instruction. Also don't let an IRQ sneak 607 * in before we execute it. 608 */ 609 cpu->cflags_next_tb = curr_cflags(cpu) | CF_MEMI_ONLY | CF_NOIRQ | n; 610 611 if (qemu_loglevel_mask(CPU_LOG_EXEC)) { 612 vaddr pc = cpu->cc->get_pc(cpu); 613 if (qemu_log_in_addr_range(pc)) { 614 qemu_log("cpu_io_recompile: rewound execution of TB to %016" 615 VADDR_PRIx "\n", pc); 616 } 617 } 618 619 cpu_loop_exit_noexc(cpu); 620 } 621 622 #endif /* CONFIG_USER_ONLY */ 623 624 /* 625 * Called by generic code at e.g. cpu reset after cpu creation, 626 * therefore we must be prepared to allocate the jump cache. 627 */ 628 void tcg_flush_jmp_cache(CPUState *cpu) 629 { 630 CPUJumpCache *jc = cpu->tb_jmp_cache; 631 632 /* During early initialization, the cache may not yet be allocated. */ 633 if (unlikely(jc == NULL)) { 634 return; 635 } 636 637 for (int i = 0; i < TB_JMP_CACHE_SIZE; i++) { 638 qatomic_set(&jc->array[i].tb, NULL); 639 } 640 } 641