1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 35 #include <sys/param.h> 36 #if __FreeBSD_version >= 700104 37 #define HAVE_KINFO_GETVMMAP 38 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 39 #include <sys/proc.h> 40 #include <machine/profile.h> 41 #define _KERNEL 42 #include <sys/user.h> 43 #undef _KERNEL 44 #undef sigqueue 45 #include <libutil.h> 46 #endif 47 #endif 48 #else 49 #include "exec/address-spaces.h" 50 #endif 51 52 #include "exec/cputlb.h" 53 #include "exec/tb-hash.h" 54 #include "translate-all.h" 55 #include "qemu/bitmap.h" 56 #include "qemu/error-report.h" 57 #include "qemu/timer.h" 58 #include "qemu/main-loop.h" 59 #include "exec/log.h" 60 #include "sysemu/cpus.h" 61 62 /* #define DEBUG_TB_INVALIDATE */ 63 /* #define DEBUG_TB_FLUSH */ 64 /* make various TB consistency checks */ 65 /* #define DEBUG_TB_CHECK */ 66 67 #ifdef DEBUG_TB_INVALIDATE 68 #define DEBUG_TB_INVALIDATE_GATE 1 69 #else 70 #define DEBUG_TB_INVALIDATE_GATE 0 71 #endif 72 73 #ifdef DEBUG_TB_FLUSH 74 #define DEBUG_TB_FLUSH_GATE 1 75 #else 76 #define DEBUG_TB_FLUSH_GATE 0 77 #endif 78 79 #if !defined(CONFIG_USER_ONLY) 80 /* TB consistency checks only implemented for usermode emulation. */ 81 #undef DEBUG_TB_CHECK 82 #endif 83 84 #ifdef DEBUG_TB_CHECK 85 #define DEBUG_TB_CHECK_GATE 1 86 #else 87 #define DEBUG_TB_CHECK_GATE 0 88 #endif 89 90 /* Access to the various translations structures need to be serialised via locks 91 * for consistency. This is automatic for SoftMMU based system 92 * emulation due to its single threaded nature. In user-mode emulation 93 * access to the memory related structures are protected with the 94 * mmap_lock. 95 */ 96 #ifdef CONFIG_SOFTMMU 97 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 98 #else 99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 100 #endif 101 102 #define SMC_BITMAP_USE_THRESHOLD 10 103 104 typedef struct PageDesc { 105 /* list of TBs intersecting this ram page */ 106 TranslationBlock *first_tb; 107 #ifdef CONFIG_SOFTMMU 108 /* in order to optimize self modifying code, we count the number 109 of lookups we do to a given page to use a bitmap */ 110 unsigned int code_write_count; 111 unsigned long *code_bitmap; 112 #else 113 unsigned long flags; 114 #endif 115 } PageDesc; 116 117 /* In system mode we want L1_MAP to be based on ram offsets, 118 while in user mode we want it to be based on virtual addresses. */ 119 #if !defined(CONFIG_USER_ONLY) 120 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 121 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 122 #else 123 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 124 #endif 125 #else 126 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 127 #endif 128 129 /* Size of the L2 (and L3, etc) page tables. */ 130 #define V_L2_BITS 10 131 #define V_L2_SIZE (1 << V_L2_BITS) 132 133 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 134 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 135 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 136 * BITS_PER_BYTE); 137 138 /* 139 * L1 Mapping properties 140 */ 141 static int v_l1_size; 142 static int v_l1_shift; 143 static int v_l2_levels; 144 145 /* The bottom level has pointers to PageDesc, and is indexed by 146 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 147 */ 148 #define V_L1_MIN_BITS 4 149 #define V_L1_MAX_BITS (V_L2_BITS + 3) 150 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 151 152 static void *l1_map[V_L1_MAX_SIZE]; 153 154 /* code generation context */ 155 TCGContext tcg_init_ctx; 156 __thread TCGContext *tcg_ctx; 157 TBContext tb_ctx; 158 bool parallel_cpus; 159 160 /* translation block context */ 161 static __thread int have_tb_lock; 162 163 static void page_table_config_init(void) 164 { 165 uint32_t v_l1_bits; 166 167 assert(TARGET_PAGE_BITS); 168 /* The bits remaining after N lower levels of page tables. */ 169 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 170 if (v_l1_bits < V_L1_MIN_BITS) { 171 v_l1_bits += V_L2_BITS; 172 } 173 174 v_l1_size = 1 << v_l1_bits; 175 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 176 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 177 178 assert(v_l1_bits <= V_L1_MAX_BITS); 179 assert(v_l1_shift % V_L2_BITS == 0); 180 assert(v_l2_levels >= 0); 181 } 182 183 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 184 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 185 186 void tb_lock(void) 187 { 188 assert_tb_unlocked(); 189 qemu_mutex_lock(&tb_ctx.tb_lock); 190 have_tb_lock++; 191 } 192 193 void tb_unlock(void) 194 { 195 assert_tb_locked(); 196 have_tb_lock--; 197 qemu_mutex_unlock(&tb_ctx.tb_lock); 198 } 199 200 void tb_lock_reset(void) 201 { 202 if (have_tb_lock) { 203 qemu_mutex_unlock(&tb_ctx.tb_lock); 204 have_tb_lock = 0; 205 } 206 } 207 208 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr); 209 210 void cpu_gen_init(void) 211 { 212 tcg_context_init(&tcg_init_ctx); 213 } 214 215 /* Encode VAL as a signed leb128 sequence at P. 216 Return P incremented past the encoded value. */ 217 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 218 { 219 int more, byte; 220 221 do { 222 byte = val & 0x7f; 223 val >>= 7; 224 more = !((val == 0 && (byte & 0x40) == 0) 225 || (val == -1 && (byte & 0x40) != 0)); 226 if (more) { 227 byte |= 0x80; 228 } 229 *p++ = byte; 230 } while (more); 231 232 return p; 233 } 234 235 /* Decode a signed leb128 sequence at *PP; increment *PP past the 236 decoded value. Return the decoded value. */ 237 static target_long decode_sleb128(uint8_t **pp) 238 { 239 uint8_t *p = *pp; 240 target_long val = 0; 241 int byte, shift = 0; 242 243 do { 244 byte = *p++; 245 val |= (target_ulong)(byte & 0x7f) << shift; 246 shift += 7; 247 } while (byte & 0x80); 248 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 249 val |= -(target_ulong)1 << shift; 250 } 251 252 *pp = p; 253 return val; 254 } 255 256 /* Encode the data collected about the instructions while compiling TB. 257 Place the data at BLOCK, and return the number of bytes consumed. 258 259 The logical table consists of TARGET_INSN_START_WORDS target_ulong's, 260 which come from the target's insn_start data, followed by a uintptr_t 261 which comes from the host pc of the end of the code implementing the insn. 262 263 Each line of the table is encoded as sleb128 deltas from the previous 264 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 265 That is, the first column is seeded with the guest pc, the last column 266 with the host pc, and the middle columns with zeros. */ 267 268 static int encode_search(TranslationBlock *tb, uint8_t *block) 269 { 270 uint8_t *highwater = tcg_ctx->code_gen_highwater; 271 uint8_t *p = block; 272 int i, j, n; 273 274 for (i = 0, n = tb->icount; i < n; ++i) { 275 target_ulong prev; 276 277 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 278 if (i == 0) { 279 prev = (j == 0 ? tb->pc : 0); 280 } else { 281 prev = tcg_ctx->gen_insn_data[i - 1][j]; 282 } 283 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 284 } 285 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 286 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 287 288 /* Test for (pending) buffer overflow. The assumption is that any 289 one row beginning below the high water mark cannot overrun 290 the buffer completely. Thus we can test for overflow after 291 encoding a row without having to check during encoding. */ 292 if (unlikely(p > highwater)) { 293 return -1; 294 } 295 } 296 297 return p - block; 298 } 299 300 /* The cpu state corresponding to 'searched_pc' is restored. 301 * Called with tb_lock held. 302 * When reset_icount is true, current TB will be interrupted and 303 * icount should be recalculated. 304 */ 305 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 306 uintptr_t searched_pc, bool reset_icount) 307 { 308 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 309 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 310 CPUArchState *env = cpu->env_ptr; 311 uint8_t *p = tb->tc.ptr + tb->tc.size; 312 int i, j, num_insns = tb->icount; 313 #ifdef CONFIG_PROFILER 314 TCGProfile *prof = &tcg_ctx->prof; 315 int64_t ti = profile_getclock(); 316 #endif 317 318 searched_pc -= GETPC_ADJ; 319 320 if (searched_pc < host_pc) { 321 return -1; 322 } 323 324 /* Reconstruct the stored insn data while looking for the point at 325 which the end of the insn exceeds the searched_pc. */ 326 for (i = 0; i < num_insns; ++i) { 327 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 328 data[j] += decode_sleb128(&p); 329 } 330 host_pc += decode_sleb128(&p); 331 if (host_pc > searched_pc) { 332 goto found; 333 } 334 } 335 return -1; 336 337 found: 338 if (reset_icount && (tb->cflags & CF_USE_ICOUNT)) { 339 assert(use_icount); 340 /* Reset the cycle counter to the start of the block 341 and shift if to the number of actually executed instructions */ 342 cpu->icount_decr.u16.low += num_insns - i; 343 } 344 restore_state_to_opc(env, tb, data); 345 346 #ifdef CONFIG_PROFILER 347 atomic_set(&prof->restore_time, 348 prof->restore_time + profile_getclock() - ti); 349 atomic_set(&prof->restore_count, prof->restore_count + 1); 350 #endif 351 return 0; 352 } 353 354 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) 355 { 356 TranslationBlock *tb; 357 bool r = false; 358 uintptr_t check_offset; 359 360 /* The host_pc has to be in the region of current code buffer. If 361 * it is not we will not be able to resolve it here. The two cases 362 * where host_pc will not be correct are: 363 * 364 * - fault during translation (instruction fetch) 365 * - fault from helper (not using GETPC() macro) 366 * 367 * Either way we need return early to avoid blowing up on a 368 * recursive tb_lock() as we can't resolve it here. 369 * 370 * We are using unsigned arithmetic so if host_pc < 371 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way 372 * above the code_gen_buffer_size 373 */ 374 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; 375 376 if (check_offset < tcg_init_ctx.code_gen_buffer_size) { 377 tb_lock(); 378 tb = tb_find_pc(host_pc); 379 if (tb) { 380 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); 381 if (tb->cflags & CF_NOCACHE) { 382 /* one-shot translation, invalidate it immediately */ 383 tb_phys_invalidate(tb, -1); 384 tb_remove(tb); 385 } 386 r = true; 387 } 388 tb_unlock(); 389 } 390 391 return r; 392 } 393 394 static void page_init(void) 395 { 396 page_size_init(); 397 page_table_config_init(); 398 399 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 400 { 401 #ifdef HAVE_KINFO_GETVMMAP 402 struct kinfo_vmentry *freep; 403 int i, cnt; 404 405 freep = kinfo_getvmmap(getpid(), &cnt); 406 if (freep) { 407 mmap_lock(); 408 for (i = 0; i < cnt; i++) { 409 unsigned long startaddr, endaddr; 410 411 startaddr = freep[i].kve_start; 412 endaddr = freep[i].kve_end; 413 if (h2g_valid(startaddr)) { 414 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 415 416 if (h2g_valid(endaddr)) { 417 endaddr = h2g(endaddr); 418 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 419 } else { 420 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 421 endaddr = ~0ul; 422 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 423 #endif 424 } 425 } 426 } 427 free(freep); 428 mmap_unlock(); 429 } 430 #else 431 FILE *f; 432 433 last_brk = (unsigned long)sbrk(0); 434 435 f = fopen("/compat/linux/proc/self/maps", "r"); 436 if (f) { 437 mmap_lock(); 438 439 do { 440 unsigned long startaddr, endaddr; 441 int n; 442 443 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 444 445 if (n == 2 && h2g_valid(startaddr)) { 446 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 447 448 if (h2g_valid(endaddr)) { 449 endaddr = h2g(endaddr); 450 } else { 451 endaddr = ~0ul; 452 } 453 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 454 } 455 } while (!feof(f)); 456 457 fclose(f); 458 mmap_unlock(); 459 } 460 #endif 461 } 462 #endif 463 } 464 465 /* If alloc=1: 466 * Called with tb_lock held for system emulation. 467 * Called with mmap_lock held for user-mode emulation. 468 */ 469 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 470 { 471 PageDesc *pd; 472 void **lp; 473 int i; 474 475 if (alloc) { 476 assert_memory_lock(); 477 } 478 479 /* Level 1. Always allocated. */ 480 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 481 482 /* Level 2..N-1. */ 483 for (i = v_l2_levels; i > 0; i--) { 484 void **p = atomic_rcu_read(lp); 485 486 if (p == NULL) { 487 if (!alloc) { 488 return NULL; 489 } 490 p = g_new0(void *, V_L2_SIZE); 491 atomic_rcu_set(lp, p); 492 } 493 494 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 495 } 496 497 pd = atomic_rcu_read(lp); 498 if (pd == NULL) { 499 if (!alloc) { 500 return NULL; 501 } 502 pd = g_new0(PageDesc, V_L2_SIZE); 503 atomic_rcu_set(lp, pd); 504 } 505 506 return pd + (index & (V_L2_SIZE - 1)); 507 } 508 509 static inline PageDesc *page_find(tb_page_addr_t index) 510 { 511 return page_find_alloc(index, 0); 512 } 513 514 #if defined(CONFIG_USER_ONLY) 515 /* Currently it is not recommended to allocate big chunks of data in 516 user mode. It will change when a dedicated libc will be used. */ 517 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 518 region in which the guest needs to run. Revisit this. */ 519 #define USE_STATIC_CODE_GEN_BUFFER 520 #endif 521 522 /* Minimum size of the code gen buffer. This number is randomly chosen, 523 but not so small that we can't have a fair number of TB's live. */ 524 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 525 526 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 527 indicated, this is constrained by the range of direct branches on the 528 host cpu, as used by the TCG implementation of goto_tb. */ 529 #if defined(__x86_64__) 530 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 531 #elif defined(__sparc__) 532 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 533 #elif defined(__powerpc64__) 534 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 535 #elif defined(__powerpc__) 536 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 537 #elif defined(__aarch64__) 538 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 539 #elif defined(__s390x__) 540 /* We have a +- 4GB range on the branches; leave some slop. */ 541 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 542 #elif defined(__mips__) 543 /* We have a 256MB branch region, but leave room to make sure the 544 main executable is also within that region. */ 545 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 546 #else 547 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 548 #endif 549 550 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 551 552 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 553 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 554 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 555 556 static inline size_t size_code_gen_buffer(size_t tb_size) 557 { 558 /* Size the buffer. */ 559 if (tb_size == 0) { 560 #ifdef USE_STATIC_CODE_GEN_BUFFER 561 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 562 #else 563 /* ??? Needs adjustments. */ 564 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 565 static buffer, we could size this on RESERVED_VA, on the text 566 segment size of the executable, or continue to use the default. */ 567 tb_size = (unsigned long)(ram_size / 4); 568 #endif 569 } 570 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 571 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 572 } 573 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 574 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 575 } 576 return tb_size; 577 } 578 579 #ifdef __mips__ 580 /* In order to use J and JAL within the code_gen_buffer, we require 581 that the buffer not cross a 256MB boundary. */ 582 static inline bool cross_256mb(void *addr, size_t size) 583 { 584 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 585 } 586 587 /* We weren't able to allocate a buffer without crossing that boundary, 588 so make do with the larger portion of the buffer that doesn't cross. 589 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 590 static inline void *split_cross_256mb(void *buf1, size_t size1) 591 { 592 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 593 size_t size2 = buf1 + size1 - buf2; 594 595 size1 = buf2 - buf1; 596 if (size1 < size2) { 597 size1 = size2; 598 buf1 = buf2; 599 } 600 601 tcg_ctx->code_gen_buffer_size = size1; 602 return buf1; 603 } 604 #endif 605 606 #ifdef USE_STATIC_CODE_GEN_BUFFER 607 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 608 __attribute__((aligned(CODE_GEN_ALIGN))); 609 610 static inline void *alloc_code_gen_buffer(void) 611 { 612 void *buf = static_code_gen_buffer; 613 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 614 size_t size; 615 616 /* page-align the beginning and end of the buffer */ 617 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 618 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 619 620 size = end - buf; 621 622 /* Honor a command-line option limiting the size of the buffer. */ 623 if (size > tcg_ctx->code_gen_buffer_size) { 624 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 625 qemu_real_host_page_size); 626 } 627 tcg_ctx->code_gen_buffer_size = size; 628 629 #ifdef __mips__ 630 if (cross_256mb(buf, size)) { 631 buf = split_cross_256mb(buf, size); 632 size = tcg_ctx->code_gen_buffer_size; 633 } 634 #endif 635 636 if (qemu_mprotect_rwx(buf, size)) { 637 abort(); 638 } 639 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 640 641 return buf; 642 } 643 #elif defined(_WIN32) 644 static inline void *alloc_code_gen_buffer(void) 645 { 646 size_t size = tcg_ctx->code_gen_buffer_size; 647 return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 648 PAGE_EXECUTE_READWRITE); 649 } 650 #else 651 static inline void *alloc_code_gen_buffer(void) 652 { 653 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 654 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 655 uintptr_t start = 0; 656 size_t size = tcg_ctx->code_gen_buffer_size; 657 void *buf; 658 659 /* Constrain the position of the buffer based on the host cpu. 660 Note that these addresses are chosen in concert with the 661 addresses assigned in the relevant linker script file. */ 662 # if defined(__PIE__) || defined(__PIC__) 663 /* Don't bother setting a preferred location if we're building 664 a position-independent executable. We're more likely to get 665 an address near the main executable if we let the kernel 666 choose the address. */ 667 # elif defined(__x86_64__) && defined(MAP_32BIT) 668 /* Force the memory down into low memory with the executable. 669 Leave the choice of exact location with the kernel. */ 670 flags |= MAP_32BIT; 671 /* Cannot expect to map more than 800MB in low memory. */ 672 if (size > 800u * 1024 * 1024) { 673 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 674 } 675 # elif defined(__sparc__) 676 start = 0x40000000ul; 677 # elif defined(__s390x__) 678 start = 0x90000000ul; 679 # elif defined(__mips__) 680 # if _MIPS_SIM == _ABI64 681 start = 0x128000000ul; 682 # else 683 start = 0x08000000ul; 684 # endif 685 # endif 686 687 buf = mmap((void *)start, size, prot, flags, -1, 0); 688 if (buf == MAP_FAILED) { 689 return NULL; 690 } 691 692 #ifdef __mips__ 693 if (cross_256mb(buf, size)) { 694 /* Try again, with the original still mapped, to avoid re-acquiring 695 that 256mb crossing. This time don't specify an address. */ 696 size_t size2; 697 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 698 switch ((int)(buf2 != MAP_FAILED)) { 699 case 1: 700 if (!cross_256mb(buf2, size)) { 701 /* Success! Use the new buffer. */ 702 munmap(buf, size); 703 break; 704 } 705 /* Failure. Work with what we had. */ 706 munmap(buf2, size); 707 /* fallthru */ 708 default: 709 /* Split the original buffer. Free the smaller half. */ 710 buf2 = split_cross_256mb(buf, size); 711 size2 = tcg_ctx->code_gen_buffer_size; 712 if (buf == buf2) { 713 munmap(buf + size2, size - size2); 714 } else { 715 munmap(buf, size - size2); 716 } 717 size = size2; 718 break; 719 } 720 buf = buf2; 721 } 722 #endif 723 724 /* Request large pages for the buffer. */ 725 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 726 727 return buf; 728 } 729 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 730 731 /* compare a pointer @ptr and a tb_tc @s */ 732 static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s) 733 { 734 if (ptr >= s->ptr + s->size) { 735 return 1; 736 } else if (ptr < s->ptr) { 737 return -1; 738 } 739 return 0; 740 } 741 742 static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp) 743 { 744 const struct tb_tc *a = ap; 745 const struct tb_tc *b = bp; 746 747 /* 748 * When both sizes are set, we know this isn't a lookup. 749 * This is the most likely case: every TB must be inserted; lookups 750 * are a lot less frequent. 751 */ 752 if (likely(a->size && b->size)) { 753 if (a->ptr > b->ptr) { 754 return 1; 755 } else if (a->ptr < b->ptr) { 756 return -1; 757 } 758 /* a->ptr == b->ptr should happen only on deletions */ 759 g_assert(a->size == b->size); 760 return 0; 761 } 762 /* 763 * All lookups have either .size field set to 0. 764 * From the glib sources we see that @ap is always the lookup key. However 765 * the docs provide no guarantee, so we just mark this case as likely. 766 */ 767 if (likely(a->size == 0)) { 768 return ptr_cmp_tb_tc(a->ptr, b); 769 } 770 return ptr_cmp_tb_tc(b->ptr, a); 771 } 772 773 static inline void code_gen_alloc(size_t tb_size) 774 { 775 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 776 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 777 if (tcg_ctx->code_gen_buffer == NULL) { 778 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 779 exit(1); 780 } 781 tb_ctx.tb_tree = g_tree_new(tb_tc_cmp); 782 qemu_mutex_init(&tb_ctx.tb_lock); 783 } 784 785 static void tb_htable_init(void) 786 { 787 unsigned int mode = QHT_MODE_AUTO_RESIZE; 788 789 qht_init(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode); 790 } 791 792 /* Must be called before using the QEMU cpus. 'tb_size' is the size 793 (in bytes) allocated to the translation buffer. Zero means default 794 size. */ 795 void tcg_exec_init(unsigned long tb_size) 796 { 797 tcg_allowed = true; 798 cpu_gen_init(); 799 page_init(); 800 tb_htable_init(); 801 code_gen_alloc(tb_size); 802 #if defined(CONFIG_SOFTMMU) 803 /* There's no guest base to take into account, so go ahead and 804 initialize the prologue now. */ 805 tcg_prologue_init(tcg_ctx); 806 #endif 807 } 808 809 /* 810 * Allocate a new translation block. Flush the translation buffer if 811 * too many translation blocks or too much generated code. 812 * 813 * Called with tb_lock held. 814 */ 815 static TranslationBlock *tb_alloc(target_ulong pc) 816 { 817 TranslationBlock *tb; 818 819 assert_tb_locked(); 820 821 tb = tcg_tb_alloc(tcg_ctx); 822 if (unlikely(tb == NULL)) { 823 return NULL; 824 } 825 return tb; 826 } 827 828 /* Called with tb_lock held. */ 829 void tb_remove(TranslationBlock *tb) 830 { 831 assert_tb_locked(); 832 833 g_tree_remove(tb_ctx.tb_tree, &tb->tc); 834 } 835 836 static inline void invalidate_page_bitmap(PageDesc *p) 837 { 838 #ifdef CONFIG_SOFTMMU 839 g_free(p->code_bitmap); 840 p->code_bitmap = NULL; 841 p->code_write_count = 0; 842 #endif 843 } 844 845 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 846 static void page_flush_tb_1(int level, void **lp) 847 { 848 int i; 849 850 if (*lp == NULL) { 851 return; 852 } 853 if (level == 0) { 854 PageDesc *pd = *lp; 855 856 for (i = 0; i < V_L2_SIZE; ++i) { 857 pd[i].first_tb = NULL; 858 invalidate_page_bitmap(pd + i); 859 } 860 } else { 861 void **pp = *lp; 862 863 for (i = 0; i < V_L2_SIZE; ++i) { 864 page_flush_tb_1(level - 1, pp + i); 865 } 866 } 867 } 868 869 static void page_flush_tb(void) 870 { 871 int i, l1_sz = v_l1_size; 872 873 for (i = 0; i < l1_sz; i++) { 874 page_flush_tb_1(v_l2_levels, l1_map + i); 875 } 876 } 877 878 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 879 { 880 const TranslationBlock *tb = value; 881 size_t *size = data; 882 883 *size += tb->tc.size; 884 return false; 885 } 886 887 /* flush all the translation blocks */ 888 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 889 { 890 tb_lock(); 891 892 /* If it is already been done on request of another CPU, 893 * just retry. 894 */ 895 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 896 goto done; 897 } 898 899 if (DEBUG_TB_FLUSH_GATE) { 900 size_t nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 901 size_t host_size = 0; 902 903 g_tree_foreach(tb_ctx.tb_tree, tb_host_size_iter, &host_size); 904 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 905 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 906 } 907 908 CPU_FOREACH(cpu) { 909 cpu_tb_jmp_cache_clear(cpu); 910 } 911 912 /* Increment the refcount first so that destroy acts as a reset */ 913 g_tree_ref(tb_ctx.tb_tree); 914 g_tree_destroy(tb_ctx.tb_tree); 915 916 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 917 page_flush_tb(); 918 919 tcg_region_reset_all(); 920 /* XXX: flush processor icache at this point if cache flush is 921 expensive */ 922 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 923 924 done: 925 tb_unlock(); 926 } 927 928 void tb_flush(CPUState *cpu) 929 { 930 if (tcg_enabled()) { 931 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 932 async_safe_run_on_cpu(cpu, do_tb_flush, 933 RUN_ON_CPU_HOST_INT(tb_flush_count)); 934 } 935 } 936 937 /* 938 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 939 * so in order to prevent bit rot we compile them unconditionally in user-mode, 940 * and let the optimizer get rid of them by wrapping their user-only callers 941 * with if (DEBUG_TB_CHECK_GATE). 942 */ 943 #ifdef CONFIG_USER_ONLY 944 945 static void 946 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 947 { 948 TranslationBlock *tb = p; 949 target_ulong addr = *(target_ulong *)userp; 950 951 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 952 printf("ERROR invalidate: address=" TARGET_FMT_lx 953 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 954 } 955 } 956 957 /* verify that all the pages have correct rights for code 958 * 959 * Called with tb_lock held. 960 */ 961 static void tb_invalidate_check(target_ulong address) 962 { 963 address &= TARGET_PAGE_MASK; 964 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 965 } 966 967 static void 968 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 969 { 970 TranslationBlock *tb = p; 971 int flags1, flags2; 972 973 flags1 = page_get_flags(tb->pc); 974 flags2 = page_get_flags(tb->pc + tb->size - 1); 975 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 976 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 977 (long)tb->pc, tb->size, flags1, flags2); 978 } 979 } 980 981 /* verify that all the pages have correct rights for code */ 982 static void tb_page_check(void) 983 { 984 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 985 } 986 987 #endif /* CONFIG_USER_ONLY */ 988 989 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) 990 { 991 TranslationBlock *tb1; 992 unsigned int n1; 993 994 for (;;) { 995 tb1 = *ptb; 996 n1 = (uintptr_t)tb1 & 3; 997 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); 998 if (tb1 == tb) { 999 *ptb = tb1->page_next[n1]; 1000 break; 1001 } 1002 ptb = &tb1->page_next[n1]; 1003 } 1004 } 1005 1006 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 1007 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 1008 { 1009 TranslationBlock *tb1; 1010 uintptr_t *ptb, ntb; 1011 unsigned int n1; 1012 1013 ptb = &tb->jmp_list_next[n]; 1014 if (*ptb) { 1015 /* find tb(n) in circular list */ 1016 for (;;) { 1017 ntb = *ptb; 1018 n1 = ntb & 3; 1019 tb1 = (TranslationBlock *)(ntb & ~3); 1020 if (n1 == n && tb1 == tb) { 1021 break; 1022 } 1023 if (n1 == 2) { 1024 ptb = &tb1->jmp_list_first; 1025 } else { 1026 ptb = &tb1->jmp_list_next[n1]; 1027 } 1028 } 1029 /* now we can suppress tb(n) from the list */ 1030 *ptb = tb->jmp_list_next[n]; 1031 1032 tb->jmp_list_next[n] = (uintptr_t)NULL; 1033 } 1034 } 1035 1036 /* reset the jump entry 'n' of a TB so that it is not chained to 1037 another TB */ 1038 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1039 { 1040 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1041 tb_set_jmp_target(tb, n, addr); 1042 } 1043 1044 /* remove any jumps to the TB */ 1045 static inline void tb_jmp_unlink(TranslationBlock *tb) 1046 { 1047 TranslationBlock *tb1; 1048 uintptr_t *ptb, ntb; 1049 unsigned int n1; 1050 1051 ptb = &tb->jmp_list_first; 1052 for (;;) { 1053 ntb = *ptb; 1054 n1 = ntb & 3; 1055 tb1 = (TranslationBlock *)(ntb & ~3); 1056 if (n1 == 2) { 1057 break; 1058 } 1059 tb_reset_jump(tb1, n1); 1060 *ptb = tb1->jmp_list_next[n1]; 1061 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1062 } 1063 } 1064 1065 /* invalidate one TB 1066 * 1067 * Called with tb_lock held. 1068 */ 1069 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1070 { 1071 CPUState *cpu; 1072 PageDesc *p; 1073 uint32_t h; 1074 tb_page_addr_t phys_pc; 1075 1076 assert_tb_locked(); 1077 1078 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1079 1080 /* remove the TB from the hash list */ 1081 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1082 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1083 tb->trace_vcpu_dstate); 1084 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1085 return; 1086 } 1087 1088 /* remove the TB from the page list */ 1089 if (tb->page_addr[0] != page_addr) { 1090 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1091 tb_page_remove(&p->first_tb, tb); 1092 invalidate_page_bitmap(p); 1093 } 1094 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1095 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1096 tb_page_remove(&p->first_tb, tb); 1097 invalidate_page_bitmap(p); 1098 } 1099 1100 /* remove the TB from the hash list */ 1101 h = tb_jmp_cache_hash_func(tb->pc); 1102 CPU_FOREACH(cpu) { 1103 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1104 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1105 } 1106 } 1107 1108 /* suppress this TB from the two jump lists */ 1109 tb_remove_from_jmp_list(tb, 0); 1110 tb_remove_from_jmp_list(tb, 1); 1111 1112 /* suppress any remaining jumps to this TB */ 1113 tb_jmp_unlink(tb); 1114 1115 tb_ctx.tb_phys_invalidate_count++; 1116 } 1117 1118 #ifdef CONFIG_SOFTMMU 1119 static void build_page_bitmap(PageDesc *p) 1120 { 1121 int n, tb_start, tb_end; 1122 TranslationBlock *tb; 1123 1124 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1125 1126 tb = p->first_tb; 1127 while (tb != NULL) { 1128 n = (uintptr_t)tb & 3; 1129 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1130 /* NOTE: this is subtle as a TB may span two physical pages */ 1131 if (n == 0) { 1132 /* NOTE: tb_end may be after the end of the page, but 1133 it is not a problem */ 1134 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1135 tb_end = tb_start + tb->size; 1136 if (tb_end > TARGET_PAGE_SIZE) { 1137 tb_end = TARGET_PAGE_SIZE; 1138 } 1139 } else { 1140 tb_start = 0; 1141 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1142 } 1143 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1144 tb = tb->page_next[n]; 1145 } 1146 } 1147 #endif 1148 1149 /* add the tb in the target page and protect it if necessary 1150 * 1151 * Called with mmap_lock held for user-mode emulation. 1152 */ 1153 static inline void tb_alloc_page(TranslationBlock *tb, 1154 unsigned int n, tb_page_addr_t page_addr) 1155 { 1156 PageDesc *p; 1157 #ifndef CONFIG_USER_ONLY 1158 bool page_already_protected; 1159 #endif 1160 1161 assert_memory_lock(); 1162 1163 tb->page_addr[n] = page_addr; 1164 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1165 tb->page_next[n] = p->first_tb; 1166 #ifndef CONFIG_USER_ONLY 1167 page_already_protected = p->first_tb != NULL; 1168 #endif 1169 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); 1170 invalidate_page_bitmap(p); 1171 1172 #if defined(CONFIG_USER_ONLY) 1173 if (p->flags & PAGE_WRITE) { 1174 target_ulong addr; 1175 PageDesc *p2; 1176 int prot; 1177 1178 /* force the host page as non writable (writes will have a 1179 page fault + mprotect overhead) */ 1180 page_addr &= qemu_host_page_mask; 1181 prot = 0; 1182 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1183 addr += TARGET_PAGE_SIZE) { 1184 1185 p2 = page_find(addr >> TARGET_PAGE_BITS); 1186 if (!p2) { 1187 continue; 1188 } 1189 prot |= p2->flags; 1190 p2->flags &= ~PAGE_WRITE; 1191 } 1192 mprotect(g2h(page_addr), qemu_host_page_size, 1193 (prot & PAGE_BITS) & ~PAGE_WRITE); 1194 if (DEBUG_TB_INVALIDATE_GATE) { 1195 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1196 } 1197 } 1198 #else 1199 /* if some code is already present, then the pages are already 1200 protected. So we handle the case where only the first TB is 1201 allocated in a physical page */ 1202 if (!page_already_protected) { 1203 tlb_protect_code(page_addr); 1204 } 1205 #endif 1206 } 1207 1208 /* add a new TB and link it to the physical page tables. phys_page2 is 1209 * (-1) to indicate that only one page contains the TB. 1210 * 1211 * Called with mmap_lock held for user-mode emulation. 1212 */ 1213 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1214 tb_page_addr_t phys_page2) 1215 { 1216 uint32_t h; 1217 1218 assert_memory_lock(); 1219 1220 /* add in the page list */ 1221 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1222 if (phys_page2 != -1) { 1223 tb_alloc_page(tb, 1, phys_page2); 1224 } else { 1225 tb->page_addr[1] = -1; 1226 } 1227 1228 /* add in the hash table */ 1229 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1230 tb->trace_vcpu_dstate); 1231 qht_insert(&tb_ctx.htable, tb, h); 1232 1233 #ifdef CONFIG_USER_ONLY 1234 if (DEBUG_TB_CHECK_GATE) { 1235 tb_page_check(); 1236 } 1237 #endif 1238 } 1239 1240 /* Called with mmap_lock held for user mode emulation. */ 1241 TranslationBlock *tb_gen_code(CPUState *cpu, 1242 target_ulong pc, target_ulong cs_base, 1243 uint32_t flags, int cflags) 1244 { 1245 CPUArchState *env = cpu->env_ptr; 1246 TranslationBlock *tb; 1247 tb_page_addr_t phys_pc, phys_page2; 1248 target_ulong virt_page2; 1249 tcg_insn_unit *gen_code_buf; 1250 int gen_code_size, search_size; 1251 #ifdef CONFIG_PROFILER 1252 TCGProfile *prof = &tcg_ctx->prof; 1253 int64_t ti; 1254 #endif 1255 assert_memory_lock(); 1256 1257 phys_pc = get_page_addr_code(env, pc); 1258 1259 buffer_overflow: 1260 tb = tb_alloc(pc); 1261 if (unlikely(!tb)) { 1262 /* flush must be done */ 1263 tb_flush(cpu); 1264 mmap_unlock(); 1265 /* Make the execution loop process the flush as soon as possible. */ 1266 cpu->exception_index = EXCP_INTERRUPT; 1267 cpu_loop_exit(cpu); 1268 } 1269 1270 gen_code_buf = tcg_ctx->code_gen_ptr; 1271 tb->tc.ptr = gen_code_buf; 1272 tb->pc = pc; 1273 tb->cs_base = cs_base; 1274 tb->flags = flags; 1275 tb->cflags = cflags; 1276 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1277 tcg_ctx->tb_cflags = cflags; 1278 1279 #ifdef CONFIG_PROFILER 1280 /* includes aborted translations because of exceptions */ 1281 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1282 ti = profile_getclock(); 1283 #endif 1284 1285 tcg_func_start(tcg_ctx); 1286 1287 tcg_ctx->cpu = ENV_GET_CPU(env); 1288 gen_intermediate_code(cpu, tb); 1289 tcg_ctx->cpu = NULL; 1290 1291 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1292 1293 /* generate machine code */ 1294 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1295 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1296 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1297 if (TCG_TARGET_HAS_direct_jump) { 1298 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1299 tcg_ctx->tb_jmp_target_addr = NULL; 1300 } else { 1301 tcg_ctx->tb_jmp_insn_offset = NULL; 1302 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1303 } 1304 1305 #ifdef CONFIG_PROFILER 1306 atomic_set(&prof->tb_count, prof->tb_count + 1); 1307 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1308 ti = profile_getclock(); 1309 #endif 1310 1311 /* ??? Overflow could be handled better here. In particular, we 1312 don't need to re-do gen_intermediate_code, nor should we re-do 1313 the tcg optimization currently hidden inside tcg_gen_code. All 1314 that should be required is to flush the TBs, allocate a new TB, 1315 re-initialize it per above, and re-do the actual code generation. */ 1316 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1317 if (unlikely(gen_code_size < 0)) { 1318 goto buffer_overflow; 1319 } 1320 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1321 if (unlikely(search_size < 0)) { 1322 goto buffer_overflow; 1323 } 1324 tb->tc.size = gen_code_size; 1325 1326 #ifdef CONFIG_PROFILER 1327 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1328 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1329 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1330 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1331 #endif 1332 1333 #ifdef DEBUG_DISAS 1334 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1335 qemu_log_in_addr_range(tb->pc)) { 1336 qemu_log_lock(); 1337 qemu_log("OUT: [size=%d]\n", gen_code_size); 1338 if (tcg_ctx->data_gen_ptr) { 1339 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1340 size_t data_size = gen_code_size - code_size; 1341 size_t i; 1342 1343 log_disas(tb->tc.ptr, code_size); 1344 1345 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1346 if (sizeof(tcg_target_ulong) == 8) { 1347 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1348 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1349 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1350 } else { 1351 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1352 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1353 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1354 } 1355 } 1356 } else { 1357 log_disas(tb->tc.ptr, gen_code_size); 1358 } 1359 qemu_log("\n"); 1360 qemu_log_flush(); 1361 qemu_log_unlock(); 1362 } 1363 #endif 1364 1365 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1366 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1367 CODE_GEN_ALIGN)); 1368 1369 /* init jump list */ 1370 assert(((uintptr_t)tb & 3) == 0); 1371 tb->jmp_list_first = (uintptr_t)tb | 2; 1372 tb->jmp_list_next[0] = (uintptr_t)NULL; 1373 tb->jmp_list_next[1] = (uintptr_t)NULL; 1374 1375 /* init original jump addresses wich has been set during tcg_gen_code() */ 1376 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1377 tb_reset_jump(tb, 0); 1378 } 1379 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1380 tb_reset_jump(tb, 1); 1381 } 1382 1383 /* check next page if needed */ 1384 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1385 phys_page2 = -1; 1386 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1387 phys_page2 = get_page_addr_code(env, virt_page2); 1388 } 1389 /* As long as consistency of the TB stuff is provided by tb_lock in user 1390 * mode and is implicit in single-threaded softmmu emulation, no explicit 1391 * memory barrier is required before tb_link_page() makes the TB visible 1392 * through the physical hash table and physical page list. 1393 */ 1394 tb_link_page(tb, phys_pc, phys_page2); 1395 g_tree_insert(tb_ctx.tb_tree, &tb->tc, tb); 1396 return tb; 1397 } 1398 1399 /* 1400 * Invalidate all TBs which intersect with the target physical address range 1401 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1402 * 'is_cpu_write_access' should be true if called from a real cpu write 1403 * access: the virtual CPU will exit the current TB if code is modified inside 1404 * this TB. 1405 * 1406 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1407 * Called with tb_lock held for system-mode emulation 1408 */ 1409 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1410 { 1411 while (start < end) { 1412 tb_invalidate_phys_page_range(start, end, 0); 1413 start &= TARGET_PAGE_MASK; 1414 start += TARGET_PAGE_SIZE; 1415 } 1416 } 1417 1418 #ifdef CONFIG_SOFTMMU 1419 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1420 { 1421 assert_tb_locked(); 1422 tb_invalidate_phys_range_1(start, end); 1423 } 1424 #else 1425 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1426 { 1427 assert_memory_lock(); 1428 tb_lock(); 1429 tb_invalidate_phys_range_1(start, end); 1430 tb_unlock(); 1431 } 1432 #endif 1433 /* 1434 * Invalidate all TBs which intersect with the target physical address range 1435 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1436 * 'is_cpu_write_access' should be true if called from a real cpu write 1437 * access: the virtual CPU will exit the current TB if code is modified inside 1438 * this TB. 1439 * 1440 * Called with tb_lock/mmap_lock held for user-mode emulation 1441 * Called with tb_lock held for system-mode emulation 1442 */ 1443 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1444 int is_cpu_write_access) 1445 { 1446 TranslationBlock *tb, *tb_next; 1447 tb_page_addr_t tb_start, tb_end; 1448 PageDesc *p; 1449 int n; 1450 #ifdef TARGET_HAS_PRECISE_SMC 1451 CPUState *cpu = current_cpu; 1452 CPUArchState *env = NULL; 1453 int current_tb_not_found = is_cpu_write_access; 1454 TranslationBlock *current_tb = NULL; 1455 int current_tb_modified = 0; 1456 target_ulong current_pc = 0; 1457 target_ulong current_cs_base = 0; 1458 uint32_t current_flags = 0; 1459 #endif /* TARGET_HAS_PRECISE_SMC */ 1460 1461 assert_memory_lock(); 1462 assert_tb_locked(); 1463 1464 p = page_find(start >> TARGET_PAGE_BITS); 1465 if (!p) { 1466 return; 1467 } 1468 #if defined(TARGET_HAS_PRECISE_SMC) 1469 if (cpu != NULL) { 1470 env = cpu->env_ptr; 1471 } 1472 #endif 1473 1474 /* we remove all the TBs in the range [start, end[ */ 1475 /* XXX: see if in some cases it could be faster to invalidate all 1476 the code */ 1477 tb = p->first_tb; 1478 while (tb != NULL) { 1479 n = (uintptr_t)tb & 3; 1480 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1481 tb_next = tb->page_next[n]; 1482 /* NOTE: this is subtle as a TB may span two physical pages */ 1483 if (n == 0) { 1484 /* NOTE: tb_end may be after the end of the page, but 1485 it is not a problem */ 1486 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1487 tb_end = tb_start + tb->size; 1488 } else { 1489 tb_start = tb->page_addr[1]; 1490 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1491 } 1492 if (!(tb_end <= start || tb_start >= end)) { 1493 #ifdef TARGET_HAS_PRECISE_SMC 1494 if (current_tb_not_found) { 1495 current_tb_not_found = 0; 1496 current_tb = NULL; 1497 if (cpu->mem_io_pc) { 1498 /* now we have a real cpu fault */ 1499 current_tb = tb_find_pc(cpu->mem_io_pc); 1500 } 1501 } 1502 if (current_tb == tb && 1503 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1504 /* If we are modifying the current TB, we must stop 1505 its execution. We could be more precise by checking 1506 that the modification is after the current PC, but it 1507 would require a specialized function to partially 1508 restore the CPU state */ 1509 1510 current_tb_modified = 1; 1511 cpu_restore_state_from_tb(cpu, current_tb, 1512 cpu->mem_io_pc, true); 1513 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1514 ¤t_flags); 1515 } 1516 #endif /* TARGET_HAS_PRECISE_SMC */ 1517 tb_phys_invalidate(tb, -1); 1518 } 1519 tb = tb_next; 1520 } 1521 #if !defined(CONFIG_USER_ONLY) 1522 /* if no code remaining, no need to continue to use slow writes */ 1523 if (!p->first_tb) { 1524 invalidate_page_bitmap(p); 1525 tlb_unprotect_code(start); 1526 } 1527 #endif 1528 #ifdef TARGET_HAS_PRECISE_SMC 1529 if (current_tb_modified) { 1530 /* Force execution of one insn next time. */ 1531 cpu->cflags_next_tb = 1 | curr_cflags(); 1532 cpu_loop_exit_noexc(cpu); 1533 } 1534 #endif 1535 } 1536 1537 #ifdef CONFIG_SOFTMMU 1538 /* len must be <= 8 and start must be a multiple of len. 1539 * Called via softmmu_template.h when code areas are written to with 1540 * iothread mutex not held. 1541 */ 1542 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1543 { 1544 PageDesc *p; 1545 1546 #if 0 1547 if (1) { 1548 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1549 cpu_single_env->mem_io_vaddr, len, 1550 cpu_single_env->eip, 1551 cpu_single_env->eip + 1552 (intptr_t)cpu_single_env->segs[R_CS].base); 1553 } 1554 #endif 1555 assert_memory_lock(); 1556 1557 p = page_find(start >> TARGET_PAGE_BITS); 1558 if (!p) { 1559 return; 1560 } 1561 if (!p->code_bitmap && 1562 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1563 /* build code bitmap. FIXME: writes should be protected by 1564 * tb_lock, reads by tb_lock or RCU. 1565 */ 1566 build_page_bitmap(p); 1567 } 1568 if (p->code_bitmap) { 1569 unsigned int nr; 1570 unsigned long b; 1571 1572 nr = start & ~TARGET_PAGE_MASK; 1573 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1574 if (b & ((1 << len) - 1)) { 1575 goto do_invalidate; 1576 } 1577 } else { 1578 do_invalidate: 1579 tb_invalidate_phys_page_range(start, start + len, 1); 1580 } 1581 } 1582 #else 1583 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1584 * host PC of the faulting store instruction that caused this invalidate. 1585 * Returns true if the caller needs to abort execution of the current 1586 * TB (because it was modified by this store and the guest CPU has 1587 * precise-SMC semantics). 1588 */ 1589 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1590 { 1591 TranslationBlock *tb; 1592 PageDesc *p; 1593 int n; 1594 #ifdef TARGET_HAS_PRECISE_SMC 1595 TranslationBlock *current_tb = NULL; 1596 CPUState *cpu = current_cpu; 1597 CPUArchState *env = NULL; 1598 int current_tb_modified = 0; 1599 target_ulong current_pc = 0; 1600 target_ulong current_cs_base = 0; 1601 uint32_t current_flags = 0; 1602 #endif 1603 1604 assert_memory_lock(); 1605 1606 addr &= TARGET_PAGE_MASK; 1607 p = page_find(addr >> TARGET_PAGE_BITS); 1608 if (!p) { 1609 return false; 1610 } 1611 1612 tb_lock(); 1613 tb = p->first_tb; 1614 #ifdef TARGET_HAS_PRECISE_SMC 1615 if (tb && pc != 0) { 1616 current_tb = tb_find_pc(pc); 1617 } 1618 if (cpu != NULL) { 1619 env = cpu->env_ptr; 1620 } 1621 #endif 1622 while (tb != NULL) { 1623 n = (uintptr_t)tb & 3; 1624 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1625 #ifdef TARGET_HAS_PRECISE_SMC 1626 if (current_tb == tb && 1627 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1628 /* If we are modifying the current TB, we must stop 1629 its execution. We could be more precise by checking 1630 that the modification is after the current PC, but it 1631 would require a specialized function to partially 1632 restore the CPU state */ 1633 1634 current_tb_modified = 1; 1635 cpu_restore_state_from_tb(cpu, current_tb, pc, true); 1636 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1637 ¤t_flags); 1638 } 1639 #endif /* TARGET_HAS_PRECISE_SMC */ 1640 tb_phys_invalidate(tb, addr); 1641 tb = tb->page_next[n]; 1642 } 1643 p->first_tb = NULL; 1644 #ifdef TARGET_HAS_PRECISE_SMC 1645 if (current_tb_modified) { 1646 /* Force execution of one insn next time. */ 1647 cpu->cflags_next_tb = 1 | curr_cflags(); 1648 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1649 * back into the cpu_exec loop. */ 1650 return true; 1651 } 1652 #endif 1653 tb_unlock(); 1654 1655 return false; 1656 } 1657 #endif 1658 1659 /* 1660 * Find the TB 'tb' such that 1661 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size 1662 * Return NULL if not found. 1663 */ 1664 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) 1665 { 1666 struct tb_tc s = { .ptr = (void *)tc_ptr }; 1667 1668 return g_tree_lookup(tb_ctx.tb_tree, &s); 1669 } 1670 1671 #if !defined(CONFIG_USER_ONLY) 1672 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs) 1673 { 1674 ram_addr_t ram_addr; 1675 MemoryRegion *mr; 1676 hwaddr l = 1; 1677 1678 rcu_read_lock(); 1679 mr = address_space_translate(as, addr, &addr, &l, false, attrs); 1680 if (!(memory_region_is_ram(mr) 1681 || memory_region_is_romd(mr))) { 1682 rcu_read_unlock(); 1683 return; 1684 } 1685 ram_addr = memory_region_get_ram_addr(mr) + addr; 1686 tb_lock(); 1687 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1688 tb_unlock(); 1689 rcu_read_unlock(); 1690 } 1691 #endif /* !defined(CONFIG_USER_ONLY) */ 1692 1693 /* Called with tb_lock held. */ 1694 void tb_check_watchpoint(CPUState *cpu) 1695 { 1696 TranslationBlock *tb; 1697 1698 tb = tb_find_pc(cpu->mem_io_pc); 1699 if (tb) { 1700 /* We can use retranslation to find the PC. */ 1701 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc, true); 1702 tb_phys_invalidate(tb, -1); 1703 } else { 1704 /* The exception probably happened in a helper. The CPU state should 1705 have been saved before calling it. Fetch the PC from there. */ 1706 CPUArchState *env = cpu->env_ptr; 1707 target_ulong pc, cs_base; 1708 tb_page_addr_t addr; 1709 uint32_t flags; 1710 1711 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1712 addr = get_page_addr_code(env, pc); 1713 tb_invalidate_phys_range(addr, addr + 1); 1714 } 1715 } 1716 1717 #ifndef CONFIG_USER_ONLY 1718 /* in deterministic execution mode, instructions doing device I/Os 1719 * must be at the end of the TB. 1720 * 1721 * Called by softmmu_template.h, with iothread mutex not held. 1722 */ 1723 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1724 { 1725 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1726 CPUArchState *env = cpu->env_ptr; 1727 #endif 1728 TranslationBlock *tb; 1729 uint32_t n; 1730 1731 tb_lock(); 1732 tb = tb_find_pc(retaddr); 1733 if (!tb) { 1734 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1735 (void *)retaddr); 1736 } 1737 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 1738 1739 /* On MIPS and SH, delay slot instructions can only be restarted if 1740 they were already the first instruction in the TB. If this is not 1741 the first instruction in a TB then re-execute the preceding 1742 branch. */ 1743 n = 1; 1744 #if defined(TARGET_MIPS) 1745 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 1746 && env->active_tc.PC != tb->pc) { 1747 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1748 cpu->icount_decr.u16.low++; 1749 env->hflags &= ~MIPS_HFLAG_BMASK; 1750 n = 2; 1751 } 1752 #elif defined(TARGET_SH4) 1753 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1754 && env->pc != tb->pc) { 1755 env->pc -= 2; 1756 cpu->icount_decr.u16.low++; 1757 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1758 n = 2; 1759 } 1760 #endif 1761 1762 /* Generate a new TB executing the I/O insn. */ 1763 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1764 1765 if (tb->cflags & CF_NOCACHE) { 1766 if (tb->orig_tb) { 1767 /* Invalidate original TB if this TB was generated in 1768 * cpu_exec_nocache() */ 1769 tb_phys_invalidate(tb->orig_tb, -1); 1770 } 1771 tb_remove(tb); 1772 } 1773 1774 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1775 * the first in the TB) then we end up generating a whole new TB and 1776 * repeating the fault, which is horribly inefficient. 1777 * Better would be to execute just this insn uncached, or generate a 1778 * second new TB. 1779 * 1780 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1781 * tb_lock gets reset. 1782 */ 1783 cpu_loop_exit_noexc(cpu); 1784 } 1785 1786 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1787 { 1788 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1789 1790 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1791 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1792 } 1793 } 1794 1795 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1796 { 1797 /* Discard jump cache entries for any tb which might potentially 1798 overlap the flushed page. */ 1799 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1800 tb_jmp_cache_clear_page(cpu, addr); 1801 } 1802 1803 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1804 struct qht_stats hst) 1805 { 1806 uint32_t hgram_opts; 1807 size_t hgram_bins; 1808 char *hgram; 1809 1810 if (!hst.head_buckets) { 1811 return; 1812 } 1813 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1814 hst.used_head_buckets, hst.head_buckets, 1815 (double)hst.used_head_buckets / hst.head_buckets * 100); 1816 1817 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1818 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1819 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1820 hgram_opts |= QDIST_PR_NODECIMAL; 1821 } 1822 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1823 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1824 qdist_avg(&hst.occupancy) * 100, hgram); 1825 g_free(hgram); 1826 1827 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1828 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1829 if (hgram_bins > 10) { 1830 hgram_bins = 10; 1831 } else { 1832 hgram_bins = 0; 1833 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1834 } 1835 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1836 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1837 qdist_avg(&hst.chain), hgram); 1838 g_free(hgram); 1839 } 1840 1841 struct tb_tree_stats { 1842 size_t host_size; 1843 size_t target_size; 1844 size_t max_target_size; 1845 size_t direct_jmp_count; 1846 size_t direct_jmp2_count; 1847 size_t cross_page; 1848 }; 1849 1850 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1851 { 1852 const TranslationBlock *tb = value; 1853 struct tb_tree_stats *tst = data; 1854 1855 tst->host_size += tb->tc.size; 1856 tst->target_size += tb->size; 1857 if (tb->size > tst->max_target_size) { 1858 tst->max_target_size = tb->size; 1859 } 1860 if (tb->page_addr[1] != -1) { 1861 tst->cross_page++; 1862 } 1863 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1864 tst->direct_jmp_count++; 1865 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1866 tst->direct_jmp2_count++; 1867 } 1868 } 1869 return false; 1870 } 1871 1872 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1873 { 1874 struct tb_tree_stats tst = {}; 1875 struct qht_stats hst; 1876 size_t nb_tbs; 1877 1878 tb_lock(); 1879 1880 nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 1881 g_tree_foreach(tb_ctx.tb_tree, tb_tree_stats_iter, &tst); 1882 /* XXX: avoid using doubles ? */ 1883 cpu_fprintf(f, "Translation buffer state:\n"); 1884 /* 1885 * Report total code size including the padding and TB structs; 1886 * otherwise users might think "-tb-size" is not honoured. 1887 * For avg host size we use the precise numbers from tb_tree_stats though. 1888 */ 1889 cpu_fprintf(f, "gen code size %zu/%zu\n", 1890 tcg_code_size(), tcg_code_capacity()); 1891 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1892 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1893 nb_tbs ? tst.target_size / nb_tbs : 0, 1894 tst.max_target_size); 1895 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1896 nb_tbs ? tst.host_size / nb_tbs : 0, 1897 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1898 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1899 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1900 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1901 tst.direct_jmp_count, 1902 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1903 tst.direct_jmp2_count, 1904 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1905 1906 qht_statistics_init(&tb_ctx.htable, &hst); 1907 print_qht_statistics(f, cpu_fprintf, hst); 1908 qht_statistics_destroy(&hst); 1909 1910 cpu_fprintf(f, "\nStatistics:\n"); 1911 cpu_fprintf(f, "TB flush count %u\n", 1912 atomic_read(&tb_ctx.tb_flush_count)); 1913 cpu_fprintf(f, "TB invalidate count %d\n", tb_ctx.tb_phys_invalidate_count); 1914 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1915 tcg_dump_info(f, cpu_fprintf); 1916 1917 tb_unlock(); 1918 } 1919 1920 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1921 { 1922 tcg_dump_op_count(f, cpu_fprintf); 1923 } 1924 1925 #else /* CONFIG_USER_ONLY */ 1926 1927 void cpu_interrupt(CPUState *cpu, int mask) 1928 { 1929 g_assert(qemu_mutex_iothread_locked()); 1930 cpu->interrupt_request |= mask; 1931 cpu->icount_decr.u16.high = -1; 1932 } 1933 1934 /* 1935 * Walks guest process memory "regions" one by one 1936 * and calls callback function 'fn' for each region. 1937 */ 1938 struct walk_memory_regions_data { 1939 walk_memory_regions_fn fn; 1940 void *priv; 1941 target_ulong start; 1942 int prot; 1943 }; 1944 1945 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1946 target_ulong end, int new_prot) 1947 { 1948 if (data->start != -1u) { 1949 int rc = data->fn(data->priv, data->start, end, data->prot); 1950 if (rc != 0) { 1951 return rc; 1952 } 1953 } 1954 1955 data->start = (new_prot ? end : -1u); 1956 data->prot = new_prot; 1957 1958 return 0; 1959 } 1960 1961 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1962 target_ulong base, int level, void **lp) 1963 { 1964 target_ulong pa; 1965 int i, rc; 1966 1967 if (*lp == NULL) { 1968 return walk_memory_regions_end(data, base, 0); 1969 } 1970 1971 if (level == 0) { 1972 PageDesc *pd = *lp; 1973 1974 for (i = 0; i < V_L2_SIZE; ++i) { 1975 int prot = pd[i].flags; 1976 1977 pa = base | (i << TARGET_PAGE_BITS); 1978 if (prot != data->prot) { 1979 rc = walk_memory_regions_end(data, pa, prot); 1980 if (rc != 0) { 1981 return rc; 1982 } 1983 } 1984 } 1985 } else { 1986 void **pp = *lp; 1987 1988 for (i = 0; i < V_L2_SIZE; ++i) { 1989 pa = base | ((target_ulong)i << 1990 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1991 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 1992 if (rc != 0) { 1993 return rc; 1994 } 1995 } 1996 } 1997 1998 return 0; 1999 } 2000 2001 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2002 { 2003 struct walk_memory_regions_data data; 2004 uintptr_t i, l1_sz = v_l1_size; 2005 2006 data.fn = fn; 2007 data.priv = priv; 2008 data.start = -1u; 2009 data.prot = 0; 2010 2011 for (i = 0; i < l1_sz; i++) { 2012 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2013 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2014 if (rc != 0) { 2015 return rc; 2016 } 2017 } 2018 2019 return walk_memory_regions_end(&data, 0, 0); 2020 } 2021 2022 static int dump_region(void *priv, target_ulong start, 2023 target_ulong end, unsigned long prot) 2024 { 2025 FILE *f = (FILE *)priv; 2026 2027 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2028 " "TARGET_FMT_lx" %c%c%c\n", 2029 start, end, end - start, 2030 ((prot & PAGE_READ) ? 'r' : '-'), 2031 ((prot & PAGE_WRITE) ? 'w' : '-'), 2032 ((prot & PAGE_EXEC) ? 'x' : '-')); 2033 2034 return 0; 2035 } 2036 2037 /* dump memory mappings */ 2038 void page_dump(FILE *f) 2039 { 2040 const int length = sizeof(target_ulong) * 2; 2041 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2042 length, "start", length, "end", length, "size", "prot"); 2043 walk_memory_regions(f, dump_region); 2044 } 2045 2046 int page_get_flags(target_ulong address) 2047 { 2048 PageDesc *p; 2049 2050 p = page_find(address >> TARGET_PAGE_BITS); 2051 if (!p) { 2052 return 0; 2053 } 2054 return p->flags; 2055 } 2056 2057 /* Modify the flags of a page and invalidate the code if necessary. 2058 The flag PAGE_WRITE_ORG is positioned automatically depending 2059 on PAGE_WRITE. The mmap_lock should already be held. */ 2060 void page_set_flags(target_ulong start, target_ulong end, int flags) 2061 { 2062 target_ulong addr, len; 2063 2064 /* This function should never be called with addresses outside the 2065 guest address space. If this assert fires, it probably indicates 2066 a missing call to h2g_valid. */ 2067 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2068 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2069 #endif 2070 assert(start < end); 2071 assert_memory_lock(); 2072 2073 start = start & TARGET_PAGE_MASK; 2074 end = TARGET_PAGE_ALIGN(end); 2075 2076 if (flags & PAGE_WRITE) { 2077 flags |= PAGE_WRITE_ORG; 2078 } 2079 2080 for (addr = start, len = end - start; 2081 len != 0; 2082 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2083 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2084 2085 /* If the write protection bit is set, then we invalidate 2086 the code inside. */ 2087 if (!(p->flags & PAGE_WRITE) && 2088 (flags & PAGE_WRITE) && 2089 p->first_tb) { 2090 tb_invalidate_phys_page(addr, 0); 2091 } 2092 p->flags = flags; 2093 } 2094 } 2095 2096 int page_check_range(target_ulong start, target_ulong len, int flags) 2097 { 2098 PageDesc *p; 2099 target_ulong end; 2100 target_ulong addr; 2101 2102 /* This function should never be called with addresses outside the 2103 guest address space. If this assert fires, it probably indicates 2104 a missing call to h2g_valid. */ 2105 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2106 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2107 #endif 2108 2109 if (len == 0) { 2110 return 0; 2111 } 2112 if (start + len - 1 < start) { 2113 /* We've wrapped around. */ 2114 return -1; 2115 } 2116 2117 /* must do before we loose bits in the next step */ 2118 end = TARGET_PAGE_ALIGN(start + len); 2119 start = start & TARGET_PAGE_MASK; 2120 2121 for (addr = start, len = end - start; 2122 len != 0; 2123 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2124 p = page_find(addr >> TARGET_PAGE_BITS); 2125 if (!p) { 2126 return -1; 2127 } 2128 if (!(p->flags & PAGE_VALID)) { 2129 return -1; 2130 } 2131 2132 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2133 return -1; 2134 } 2135 if (flags & PAGE_WRITE) { 2136 if (!(p->flags & PAGE_WRITE_ORG)) { 2137 return -1; 2138 } 2139 /* unprotect the page if it was put read-only because it 2140 contains translated code */ 2141 if (!(p->flags & PAGE_WRITE)) { 2142 if (!page_unprotect(addr, 0)) { 2143 return -1; 2144 } 2145 } 2146 } 2147 } 2148 return 0; 2149 } 2150 2151 /* called from signal handler: invalidate the code and unprotect the 2152 * page. Return 0 if the fault was not handled, 1 if it was handled, 2153 * and 2 if it was handled but the caller must cause the TB to be 2154 * immediately exited. (We can only return 2 if the 'pc' argument is 2155 * non-zero.) 2156 */ 2157 int page_unprotect(target_ulong address, uintptr_t pc) 2158 { 2159 unsigned int prot; 2160 bool current_tb_invalidated; 2161 PageDesc *p; 2162 target_ulong host_start, host_end, addr; 2163 2164 /* Technically this isn't safe inside a signal handler. However we 2165 know this only ever happens in a synchronous SEGV handler, so in 2166 practice it seems to be ok. */ 2167 mmap_lock(); 2168 2169 p = page_find(address >> TARGET_PAGE_BITS); 2170 if (!p) { 2171 mmap_unlock(); 2172 return 0; 2173 } 2174 2175 /* if the page was really writable, then we change its 2176 protection back to writable */ 2177 if (p->flags & PAGE_WRITE_ORG) { 2178 current_tb_invalidated = false; 2179 if (p->flags & PAGE_WRITE) { 2180 /* If the page is actually marked WRITE then assume this is because 2181 * this thread raced with another one which got here first and 2182 * set the page to PAGE_WRITE and did the TB invalidate for us. 2183 */ 2184 #ifdef TARGET_HAS_PRECISE_SMC 2185 TranslationBlock *current_tb = tb_find_pc(pc); 2186 if (current_tb) { 2187 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; 2188 } 2189 #endif 2190 } else { 2191 host_start = address & qemu_host_page_mask; 2192 host_end = host_start + qemu_host_page_size; 2193 2194 prot = 0; 2195 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) { 2196 p = page_find(addr >> TARGET_PAGE_BITS); 2197 p->flags |= PAGE_WRITE; 2198 prot |= p->flags; 2199 2200 /* and since the content will be modified, we must invalidate 2201 the corresponding translated code. */ 2202 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2203 #ifdef CONFIG_USER_ONLY 2204 if (DEBUG_TB_CHECK_GATE) { 2205 tb_invalidate_check(addr); 2206 } 2207 #endif 2208 } 2209 mprotect((void *)g2h(host_start), qemu_host_page_size, 2210 prot & PAGE_BITS); 2211 } 2212 mmap_unlock(); 2213 /* If current TB was invalidated return to main loop */ 2214 return current_tb_invalidated ? 2 : 1; 2215 } 2216 mmap_unlock(); 2217 return 0; 2218 } 2219 #endif /* CONFIG_USER_ONLY */ 2220 2221 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2222 void tcg_flush_softmmu_tlb(CPUState *cs) 2223 { 2224 #ifdef CONFIG_SOFTMMU 2225 tlb_flush(cs); 2226 #endif 2227 } 2228