1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #include "exec/exec-all.h" 35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 36 #include <sys/param.h> 37 #if __FreeBSD_version >= 700104 38 #define HAVE_KINFO_GETVMMAP 39 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 40 #include <sys/proc.h> 41 #include <machine/profile.h> 42 #define _KERNEL 43 #include <sys/user.h> 44 #undef _KERNEL 45 #undef sigqueue 46 #include <libutil.h> 47 #endif 48 #endif 49 #else 50 #include "exec/address-spaces.h" 51 #endif 52 53 #include "exec/cputlb.h" 54 #include "exec/tb-hash.h" 55 #include "translate-all.h" 56 #include "qemu/bitmap.h" 57 #include "qemu/error-report.h" 58 #include "qemu/timer.h" 59 #include "qemu/main-loop.h" 60 #include "exec/log.h" 61 #include "sysemu/cpus.h" 62 63 /* #define DEBUG_TB_INVALIDATE */ 64 /* #define DEBUG_TB_FLUSH */ 65 /* make various TB consistency checks */ 66 /* #define DEBUG_TB_CHECK */ 67 68 #ifdef DEBUG_TB_INVALIDATE 69 #define DEBUG_TB_INVALIDATE_GATE 1 70 #else 71 #define DEBUG_TB_INVALIDATE_GATE 0 72 #endif 73 74 #ifdef DEBUG_TB_FLUSH 75 #define DEBUG_TB_FLUSH_GATE 1 76 #else 77 #define DEBUG_TB_FLUSH_GATE 0 78 #endif 79 80 #if !defined(CONFIG_USER_ONLY) 81 /* TB consistency checks only implemented for usermode emulation. */ 82 #undef DEBUG_TB_CHECK 83 #endif 84 85 #ifdef DEBUG_TB_CHECK 86 #define DEBUG_TB_CHECK_GATE 1 87 #else 88 #define DEBUG_TB_CHECK_GATE 0 89 #endif 90 91 /* Access to the various translations structures need to be serialised via locks 92 * for consistency. This is automatic for SoftMMU based system 93 * emulation due to its single threaded nature. In user-mode emulation 94 * access to the memory related structures are protected with the 95 * mmap_lock. 96 */ 97 #ifdef CONFIG_SOFTMMU 98 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 99 #else 100 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 101 #endif 102 103 #define SMC_BITMAP_USE_THRESHOLD 10 104 105 typedef struct PageDesc { 106 /* list of TBs intersecting this ram page */ 107 TranslationBlock *first_tb; 108 #ifdef CONFIG_SOFTMMU 109 /* in order to optimize self modifying code, we count the number 110 of lookups we do to a given page to use a bitmap */ 111 unsigned int code_write_count; 112 unsigned long *code_bitmap; 113 #else 114 unsigned long flags; 115 #endif 116 } PageDesc; 117 118 /* In system mode we want L1_MAP to be based on ram offsets, 119 while in user mode we want it to be based on virtual addresses. */ 120 #if !defined(CONFIG_USER_ONLY) 121 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 122 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 123 #else 124 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 125 #endif 126 #else 127 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 128 #endif 129 130 /* Size of the L2 (and L3, etc) page tables. */ 131 #define V_L2_BITS 10 132 #define V_L2_SIZE (1 << V_L2_BITS) 133 134 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 135 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 136 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 137 * BITS_PER_BYTE); 138 139 /* 140 * L1 Mapping properties 141 */ 142 static int v_l1_size; 143 static int v_l1_shift; 144 static int v_l2_levels; 145 146 /* The bottom level has pointers to PageDesc, and is indexed by 147 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 148 */ 149 #define V_L1_MIN_BITS 4 150 #define V_L1_MAX_BITS (V_L2_BITS + 3) 151 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 152 153 static void *l1_map[V_L1_MAX_SIZE]; 154 155 /* code generation context */ 156 TCGContext tcg_ctx; 157 bool parallel_cpus; 158 159 /* translation block context */ 160 static __thread int have_tb_lock; 161 162 static void page_table_config_init(void) 163 { 164 uint32_t v_l1_bits; 165 166 assert(TARGET_PAGE_BITS); 167 /* The bits remaining after N lower levels of page tables. */ 168 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 169 if (v_l1_bits < V_L1_MIN_BITS) { 170 v_l1_bits += V_L2_BITS; 171 } 172 173 v_l1_size = 1 << v_l1_bits; 174 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 175 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 176 177 assert(v_l1_bits <= V_L1_MAX_BITS); 178 assert(v_l1_shift % V_L2_BITS == 0); 179 assert(v_l2_levels >= 0); 180 } 181 182 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 183 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 184 185 void tb_lock(void) 186 { 187 assert_tb_unlocked(); 188 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock); 189 have_tb_lock++; 190 } 191 192 void tb_unlock(void) 193 { 194 assert_tb_locked(); 195 have_tb_lock--; 196 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock); 197 } 198 199 void tb_lock_reset(void) 200 { 201 if (have_tb_lock) { 202 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock); 203 have_tb_lock = 0; 204 } 205 } 206 207 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr); 208 209 void cpu_gen_init(void) 210 { 211 tcg_context_init(&tcg_ctx); 212 } 213 214 /* Encode VAL as a signed leb128 sequence at P. 215 Return P incremented past the encoded value. */ 216 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 217 { 218 int more, byte; 219 220 do { 221 byte = val & 0x7f; 222 val >>= 7; 223 more = !((val == 0 && (byte & 0x40) == 0) 224 || (val == -1 && (byte & 0x40) != 0)); 225 if (more) { 226 byte |= 0x80; 227 } 228 *p++ = byte; 229 } while (more); 230 231 return p; 232 } 233 234 /* Decode a signed leb128 sequence at *PP; increment *PP past the 235 decoded value. Return the decoded value. */ 236 static target_long decode_sleb128(uint8_t **pp) 237 { 238 uint8_t *p = *pp; 239 target_long val = 0; 240 int byte, shift = 0; 241 242 do { 243 byte = *p++; 244 val |= (target_ulong)(byte & 0x7f) << shift; 245 shift += 7; 246 } while (byte & 0x80); 247 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 248 val |= -(target_ulong)1 << shift; 249 } 250 251 *pp = p; 252 return val; 253 } 254 255 /* Encode the data collected about the instructions while compiling TB. 256 Place the data at BLOCK, and return the number of bytes consumed. 257 258 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's, 259 which come from the target's insn_start data, followed by a uintptr_t 260 which comes from the host pc of the end of the code implementing the insn. 261 262 Each line of the table is encoded as sleb128 deltas from the previous 263 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 264 That is, the first column is seeded with the guest pc, the last column 265 with the host pc, and the middle columns with zeros. */ 266 267 static int encode_search(TranslationBlock *tb, uint8_t *block) 268 { 269 uint8_t *highwater = tcg_ctx.code_gen_highwater; 270 uint8_t *p = block; 271 int i, j, n; 272 273 tb->tc.search = block; 274 275 for (i = 0, n = tb->icount; i < n; ++i) { 276 target_ulong prev; 277 278 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 279 if (i == 0) { 280 prev = (j == 0 ? tb->pc : 0); 281 } else { 282 prev = tcg_ctx.gen_insn_data[i - 1][j]; 283 } 284 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev); 285 } 286 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]); 287 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev); 288 289 /* Test for (pending) buffer overflow. The assumption is that any 290 one row beginning below the high water mark cannot overrun 291 the buffer completely. Thus we can test for overflow after 292 encoding a row without having to check during encoding. */ 293 if (unlikely(p > highwater)) { 294 return -1; 295 } 296 } 297 298 return p - block; 299 } 300 301 /* The cpu state corresponding to 'searched_pc' is restored. 302 * Called with tb_lock held. 303 */ 304 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 305 uintptr_t searched_pc) 306 { 307 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 308 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 309 CPUArchState *env = cpu->env_ptr; 310 uint8_t *p = tb->tc.search; 311 int i, j, num_insns = tb->icount; 312 #ifdef CONFIG_PROFILER 313 int64_t ti = profile_getclock(); 314 #endif 315 316 searched_pc -= GETPC_ADJ; 317 318 if (searched_pc < host_pc) { 319 return -1; 320 } 321 322 /* Reconstruct the stored insn data while looking for the point at 323 which the end of the insn exceeds the searched_pc. */ 324 for (i = 0; i < num_insns; ++i) { 325 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 326 data[j] += decode_sleb128(&p); 327 } 328 host_pc += decode_sleb128(&p); 329 if (host_pc > searched_pc) { 330 goto found; 331 } 332 } 333 return -1; 334 335 found: 336 if (tb->cflags & CF_USE_ICOUNT) { 337 assert(use_icount); 338 /* Reset the cycle counter to the start of the block. */ 339 cpu->icount_decr.u16.low += num_insns; 340 /* Clear the IO flag. */ 341 cpu->can_do_io = 0; 342 } 343 cpu->icount_decr.u16.low -= i; 344 restore_state_to_opc(env, tb, data); 345 346 #ifdef CONFIG_PROFILER 347 tcg_ctx.restore_time += profile_getclock() - ti; 348 tcg_ctx.restore_count++; 349 #endif 350 return 0; 351 } 352 353 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr) 354 { 355 TranslationBlock *tb; 356 bool r = false; 357 358 /* A retaddr of zero is invalid so we really shouldn't have ended 359 * up here. The target code has likely forgotten to check retaddr 360 * != 0 before attempting to restore state. We return early to 361 * avoid blowing up on a recursive tb_lock(). The target must have 362 * previously survived a failed cpu_restore_state because 363 * tb_find_pc(0) would have failed anyway. It still should be 364 * fixed though. 365 */ 366 367 if (!retaddr) { 368 return r; 369 } 370 371 tb_lock(); 372 tb = tb_find_pc(retaddr); 373 if (tb) { 374 cpu_restore_state_from_tb(cpu, tb, retaddr); 375 if (tb->cflags & CF_NOCACHE) { 376 /* one-shot translation, invalidate it immediately */ 377 tb_phys_invalidate(tb, -1); 378 tb_free(tb); 379 } 380 r = true; 381 } 382 tb_unlock(); 383 384 return r; 385 } 386 387 static void page_init(void) 388 { 389 page_size_init(); 390 page_table_config_init(); 391 392 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 393 { 394 #ifdef HAVE_KINFO_GETVMMAP 395 struct kinfo_vmentry *freep; 396 int i, cnt; 397 398 freep = kinfo_getvmmap(getpid(), &cnt); 399 if (freep) { 400 mmap_lock(); 401 for (i = 0; i < cnt; i++) { 402 unsigned long startaddr, endaddr; 403 404 startaddr = freep[i].kve_start; 405 endaddr = freep[i].kve_end; 406 if (h2g_valid(startaddr)) { 407 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 408 409 if (h2g_valid(endaddr)) { 410 endaddr = h2g(endaddr); 411 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 412 } else { 413 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 414 endaddr = ~0ul; 415 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 416 #endif 417 } 418 } 419 } 420 free(freep); 421 mmap_unlock(); 422 } 423 #else 424 FILE *f; 425 426 last_brk = (unsigned long)sbrk(0); 427 428 f = fopen("/compat/linux/proc/self/maps", "r"); 429 if (f) { 430 mmap_lock(); 431 432 do { 433 unsigned long startaddr, endaddr; 434 int n; 435 436 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 437 438 if (n == 2 && h2g_valid(startaddr)) { 439 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 440 441 if (h2g_valid(endaddr)) { 442 endaddr = h2g(endaddr); 443 } else { 444 endaddr = ~0ul; 445 } 446 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 447 } 448 } while (!feof(f)); 449 450 fclose(f); 451 mmap_unlock(); 452 } 453 #endif 454 } 455 #endif 456 } 457 458 /* If alloc=1: 459 * Called with tb_lock held for system emulation. 460 * Called with mmap_lock held for user-mode emulation. 461 */ 462 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 463 { 464 PageDesc *pd; 465 void **lp; 466 int i; 467 468 if (alloc) { 469 assert_memory_lock(); 470 } 471 472 /* Level 1. Always allocated. */ 473 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 474 475 /* Level 2..N-1. */ 476 for (i = v_l2_levels; i > 0; i--) { 477 void **p = atomic_rcu_read(lp); 478 479 if (p == NULL) { 480 if (!alloc) { 481 return NULL; 482 } 483 p = g_new0(void *, V_L2_SIZE); 484 atomic_rcu_set(lp, p); 485 } 486 487 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 488 } 489 490 pd = atomic_rcu_read(lp); 491 if (pd == NULL) { 492 if (!alloc) { 493 return NULL; 494 } 495 pd = g_new0(PageDesc, V_L2_SIZE); 496 atomic_rcu_set(lp, pd); 497 } 498 499 return pd + (index & (V_L2_SIZE - 1)); 500 } 501 502 static inline PageDesc *page_find(tb_page_addr_t index) 503 { 504 return page_find_alloc(index, 0); 505 } 506 507 #if defined(CONFIG_USER_ONLY) 508 /* Currently it is not recommended to allocate big chunks of data in 509 user mode. It will change when a dedicated libc will be used. */ 510 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 511 region in which the guest needs to run. Revisit this. */ 512 #define USE_STATIC_CODE_GEN_BUFFER 513 #endif 514 515 /* Minimum size of the code gen buffer. This number is randomly chosen, 516 but not so small that we can't have a fair number of TB's live. */ 517 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 518 519 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 520 indicated, this is constrained by the range of direct branches on the 521 host cpu, as used by the TCG implementation of goto_tb. */ 522 #if defined(__x86_64__) 523 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 524 #elif defined(__sparc__) 525 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 526 #elif defined(__powerpc64__) 527 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 528 #elif defined(__powerpc__) 529 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 530 #elif defined(__aarch64__) 531 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 532 #elif defined(__s390x__) 533 /* We have a +- 4GB range on the branches; leave some slop. */ 534 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 535 #elif defined(__mips__) 536 /* We have a 256MB branch region, but leave room to make sure the 537 main executable is also within that region. */ 538 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 539 #else 540 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 541 #endif 542 543 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 544 545 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 546 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 547 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 548 549 static inline size_t size_code_gen_buffer(size_t tb_size) 550 { 551 /* Size the buffer. */ 552 if (tb_size == 0) { 553 #ifdef USE_STATIC_CODE_GEN_BUFFER 554 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 555 #else 556 /* ??? Needs adjustments. */ 557 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 558 static buffer, we could size this on RESERVED_VA, on the text 559 segment size of the executable, or continue to use the default. */ 560 tb_size = (unsigned long)(ram_size / 4); 561 #endif 562 } 563 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 564 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 565 } 566 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 567 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 568 } 569 return tb_size; 570 } 571 572 #ifdef __mips__ 573 /* In order to use J and JAL within the code_gen_buffer, we require 574 that the buffer not cross a 256MB boundary. */ 575 static inline bool cross_256mb(void *addr, size_t size) 576 { 577 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 578 } 579 580 /* We weren't able to allocate a buffer without crossing that boundary, 581 so make do with the larger portion of the buffer that doesn't cross. 582 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 583 static inline void *split_cross_256mb(void *buf1, size_t size1) 584 { 585 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 586 size_t size2 = buf1 + size1 - buf2; 587 588 size1 = buf2 - buf1; 589 if (size1 < size2) { 590 size1 = size2; 591 buf1 = buf2; 592 } 593 594 tcg_ctx.code_gen_buffer_size = size1; 595 return buf1; 596 } 597 #endif 598 599 #ifdef USE_STATIC_CODE_GEN_BUFFER 600 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 601 __attribute__((aligned(CODE_GEN_ALIGN))); 602 603 # ifdef _WIN32 604 static inline void do_protect(void *addr, long size, int prot) 605 { 606 DWORD old_protect; 607 VirtualProtect(addr, size, prot, &old_protect); 608 } 609 610 static inline void map_exec(void *addr, long size) 611 { 612 do_protect(addr, size, PAGE_EXECUTE_READWRITE); 613 } 614 615 static inline void map_none(void *addr, long size) 616 { 617 do_protect(addr, size, PAGE_NOACCESS); 618 } 619 # else 620 static inline void do_protect(void *addr, long size, int prot) 621 { 622 uintptr_t start, end; 623 624 start = (uintptr_t)addr; 625 start &= qemu_real_host_page_mask; 626 627 end = (uintptr_t)addr + size; 628 end = ROUND_UP(end, qemu_real_host_page_size); 629 630 mprotect((void *)start, end - start, prot); 631 } 632 633 static inline void map_exec(void *addr, long size) 634 { 635 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC); 636 } 637 638 static inline void map_none(void *addr, long size) 639 { 640 do_protect(addr, size, PROT_NONE); 641 } 642 # endif /* WIN32 */ 643 644 static inline void *alloc_code_gen_buffer(void) 645 { 646 void *buf = static_code_gen_buffer; 647 size_t full_size, size; 648 649 /* The size of the buffer, rounded down to end on a page boundary. */ 650 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer)) 651 & qemu_real_host_page_mask) - (uintptr_t)buf; 652 653 /* Reserve a guard page. */ 654 size = full_size - qemu_real_host_page_size; 655 656 /* Honor a command-line option limiting the size of the buffer. */ 657 if (size > tcg_ctx.code_gen_buffer_size) { 658 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size) 659 & qemu_real_host_page_mask) - (uintptr_t)buf; 660 } 661 tcg_ctx.code_gen_buffer_size = size; 662 663 #ifdef __mips__ 664 if (cross_256mb(buf, size)) { 665 buf = split_cross_256mb(buf, size); 666 size = tcg_ctx.code_gen_buffer_size; 667 } 668 #endif 669 670 map_exec(buf, size); 671 map_none(buf + size, qemu_real_host_page_size); 672 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 673 674 return buf; 675 } 676 #elif defined(_WIN32) 677 static inline void *alloc_code_gen_buffer(void) 678 { 679 size_t size = tcg_ctx.code_gen_buffer_size; 680 void *buf1, *buf2; 681 682 /* Perform the allocation in two steps, so that the guard page 683 is reserved but uncommitted. */ 684 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size, 685 MEM_RESERVE, PAGE_NOACCESS); 686 if (buf1 != NULL) { 687 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE); 688 assert(buf1 == buf2); 689 } 690 691 return buf1; 692 } 693 #else 694 static inline void *alloc_code_gen_buffer(void) 695 { 696 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 697 uintptr_t start = 0; 698 size_t size = tcg_ctx.code_gen_buffer_size; 699 void *buf; 700 701 /* Constrain the position of the buffer based on the host cpu. 702 Note that these addresses are chosen in concert with the 703 addresses assigned in the relevant linker script file. */ 704 # if defined(__PIE__) || defined(__PIC__) 705 /* Don't bother setting a preferred location if we're building 706 a position-independent executable. We're more likely to get 707 an address near the main executable if we let the kernel 708 choose the address. */ 709 # elif defined(__x86_64__) && defined(MAP_32BIT) 710 /* Force the memory down into low memory with the executable. 711 Leave the choice of exact location with the kernel. */ 712 flags |= MAP_32BIT; 713 /* Cannot expect to map more than 800MB in low memory. */ 714 if (size > 800u * 1024 * 1024) { 715 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024; 716 } 717 # elif defined(__sparc__) 718 start = 0x40000000ul; 719 # elif defined(__s390x__) 720 start = 0x90000000ul; 721 # elif defined(__mips__) 722 # if _MIPS_SIM == _ABI64 723 start = 0x128000000ul; 724 # else 725 start = 0x08000000ul; 726 # endif 727 # endif 728 729 buf = mmap((void *)start, size + qemu_real_host_page_size, 730 PROT_NONE, flags, -1, 0); 731 if (buf == MAP_FAILED) { 732 return NULL; 733 } 734 735 #ifdef __mips__ 736 if (cross_256mb(buf, size)) { 737 /* Try again, with the original still mapped, to avoid re-acquiring 738 that 256mb crossing. This time don't specify an address. */ 739 size_t size2; 740 void *buf2 = mmap(NULL, size + qemu_real_host_page_size, 741 PROT_NONE, flags, -1, 0); 742 switch ((int)(buf2 != MAP_FAILED)) { 743 case 1: 744 if (!cross_256mb(buf2, size)) { 745 /* Success! Use the new buffer. */ 746 munmap(buf, size + qemu_real_host_page_size); 747 break; 748 } 749 /* Failure. Work with what we had. */ 750 munmap(buf2, size + qemu_real_host_page_size); 751 /* fallthru */ 752 default: 753 /* Split the original buffer. Free the smaller half. */ 754 buf2 = split_cross_256mb(buf, size); 755 size2 = tcg_ctx.code_gen_buffer_size; 756 if (buf == buf2) { 757 munmap(buf + size2 + qemu_real_host_page_size, size - size2); 758 } else { 759 munmap(buf, size - size2); 760 } 761 size = size2; 762 break; 763 } 764 buf = buf2; 765 } 766 #endif 767 768 /* Make the final buffer accessible. The guard page at the end 769 will remain inaccessible with PROT_NONE. */ 770 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC); 771 772 /* Request large pages for the buffer. */ 773 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 774 775 return buf; 776 } 777 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 778 779 static inline void code_gen_alloc(size_t tb_size) 780 { 781 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size); 782 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer(); 783 if (tcg_ctx.code_gen_buffer == NULL) { 784 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 785 exit(1); 786 } 787 788 /* size this conservatively -- realloc later if needed */ 789 tcg_ctx.tb_ctx.tbs_size = 790 tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE / 8; 791 if (unlikely(!tcg_ctx.tb_ctx.tbs_size)) { 792 tcg_ctx.tb_ctx.tbs_size = 64 * 1024; 793 } 794 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock *, tcg_ctx.tb_ctx.tbs_size); 795 796 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock); 797 } 798 799 static void tb_htable_init(void) 800 { 801 unsigned int mode = QHT_MODE_AUTO_RESIZE; 802 803 qht_init(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode); 804 } 805 806 /* Must be called before using the QEMU cpus. 'tb_size' is the size 807 (in bytes) allocated to the translation buffer. Zero means default 808 size. */ 809 void tcg_exec_init(unsigned long tb_size) 810 { 811 tcg_allowed = true; 812 cpu_gen_init(); 813 page_init(); 814 tb_htable_init(); 815 code_gen_alloc(tb_size); 816 #if defined(CONFIG_SOFTMMU) 817 /* There's no guest base to take into account, so go ahead and 818 initialize the prologue now. */ 819 tcg_prologue_init(&tcg_ctx); 820 #endif 821 } 822 823 /* 824 * Allocate a new translation block. Flush the translation buffer if 825 * too many translation blocks or too much generated code. 826 * 827 * Called with tb_lock held. 828 */ 829 static TranslationBlock *tb_alloc(target_ulong pc) 830 { 831 TranslationBlock *tb; 832 TBContext *ctx; 833 834 assert_tb_locked(); 835 836 tb = tcg_tb_alloc(&tcg_ctx); 837 if (unlikely(tb == NULL)) { 838 return NULL; 839 } 840 ctx = &tcg_ctx.tb_ctx; 841 if (unlikely(ctx->nb_tbs == ctx->tbs_size)) { 842 ctx->tbs_size *= 2; 843 ctx->tbs = g_renew(TranslationBlock *, ctx->tbs, ctx->tbs_size); 844 } 845 ctx->tbs[ctx->nb_tbs++] = tb; 846 return tb; 847 } 848 849 /* Called with tb_lock held. */ 850 void tb_free(TranslationBlock *tb) 851 { 852 assert_tb_locked(); 853 854 /* In practice this is mostly used for single use temporary TB 855 Ignore the hard cases and just back up if this TB happens to 856 be the last one generated. */ 857 if (tcg_ctx.tb_ctx.nb_tbs > 0 && 858 tb == tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) { 859 size_t struct_size = ROUND_UP(sizeof(*tb), qemu_icache_linesize); 860 861 tcg_ctx.code_gen_ptr = tb->tc.ptr - struct_size; 862 tcg_ctx.tb_ctx.nb_tbs--; 863 } 864 } 865 866 static inline void invalidate_page_bitmap(PageDesc *p) 867 { 868 #ifdef CONFIG_SOFTMMU 869 g_free(p->code_bitmap); 870 p->code_bitmap = NULL; 871 p->code_write_count = 0; 872 #endif 873 } 874 875 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 876 static void page_flush_tb_1(int level, void **lp) 877 { 878 int i; 879 880 if (*lp == NULL) { 881 return; 882 } 883 if (level == 0) { 884 PageDesc *pd = *lp; 885 886 for (i = 0; i < V_L2_SIZE; ++i) { 887 pd[i].first_tb = NULL; 888 invalidate_page_bitmap(pd + i); 889 } 890 } else { 891 void **pp = *lp; 892 893 for (i = 0; i < V_L2_SIZE; ++i) { 894 page_flush_tb_1(level - 1, pp + i); 895 } 896 } 897 } 898 899 static void page_flush_tb(void) 900 { 901 int i, l1_sz = v_l1_size; 902 903 for (i = 0; i < l1_sz; i++) { 904 page_flush_tb_1(v_l2_levels, l1_map + i); 905 } 906 } 907 908 /* flush all the translation blocks */ 909 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 910 { 911 tb_lock(); 912 913 /* If it is already been done on request of another CPU, 914 * just retry. 915 */ 916 if (tcg_ctx.tb_ctx.tb_flush_count != tb_flush_count.host_int) { 917 goto done; 918 } 919 920 if (DEBUG_TB_FLUSH_GATE) { 921 printf("qemu: flush code_size=%td nb_tbs=%d avg_tb_size=%td\n", 922 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer, 923 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ? 924 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) / 925 tcg_ctx.tb_ctx.nb_tbs : 0); 926 } 927 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) 928 > tcg_ctx.code_gen_buffer_size) { 929 cpu_abort(cpu, "Internal error: code buffer overflow\n"); 930 } 931 932 CPU_FOREACH(cpu) { 933 cpu_tb_jmp_cache_clear(cpu); 934 } 935 936 tcg_ctx.tb_ctx.nb_tbs = 0; 937 qht_reset_size(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 938 page_flush_tb(); 939 940 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer; 941 /* XXX: flush processor icache at this point if cache flush is 942 expensive */ 943 atomic_mb_set(&tcg_ctx.tb_ctx.tb_flush_count, 944 tcg_ctx.tb_ctx.tb_flush_count + 1); 945 946 done: 947 tb_unlock(); 948 } 949 950 void tb_flush(CPUState *cpu) 951 { 952 if (tcg_enabled()) { 953 unsigned tb_flush_count = atomic_mb_read(&tcg_ctx.tb_ctx.tb_flush_count); 954 async_safe_run_on_cpu(cpu, do_tb_flush, 955 RUN_ON_CPU_HOST_INT(tb_flush_count)); 956 } 957 } 958 959 /* 960 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 961 * so in order to prevent bit rot we compile them unconditionally in user-mode, 962 * and let the optimizer get rid of them by wrapping their user-only callers 963 * with if (DEBUG_TB_CHECK_GATE). 964 */ 965 #ifdef CONFIG_USER_ONLY 966 967 static void 968 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 969 { 970 TranslationBlock *tb = p; 971 target_ulong addr = *(target_ulong *)userp; 972 973 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 974 printf("ERROR invalidate: address=" TARGET_FMT_lx 975 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 976 } 977 } 978 979 /* verify that all the pages have correct rights for code 980 * 981 * Called with tb_lock held. 982 */ 983 static void tb_invalidate_check(target_ulong address) 984 { 985 address &= TARGET_PAGE_MASK; 986 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_invalidate_check, &address); 987 } 988 989 static void 990 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 991 { 992 TranslationBlock *tb = p; 993 int flags1, flags2; 994 995 flags1 = page_get_flags(tb->pc); 996 flags2 = page_get_flags(tb->pc + tb->size - 1); 997 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 998 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 999 (long)tb->pc, tb->size, flags1, flags2); 1000 } 1001 } 1002 1003 /* verify that all the pages have correct rights for code */ 1004 static void tb_page_check(void) 1005 { 1006 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_page_check, NULL); 1007 } 1008 1009 #endif /* CONFIG_USER_ONLY */ 1010 1011 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) 1012 { 1013 TranslationBlock *tb1; 1014 unsigned int n1; 1015 1016 for (;;) { 1017 tb1 = *ptb; 1018 n1 = (uintptr_t)tb1 & 3; 1019 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); 1020 if (tb1 == tb) { 1021 *ptb = tb1->page_next[n1]; 1022 break; 1023 } 1024 ptb = &tb1->page_next[n1]; 1025 } 1026 } 1027 1028 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 1029 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 1030 { 1031 TranslationBlock *tb1; 1032 uintptr_t *ptb, ntb; 1033 unsigned int n1; 1034 1035 ptb = &tb->jmp_list_next[n]; 1036 if (*ptb) { 1037 /* find tb(n) in circular list */ 1038 for (;;) { 1039 ntb = *ptb; 1040 n1 = ntb & 3; 1041 tb1 = (TranslationBlock *)(ntb & ~3); 1042 if (n1 == n && tb1 == tb) { 1043 break; 1044 } 1045 if (n1 == 2) { 1046 ptb = &tb1->jmp_list_first; 1047 } else { 1048 ptb = &tb1->jmp_list_next[n1]; 1049 } 1050 } 1051 /* now we can suppress tb(n) from the list */ 1052 *ptb = tb->jmp_list_next[n]; 1053 1054 tb->jmp_list_next[n] = (uintptr_t)NULL; 1055 } 1056 } 1057 1058 /* reset the jump entry 'n' of a TB so that it is not chained to 1059 another TB */ 1060 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1061 { 1062 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1063 tb_set_jmp_target(tb, n, addr); 1064 } 1065 1066 /* remove any jumps to the TB */ 1067 static inline void tb_jmp_unlink(TranslationBlock *tb) 1068 { 1069 TranslationBlock *tb1; 1070 uintptr_t *ptb, ntb; 1071 unsigned int n1; 1072 1073 ptb = &tb->jmp_list_first; 1074 for (;;) { 1075 ntb = *ptb; 1076 n1 = ntb & 3; 1077 tb1 = (TranslationBlock *)(ntb & ~3); 1078 if (n1 == 2) { 1079 break; 1080 } 1081 tb_reset_jump(tb1, n1); 1082 *ptb = tb1->jmp_list_next[n1]; 1083 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1084 } 1085 } 1086 1087 /* invalidate one TB 1088 * 1089 * Called with tb_lock held. 1090 */ 1091 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1092 { 1093 CPUState *cpu; 1094 PageDesc *p; 1095 uint32_t h; 1096 tb_page_addr_t phys_pc; 1097 1098 assert_tb_locked(); 1099 1100 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1101 1102 /* remove the TB from the hash list */ 1103 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1104 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1105 tb->trace_vcpu_dstate); 1106 qht_remove(&tcg_ctx.tb_ctx.htable, tb, h); 1107 1108 /* remove the TB from the page list */ 1109 if (tb->page_addr[0] != page_addr) { 1110 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1111 tb_page_remove(&p->first_tb, tb); 1112 invalidate_page_bitmap(p); 1113 } 1114 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1115 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1116 tb_page_remove(&p->first_tb, tb); 1117 invalidate_page_bitmap(p); 1118 } 1119 1120 /* remove the TB from the hash list */ 1121 h = tb_jmp_cache_hash_func(tb->pc); 1122 CPU_FOREACH(cpu) { 1123 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1124 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1125 } 1126 } 1127 1128 /* suppress this TB from the two jump lists */ 1129 tb_remove_from_jmp_list(tb, 0); 1130 tb_remove_from_jmp_list(tb, 1); 1131 1132 /* suppress any remaining jumps to this TB */ 1133 tb_jmp_unlink(tb); 1134 1135 tcg_ctx.tb_ctx.tb_phys_invalidate_count++; 1136 } 1137 1138 #ifdef CONFIG_SOFTMMU 1139 static void build_page_bitmap(PageDesc *p) 1140 { 1141 int n, tb_start, tb_end; 1142 TranslationBlock *tb; 1143 1144 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1145 1146 tb = p->first_tb; 1147 while (tb != NULL) { 1148 n = (uintptr_t)tb & 3; 1149 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1150 /* NOTE: this is subtle as a TB may span two physical pages */ 1151 if (n == 0) { 1152 /* NOTE: tb_end may be after the end of the page, but 1153 it is not a problem */ 1154 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1155 tb_end = tb_start + tb->size; 1156 if (tb_end > TARGET_PAGE_SIZE) { 1157 tb_end = TARGET_PAGE_SIZE; 1158 } 1159 } else { 1160 tb_start = 0; 1161 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1162 } 1163 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1164 tb = tb->page_next[n]; 1165 } 1166 } 1167 #endif 1168 1169 /* add the tb in the target page and protect it if necessary 1170 * 1171 * Called with mmap_lock held for user-mode emulation. 1172 */ 1173 static inline void tb_alloc_page(TranslationBlock *tb, 1174 unsigned int n, tb_page_addr_t page_addr) 1175 { 1176 PageDesc *p; 1177 #ifndef CONFIG_USER_ONLY 1178 bool page_already_protected; 1179 #endif 1180 1181 assert_memory_lock(); 1182 1183 tb->page_addr[n] = page_addr; 1184 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1185 tb->page_next[n] = p->first_tb; 1186 #ifndef CONFIG_USER_ONLY 1187 page_already_protected = p->first_tb != NULL; 1188 #endif 1189 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); 1190 invalidate_page_bitmap(p); 1191 1192 #if defined(CONFIG_USER_ONLY) 1193 if (p->flags & PAGE_WRITE) { 1194 target_ulong addr; 1195 PageDesc *p2; 1196 int prot; 1197 1198 /* force the host page as non writable (writes will have a 1199 page fault + mprotect overhead) */ 1200 page_addr &= qemu_host_page_mask; 1201 prot = 0; 1202 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1203 addr += TARGET_PAGE_SIZE) { 1204 1205 p2 = page_find(addr >> TARGET_PAGE_BITS); 1206 if (!p2) { 1207 continue; 1208 } 1209 prot |= p2->flags; 1210 p2->flags &= ~PAGE_WRITE; 1211 } 1212 mprotect(g2h(page_addr), qemu_host_page_size, 1213 (prot & PAGE_BITS) & ~PAGE_WRITE); 1214 if (DEBUG_TB_INVALIDATE_GATE) { 1215 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1216 } 1217 } 1218 #else 1219 /* if some code is already present, then the pages are already 1220 protected. So we handle the case where only the first TB is 1221 allocated in a physical page */ 1222 if (!page_already_protected) { 1223 tlb_protect_code(page_addr); 1224 } 1225 #endif 1226 } 1227 1228 /* add a new TB and link it to the physical page tables. phys_page2 is 1229 * (-1) to indicate that only one page contains the TB. 1230 * 1231 * Called with mmap_lock held for user-mode emulation. 1232 */ 1233 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1234 tb_page_addr_t phys_page2) 1235 { 1236 uint32_t h; 1237 1238 assert_memory_lock(); 1239 1240 /* add in the page list */ 1241 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1242 if (phys_page2 != -1) { 1243 tb_alloc_page(tb, 1, phys_page2); 1244 } else { 1245 tb->page_addr[1] = -1; 1246 } 1247 1248 /* add in the hash table */ 1249 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1250 tb->trace_vcpu_dstate); 1251 qht_insert(&tcg_ctx.tb_ctx.htable, tb, h); 1252 1253 #ifdef CONFIG_USER_ONLY 1254 if (DEBUG_TB_CHECK_GATE) { 1255 tb_page_check(); 1256 } 1257 #endif 1258 } 1259 1260 /* Called with mmap_lock held for user mode emulation. */ 1261 TranslationBlock *tb_gen_code(CPUState *cpu, 1262 target_ulong pc, target_ulong cs_base, 1263 uint32_t flags, int cflags) 1264 { 1265 CPUArchState *env = cpu->env_ptr; 1266 TranslationBlock *tb; 1267 tb_page_addr_t phys_pc, phys_page2; 1268 target_ulong virt_page2; 1269 tcg_insn_unit *gen_code_buf; 1270 int gen_code_size, search_size; 1271 #ifdef CONFIG_PROFILER 1272 int64_t ti; 1273 #endif 1274 assert_memory_lock(); 1275 1276 phys_pc = get_page_addr_code(env, pc); 1277 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) { 1278 cflags |= CF_USE_ICOUNT; 1279 } 1280 1281 tb = tb_alloc(pc); 1282 if (unlikely(!tb)) { 1283 buffer_overflow: 1284 /* flush must be done */ 1285 tb_flush(cpu); 1286 mmap_unlock(); 1287 /* Make the execution loop process the flush as soon as possible. */ 1288 cpu->exception_index = EXCP_INTERRUPT; 1289 cpu_loop_exit(cpu); 1290 } 1291 1292 gen_code_buf = tcg_ctx.code_gen_ptr; 1293 tb->tc.ptr = gen_code_buf; 1294 tb->pc = pc; 1295 tb->cs_base = cs_base; 1296 tb->flags = flags; 1297 tb->cflags = cflags; 1298 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1299 1300 #ifdef CONFIG_PROFILER 1301 tcg_ctx.tb_count1++; /* includes aborted translations because of 1302 exceptions */ 1303 ti = profile_getclock(); 1304 #endif 1305 1306 tcg_func_start(&tcg_ctx); 1307 1308 tcg_ctx.cpu = ENV_GET_CPU(env); 1309 gen_intermediate_code(cpu, tb); 1310 tcg_ctx.cpu = NULL; 1311 1312 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1313 1314 /* generate machine code */ 1315 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1316 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1317 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset; 1318 if (TCG_TARGET_HAS_direct_jump) { 1319 tcg_ctx.tb_jmp_insn_offset = tb->jmp_target_arg; 1320 tcg_ctx.tb_jmp_target_addr = NULL; 1321 } else { 1322 tcg_ctx.tb_jmp_insn_offset = NULL; 1323 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_arg; 1324 } 1325 1326 #ifdef CONFIG_PROFILER 1327 tcg_ctx.tb_count++; 1328 tcg_ctx.interm_time += profile_getclock() - ti; 1329 ti = profile_getclock(); 1330 #endif 1331 1332 /* ??? Overflow could be handled better here. In particular, we 1333 don't need to re-do gen_intermediate_code, nor should we re-do 1334 the tcg optimization currently hidden inside tcg_gen_code. All 1335 that should be required is to flush the TBs, allocate a new TB, 1336 re-initialize it per above, and re-do the actual code generation. */ 1337 gen_code_size = tcg_gen_code(&tcg_ctx, tb); 1338 if (unlikely(gen_code_size < 0)) { 1339 goto buffer_overflow; 1340 } 1341 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1342 if (unlikely(search_size < 0)) { 1343 goto buffer_overflow; 1344 } 1345 1346 #ifdef CONFIG_PROFILER 1347 tcg_ctx.code_time += profile_getclock() - ti; 1348 tcg_ctx.code_in_len += tb->size; 1349 tcg_ctx.code_out_len += gen_code_size; 1350 tcg_ctx.search_out_len += search_size; 1351 #endif 1352 1353 #ifdef DEBUG_DISAS 1354 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1355 qemu_log_in_addr_range(tb->pc)) { 1356 qemu_log_lock(); 1357 qemu_log("OUT: [size=%d]\n", gen_code_size); 1358 if (tcg_ctx.data_gen_ptr) { 1359 size_t code_size = tcg_ctx.data_gen_ptr - tb->tc.ptr; 1360 size_t data_size = gen_code_size - code_size; 1361 size_t i; 1362 1363 log_disas(tb->tc.ptr, code_size); 1364 1365 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1366 if (sizeof(tcg_target_ulong) == 8) { 1367 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1368 (uintptr_t)tcg_ctx.data_gen_ptr + i, 1369 *(uint64_t *)(tcg_ctx.data_gen_ptr + i)); 1370 } else { 1371 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1372 (uintptr_t)tcg_ctx.data_gen_ptr + i, 1373 *(uint32_t *)(tcg_ctx.data_gen_ptr + i)); 1374 } 1375 } 1376 } else { 1377 log_disas(tb->tc.ptr, gen_code_size); 1378 } 1379 qemu_log("\n"); 1380 qemu_log_flush(); 1381 qemu_log_unlock(); 1382 } 1383 #endif 1384 1385 tcg_ctx.code_gen_ptr = (void *) 1386 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1387 CODE_GEN_ALIGN); 1388 1389 /* init jump list */ 1390 assert(((uintptr_t)tb & 3) == 0); 1391 tb->jmp_list_first = (uintptr_t)tb | 2; 1392 tb->jmp_list_next[0] = (uintptr_t)NULL; 1393 tb->jmp_list_next[1] = (uintptr_t)NULL; 1394 1395 /* init original jump addresses wich has been set during tcg_gen_code() */ 1396 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1397 tb_reset_jump(tb, 0); 1398 } 1399 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1400 tb_reset_jump(tb, 1); 1401 } 1402 1403 /* check next page if needed */ 1404 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1405 phys_page2 = -1; 1406 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1407 phys_page2 = get_page_addr_code(env, virt_page2); 1408 } 1409 /* As long as consistency of the TB stuff is provided by tb_lock in user 1410 * mode and is implicit in single-threaded softmmu emulation, no explicit 1411 * memory barrier is required before tb_link_page() makes the TB visible 1412 * through the physical hash table and physical page list. 1413 */ 1414 tb_link_page(tb, phys_pc, phys_page2); 1415 return tb; 1416 } 1417 1418 /* 1419 * Invalidate all TBs which intersect with the target physical address range 1420 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1421 * 'is_cpu_write_access' should be true if called from a real cpu write 1422 * access: the virtual CPU will exit the current TB if code is modified inside 1423 * this TB. 1424 * 1425 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1426 * Called with tb_lock held for system-mode emulation 1427 */ 1428 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1429 { 1430 while (start < end) { 1431 tb_invalidate_phys_page_range(start, end, 0); 1432 start &= TARGET_PAGE_MASK; 1433 start += TARGET_PAGE_SIZE; 1434 } 1435 } 1436 1437 #ifdef CONFIG_SOFTMMU 1438 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1439 { 1440 assert_tb_locked(); 1441 tb_invalidate_phys_range_1(start, end); 1442 } 1443 #else 1444 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1445 { 1446 assert_memory_lock(); 1447 tb_lock(); 1448 tb_invalidate_phys_range_1(start, end); 1449 tb_unlock(); 1450 } 1451 #endif 1452 /* 1453 * Invalidate all TBs which intersect with the target physical address range 1454 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1455 * 'is_cpu_write_access' should be true if called from a real cpu write 1456 * access: the virtual CPU will exit the current TB if code is modified inside 1457 * this TB. 1458 * 1459 * Called with tb_lock/mmap_lock held for user-mode emulation 1460 * Called with tb_lock held for system-mode emulation 1461 */ 1462 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1463 int is_cpu_write_access) 1464 { 1465 TranslationBlock *tb, *tb_next; 1466 tb_page_addr_t tb_start, tb_end; 1467 PageDesc *p; 1468 int n; 1469 #ifdef TARGET_HAS_PRECISE_SMC 1470 CPUState *cpu = current_cpu; 1471 CPUArchState *env = NULL; 1472 int current_tb_not_found = is_cpu_write_access; 1473 TranslationBlock *current_tb = NULL; 1474 int current_tb_modified = 0; 1475 target_ulong current_pc = 0; 1476 target_ulong current_cs_base = 0; 1477 uint32_t current_flags = 0; 1478 #endif /* TARGET_HAS_PRECISE_SMC */ 1479 1480 assert_memory_lock(); 1481 assert_tb_locked(); 1482 1483 p = page_find(start >> TARGET_PAGE_BITS); 1484 if (!p) { 1485 return; 1486 } 1487 #if defined(TARGET_HAS_PRECISE_SMC) 1488 if (cpu != NULL) { 1489 env = cpu->env_ptr; 1490 } 1491 #endif 1492 1493 /* we remove all the TBs in the range [start, end[ */ 1494 /* XXX: see if in some cases it could be faster to invalidate all 1495 the code */ 1496 tb = p->first_tb; 1497 while (tb != NULL) { 1498 n = (uintptr_t)tb & 3; 1499 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1500 tb_next = tb->page_next[n]; 1501 /* NOTE: this is subtle as a TB may span two physical pages */ 1502 if (n == 0) { 1503 /* NOTE: tb_end may be after the end of the page, but 1504 it is not a problem */ 1505 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1506 tb_end = tb_start + tb->size; 1507 } else { 1508 tb_start = tb->page_addr[1]; 1509 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1510 } 1511 if (!(tb_end <= start || tb_start >= end)) { 1512 #ifdef TARGET_HAS_PRECISE_SMC 1513 if (current_tb_not_found) { 1514 current_tb_not_found = 0; 1515 current_tb = NULL; 1516 if (cpu->mem_io_pc) { 1517 /* now we have a real cpu fault */ 1518 current_tb = tb_find_pc(cpu->mem_io_pc); 1519 } 1520 } 1521 if (current_tb == tb && 1522 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1523 /* If we are modifying the current TB, we must stop 1524 its execution. We could be more precise by checking 1525 that the modification is after the current PC, but it 1526 would require a specialized function to partially 1527 restore the CPU state */ 1528 1529 current_tb_modified = 1; 1530 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc); 1531 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1532 ¤t_flags); 1533 } 1534 #endif /* TARGET_HAS_PRECISE_SMC */ 1535 tb_phys_invalidate(tb, -1); 1536 } 1537 tb = tb_next; 1538 } 1539 #if !defined(CONFIG_USER_ONLY) 1540 /* if no code remaining, no need to continue to use slow writes */ 1541 if (!p->first_tb) { 1542 invalidate_page_bitmap(p); 1543 tlb_unprotect_code(start); 1544 } 1545 #endif 1546 #ifdef TARGET_HAS_PRECISE_SMC 1547 if (current_tb_modified) { 1548 /* Force execution of one insn next time. */ 1549 cpu->cflags_next_tb = 1 | curr_cflags(); 1550 cpu_loop_exit_noexc(cpu); 1551 } 1552 #endif 1553 } 1554 1555 #ifdef CONFIG_SOFTMMU 1556 /* len must be <= 8 and start must be a multiple of len. 1557 * Called via softmmu_template.h when code areas are written to with 1558 * iothread mutex not held. 1559 */ 1560 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1561 { 1562 PageDesc *p; 1563 1564 #if 0 1565 if (1) { 1566 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1567 cpu_single_env->mem_io_vaddr, len, 1568 cpu_single_env->eip, 1569 cpu_single_env->eip + 1570 (intptr_t)cpu_single_env->segs[R_CS].base); 1571 } 1572 #endif 1573 assert_memory_lock(); 1574 1575 p = page_find(start >> TARGET_PAGE_BITS); 1576 if (!p) { 1577 return; 1578 } 1579 if (!p->code_bitmap && 1580 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1581 /* build code bitmap. FIXME: writes should be protected by 1582 * tb_lock, reads by tb_lock or RCU. 1583 */ 1584 build_page_bitmap(p); 1585 } 1586 if (p->code_bitmap) { 1587 unsigned int nr; 1588 unsigned long b; 1589 1590 nr = start & ~TARGET_PAGE_MASK; 1591 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1592 if (b & ((1 << len) - 1)) { 1593 goto do_invalidate; 1594 } 1595 } else { 1596 do_invalidate: 1597 tb_invalidate_phys_page_range(start, start + len, 1); 1598 } 1599 } 1600 #else 1601 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1602 * host PC of the faulting store instruction that caused this invalidate. 1603 * Returns true if the caller needs to abort execution of the current 1604 * TB (because it was modified by this store and the guest CPU has 1605 * precise-SMC semantics). 1606 */ 1607 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1608 { 1609 TranslationBlock *tb; 1610 PageDesc *p; 1611 int n; 1612 #ifdef TARGET_HAS_PRECISE_SMC 1613 TranslationBlock *current_tb = NULL; 1614 CPUState *cpu = current_cpu; 1615 CPUArchState *env = NULL; 1616 int current_tb_modified = 0; 1617 target_ulong current_pc = 0; 1618 target_ulong current_cs_base = 0; 1619 uint32_t current_flags = 0; 1620 #endif 1621 1622 assert_memory_lock(); 1623 1624 addr &= TARGET_PAGE_MASK; 1625 p = page_find(addr >> TARGET_PAGE_BITS); 1626 if (!p) { 1627 return false; 1628 } 1629 1630 tb_lock(); 1631 tb = p->first_tb; 1632 #ifdef TARGET_HAS_PRECISE_SMC 1633 if (tb && pc != 0) { 1634 current_tb = tb_find_pc(pc); 1635 } 1636 if (cpu != NULL) { 1637 env = cpu->env_ptr; 1638 } 1639 #endif 1640 while (tb != NULL) { 1641 n = (uintptr_t)tb & 3; 1642 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1643 #ifdef TARGET_HAS_PRECISE_SMC 1644 if (current_tb == tb && 1645 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1646 /* If we are modifying the current TB, we must stop 1647 its execution. We could be more precise by checking 1648 that the modification is after the current PC, but it 1649 would require a specialized function to partially 1650 restore the CPU state */ 1651 1652 current_tb_modified = 1; 1653 cpu_restore_state_from_tb(cpu, current_tb, pc); 1654 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1655 ¤t_flags); 1656 } 1657 #endif /* TARGET_HAS_PRECISE_SMC */ 1658 tb_phys_invalidate(tb, addr); 1659 tb = tb->page_next[n]; 1660 } 1661 p->first_tb = NULL; 1662 #ifdef TARGET_HAS_PRECISE_SMC 1663 if (current_tb_modified) { 1664 /* Force execution of one insn next time. */ 1665 cpu->cflags_next_tb = 1 | curr_cflags(); 1666 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1667 * back into the cpu_exec loop. */ 1668 return true; 1669 } 1670 #endif 1671 tb_unlock(); 1672 1673 return false; 1674 } 1675 #endif 1676 1677 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr < 1678 tb[1].tc_ptr. Return NULL if not found */ 1679 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) 1680 { 1681 int m_min, m_max, m; 1682 uintptr_t v; 1683 TranslationBlock *tb; 1684 1685 if (tcg_ctx.tb_ctx.nb_tbs <= 0) { 1686 return NULL; 1687 } 1688 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer || 1689 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) { 1690 return NULL; 1691 } 1692 /* binary search (cf Knuth) */ 1693 m_min = 0; 1694 m_max = tcg_ctx.tb_ctx.nb_tbs - 1; 1695 while (m_min <= m_max) { 1696 m = (m_min + m_max) >> 1; 1697 tb = tcg_ctx.tb_ctx.tbs[m]; 1698 v = (uintptr_t)tb->tc.ptr; 1699 if (v == tc_ptr) { 1700 return tb; 1701 } else if (tc_ptr < v) { 1702 m_max = m - 1; 1703 } else { 1704 m_min = m + 1; 1705 } 1706 } 1707 return tcg_ctx.tb_ctx.tbs[m_max]; 1708 } 1709 1710 #if !defined(CONFIG_USER_ONLY) 1711 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr) 1712 { 1713 ram_addr_t ram_addr; 1714 MemoryRegion *mr; 1715 hwaddr l = 1; 1716 1717 rcu_read_lock(); 1718 mr = address_space_translate(as, addr, &addr, &l, false); 1719 if (!(memory_region_is_ram(mr) 1720 || memory_region_is_romd(mr))) { 1721 rcu_read_unlock(); 1722 return; 1723 } 1724 ram_addr = memory_region_get_ram_addr(mr) + addr; 1725 tb_lock(); 1726 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1727 tb_unlock(); 1728 rcu_read_unlock(); 1729 } 1730 #endif /* !defined(CONFIG_USER_ONLY) */ 1731 1732 /* Called with tb_lock held. */ 1733 void tb_check_watchpoint(CPUState *cpu) 1734 { 1735 TranslationBlock *tb; 1736 1737 tb = tb_find_pc(cpu->mem_io_pc); 1738 if (tb) { 1739 /* We can use retranslation to find the PC. */ 1740 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc); 1741 tb_phys_invalidate(tb, -1); 1742 } else { 1743 /* The exception probably happened in a helper. The CPU state should 1744 have been saved before calling it. Fetch the PC from there. */ 1745 CPUArchState *env = cpu->env_ptr; 1746 target_ulong pc, cs_base; 1747 tb_page_addr_t addr; 1748 uint32_t flags; 1749 1750 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1751 addr = get_page_addr_code(env, pc); 1752 tb_invalidate_phys_range(addr, addr + 1); 1753 } 1754 } 1755 1756 #ifndef CONFIG_USER_ONLY 1757 /* in deterministic execution mode, instructions doing device I/Os 1758 * must be at the end of the TB. 1759 * 1760 * Called by softmmu_template.h, with iothread mutex not held. 1761 */ 1762 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1763 { 1764 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1765 CPUArchState *env = cpu->env_ptr; 1766 #endif 1767 TranslationBlock *tb; 1768 uint32_t n; 1769 1770 tb_lock(); 1771 tb = tb_find_pc(retaddr); 1772 if (!tb) { 1773 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1774 (void *)retaddr); 1775 } 1776 n = cpu->icount_decr.u16.low + tb->icount; 1777 cpu_restore_state_from_tb(cpu, tb, retaddr); 1778 /* Calculate how many instructions had been executed before the fault 1779 occurred. */ 1780 n = n - cpu->icount_decr.u16.low; 1781 /* Generate a new TB ending on the I/O insn. */ 1782 n++; 1783 /* On MIPS and SH, delay slot instructions can only be restarted if 1784 they were already the first instruction in the TB. If this is not 1785 the first instruction in a TB then re-execute the preceding 1786 branch. */ 1787 #if defined(TARGET_MIPS) 1788 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) { 1789 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1790 cpu->icount_decr.u16.low++; 1791 env->hflags &= ~MIPS_HFLAG_BMASK; 1792 } 1793 #elif defined(TARGET_SH4) 1794 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1795 && n > 1) { 1796 env->pc -= 2; 1797 cpu->icount_decr.u16.low++; 1798 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1799 } 1800 #endif 1801 /* This should never happen. */ 1802 if (n > CF_COUNT_MASK) { 1803 cpu_abort(cpu, "TB too big during recompile"); 1804 } 1805 1806 /* Adjust the execution state of the next TB. */ 1807 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1808 1809 if (tb->cflags & CF_NOCACHE) { 1810 if (tb->orig_tb) { 1811 /* Invalidate original TB if this TB was generated in 1812 * cpu_exec_nocache() */ 1813 tb_phys_invalidate(tb->orig_tb, -1); 1814 } 1815 tb_free(tb); 1816 } 1817 1818 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1819 * the first in the TB) then we end up generating a whole new TB and 1820 * repeating the fault, which is horribly inefficient. 1821 * Better would be to execute just this insn uncached, or generate a 1822 * second new TB. 1823 * 1824 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1825 * tb_lock gets reset. 1826 */ 1827 cpu_loop_exit_noexc(cpu); 1828 } 1829 1830 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1831 { 1832 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1833 1834 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1835 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1836 } 1837 } 1838 1839 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1840 { 1841 /* Discard jump cache entries for any tb which might potentially 1842 overlap the flushed page. */ 1843 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1844 tb_jmp_cache_clear_page(cpu, addr); 1845 } 1846 1847 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1848 struct qht_stats hst) 1849 { 1850 uint32_t hgram_opts; 1851 size_t hgram_bins; 1852 char *hgram; 1853 1854 if (!hst.head_buckets) { 1855 return; 1856 } 1857 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1858 hst.used_head_buckets, hst.head_buckets, 1859 (double)hst.used_head_buckets / hst.head_buckets * 100); 1860 1861 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1862 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1863 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1864 hgram_opts |= QDIST_PR_NODECIMAL; 1865 } 1866 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1867 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1868 qdist_avg(&hst.occupancy) * 100, hgram); 1869 g_free(hgram); 1870 1871 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1872 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1873 if (hgram_bins > 10) { 1874 hgram_bins = 10; 1875 } else { 1876 hgram_bins = 0; 1877 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1878 } 1879 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1880 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1881 qdist_avg(&hst.chain), hgram); 1882 g_free(hgram); 1883 } 1884 1885 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1886 { 1887 int i, target_code_size, max_target_code_size; 1888 int direct_jmp_count, direct_jmp2_count, cross_page; 1889 TranslationBlock *tb; 1890 struct qht_stats hst; 1891 1892 tb_lock(); 1893 1894 target_code_size = 0; 1895 max_target_code_size = 0; 1896 cross_page = 0; 1897 direct_jmp_count = 0; 1898 direct_jmp2_count = 0; 1899 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) { 1900 tb = tcg_ctx.tb_ctx.tbs[i]; 1901 target_code_size += tb->size; 1902 if (tb->size > max_target_code_size) { 1903 max_target_code_size = tb->size; 1904 } 1905 if (tb->page_addr[1] != -1) { 1906 cross_page++; 1907 } 1908 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1909 direct_jmp_count++; 1910 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1911 direct_jmp2_count++; 1912 } 1913 } 1914 } 1915 /* XXX: avoid using doubles ? */ 1916 cpu_fprintf(f, "Translation buffer state:\n"); 1917 cpu_fprintf(f, "gen code size %td/%zd\n", 1918 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer, 1919 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer); 1920 cpu_fprintf(f, "TB count %d\n", tcg_ctx.tb_ctx.nb_tbs); 1921 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n", 1922 tcg_ctx.tb_ctx.nb_tbs ? target_code_size / 1923 tcg_ctx.tb_ctx.nb_tbs : 0, 1924 max_target_code_size); 1925 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n", 1926 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr - 1927 tcg_ctx.code_gen_buffer) / 1928 tcg_ctx.tb_ctx.nb_tbs : 0, 1929 target_code_size ? (double) (tcg_ctx.code_gen_ptr - 1930 tcg_ctx.code_gen_buffer) / 1931 target_code_size : 0); 1932 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page, 1933 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) / 1934 tcg_ctx.tb_ctx.nb_tbs : 0); 1935 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n", 1936 direct_jmp_count, 1937 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) / 1938 tcg_ctx.tb_ctx.nb_tbs : 0, 1939 direct_jmp2_count, 1940 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) / 1941 tcg_ctx.tb_ctx.nb_tbs : 0); 1942 1943 qht_statistics_init(&tcg_ctx.tb_ctx.htable, &hst); 1944 print_qht_statistics(f, cpu_fprintf, hst); 1945 qht_statistics_destroy(&hst); 1946 1947 cpu_fprintf(f, "\nStatistics:\n"); 1948 cpu_fprintf(f, "TB flush count %u\n", 1949 atomic_read(&tcg_ctx.tb_ctx.tb_flush_count)); 1950 cpu_fprintf(f, "TB invalidate count %d\n", 1951 tcg_ctx.tb_ctx.tb_phys_invalidate_count); 1952 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1953 tcg_dump_info(f, cpu_fprintf); 1954 1955 tb_unlock(); 1956 } 1957 1958 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1959 { 1960 tcg_dump_op_count(f, cpu_fprintf); 1961 } 1962 1963 #else /* CONFIG_USER_ONLY */ 1964 1965 void cpu_interrupt(CPUState *cpu, int mask) 1966 { 1967 g_assert(qemu_mutex_iothread_locked()); 1968 cpu->interrupt_request |= mask; 1969 cpu->icount_decr.u16.high = -1; 1970 } 1971 1972 /* 1973 * Walks guest process memory "regions" one by one 1974 * and calls callback function 'fn' for each region. 1975 */ 1976 struct walk_memory_regions_data { 1977 walk_memory_regions_fn fn; 1978 void *priv; 1979 target_ulong start; 1980 int prot; 1981 }; 1982 1983 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1984 target_ulong end, int new_prot) 1985 { 1986 if (data->start != -1u) { 1987 int rc = data->fn(data->priv, data->start, end, data->prot); 1988 if (rc != 0) { 1989 return rc; 1990 } 1991 } 1992 1993 data->start = (new_prot ? end : -1u); 1994 data->prot = new_prot; 1995 1996 return 0; 1997 } 1998 1999 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 2000 target_ulong base, int level, void **lp) 2001 { 2002 target_ulong pa; 2003 int i, rc; 2004 2005 if (*lp == NULL) { 2006 return walk_memory_regions_end(data, base, 0); 2007 } 2008 2009 if (level == 0) { 2010 PageDesc *pd = *lp; 2011 2012 for (i = 0; i < V_L2_SIZE; ++i) { 2013 int prot = pd[i].flags; 2014 2015 pa = base | (i << TARGET_PAGE_BITS); 2016 if (prot != data->prot) { 2017 rc = walk_memory_regions_end(data, pa, prot); 2018 if (rc != 0) { 2019 return rc; 2020 } 2021 } 2022 } 2023 } else { 2024 void **pp = *lp; 2025 2026 for (i = 0; i < V_L2_SIZE; ++i) { 2027 pa = base | ((target_ulong)i << 2028 (TARGET_PAGE_BITS + V_L2_BITS * level)); 2029 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 2030 if (rc != 0) { 2031 return rc; 2032 } 2033 } 2034 } 2035 2036 return 0; 2037 } 2038 2039 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2040 { 2041 struct walk_memory_regions_data data; 2042 uintptr_t i, l1_sz = v_l1_size; 2043 2044 data.fn = fn; 2045 data.priv = priv; 2046 data.start = -1u; 2047 data.prot = 0; 2048 2049 for (i = 0; i < l1_sz; i++) { 2050 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2051 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2052 if (rc != 0) { 2053 return rc; 2054 } 2055 } 2056 2057 return walk_memory_regions_end(&data, 0, 0); 2058 } 2059 2060 static int dump_region(void *priv, target_ulong start, 2061 target_ulong end, unsigned long prot) 2062 { 2063 FILE *f = (FILE *)priv; 2064 2065 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2066 " "TARGET_FMT_lx" %c%c%c\n", 2067 start, end, end - start, 2068 ((prot & PAGE_READ) ? 'r' : '-'), 2069 ((prot & PAGE_WRITE) ? 'w' : '-'), 2070 ((prot & PAGE_EXEC) ? 'x' : '-')); 2071 2072 return 0; 2073 } 2074 2075 /* dump memory mappings */ 2076 void page_dump(FILE *f) 2077 { 2078 const int length = sizeof(target_ulong) * 2; 2079 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2080 length, "start", length, "end", length, "size", "prot"); 2081 walk_memory_regions(f, dump_region); 2082 } 2083 2084 int page_get_flags(target_ulong address) 2085 { 2086 PageDesc *p; 2087 2088 p = page_find(address >> TARGET_PAGE_BITS); 2089 if (!p) { 2090 return 0; 2091 } 2092 return p->flags; 2093 } 2094 2095 /* Modify the flags of a page and invalidate the code if necessary. 2096 The flag PAGE_WRITE_ORG is positioned automatically depending 2097 on PAGE_WRITE. The mmap_lock should already be held. */ 2098 void page_set_flags(target_ulong start, target_ulong end, int flags) 2099 { 2100 target_ulong addr, len; 2101 2102 /* This function should never be called with addresses outside the 2103 guest address space. If this assert fires, it probably indicates 2104 a missing call to h2g_valid. */ 2105 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2106 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2107 #endif 2108 assert(start < end); 2109 assert_memory_lock(); 2110 2111 start = start & TARGET_PAGE_MASK; 2112 end = TARGET_PAGE_ALIGN(end); 2113 2114 if (flags & PAGE_WRITE) { 2115 flags |= PAGE_WRITE_ORG; 2116 } 2117 2118 for (addr = start, len = end - start; 2119 len != 0; 2120 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2121 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2122 2123 /* If the write protection bit is set, then we invalidate 2124 the code inside. */ 2125 if (!(p->flags & PAGE_WRITE) && 2126 (flags & PAGE_WRITE) && 2127 p->first_tb) { 2128 tb_invalidate_phys_page(addr, 0); 2129 } 2130 p->flags = flags; 2131 } 2132 } 2133 2134 int page_check_range(target_ulong start, target_ulong len, int flags) 2135 { 2136 PageDesc *p; 2137 target_ulong end; 2138 target_ulong addr; 2139 2140 /* This function should never be called with addresses outside the 2141 guest address space. If this assert fires, it probably indicates 2142 a missing call to h2g_valid. */ 2143 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2144 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2145 #endif 2146 2147 if (len == 0) { 2148 return 0; 2149 } 2150 if (start + len - 1 < start) { 2151 /* We've wrapped around. */ 2152 return -1; 2153 } 2154 2155 /* must do before we loose bits in the next step */ 2156 end = TARGET_PAGE_ALIGN(start + len); 2157 start = start & TARGET_PAGE_MASK; 2158 2159 for (addr = start, len = end - start; 2160 len != 0; 2161 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2162 p = page_find(addr >> TARGET_PAGE_BITS); 2163 if (!p) { 2164 return -1; 2165 } 2166 if (!(p->flags & PAGE_VALID)) { 2167 return -1; 2168 } 2169 2170 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2171 return -1; 2172 } 2173 if (flags & PAGE_WRITE) { 2174 if (!(p->flags & PAGE_WRITE_ORG)) { 2175 return -1; 2176 } 2177 /* unprotect the page if it was put read-only because it 2178 contains translated code */ 2179 if (!(p->flags & PAGE_WRITE)) { 2180 if (!page_unprotect(addr, 0)) { 2181 return -1; 2182 } 2183 } 2184 } 2185 } 2186 return 0; 2187 } 2188 2189 /* called from signal handler: invalidate the code and unprotect the 2190 * page. Return 0 if the fault was not handled, 1 if it was handled, 2191 * and 2 if it was handled but the caller must cause the TB to be 2192 * immediately exited. (We can only return 2 if the 'pc' argument is 2193 * non-zero.) 2194 */ 2195 int page_unprotect(target_ulong address, uintptr_t pc) 2196 { 2197 unsigned int prot; 2198 bool current_tb_invalidated; 2199 PageDesc *p; 2200 target_ulong host_start, host_end, addr; 2201 2202 /* Technically this isn't safe inside a signal handler. However we 2203 know this only ever happens in a synchronous SEGV handler, so in 2204 practice it seems to be ok. */ 2205 mmap_lock(); 2206 2207 p = page_find(address >> TARGET_PAGE_BITS); 2208 if (!p) { 2209 mmap_unlock(); 2210 return 0; 2211 } 2212 2213 /* if the page was really writable, then we change its 2214 protection back to writable */ 2215 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) { 2216 host_start = address & qemu_host_page_mask; 2217 host_end = host_start + qemu_host_page_size; 2218 2219 prot = 0; 2220 current_tb_invalidated = false; 2221 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) { 2222 p = page_find(addr >> TARGET_PAGE_BITS); 2223 p->flags |= PAGE_WRITE; 2224 prot |= p->flags; 2225 2226 /* and since the content will be modified, we must invalidate 2227 the corresponding translated code. */ 2228 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2229 #ifdef CONFIG_USER_ONLY 2230 if (DEBUG_TB_CHECK_GATE) { 2231 tb_invalidate_check(addr); 2232 } 2233 #endif 2234 } 2235 mprotect((void *)g2h(host_start), qemu_host_page_size, 2236 prot & PAGE_BITS); 2237 2238 mmap_unlock(); 2239 /* If current TB was invalidated return to main loop */ 2240 return current_tb_invalidated ? 2 : 1; 2241 } 2242 mmap_unlock(); 2243 return 0; 2244 } 2245 #endif /* CONFIG_USER_ONLY */ 2246 2247 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2248 void tcg_flush_softmmu_tlb(CPUState *cs) 2249 { 2250 #ifdef CONFIG_SOFTMMU 2251 tlb_flush(cs); 2252 #endif 2253 } 2254