1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #include "exec/exec-all.h" 35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 36 #include <sys/param.h> 37 #if __FreeBSD_version >= 700104 38 #define HAVE_KINFO_GETVMMAP 39 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 40 #include <sys/proc.h> 41 #include <machine/profile.h> 42 #define _KERNEL 43 #include <sys/user.h> 44 #undef _KERNEL 45 #undef sigqueue 46 #include <libutil.h> 47 #endif 48 #endif 49 #else 50 #include "exec/address-spaces.h" 51 #endif 52 53 #include "exec/cputlb.h" 54 #include "exec/tb-hash.h" 55 #include "translate-all.h" 56 #include "qemu/bitmap.h" 57 #include "qemu/error-report.h" 58 #include "qemu/timer.h" 59 #include "qemu/main-loop.h" 60 #include "exec/log.h" 61 #include "sysemu/cpus.h" 62 63 /* #define DEBUG_TB_INVALIDATE */ 64 /* #define DEBUG_TB_FLUSH */ 65 /* make various TB consistency checks */ 66 /* #define DEBUG_TB_CHECK */ 67 68 #ifdef DEBUG_TB_INVALIDATE 69 #define DEBUG_TB_INVALIDATE_GATE 1 70 #else 71 #define DEBUG_TB_INVALIDATE_GATE 0 72 #endif 73 74 #ifdef DEBUG_TB_FLUSH 75 #define DEBUG_TB_FLUSH_GATE 1 76 #else 77 #define DEBUG_TB_FLUSH_GATE 0 78 #endif 79 80 #if !defined(CONFIG_USER_ONLY) 81 /* TB consistency checks only implemented for usermode emulation. */ 82 #undef DEBUG_TB_CHECK 83 #endif 84 85 /* Access to the various translations structures need to be serialised via locks 86 * for consistency. This is automatic for SoftMMU based system 87 * emulation due to its single threaded nature. In user-mode emulation 88 * access to the memory related structures are protected with the 89 * mmap_lock. 90 */ 91 #ifdef CONFIG_SOFTMMU 92 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 93 #else 94 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 95 #endif 96 97 #define SMC_BITMAP_USE_THRESHOLD 10 98 99 typedef struct PageDesc { 100 /* list of TBs intersecting this ram page */ 101 TranslationBlock *first_tb; 102 #ifdef CONFIG_SOFTMMU 103 /* in order to optimize self modifying code, we count the number 104 of lookups we do to a given page to use a bitmap */ 105 unsigned int code_write_count; 106 unsigned long *code_bitmap; 107 #else 108 unsigned long flags; 109 #endif 110 } PageDesc; 111 112 /* In system mode we want L1_MAP to be based on ram offsets, 113 while in user mode we want it to be based on virtual addresses. */ 114 #if !defined(CONFIG_USER_ONLY) 115 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 116 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 117 #else 118 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 119 #endif 120 #else 121 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 122 #endif 123 124 /* Size of the L2 (and L3, etc) page tables. */ 125 #define V_L2_BITS 10 126 #define V_L2_SIZE (1 << V_L2_BITS) 127 128 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 129 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 130 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 131 * BITS_PER_BYTE); 132 133 /* 134 * L1 Mapping properties 135 */ 136 static int v_l1_size; 137 static int v_l1_shift; 138 static int v_l2_levels; 139 140 /* The bottom level has pointers to PageDesc, and is indexed by 141 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 142 */ 143 #define V_L1_MIN_BITS 4 144 #define V_L1_MAX_BITS (V_L2_BITS + 3) 145 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 146 147 static void *l1_map[V_L1_MAX_SIZE]; 148 149 /* code generation context */ 150 TCGContext tcg_ctx; 151 bool parallel_cpus; 152 153 /* translation block context */ 154 static __thread int have_tb_lock; 155 156 static void page_table_config_init(void) 157 { 158 uint32_t v_l1_bits; 159 160 assert(TARGET_PAGE_BITS); 161 /* The bits remaining after N lower levels of page tables. */ 162 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 163 if (v_l1_bits < V_L1_MIN_BITS) { 164 v_l1_bits += V_L2_BITS; 165 } 166 167 v_l1_size = 1 << v_l1_bits; 168 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 169 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 170 171 assert(v_l1_bits <= V_L1_MAX_BITS); 172 assert(v_l1_shift % V_L2_BITS == 0); 173 assert(v_l2_levels >= 0); 174 } 175 176 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 177 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 178 179 void tb_lock(void) 180 { 181 assert_tb_unlocked(); 182 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock); 183 have_tb_lock++; 184 } 185 186 void tb_unlock(void) 187 { 188 assert_tb_locked(); 189 have_tb_lock--; 190 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock); 191 } 192 193 void tb_lock_reset(void) 194 { 195 if (have_tb_lock) { 196 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock); 197 have_tb_lock = 0; 198 } 199 } 200 201 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr); 202 203 void cpu_gen_init(void) 204 { 205 tcg_context_init(&tcg_ctx); 206 } 207 208 /* Encode VAL as a signed leb128 sequence at P. 209 Return P incremented past the encoded value. */ 210 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 211 { 212 int more, byte; 213 214 do { 215 byte = val & 0x7f; 216 val >>= 7; 217 more = !((val == 0 && (byte & 0x40) == 0) 218 || (val == -1 && (byte & 0x40) != 0)); 219 if (more) { 220 byte |= 0x80; 221 } 222 *p++ = byte; 223 } while (more); 224 225 return p; 226 } 227 228 /* Decode a signed leb128 sequence at *PP; increment *PP past the 229 decoded value. Return the decoded value. */ 230 static target_long decode_sleb128(uint8_t **pp) 231 { 232 uint8_t *p = *pp; 233 target_long val = 0; 234 int byte, shift = 0; 235 236 do { 237 byte = *p++; 238 val |= (target_ulong)(byte & 0x7f) << shift; 239 shift += 7; 240 } while (byte & 0x80); 241 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 242 val |= -(target_ulong)1 << shift; 243 } 244 245 *pp = p; 246 return val; 247 } 248 249 /* Encode the data collected about the instructions while compiling TB. 250 Place the data at BLOCK, and return the number of bytes consumed. 251 252 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's, 253 which come from the target's insn_start data, followed by a uintptr_t 254 which comes from the host pc of the end of the code implementing the insn. 255 256 Each line of the table is encoded as sleb128 deltas from the previous 257 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }. 258 That is, the first column is seeded with the guest pc, the last column 259 with the host pc, and the middle columns with zeros. */ 260 261 static int encode_search(TranslationBlock *tb, uint8_t *block) 262 { 263 uint8_t *highwater = tcg_ctx.code_gen_highwater; 264 uint8_t *p = block; 265 int i, j, n; 266 267 tb->tc_search = block; 268 269 for (i = 0, n = tb->icount; i < n; ++i) { 270 target_ulong prev; 271 272 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 273 if (i == 0) { 274 prev = (j == 0 ? tb->pc : 0); 275 } else { 276 prev = tcg_ctx.gen_insn_data[i - 1][j]; 277 } 278 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev); 279 } 280 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]); 281 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev); 282 283 /* Test for (pending) buffer overflow. The assumption is that any 284 one row beginning below the high water mark cannot overrun 285 the buffer completely. Thus we can test for overflow after 286 encoding a row without having to check during encoding. */ 287 if (unlikely(p > highwater)) { 288 return -1; 289 } 290 } 291 292 return p - block; 293 } 294 295 /* The cpu state corresponding to 'searched_pc' is restored. 296 * Called with tb_lock held. 297 */ 298 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 299 uintptr_t searched_pc) 300 { 301 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 302 uintptr_t host_pc = (uintptr_t)tb->tc_ptr; 303 CPUArchState *env = cpu->env_ptr; 304 uint8_t *p = tb->tc_search; 305 int i, j, num_insns = tb->icount; 306 #ifdef CONFIG_PROFILER 307 int64_t ti = profile_getclock(); 308 #endif 309 310 searched_pc -= GETPC_ADJ; 311 312 if (searched_pc < host_pc) { 313 return -1; 314 } 315 316 /* Reconstruct the stored insn data while looking for the point at 317 which the end of the insn exceeds the searched_pc. */ 318 for (i = 0; i < num_insns; ++i) { 319 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 320 data[j] += decode_sleb128(&p); 321 } 322 host_pc += decode_sleb128(&p); 323 if (host_pc > searched_pc) { 324 goto found; 325 } 326 } 327 return -1; 328 329 found: 330 if (tb->cflags & CF_USE_ICOUNT) { 331 assert(use_icount); 332 /* Reset the cycle counter to the start of the block. */ 333 cpu->icount_decr.u16.low += num_insns; 334 /* Clear the IO flag. */ 335 cpu->can_do_io = 0; 336 } 337 cpu->icount_decr.u16.low -= i; 338 restore_state_to_opc(env, tb, data); 339 340 #ifdef CONFIG_PROFILER 341 tcg_ctx.restore_time += profile_getclock() - ti; 342 tcg_ctx.restore_count++; 343 #endif 344 return 0; 345 } 346 347 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr) 348 { 349 TranslationBlock *tb; 350 bool r = false; 351 352 /* A retaddr of zero is invalid so we really shouldn't have ended 353 * up here. The target code has likely forgotten to check retaddr 354 * != 0 before attempting to restore state. We return early to 355 * avoid blowing up on a recursive tb_lock(). The target must have 356 * previously survived a failed cpu_restore_state because 357 * tb_find_pc(0) would have failed anyway. It still should be 358 * fixed though. 359 */ 360 361 if (!retaddr) { 362 return r; 363 } 364 365 tb_lock(); 366 tb = tb_find_pc(retaddr); 367 if (tb) { 368 cpu_restore_state_from_tb(cpu, tb, retaddr); 369 if (tb->cflags & CF_NOCACHE) { 370 /* one-shot translation, invalidate it immediately */ 371 tb_phys_invalidate(tb, -1); 372 tb_free(tb); 373 } 374 r = true; 375 } 376 tb_unlock(); 377 378 return r; 379 } 380 381 static void page_init(void) 382 { 383 page_size_init(); 384 page_table_config_init(); 385 386 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 387 { 388 #ifdef HAVE_KINFO_GETVMMAP 389 struct kinfo_vmentry *freep; 390 int i, cnt; 391 392 freep = kinfo_getvmmap(getpid(), &cnt); 393 if (freep) { 394 mmap_lock(); 395 for (i = 0; i < cnt; i++) { 396 unsigned long startaddr, endaddr; 397 398 startaddr = freep[i].kve_start; 399 endaddr = freep[i].kve_end; 400 if (h2g_valid(startaddr)) { 401 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 402 403 if (h2g_valid(endaddr)) { 404 endaddr = h2g(endaddr); 405 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 406 } else { 407 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 408 endaddr = ~0ul; 409 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 410 #endif 411 } 412 } 413 } 414 free(freep); 415 mmap_unlock(); 416 } 417 #else 418 FILE *f; 419 420 last_brk = (unsigned long)sbrk(0); 421 422 f = fopen("/compat/linux/proc/self/maps", "r"); 423 if (f) { 424 mmap_lock(); 425 426 do { 427 unsigned long startaddr, endaddr; 428 int n; 429 430 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 431 432 if (n == 2 && h2g_valid(startaddr)) { 433 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 434 435 if (h2g_valid(endaddr)) { 436 endaddr = h2g(endaddr); 437 } else { 438 endaddr = ~0ul; 439 } 440 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 441 } 442 } while (!feof(f)); 443 444 fclose(f); 445 mmap_unlock(); 446 } 447 #endif 448 } 449 #endif 450 } 451 452 /* If alloc=1: 453 * Called with tb_lock held for system emulation. 454 * Called with mmap_lock held for user-mode emulation. 455 */ 456 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 457 { 458 PageDesc *pd; 459 void **lp; 460 int i; 461 462 if (alloc) { 463 assert_memory_lock(); 464 } 465 466 /* Level 1. Always allocated. */ 467 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 468 469 /* Level 2..N-1. */ 470 for (i = v_l2_levels; i > 0; i--) { 471 void **p = atomic_rcu_read(lp); 472 473 if (p == NULL) { 474 if (!alloc) { 475 return NULL; 476 } 477 p = g_new0(void *, V_L2_SIZE); 478 atomic_rcu_set(lp, p); 479 } 480 481 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 482 } 483 484 pd = atomic_rcu_read(lp); 485 if (pd == NULL) { 486 if (!alloc) { 487 return NULL; 488 } 489 pd = g_new0(PageDesc, V_L2_SIZE); 490 atomic_rcu_set(lp, pd); 491 } 492 493 return pd + (index & (V_L2_SIZE - 1)); 494 } 495 496 static inline PageDesc *page_find(tb_page_addr_t index) 497 { 498 return page_find_alloc(index, 0); 499 } 500 501 #if defined(CONFIG_USER_ONLY) 502 /* Currently it is not recommended to allocate big chunks of data in 503 user mode. It will change when a dedicated libc will be used. */ 504 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 505 region in which the guest needs to run. Revisit this. */ 506 #define USE_STATIC_CODE_GEN_BUFFER 507 #endif 508 509 /* Minimum size of the code gen buffer. This number is randomly chosen, 510 but not so small that we can't have a fair number of TB's live. */ 511 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 512 513 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 514 indicated, this is constrained by the range of direct branches on the 515 host cpu, as used by the TCG implementation of goto_tb. */ 516 #if defined(__x86_64__) 517 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 518 #elif defined(__sparc__) 519 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 520 #elif defined(__powerpc64__) 521 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 522 #elif defined(__powerpc__) 523 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 524 #elif defined(__aarch64__) 525 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 526 #elif defined(__s390x__) 527 /* We have a +- 4GB range on the branches; leave some slop. */ 528 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 529 #elif defined(__mips__) 530 /* We have a 256MB branch region, but leave room to make sure the 531 main executable is also within that region. */ 532 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 533 #else 534 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 535 #endif 536 537 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 538 539 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 540 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 541 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 542 543 static inline size_t size_code_gen_buffer(size_t tb_size) 544 { 545 /* Size the buffer. */ 546 if (tb_size == 0) { 547 #ifdef USE_STATIC_CODE_GEN_BUFFER 548 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 549 #else 550 /* ??? Needs adjustments. */ 551 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 552 static buffer, we could size this on RESERVED_VA, on the text 553 segment size of the executable, or continue to use the default. */ 554 tb_size = (unsigned long)(ram_size / 4); 555 #endif 556 } 557 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 558 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 559 } 560 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 561 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 562 } 563 return tb_size; 564 } 565 566 #ifdef __mips__ 567 /* In order to use J and JAL within the code_gen_buffer, we require 568 that the buffer not cross a 256MB boundary. */ 569 static inline bool cross_256mb(void *addr, size_t size) 570 { 571 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 572 } 573 574 /* We weren't able to allocate a buffer without crossing that boundary, 575 so make do with the larger portion of the buffer that doesn't cross. 576 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 577 static inline void *split_cross_256mb(void *buf1, size_t size1) 578 { 579 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 580 size_t size2 = buf1 + size1 - buf2; 581 582 size1 = buf2 - buf1; 583 if (size1 < size2) { 584 size1 = size2; 585 buf1 = buf2; 586 } 587 588 tcg_ctx.code_gen_buffer_size = size1; 589 return buf1; 590 } 591 #endif 592 593 #ifdef USE_STATIC_CODE_GEN_BUFFER 594 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 595 __attribute__((aligned(CODE_GEN_ALIGN))); 596 597 # ifdef _WIN32 598 static inline void do_protect(void *addr, long size, int prot) 599 { 600 DWORD old_protect; 601 VirtualProtect(addr, size, prot, &old_protect); 602 } 603 604 static inline void map_exec(void *addr, long size) 605 { 606 do_protect(addr, size, PAGE_EXECUTE_READWRITE); 607 } 608 609 static inline void map_none(void *addr, long size) 610 { 611 do_protect(addr, size, PAGE_NOACCESS); 612 } 613 # else 614 static inline void do_protect(void *addr, long size, int prot) 615 { 616 uintptr_t start, end; 617 618 start = (uintptr_t)addr; 619 start &= qemu_real_host_page_mask; 620 621 end = (uintptr_t)addr + size; 622 end = ROUND_UP(end, qemu_real_host_page_size); 623 624 mprotect((void *)start, end - start, prot); 625 } 626 627 static inline void map_exec(void *addr, long size) 628 { 629 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC); 630 } 631 632 static inline void map_none(void *addr, long size) 633 { 634 do_protect(addr, size, PROT_NONE); 635 } 636 # endif /* WIN32 */ 637 638 static inline void *alloc_code_gen_buffer(void) 639 { 640 void *buf = static_code_gen_buffer; 641 size_t full_size, size; 642 643 /* The size of the buffer, rounded down to end on a page boundary. */ 644 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer)) 645 & qemu_real_host_page_mask) - (uintptr_t)buf; 646 647 /* Reserve a guard page. */ 648 size = full_size - qemu_real_host_page_size; 649 650 /* Honor a command-line option limiting the size of the buffer. */ 651 if (size > tcg_ctx.code_gen_buffer_size) { 652 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size) 653 & qemu_real_host_page_mask) - (uintptr_t)buf; 654 } 655 tcg_ctx.code_gen_buffer_size = size; 656 657 #ifdef __mips__ 658 if (cross_256mb(buf, size)) { 659 buf = split_cross_256mb(buf, size); 660 size = tcg_ctx.code_gen_buffer_size; 661 } 662 #endif 663 664 map_exec(buf, size); 665 map_none(buf + size, qemu_real_host_page_size); 666 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 667 668 return buf; 669 } 670 #elif defined(_WIN32) 671 static inline void *alloc_code_gen_buffer(void) 672 { 673 size_t size = tcg_ctx.code_gen_buffer_size; 674 void *buf1, *buf2; 675 676 /* Perform the allocation in two steps, so that the guard page 677 is reserved but uncommitted. */ 678 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size, 679 MEM_RESERVE, PAGE_NOACCESS); 680 if (buf1 != NULL) { 681 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE); 682 assert(buf1 == buf2); 683 } 684 685 return buf1; 686 } 687 #else 688 static inline void *alloc_code_gen_buffer(void) 689 { 690 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 691 uintptr_t start = 0; 692 size_t size = tcg_ctx.code_gen_buffer_size; 693 void *buf; 694 695 /* Constrain the position of the buffer based on the host cpu. 696 Note that these addresses are chosen in concert with the 697 addresses assigned in the relevant linker script file. */ 698 # if defined(__PIE__) || defined(__PIC__) 699 /* Don't bother setting a preferred location if we're building 700 a position-independent executable. We're more likely to get 701 an address near the main executable if we let the kernel 702 choose the address. */ 703 # elif defined(__x86_64__) && defined(MAP_32BIT) 704 /* Force the memory down into low memory with the executable. 705 Leave the choice of exact location with the kernel. */ 706 flags |= MAP_32BIT; 707 /* Cannot expect to map more than 800MB in low memory. */ 708 if (size > 800u * 1024 * 1024) { 709 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024; 710 } 711 # elif defined(__sparc__) 712 start = 0x40000000ul; 713 # elif defined(__s390x__) 714 start = 0x90000000ul; 715 # elif defined(__mips__) 716 # if _MIPS_SIM == _ABI64 717 start = 0x128000000ul; 718 # else 719 start = 0x08000000ul; 720 # endif 721 # endif 722 723 buf = mmap((void *)start, size + qemu_real_host_page_size, 724 PROT_NONE, flags, -1, 0); 725 if (buf == MAP_FAILED) { 726 return NULL; 727 } 728 729 #ifdef __mips__ 730 if (cross_256mb(buf, size)) { 731 /* Try again, with the original still mapped, to avoid re-acquiring 732 that 256mb crossing. This time don't specify an address. */ 733 size_t size2; 734 void *buf2 = mmap(NULL, size + qemu_real_host_page_size, 735 PROT_NONE, flags, -1, 0); 736 switch ((int)(buf2 != MAP_FAILED)) { 737 case 1: 738 if (!cross_256mb(buf2, size)) { 739 /* Success! Use the new buffer. */ 740 munmap(buf, size + qemu_real_host_page_size); 741 break; 742 } 743 /* Failure. Work with what we had. */ 744 munmap(buf2, size + qemu_real_host_page_size); 745 /* fallthru */ 746 default: 747 /* Split the original buffer. Free the smaller half. */ 748 buf2 = split_cross_256mb(buf, size); 749 size2 = tcg_ctx.code_gen_buffer_size; 750 if (buf == buf2) { 751 munmap(buf + size2 + qemu_real_host_page_size, size - size2); 752 } else { 753 munmap(buf, size - size2); 754 } 755 size = size2; 756 break; 757 } 758 buf = buf2; 759 } 760 #endif 761 762 /* Make the final buffer accessible. The guard page at the end 763 will remain inaccessible with PROT_NONE. */ 764 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC); 765 766 /* Request large pages for the buffer. */ 767 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 768 769 return buf; 770 } 771 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 772 773 static inline void code_gen_alloc(size_t tb_size) 774 { 775 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size); 776 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer(); 777 if (tcg_ctx.code_gen_buffer == NULL) { 778 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 779 exit(1); 780 } 781 782 /* size this conservatively -- realloc later if needed */ 783 tcg_ctx.tb_ctx.tbs_size = 784 tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE / 8; 785 if (unlikely(!tcg_ctx.tb_ctx.tbs_size)) { 786 tcg_ctx.tb_ctx.tbs_size = 64 * 1024; 787 } 788 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock *, tcg_ctx.tb_ctx.tbs_size); 789 790 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock); 791 } 792 793 static void tb_htable_init(void) 794 { 795 unsigned int mode = QHT_MODE_AUTO_RESIZE; 796 797 qht_init(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode); 798 } 799 800 /* Must be called before using the QEMU cpus. 'tb_size' is the size 801 (in bytes) allocated to the translation buffer. Zero means default 802 size. */ 803 void tcg_exec_init(unsigned long tb_size) 804 { 805 tcg_allowed = true; 806 cpu_gen_init(); 807 page_init(); 808 tb_htable_init(); 809 code_gen_alloc(tb_size); 810 #if defined(CONFIG_SOFTMMU) 811 /* There's no guest base to take into account, so go ahead and 812 initialize the prologue now. */ 813 tcg_prologue_init(&tcg_ctx); 814 #endif 815 } 816 817 /* 818 * Allocate a new translation block. Flush the translation buffer if 819 * too many translation blocks or too much generated code. 820 * 821 * Called with tb_lock held. 822 */ 823 static TranslationBlock *tb_alloc(target_ulong pc) 824 { 825 TranslationBlock *tb; 826 TBContext *ctx; 827 828 assert_tb_locked(); 829 830 tb = tcg_tb_alloc(&tcg_ctx); 831 if (unlikely(tb == NULL)) { 832 return NULL; 833 } 834 ctx = &tcg_ctx.tb_ctx; 835 if (unlikely(ctx->nb_tbs == ctx->tbs_size)) { 836 ctx->tbs_size *= 2; 837 ctx->tbs = g_renew(TranslationBlock *, ctx->tbs, ctx->tbs_size); 838 } 839 ctx->tbs[ctx->nb_tbs++] = tb; 840 return tb; 841 } 842 843 /* Called with tb_lock held. */ 844 void tb_free(TranslationBlock *tb) 845 { 846 assert_tb_locked(); 847 848 /* In practice this is mostly used for single use temporary TB 849 Ignore the hard cases and just back up if this TB happens to 850 be the last one generated. */ 851 if (tcg_ctx.tb_ctx.nb_tbs > 0 && 852 tb == tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) { 853 size_t struct_size = ROUND_UP(sizeof(*tb), qemu_icache_linesize); 854 855 tcg_ctx.code_gen_ptr = tb->tc_ptr - struct_size; 856 tcg_ctx.tb_ctx.nb_tbs--; 857 } 858 } 859 860 static inline void invalidate_page_bitmap(PageDesc *p) 861 { 862 #ifdef CONFIG_SOFTMMU 863 g_free(p->code_bitmap); 864 p->code_bitmap = NULL; 865 p->code_write_count = 0; 866 #endif 867 } 868 869 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 870 static void page_flush_tb_1(int level, void **lp) 871 { 872 int i; 873 874 if (*lp == NULL) { 875 return; 876 } 877 if (level == 0) { 878 PageDesc *pd = *lp; 879 880 for (i = 0; i < V_L2_SIZE; ++i) { 881 pd[i].first_tb = NULL; 882 invalidate_page_bitmap(pd + i); 883 } 884 } else { 885 void **pp = *lp; 886 887 for (i = 0; i < V_L2_SIZE; ++i) { 888 page_flush_tb_1(level - 1, pp + i); 889 } 890 } 891 } 892 893 static void page_flush_tb(void) 894 { 895 int i, l1_sz = v_l1_size; 896 897 for (i = 0; i < l1_sz; i++) { 898 page_flush_tb_1(v_l2_levels, l1_map + i); 899 } 900 } 901 902 /* flush all the translation blocks */ 903 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 904 { 905 tb_lock(); 906 907 /* If it is already been done on request of another CPU, 908 * just retry. 909 */ 910 if (tcg_ctx.tb_ctx.tb_flush_count != tb_flush_count.host_int) { 911 goto done; 912 } 913 914 if (DEBUG_TB_FLUSH_GATE) { 915 printf("qemu: flush code_size=%td nb_tbs=%d avg_tb_size=%td\n", 916 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer, 917 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ? 918 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) / 919 tcg_ctx.tb_ctx.nb_tbs : 0); 920 } 921 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) 922 > tcg_ctx.code_gen_buffer_size) { 923 cpu_abort(cpu, "Internal error: code buffer overflow\n"); 924 } 925 926 CPU_FOREACH(cpu) { 927 cpu_tb_jmp_cache_clear(cpu); 928 } 929 930 tcg_ctx.tb_ctx.nb_tbs = 0; 931 qht_reset_size(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 932 page_flush_tb(); 933 934 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer; 935 /* XXX: flush processor icache at this point if cache flush is 936 expensive */ 937 atomic_mb_set(&tcg_ctx.tb_ctx.tb_flush_count, 938 tcg_ctx.tb_ctx.tb_flush_count + 1); 939 940 done: 941 tb_unlock(); 942 } 943 944 void tb_flush(CPUState *cpu) 945 { 946 if (tcg_enabled()) { 947 unsigned tb_flush_count = atomic_mb_read(&tcg_ctx.tb_ctx.tb_flush_count); 948 async_safe_run_on_cpu(cpu, do_tb_flush, 949 RUN_ON_CPU_HOST_INT(tb_flush_count)); 950 } 951 } 952 953 #ifdef DEBUG_TB_CHECK 954 955 static void 956 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 957 { 958 TranslationBlock *tb = p; 959 target_ulong addr = *(target_ulong *)userp; 960 961 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 962 printf("ERROR invalidate: address=" TARGET_FMT_lx 963 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 964 } 965 } 966 967 /* verify that all the pages have correct rights for code 968 * 969 * Called with tb_lock held. 970 */ 971 static void tb_invalidate_check(target_ulong address) 972 { 973 address &= TARGET_PAGE_MASK; 974 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_invalidate_check, &address); 975 } 976 977 static void 978 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 979 { 980 TranslationBlock *tb = p; 981 int flags1, flags2; 982 983 flags1 = page_get_flags(tb->pc); 984 flags2 = page_get_flags(tb->pc + tb->size - 1); 985 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 986 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 987 (long)tb->pc, tb->size, flags1, flags2); 988 } 989 } 990 991 /* verify that all the pages have correct rights for code */ 992 static void tb_page_check(void) 993 { 994 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_page_check, NULL); 995 } 996 997 #endif 998 999 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) 1000 { 1001 TranslationBlock *tb1; 1002 unsigned int n1; 1003 1004 for (;;) { 1005 tb1 = *ptb; 1006 n1 = (uintptr_t)tb1 & 3; 1007 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); 1008 if (tb1 == tb) { 1009 *ptb = tb1->page_next[n1]; 1010 break; 1011 } 1012 ptb = &tb1->page_next[n1]; 1013 } 1014 } 1015 1016 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 1017 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 1018 { 1019 TranslationBlock *tb1; 1020 uintptr_t *ptb, ntb; 1021 unsigned int n1; 1022 1023 ptb = &tb->jmp_list_next[n]; 1024 if (*ptb) { 1025 /* find tb(n) in circular list */ 1026 for (;;) { 1027 ntb = *ptb; 1028 n1 = ntb & 3; 1029 tb1 = (TranslationBlock *)(ntb & ~3); 1030 if (n1 == n && tb1 == tb) { 1031 break; 1032 } 1033 if (n1 == 2) { 1034 ptb = &tb1->jmp_list_first; 1035 } else { 1036 ptb = &tb1->jmp_list_next[n1]; 1037 } 1038 } 1039 /* now we can suppress tb(n) from the list */ 1040 *ptb = tb->jmp_list_next[n]; 1041 1042 tb->jmp_list_next[n] = (uintptr_t)NULL; 1043 } 1044 } 1045 1046 /* reset the jump entry 'n' of a TB so that it is not chained to 1047 another TB */ 1048 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1049 { 1050 uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]); 1051 tb_set_jmp_target(tb, n, addr); 1052 } 1053 1054 /* remove any jumps to the TB */ 1055 static inline void tb_jmp_unlink(TranslationBlock *tb) 1056 { 1057 TranslationBlock *tb1; 1058 uintptr_t *ptb, ntb; 1059 unsigned int n1; 1060 1061 ptb = &tb->jmp_list_first; 1062 for (;;) { 1063 ntb = *ptb; 1064 n1 = ntb & 3; 1065 tb1 = (TranslationBlock *)(ntb & ~3); 1066 if (n1 == 2) { 1067 break; 1068 } 1069 tb_reset_jump(tb1, n1); 1070 *ptb = tb1->jmp_list_next[n1]; 1071 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1072 } 1073 } 1074 1075 /* invalidate one TB 1076 * 1077 * Called with tb_lock held. 1078 */ 1079 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1080 { 1081 CPUState *cpu; 1082 PageDesc *p; 1083 uint32_t h; 1084 tb_page_addr_t phys_pc; 1085 1086 assert_tb_locked(); 1087 1088 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1089 1090 /* remove the TB from the hash list */ 1091 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1092 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->trace_vcpu_dstate); 1093 qht_remove(&tcg_ctx.tb_ctx.htable, tb, h); 1094 1095 /* remove the TB from the page list */ 1096 if (tb->page_addr[0] != page_addr) { 1097 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1098 tb_page_remove(&p->first_tb, tb); 1099 invalidate_page_bitmap(p); 1100 } 1101 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1102 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1103 tb_page_remove(&p->first_tb, tb); 1104 invalidate_page_bitmap(p); 1105 } 1106 1107 /* remove the TB from the hash list */ 1108 h = tb_jmp_cache_hash_func(tb->pc); 1109 CPU_FOREACH(cpu) { 1110 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1111 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1112 } 1113 } 1114 1115 /* suppress this TB from the two jump lists */ 1116 tb_remove_from_jmp_list(tb, 0); 1117 tb_remove_from_jmp_list(tb, 1); 1118 1119 /* suppress any remaining jumps to this TB */ 1120 tb_jmp_unlink(tb); 1121 1122 tcg_ctx.tb_ctx.tb_phys_invalidate_count++; 1123 } 1124 1125 #ifdef CONFIG_SOFTMMU 1126 static void build_page_bitmap(PageDesc *p) 1127 { 1128 int n, tb_start, tb_end; 1129 TranslationBlock *tb; 1130 1131 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1132 1133 tb = p->first_tb; 1134 while (tb != NULL) { 1135 n = (uintptr_t)tb & 3; 1136 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1137 /* NOTE: this is subtle as a TB may span two physical pages */ 1138 if (n == 0) { 1139 /* NOTE: tb_end may be after the end of the page, but 1140 it is not a problem */ 1141 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1142 tb_end = tb_start + tb->size; 1143 if (tb_end > TARGET_PAGE_SIZE) { 1144 tb_end = TARGET_PAGE_SIZE; 1145 } 1146 } else { 1147 tb_start = 0; 1148 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1149 } 1150 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1151 tb = tb->page_next[n]; 1152 } 1153 } 1154 #endif 1155 1156 /* add the tb in the target page and protect it if necessary 1157 * 1158 * Called with mmap_lock held for user-mode emulation. 1159 */ 1160 static inline void tb_alloc_page(TranslationBlock *tb, 1161 unsigned int n, tb_page_addr_t page_addr) 1162 { 1163 PageDesc *p; 1164 #ifndef CONFIG_USER_ONLY 1165 bool page_already_protected; 1166 #endif 1167 1168 assert_memory_lock(); 1169 1170 tb->page_addr[n] = page_addr; 1171 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1172 tb->page_next[n] = p->first_tb; 1173 #ifndef CONFIG_USER_ONLY 1174 page_already_protected = p->first_tb != NULL; 1175 #endif 1176 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); 1177 invalidate_page_bitmap(p); 1178 1179 #if defined(CONFIG_USER_ONLY) 1180 if (p->flags & PAGE_WRITE) { 1181 target_ulong addr; 1182 PageDesc *p2; 1183 int prot; 1184 1185 /* force the host page as non writable (writes will have a 1186 page fault + mprotect overhead) */ 1187 page_addr &= qemu_host_page_mask; 1188 prot = 0; 1189 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1190 addr += TARGET_PAGE_SIZE) { 1191 1192 p2 = page_find(addr >> TARGET_PAGE_BITS); 1193 if (!p2) { 1194 continue; 1195 } 1196 prot |= p2->flags; 1197 p2->flags &= ~PAGE_WRITE; 1198 } 1199 mprotect(g2h(page_addr), qemu_host_page_size, 1200 (prot & PAGE_BITS) & ~PAGE_WRITE); 1201 if (DEBUG_TB_INVALIDATE_GATE) { 1202 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1203 } 1204 } 1205 #else 1206 /* if some code is already present, then the pages are already 1207 protected. So we handle the case where only the first TB is 1208 allocated in a physical page */ 1209 if (!page_already_protected) { 1210 tlb_protect_code(page_addr); 1211 } 1212 #endif 1213 } 1214 1215 /* add a new TB and link it to the physical page tables. phys_page2 is 1216 * (-1) to indicate that only one page contains the TB. 1217 * 1218 * Called with mmap_lock held for user-mode emulation. 1219 */ 1220 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1221 tb_page_addr_t phys_page2) 1222 { 1223 uint32_t h; 1224 1225 assert_memory_lock(); 1226 1227 /* add in the page list */ 1228 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1229 if (phys_page2 != -1) { 1230 tb_alloc_page(tb, 1, phys_page2); 1231 } else { 1232 tb->page_addr[1] = -1; 1233 } 1234 1235 /* add in the hash table */ 1236 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->trace_vcpu_dstate); 1237 qht_insert(&tcg_ctx.tb_ctx.htable, tb, h); 1238 1239 #ifdef DEBUG_TB_CHECK 1240 tb_page_check(); 1241 #endif 1242 } 1243 1244 /* Called with mmap_lock held for user mode emulation. */ 1245 TranslationBlock *tb_gen_code(CPUState *cpu, 1246 target_ulong pc, target_ulong cs_base, 1247 uint32_t flags, int cflags) 1248 { 1249 CPUArchState *env = cpu->env_ptr; 1250 TranslationBlock *tb; 1251 tb_page_addr_t phys_pc, phys_page2; 1252 target_ulong virt_page2; 1253 tcg_insn_unit *gen_code_buf; 1254 int gen_code_size, search_size; 1255 #ifdef CONFIG_PROFILER 1256 int64_t ti; 1257 #endif 1258 assert_memory_lock(); 1259 1260 phys_pc = get_page_addr_code(env, pc); 1261 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) { 1262 cflags |= CF_USE_ICOUNT; 1263 } 1264 1265 tb = tb_alloc(pc); 1266 if (unlikely(!tb)) { 1267 buffer_overflow: 1268 /* flush must be done */ 1269 tb_flush(cpu); 1270 mmap_unlock(); 1271 /* Make the execution loop process the flush as soon as possible. */ 1272 cpu->exception_index = EXCP_INTERRUPT; 1273 cpu_loop_exit(cpu); 1274 } 1275 1276 gen_code_buf = tcg_ctx.code_gen_ptr; 1277 tb->tc_ptr = gen_code_buf; 1278 tb->pc = pc; 1279 tb->cs_base = cs_base; 1280 tb->flags = flags; 1281 tb->cflags = cflags; 1282 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1283 1284 #ifdef CONFIG_PROFILER 1285 tcg_ctx.tb_count1++; /* includes aborted translations because of 1286 exceptions */ 1287 ti = profile_getclock(); 1288 #endif 1289 1290 tcg_func_start(&tcg_ctx); 1291 1292 tcg_ctx.cpu = ENV_GET_CPU(env); 1293 gen_intermediate_code(cpu, tb); 1294 tcg_ctx.cpu = NULL; 1295 1296 trace_translate_block(tb, tb->pc, tb->tc_ptr); 1297 1298 /* generate machine code */ 1299 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1300 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1301 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset; 1302 if (TCG_TARGET_HAS_direct_jump) { 1303 tcg_ctx.tb_jmp_insn_offset = tb->jmp_target_arg; 1304 tcg_ctx.tb_jmp_target_addr = NULL; 1305 } else { 1306 tcg_ctx.tb_jmp_insn_offset = NULL; 1307 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_arg; 1308 } 1309 1310 #ifdef CONFIG_PROFILER 1311 tcg_ctx.tb_count++; 1312 tcg_ctx.interm_time += profile_getclock() - ti; 1313 ti = profile_getclock(); 1314 #endif 1315 1316 /* ??? Overflow could be handled better here. In particular, we 1317 don't need to re-do gen_intermediate_code, nor should we re-do 1318 the tcg optimization currently hidden inside tcg_gen_code. All 1319 that should be required is to flush the TBs, allocate a new TB, 1320 re-initialize it per above, and re-do the actual code generation. */ 1321 gen_code_size = tcg_gen_code(&tcg_ctx, tb); 1322 if (unlikely(gen_code_size < 0)) { 1323 goto buffer_overflow; 1324 } 1325 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1326 if (unlikely(search_size < 0)) { 1327 goto buffer_overflow; 1328 } 1329 1330 #ifdef CONFIG_PROFILER 1331 tcg_ctx.code_time += profile_getclock() - ti; 1332 tcg_ctx.code_in_len += tb->size; 1333 tcg_ctx.code_out_len += gen_code_size; 1334 tcg_ctx.search_out_len += search_size; 1335 #endif 1336 1337 #ifdef DEBUG_DISAS 1338 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1339 qemu_log_in_addr_range(tb->pc)) { 1340 qemu_log_lock(); 1341 qemu_log("OUT: [size=%d]\n", gen_code_size); 1342 if (tcg_ctx.data_gen_ptr) { 1343 size_t code_size = tcg_ctx.data_gen_ptr - tb->tc_ptr; 1344 size_t data_size = gen_code_size - code_size; 1345 size_t i; 1346 1347 log_disas(tb->tc_ptr, code_size); 1348 1349 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1350 if (sizeof(tcg_target_ulong) == 8) { 1351 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1352 (uintptr_t)tcg_ctx.data_gen_ptr + i, 1353 *(uint64_t *)(tcg_ctx.data_gen_ptr + i)); 1354 } else { 1355 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1356 (uintptr_t)tcg_ctx.data_gen_ptr + i, 1357 *(uint32_t *)(tcg_ctx.data_gen_ptr + i)); 1358 } 1359 } 1360 } else { 1361 log_disas(tb->tc_ptr, gen_code_size); 1362 } 1363 qemu_log("\n"); 1364 qemu_log_flush(); 1365 qemu_log_unlock(); 1366 } 1367 #endif 1368 1369 tcg_ctx.code_gen_ptr = (void *) 1370 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1371 CODE_GEN_ALIGN); 1372 1373 /* init jump list */ 1374 assert(((uintptr_t)tb & 3) == 0); 1375 tb->jmp_list_first = (uintptr_t)tb | 2; 1376 tb->jmp_list_next[0] = (uintptr_t)NULL; 1377 tb->jmp_list_next[1] = (uintptr_t)NULL; 1378 1379 /* init original jump addresses wich has been set during tcg_gen_code() */ 1380 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1381 tb_reset_jump(tb, 0); 1382 } 1383 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1384 tb_reset_jump(tb, 1); 1385 } 1386 1387 /* check next page if needed */ 1388 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1389 phys_page2 = -1; 1390 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1391 phys_page2 = get_page_addr_code(env, virt_page2); 1392 } 1393 /* As long as consistency of the TB stuff is provided by tb_lock in user 1394 * mode and is implicit in single-threaded softmmu emulation, no explicit 1395 * memory barrier is required before tb_link_page() makes the TB visible 1396 * through the physical hash table and physical page list. 1397 */ 1398 tb_link_page(tb, phys_pc, phys_page2); 1399 return tb; 1400 } 1401 1402 /* 1403 * Invalidate all TBs which intersect with the target physical address range 1404 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1405 * 'is_cpu_write_access' should be true if called from a real cpu write 1406 * access: the virtual CPU will exit the current TB if code is modified inside 1407 * this TB. 1408 * 1409 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1410 * Called with tb_lock held for system-mode emulation 1411 */ 1412 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1413 { 1414 while (start < end) { 1415 tb_invalidate_phys_page_range(start, end, 0); 1416 start &= TARGET_PAGE_MASK; 1417 start += TARGET_PAGE_SIZE; 1418 } 1419 } 1420 1421 #ifdef CONFIG_SOFTMMU 1422 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1423 { 1424 assert_tb_locked(); 1425 tb_invalidate_phys_range_1(start, end); 1426 } 1427 #else 1428 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1429 { 1430 assert_memory_lock(); 1431 tb_lock(); 1432 tb_invalidate_phys_range_1(start, end); 1433 tb_unlock(); 1434 } 1435 #endif 1436 /* 1437 * Invalidate all TBs which intersect with the target physical address range 1438 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1439 * 'is_cpu_write_access' should be true if called from a real cpu write 1440 * access: the virtual CPU will exit the current TB if code is modified inside 1441 * this TB. 1442 * 1443 * Called with tb_lock/mmap_lock held for user-mode emulation 1444 * Called with tb_lock held for system-mode emulation 1445 */ 1446 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1447 int is_cpu_write_access) 1448 { 1449 TranslationBlock *tb, *tb_next; 1450 #if defined(TARGET_HAS_PRECISE_SMC) 1451 CPUState *cpu = current_cpu; 1452 CPUArchState *env = NULL; 1453 #endif 1454 tb_page_addr_t tb_start, tb_end; 1455 PageDesc *p; 1456 int n; 1457 #ifdef TARGET_HAS_PRECISE_SMC 1458 int current_tb_not_found = is_cpu_write_access; 1459 TranslationBlock *current_tb = NULL; 1460 int current_tb_modified = 0; 1461 target_ulong current_pc = 0; 1462 target_ulong current_cs_base = 0; 1463 uint32_t current_flags = 0; 1464 #endif /* TARGET_HAS_PRECISE_SMC */ 1465 1466 assert_memory_lock(); 1467 assert_tb_locked(); 1468 1469 p = page_find(start >> TARGET_PAGE_BITS); 1470 if (!p) { 1471 return; 1472 } 1473 #if defined(TARGET_HAS_PRECISE_SMC) 1474 if (cpu != NULL) { 1475 env = cpu->env_ptr; 1476 } 1477 #endif 1478 1479 /* we remove all the TBs in the range [start, end[ */ 1480 /* XXX: see if in some cases it could be faster to invalidate all 1481 the code */ 1482 tb = p->first_tb; 1483 while (tb != NULL) { 1484 n = (uintptr_t)tb & 3; 1485 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1486 tb_next = tb->page_next[n]; 1487 /* NOTE: this is subtle as a TB may span two physical pages */ 1488 if (n == 0) { 1489 /* NOTE: tb_end may be after the end of the page, but 1490 it is not a problem */ 1491 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1492 tb_end = tb_start + tb->size; 1493 } else { 1494 tb_start = tb->page_addr[1]; 1495 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1496 } 1497 if (!(tb_end <= start || tb_start >= end)) { 1498 #ifdef TARGET_HAS_PRECISE_SMC 1499 if (current_tb_not_found) { 1500 current_tb_not_found = 0; 1501 current_tb = NULL; 1502 if (cpu->mem_io_pc) { 1503 /* now we have a real cpu fault */ 1504 current_tb = tb_find_pc(cpu->mem_io_pc); 1505 } 1506 } 1507 if (current_tb == tb && 1508 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1509 /* If we are modifying the current TB, we must stop 1510 its execution. We could be more precise by checking 1511 that the modification is after the current PC, but it 1512 would require a specialized function to partially 1513 restore the CPU state */ 1514 1515 current_tb_modified = 1; 1516 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc); 1517 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1518 ¤t_flags); 1519 } 1520 #endif /* TARGET_HAS_PRECISE_SMC */ 1521 tb_phys_invalidate(tb, -1); 1522 } 1523 tb = tb_next; 1524 } 1525 #if !defined(CONFIG_USER_ONLY) 1526 /* if no code remaining, no need to continue to use slow writes */ 1527 if (!p->first_tb) { 1528 invalidate_page_bitmap(p); 1529 tlb_unprotect_code(start); 1530 } 1531 #endif 1532 #ifdef TARGET_HAS_PRECISE_SMC 1533 if (current_tb_modified) { 1534 /* we generate a block containing just the instruction 1535 modifying the memory. It will ensure that it cannot modify 1536 itself */ 1537 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1); 1538 cpu_loop_exit_noexc(cpu); 1539 } 1540 #endif 1541 } 1542 1543 #ifdef CONFIG_SOFTMMU 1544 /* len must be <= 8 and start must be a multiple of len. 1545 * Called via softmmu_template.h when code areas are written to with 1546 * iothread mutex not held. 1547 */ 1548 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1549 { 1550 PageDesc *p; 1551 1552 #if 0 1553 if (1) { 1554 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1555 cpu_single_env->mem_io_vaddr, len, 1556 cpu_single_env->eip, 1557 cpu_single_env->eip + 1558 (intptr_t)cpu_single_env->segs[R_CS].base); 1559 } 1560 #endif 1561 assert_memory_lock(); 1562 1563 p = page_find(start >> TARGET_PAGE_BITS); 1564 if (!p) { 1565 return; 1566 } 1567 if (!p->code_bitmap && 1568 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1569 /* build code bitmap. FIXME: writes should be protected by 1570 * tb_lock, reads by tb_lock or RCU. 1571 */ 1572 build_page_bitmap(p); 1573 } 1574 if (p->code_bitmap) { 1575 unsigned int nr; 1576 unsigned long b; 1577 1578 nr = start & ~TARGET_PAGE_MASK; 1579 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1580 if (b & ((1 << len) - 1)) { 1581 goto do_invalidate; 1582 } 1583 } else { 1584 do_invalidate: 1585 tb_invalidate_phys_page_range(start, start + len, 1); 1586 } 1587 } 1588 #else 1589 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1590 * host PC of the faulting store instruction that caused this invalidate. 1591 * Returns true if the caller needs to abort execution of the current 1592 * TB (because it was modified by this store and the guest CPU has 1593 * precise-SMC semantics). 1594 */ 1595 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1596 { 1597 TranslationBlock *tb; 1598 PageDesc *p; 1599 int n; 1600 #ifdef TARGET_HAS_PRECISE_SMC 1601 TranslationBlock *current_tb = NULL; 1602 CPUState *cpu = current_cpu; 1603 CPUArchState *env = NULL; 1604 int current_tb_modified = 0; 1605 target_ulong current_pc = 0; 1606 target_ulong current_cs_base = 0; 1607 uint32_t current_flags = 0; 1608 #endif 1609 1610 assert_memory_lock(); 1611 1612 addr &= TARGET_PAGE_MASK; 1613 p = page_find(addr >> TARGET_PAGE_BITS); 1614 if (!p) { 1615 return false; 1616 } 1617 1618 tb_lock(); 1619 tb = p->first_tb; 1620 #ifdef TARGET_HAS_PRECISE_SMC 1621 if (tb && pc != 0) { 1622 current_tb = tb_find_pc(pc); 1623 } 1624 if (cpu != NULL) { 1625 env = cpu->env_ptr; 1626 } 1627 #endif 1628 while (tb != NULL) { 1629 n = (uintptr_t)tb & 3; 1630 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1631 #ifdef TARGET_HAS_PRECISE_SMC 1632 if (current_tb == tb && 1633 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1634 /* If we are modifying the current TB, we must stop 1635 its execution. We could be more precise by checking 1636 that the modification is after the current PC, but it 1637 would require a specialized function to partially 1638 restore the CPU state */ 1639 1640 current_tb_modified = 1; 1641 cpu_restore_state_from_tb(cpu, current_tb, pc); 1642 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1643 ¤t_flags); 1644 } 1645 #endif /* TARGET_HAS_PRECISE_SMC */ 1646 tb_phys_invalidate(tb, addr); 1647 tb = tb->page_next[n]; 1648 } 1649 p->first_tb = NULL; 1650 #ifdef TARGET_HAS_PRECISE_SMC 1651 if (current_tb_modified) { 1652 /* we generate a block containing just the instruction 1653 modifying the memory. It will ensure that it cannot modify 1654 itself */ 1655 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1); 1656 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1657 * back into the cpu_exec loop. */ 1658 return true; 1659 } 1660 #endif 1661 tb_unlock(); 1662 1663 return false; 1664 } 1665 #endif 1666 1667 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr < 1668 tb[1].tc_ptr. Return NULL if not found */ 1669 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) 1670 { 1671 int m_min, m_max, m; 1672 uintptr_t v; 1673 TranslationBlock *tb; 1674 1675 if (tcg_ctx.tb_ctx.nb_tbs <= 0) { 1676 return NULL; 1677 } 1678 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer || 1679 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) { 1680 return NULL; 1681 } 1682 /* binary search (cf Knuth) */ 1683 m_min = 0; 1684 m_max = tcg_ctx.tb_ctx.nb_tbs - 1; 1685 while (m_min <= m_max) { 1686 m = (m_min + m_max) >> 1; 1687 tb = tcg_ctx.tb_ctx.tbs[m]; 1688 v = (uintptr_t)tb->tc_ptr; 1689 if (v == tc_ptr) { 1690 return tb; 1691 } else if (tc_ptr < v) { 1692 m_max = m - 1; 1693 } else { 1694 m_min = m + 1; 1695 } 1696 } 1697 return tcg_ctx.tb_ctx.tbs[m_max]; 1698 } 1699 1700 #if !defined(CONFIG_USER_ONLY) 1701 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr) 1702 { 1703 ram_addr_t ram_addr; 1704 MemoryRegion *mr; 1705 hwaddr l = 1; 1706 1707 rcu_read_lock(); 1708 mr = address_space_translate(as, addr, &addr, &l, false); 1709 if (!(memory_region_is_ram(mr) 1710 || memory_region_is_romd(mr))) { 1711 rcu_read_unlock(); 1712 return; 1713 } 1714 ram_addr = memory_region_get_ram_addr(mr) + addr; 1715 tb_lock(); 1716 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1717 tb_unlock(); 1718 rcu_read_unlock(); 1719 } 1720 #endif /* !defined(CONFIG_USER_ONLY) */ 1721 1722 /* Called with tb_lock held. */ 1723 void tb_check_watchpoint(CPUState *cpu) 1724 { 1725 TranslationBlock *tb; 1726 1727 tb = tb_find_pc(cpu->mem_io_pc); 1728 if (tb) { 1729 /* We can use retranslation to find the PC. */ 1730 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc); 1731 tb_phys_invalidate(tb, -1); 1732 } else { 1733 /* The exception probably happened in a helper. The CPU state should 1734 have been saved before calling it. Fetch the PC from there. */ 1735 CPUArchState *env = cpu->env_ptr; 1736 target_ulong pc, cs_base; 1737 tb_page_addr_t addr; 1738 uint32_t flags; 1739 1740 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1741 addr = get_page_addr_code(env, pc); 1742 tb_invalidate_phys_range(addr, addr + 1); 1743 } 1744 } 1745 1746 #ifndef CONFIG_USER_ONLY 1747 /* in deterministic execution mode, instructions doing device I/Os 1748 * must be at the end of the TB. 1749 * 1750 * Called by softmmu_template.h, with iothread mutex not held. 1751 */ 1752 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1753 { 1754 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1755 CPUArchState *env = cpu->env_ptr; 1756 #endif 1757 TranslationBlock *tb; 1758 uint32_t n, cflags; 1759 target_ulong pc, cs_base; 1760 uint32_t flags; 1761 1762 tb_lock(); 1763 tb = tb_find_pc(retaddr); 1764 if (!tb) { 1765 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1766 (void *)retaddr); 1767 } 1768 n = cpu->icount_decr.u16.low + tb->icount; 1769 cpu_restore_state_from_tb(cpu, tb, retaddr); 1770 /* Calculate how many instructions had been executed before the fault 1771 occurred. */ 1772 n = n - cpu->icount_decr.u16.low; 1773 /* Generate a new TB ending on the I/O insn. */ 1774 n++; 1775 /* On MIPS and SH, delay slot instructions can only be restarted if 1776 they were already the first instruction in the TB. If this is not 1777 the first instruction in a TB then re-execute the preceding 1778 branch. */ 1779 #if defined(TARGET_MIPS) 1780 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) { 1781 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1782 cpu->icount_decr.u16.low++; 1783 env->hflags &= ~MIPS_HFLAG_BMASK; 1784 } 1785 #elif defined(TARGET_SH4) 1786 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1787 && n > 1) { 1788 env->pc -= 2; 1789 cpu->icount_decr.u16.low++; 1790 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1791 } 1792 #endif 1793 /* This should never happen. */ 1794 if (n > CF_COUNT_MASK) { 1795 cpu_abort(cpu, "TB too big during recompile"); 1796 } 1797 1798 cflags = n | CF_LAST_IO; 1799 pc = tb->pc; 1800 cs_base = tb->cs_base; 1801 flags = tb->flags; 1802 tb_phys_invalidate(tb, -1); 1803 if (tb->cflags & CF_NOCACHE) { 1804 if (tb->orig_tb) { 1805 /* Invalidate original TB if this TB was generated in 1806 * cpu_exec_nocache() */ 1807 tb_phys_invalidate(tb->orig_tb, -1); 1808 } 1809 tb_free(tb); 1810 } 1811 /* FIXME: In theory this could raise an exception. In practice 1812 we have already translated the block once so it's probably ok. */ 1813 tb_gen_code(cpu, pc, cs_base, flags, cflags); 1814 1815 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1816 * the first in the TB) then we end up generating a whole new TB and 1817 * repeating the fault, which is horribly inefficient. 1818 * Better would be to execute just this insn uncached, or generate a 1819 * second new TB. 1820 * 1821 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1822 * tb_lock gets reset. 1823 */ 1824 cpu_loop_exit_noexc(cpu); 1825 } 1826 1827 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1828 { 1829 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1830 1831 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1832 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1833 } 1834 } 1835 1836 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1837 { 1838 /* Discard jump cache entries for any tb which might potentially 1839 overlap the flushed page. */ 1840 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1841 tb_jmp_cache_clear_page(cpu, addr); 1842 } 1843 1844 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1845 struct qht_stats hst) 1846 { 1847 uint32_t hgram_opts; 1848 size_t hgram_bins; 1849 char *hgram; 1850 1851 if (!hst.head_buckets) { 1852 return; 1853 } 1854 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1855 hst.used_head_buckets, hst.head_buckets, 1856 (double)hst.used_head_buckets / hst.head_buckets * 100); 1857 1858 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1859 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1860 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1861 hgram_opts |= QDIST_PR_NODECIMAL; 1862 } 1863 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1864 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1865 qdist_avg(&hst.occupancy) * 100, hgram); 1866 g_free(hgram); 1867 1868 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1869 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1870 if (hgram_bins > 10) { 1871 hgram_bins = 10; 1872 } else { 1873 hgram_bins = 0; 1874 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1875 } 1876 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1877 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1878 qdist_avg(&hst.chain), hgram); 1879 g_free(hgram); 1880 } 1881 1882 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1883 { 1884 int i, target_code_size, max_target_code_size; 1885 int direct_jmp_count, direct_jmp2_count, cross_page; 1886 TranslationBlock *tb; 1887 struct qht_stats hst; 1888 1889 tb_lock(); 1890 1891 target_code_size = 0; 1892 max_target_code_size = 0; 1893 cross_page = 0; 1894 direct_jmp_count = 0; 1895 direct_jmp2_count = 0; 1896 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) { 1897 tb = tcg_ctx.tb_ctx.tbs[i]; 1898 target_code_size += tb->size; 1899 if (tb->size > max_target_code_size) { 1900 max_target_code_size = tb->size; 1901 } 1902 if (tb->page_addr[1] != -1) { 1903 cross_page++; 1904 } 1905 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1906 direct_jmp_count++; 1907 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1908 direct_jmp2_count++; 1909 } 1910 } 1911 } 1912 /* XXX: avoid using doubles ? */ 1913 cpu_fprintf(f, "Translation buffer state:\n"); 1914 cpu_fprintf(f, "gen code size %td/%zd\n", 1915 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer, 1916 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer); 1917 cpu_fprintf(f, "TB count %d\n", tcg_ctx.tb_ctx.nb_tbs); 1918 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n", 1919 tcg_ctx.tb_ctx.nb_tbs ? target_code_size / 1920 tcg_ctx.tb_ctx.nb_tbs : 0, 1921 max_target_code_size); 1922 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n", 1923 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr - 1924 tcg_ctx.code_gen_buffer) / 1925 tcg_ctx.tb_ctx.nb_tbs : 0, 1926 target_code_size ? (double) (tcg_ctx.code_gen_ptr - 1927 tcg_ctx.code_gen_buffer) / 1928 target_code_size : 0); 1929 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page, 1930 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) / 1931 tcg_ctx.tb_ctx.nb_tbs : 0); 1932 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n", 1933 direct_jmp_count, 1934 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) / 1935 tcg_ctx.tb_ctx.nb_tbs : 0, 1936 direct_jmp2_count, 1937 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) / 1938 tcg_ctx.tb_ctx.nb_tbs : 0); 1939 1940 qht_statistics_init(&tcg_ctx.tb_ctx.htable, &hst); 1941 print_qht_statistics(f, cpu_fprintf, hst); 1942 qht_statistics_destroy(&hst); 1943 1944 cpu_fprintf(f, "\nStatistics:\n"); 1945 cpu_fprintf(f, "TB flush count %u\n", 1946 atomic_read(&tcg_ctx.tb_ctx.tb_flush_count)); 1947 cpu_fprintf(f, "TB invalidate count %d\n", 1948 tcg_ctx.tb_ctx.tb_phys_invalidate_count); 1949 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1950 tcg_dump_info(f, cpu_fprintf); 1951 1952 tb_unlock(); 1953 } 1954 1955 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1956 { 1957 tcg_dump_op_count(f, cpu_fprintf); 1958 } 1959 1960 #else /* CONFIG_USER_ONLY */ 1961 1962 void cpu_interrupt(CPUState *cpu, int mask) 1963 { 1964 g_assert(qemu_mutex_iothread_locked()); 1965 cpu->interrupt_request |= mask; 1966 cpu->icount_decr.u16.high = -1; 1967 } 1968 1969 /* 1970 * Walks guest process memory "regions" one by one 1971 * and calls callback function 'fn' for each region. 1972 */ 1973 struct walk_memory_regions_data { 1974 walk_memory_regions_fn fn; 1975 void *priv; 1976 target_ulong start; 1977 int prot; 1978 }; 1979 1980 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1981 target_ulong end, int new_prot) 1982 { 1983 if (data->start != -1u) { 1984 int rc = data->fn(data->priv, data->start, end, data->prot); 1985 if (rc != 0) { 1986 return rc; 1987 } 1988 } 1989 1990 data->start = (new_prot ? end : -1u); 1991 data->prot = new_prot; 1992 1993 return 0; 1994 } 1995 1996 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1997 target_ulong base, int level, void **lp) 1998 { 1999 target_ulong pa; 2000 int i, rc; 2001 2002 if (*lp == NULL) { 2003 return walk_memory_regions_end(data, base, 0); 2004 } 2005 2006 if (level == 0) { 2007 PageDesc *pd = *lp; 2008 2009 for (i = 0; i < V_L2_SIZE; ++i) { 2010 int prot = pd[i].flags; 2011 2012 pa = base | (i << TARGET_PAGE_BITS); 2013 if (prot != data->prot) { 2014 rc = walk_memory_regions_end(data, pa, prot); 2015 if (rc != 0) { 2016 return rc; 2017 } 2018 } 2019 } 2020 } else { 2021 void **pp = *lp; 2022 2023 for (i = 0; i < V_L2_SIZE; ++i) { 2024 pa = base | ((target_ulong)i << 2025 (TARGET_PAGE_BITS + V_L2_BITS * level)); 2026 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 2027 if (rc != 0) { 2028 return rc; 2029 } 2030 } 2031 } 2032 2033 return 0; 2034 } 2035 2036 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2037 { 2038 struct walk_memory_regions_data data; 2039 uintptr_t i, l1_sz = v_l1_size; 2040 2041 data.fn = fn; 2042 data.priv = priv; 2043 data.start = -1u; 2044 data.prot = 0; 2045 2046 for (i = 0; i < l1_sz; i++) { 2047 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2048 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2049 if (rc != 0) { 2050 return rc; 2051 } 2052 } 2053 2054 return walk_memory_regions_end(&data, 0, 0); 2055 } 2056 2057 static int dump_region(void *priv, target_ulong start, 2058 target_ulong end, unsigned long prot) 2059 { 2060 FILE *f = (FILE *)priv; 2061 2062 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2063 " "TARGET_FMT_lx" %c%c%c\n", 2064 start, end, end - start, 2065 ((prot & PAGE_READ) ? 'r' : '-'), 2066 ((prot & PAGE_WRITE) ? 'w' : '-'), 2067 ((prot & PAGE_EXEC) ? 'x' : '-')); 2068 2069 return 0; 2070 } 2071 2072 /* dump memory mappings */ 2073 void page_dump(FILE *f) 2074 { 2075 const int length = sizeof(target_ulong) * 2; 2076 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2077 length, "start", length, "end", length, "size", "prot"); 2078 walk_memory_regions(f, dump_region); 2079 } 2080 2081 int page_get_flags(target_ulong address) 2082 { 2083 PageDesc *p; 2084 2085 p = page_find(address >> TARGET_PAGE_BITS); 2086 if (!p) { 2087 return 0; 2088 } 2089 return p->flags; 2090 } 2091 2092 /* Modify the flags of a page and invalidate the code if necessary. 2093 The flag PAGE_WRITE_ORG is positioned automatically depending 2094 on PAGE_WRITE. The mmap_lock should already be held. */ 2095 void page_set_flags(target_ulong start, target_ulong end, int flags) 2096 { 2097 target_ulong addr, len; 2098 2099 /* This function should never be called with addresses outside the 2100 guest address space. If this assert fires, it probably indicates 2101 a missing call to h2g_valid. */ 2102 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2103 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2104 #endif 2105 assert(start < end); 2106 assert_memory_lock(); 2107 2108 start = start & TARGET_PAGE_MASK; 2109 end = TARGET_PAGE_ALIGN(end); 2110 2111 if (flags & PAGE_WRITE) { 2112 flags |= PAGE_WRITE_ORG; 2113 } 2114 2115 for (addr = start, len = end - start; 2116 len != 0; 2117 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2118 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2119 2120 /* If the write protection bit is set, then we invalidate 2121 the code inside. */ 2122 if (!(p->flags & PAGE_WRITE) && 2123 (flags & PAGE_WRITE) && 2124 p->first_tb) { 2125 tb_invalidate_phys_page(addr, 0); 2126 } 2127 p->flags = flags; 2128 } 2129 } 2130 2131 int page_check_range(target_ulong start, target_ulong len, int flags) 2132 { 2133 PageDesc *p; 2134 target_ulong end; 2135 target_ulong addr; 2136 2137 /* This function should never be called with addresses outside the 2138 guest address space. If this assert fires, it probably indicates 2139 a missing call to h2g_valid. */ 2140 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2141 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2142 #endif 2143 2144 if (len == 0) { 2145 return 0; 2146 } 2147 if (start + len - 1 < start) { 2148 /* We've wrapped around. */ 2149 return -1; 2150 } 2151 2152 /* must do before we loose bits in the next step */ 2153 end = TARGET_PAGE_ALIGN(start + len); 2154 start = start & TARGET_PAGE_MASK; 2155 2156 for (addr = start, len = end - start; 2157 len != 0; 2158 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2159 p = page_find(addr >> TARGET_PAGE_BITS); 2160 if (!p) { 2161 return -1; 2162 } 2163 if (!(p->flags & PAGE_VALID)) { 2164 return -1; 2165 } 2166 2167 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2168 return -1; 2169 } 2170 if (flags & PAGE_WRITE) { 2171 if (!(p->flags & PAGE_WRITE_ORG)) { 2172 return -1; 2173 } 2174 /* unprotect the page if it was put read-only because it 2175 contains translated code */ 2176 if (!(p->flags & PAGE_WRITE)) { 2177 if (!page_unprotect(addr, 0)) { 2178 return -1; 2179 } 2180 } 2181 } 2182 } 2183 return 0; 2184 } 2185 2186 /* called from signal handler: invalidate the code and unprotect the 2187 * page. Return 0 if the fault was not handled, 1 if it was handled, 2188 * and 2 if it was handled but the caller must cause the TB to be 2189 * immediately exited. (We can only return 2 if the 'pc' argument is 2190 * non-zero.) 2191 */ 2192 int page_unprotect(target_ulong address, uintptr_t pc) 2193 { 2194 unsigned int prot; 2195 bool current_tb_invalidated; 2196 PageDesc *p; 2197 target_ulong host_start, host_end, addr; 2198 2199 /* Technically this isn't safe inside a signal handler. However we 2200 know this only ever happens in a synchronous SEGV handler, so in 2201 practice it seems to be ok. */ 2202 mmap_lock(); 2203 2204 p = page_find(address >> TARGET_PAGE_BITS); 2205 if (!p) { 2206 mmap_unlock(); 2207 return 0; 2208 } 2209 2210 /* if the page was really writable, then we change its 2211 protection back to writable */ 2212 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) { 2213 host_start = address & qemu_host_page_mask; 2214 host_end = host_start + qemu_host_page_size; 2215 2216 prot = 0; 2217 current_tb_invalidated = false; 2218 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) { 2219 p = page_find(addr >> TARGET_PAGE_BITS); 2220 p->flags |= PAGE_WRITE; 2221 prot |= p->flags; 2222 2223 /* and since the content will be modified, we must invalidate 2224 the corresponding translated code. */ 2225 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2226 #ifdef DEBUG_TB_CHECK 2227 tb_invalidate_check(addr); 2228 #endif 2229 } 2230 mprotect((void *)g2h(host_start), qemu_host_page_size, 2231 prot & PAGE_BITS); 2232 2233 mmap_unlock(); 2234 /* If current TB was invalidated return to main loop */ 2235 return current_tb_invalidated ? 2 : 1; 2236 } 2237 mmap_unlock(); 2238 return 0; 2239 } 2240 #endif /* CONFIG_USER_ONLY */ 2241 2242 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2243 void tcg_flush_softmmu_tlb(CPUState *cs) 2244 { 2245 #ifdef CONFIG_SOFTMMU 2246 tlb_flush(cs); 2247 #endif 2248 } 2249