1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #include "exec/exec-all.h" 35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 36 #include <sys/param.h> 37 #if __FreeBSD_version >= 700104 38 #define HAVE_KINFO_GETVMMAP 39 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 40 #include <sys/proc.h> 41 #include <machine/profile.h> 42 #define _KERNEL 43 #include <sys/user.h> 44 #undef _KERNEL 45 #undef sigqueue 46 #include <libutil.h> 47 #endif 48 #endif 49 #else 50 #include "exec/address-spaces.h" 51 #endif 52 53 #include "exec/cputlb.h" 54 #include "exec/tb-hash.h" 55 #include "translate-all.h" 56 #include "qemu/bitmap.h" 57 #include "qemu/error-report.h" 58 #include "qemu/timer.h" 59 #include "qemu/main-loop.h" 60 #include "exec/log.h" 61 #include "sysemu/cpus.h" 62 63 /* #define DEBUG_TB_INVALIDATE */ 64 /* #define DEBUG_TB_FLUSH */ 65 /* make various TB consistency checks */ 66 /* #define DEBUG_TB_CHECK */ 67 68 #ifdef DEBUG_TB_INVALIDATE 69 #define DEBUG_TB_INVALIDATE_GATE 1 70 #else 71 #define DEBUG_TB_INVALIDATE_GATE 0 72 #endif 73 74 #ifdef DEBUG_TB_FLUSH 75 #define DEBUG_TB_FLUSH_GATE 1 76 #else 77 #define DEBUG_TB_FLUSH_GATE 0 78 #endif 79 80 #if !defined(CONFIG_USER_ONLY) 81 /* TB consistency checks only implemented for usermode emulation. */ 82 #undef DEBUG_TB_CHECK 83 #endif 84 85 #ifdef DEBUG_TB_CHECK 86 #define DEBUG_TB_CHECK_GATE 1 87 #else 88 #define DEBUG_TB_CHECK_GATE 0 89 #endif 90 91 /* Access to the various translations structures need to be serialised via locks 92 * for consistency. This is automatic for SoftMMU based system 93 * emulation due to its single threaded nature. In user-mode emulation 94 * access to the memory related structures are protected with the 95 * mmap_lock. 96 */ 97 #ifdef CONFIG_SOFTMMU 98 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 99 #else 100 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 101 #endif 102 103 #define SMC_BITMAP_USE_THRESHOLD 10 104 105 typedef struct PageDesc { 106 /* list of TBs intersecting this ram page */ 107 TranslationBlock *first_tb; 108 #ifdef CONFIG_SOFTMMU 109 /* in order to optimize self modifying code, we count the number 110 of lookups we do to a given page to use a bitmap */ 111 unsigned int code_write_count; 112 unsigned long *code_bitmap; 113 #else 114 unsigned long flags; 115 #endif 116 } PageDesc; 117 118 /* In system mode we want L1_MAP to be based on ram offsets, 119 while in user mode we want it to be based on virtual addresses. */ 120 #if !defined(CONFIG_USER_ONLY) 121 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 122 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 123 #else 124 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 125 #endif 126 #else 127 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 128 #endif 129 130 /* Size of the L2 (and L3, etc) page tables. */ 131 #define V_L2_BITS 10 132 #define V_L2_SIZE (1 << V_L2_BITS) 133 134 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 135 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 136 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 137 * BITS_PER_BYTE); 138 139 /* 140 * L1 Mapping properties 141 */ 142 static int v_l1_size; 143 static int v_l1_shift; 144 static int v_l2_levels; 145 146 /* The bottom level has pointers to PageDesc, and is indexed by 147 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 148 */ 149 #define V_L1_MIN_BITS 4 150 #define V_L1_MAX_BITS (V_L2_BITS + 3) 151 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 152 153 static void *l1_map[V_L1_MAX_SIZE]; 154 155 /* code generation context */ 156 TCGContext tcg_init_ctx; 157 __thread TCGContext *tcg_ctx; 158 TBContext tb_ctx; 159 bool parallel_cpus; 160 161 /* translation block context */ 162 static __thread int have_tb_lock; 163 164 static void page_table_config_init(void) 165 { 166 uint32_t v_l1_bits; 167 168 assert(TARGET_PAGE_BITS); 169 /* The bits remaining after N lower levels of page tables. */ 170 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 171 if (v_l1_bits < V_L1_MIN_BITS) { 172 v_l1_bits += V_L2_BITS; 173 } 174 175 v_l1_size = 1 << v_l1_bits; 176 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 177 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 178 179 assert(v_l1_bits <= V_L1_MAX_BITS); 180 assert(v_l1_shift % V_L2_BITS == 0); 181 assert(v_l2_levels >= 0); 182 } 183 184 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 185 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 186 187 void tb_lock(void) 188 { 189 assert_tb_unlocked(); 190 qemu_mutex_lock(&tb_ctx.tb_lock); 191 have_tb_lock++; 192 } 193 194 void tb_unlock(void) 195 { 196 assert_tb_locked(); 197 have_tb_lock--; 198 qemu_mutex_unlock(&tb_ctx.tb_lock); 199 } 200 201 void tb_lock_reset(void) 202 { 203 if (have_tb_lock) { 204 qemu_mutex_unlock(&tb_ctx.tb_lock); 205 have_tb_lock = 0; 206 } 207 } 208 209 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr); 210 211 void cpu_gen_init(void) 212 { 213 tcg_context_init(&tcg_init_ctx); 214 } 215 216 /* Encode VAL as a signed leb128 sequence at P. 217 Return P incremented past the encoded value. */ 218 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 219 { 220 int more, byte; 221 222 do { 223 byte = val & 0x7f; 224 val >>= 7; 225 more = !((val == 0 && (byte & 0x40) == 0) 226 || (val == -1 && (byte & 0x40) != 0)); 227 if (more) { 228 byte |= 0x80; 229 } 230 *p++ = byte; 231 } while (more); 232 233 return p; 234 } 235 236 /* Decode a signed leb128 sequence at *PP; increment *PP past the 237 decoded value. Return the decoded value. */ 238 static target_long decode_sleb128(uint8_t **pp) 239 { 240 uint8_t *p = *pp; 241 target_long val = 0; 242 int byte, shift = 0; 243 244 do { 245 byte = *p++; 246 val |= (target_ulong)(byte & 0x7f) << shift; 247 shift += 7; 248 } while (byte & 0x80); 249 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 250 val |= -(target_ulong)1 << shift; 251 } 252 253 *pp = p; 254 return val; 255 } 256 257 /* Encode the data collected about the instructions while compiling TB. 258 Place the data at BLOCK, and return the number of bytes consumed. 259 260 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's, 261 which come from the target's insn_start data, followed by a uintptr_t 262 which comes from the host pc of the end of the code implementing the insn. 263 264 Each line of the table is encoded as sleb128 deltas from the previous 265 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 266 That is, the first column is seeded with the guest pc, the last column 267 with the host pc, and the middle columns with zeros. */ 268 269 static int encode_search(TranslationBlock *tb, uint8_t *block) 270 { 271 uint8_t *highwater = tcg_ctx->code_gen_highwater; 272 uint8_t *p = block; 273 int i, j, n; 274 275 for (i = 0, n = tb->icount; i < n; ++i) { 276 target_ulong prev; 277 278 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 279 if (i == 0) { 280 prev = (j == 0 ? tb->pc : 0); 281 } else { 282 prev = tcg_ctx->gen_insn_data[i - 1][j]; 283 } 284 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 285 } 286 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 287 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 288 289 /* Test for (pending) buffer overflow. The assumption is that any 290 one row beginning below the high water mark cannot overrun 291 the buffer completely. Thus we can test for overflow after 292 encoding a row without having to check during encoding. */ 293 if (unlikely(p > highwater)) { 294 return -1; 295 } 296 } 297 298 return p - block; 299 } 300 301 /* The cpu state corresponding to 'searched_pc' is restored. 302 * Called with tb_lock held. 303 */ 304 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 305 uintptr_t searched_pc) 306 { 307 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 308 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 309 CPUArchState *env = cpu->env_ptr; 310 uint8_t *p = tb->tc.ptr + tb->tc.size; 311 int i, j, num_insns = tb->icount; 312 #ifdef CONFIG_PROFILER 313 TCGProfile *prof = &tcg_ctx->prof; 314 int64_t ti = profile_getclock(); 315 #endif 316 317 searched_pc -= GETPC_ADJ; 318 319 if (searched_pc < host_pc) { 320 return -1; 321 } 322 323 /* Reconstruct the stored insn data while looking for the point at 324 which the end of the insn exceeds the searched_pc. */ 325 for (i = 0; i < num_insns; ++i) { 326 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 327 data[j] += decode_sleb128(&p); 328 } 329 host_pc += decode_sleb128(&p); 330 if (host_pc > searched_pc) { 331 goto found; 332 } 333 } 334 return -1; 335 336 found: 337 if (tb->cflags & CF_USE_ICOUNT) { 338 assert(use_icount); 339 /* Reset the cycle counter to the start of the block. */ 340 cpu->icount_decr.u16.low += num_insns; 341 /* Clear the IO flag. */ 342 cpu->can_do_io = 0; 343 } 344 cpu->icount_decr.u16.low -= i; 345 restore_state_to_opc(env, tb, data); 346 347 #ifdef CONFIG_PROFILER 348 atomic_set(&prof->restore_time, 349 prof->restore_time + profile_getclock() - ti); 350 atomic_set(&prof->restore_count, prof->restore_count + 1); 351 #endif 352 return 0; 353 } 354 355 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr) 356 { 357 TranslationBlock *tb; 358 bool r = false; 359 360 /* A retaddr of zero is invalid so we really shouldn't have ended 361 * up here. The target code has likely forgotten to check retaddr 362 * != 0 before attempting to restore state. We return early to 363 * avoid blowing up on a recursive tb_lock(). The target must have 364 * previously survived a failed cpu_restore_state because 365 * tb_find_pc(0) would have failed anyway. It still should be 366 * fixed though. 367 */ 368 369 if (!retaddr) { 370 return r; 371 } 372 373 tb_lock(); 374 tb = tb_find_pc(retaddr); 375 if (tb) { 376 cpu_restore_state_from_tb(cpu, tb, retaddr); 377 if (tb->cflags & CF_NOCACHE) { 378 /* one-shot translation, invalidate it immediately */ 379 tb_phys_invalidate(tb, -1); 380 tb_remove(tb); 381 } 382 r = true; 383 } 384 tb_unlock(); 385 386 return r; 387 } 388 389 static void page_init(void) 390 { 391 page_size_init(); 392 page_table_config_init(); 393 394 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 395 { 396 #ifdef HAVE_KINFO_GETVMMAP 397 struct kinfo_vmentry *freep; 398 int i, cnt; 399 400 freep = kinfo_getvmmap(getpid(), &cnt); 401 if (freep) { 402 mmap_lock(); 403 for (i = 0; i < cnt; i++) { 404 unsigned long startaddr, endaddr; 405 406 startaddr = freep[i].kve_start; 407 endaddr = freep[i].kve_end; 408 if (h2g_valid(startaddr)) { 409 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 410 411 if (h2g_valid(endaddr)) { 412 endaddr = h2g(endaddr); 413 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 414 } else { 415 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 416 endaddr = ~0ul; 417 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 418 #endif 419 } 420 } 421 } 422 free(freep); 423 mmap_unlock(); 424 } 425 #else 426 FILE *f; 427 428 last_brk = (unsigned long)sbrk(0); 429 430 f = fopen("/compat/linux/proc/self/maps", "r"); 431 if (f) { 432 mmap_lock(); 433 434 do { 435 unsigned long startaddr, endaddr; 436 int n; 437 438 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 439 440 if (n == 2 && h2g_valid(startaddr)) { 441 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 442 443 if (h2g_valid(endaddr)) { 444 endaddr = h2g(endaddr); 445 } else { 446 endaddr = ~0ul; 447 } 448 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 449 } 450 } while (!feof(f)); 451 452 fclose(f); 453 mmap_unlock(); 454 } 455 #endif 456 } 457 #endif 458 } 459 460 /* If alloc=1: 461 * Called with tb_lock held for system emulation. 462 * Called with mmap_lock held for user-mode emulation. 463 */ 464 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 465 { 466 PageDesc *pd; 467 void **lp; 468 int i; 469 470 if (alloc) { 471 assert_memory_lock(); 472 } 473 474 /* Level 1. Always allocated. */ 475 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 476 477 /* Level 2..N-1. */ 478 for (i = v_l2_levels; i > 0; i--) { 479 void **p = atomic_rcu_read(lp); 480 481 if (p == NULL) { 482 if (!alloc) { 483 return NULL; 484 } 485 p = g_new0(void *, V_L2_SIZE); 486 atomic_rcu_set(lp, p); 487 } 488 489 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 490 } 491 492 pd = atomic_rcu_read(lp); 493 if (pd == NULL) { 494 if (!alloc) { 495 return NULL; 496 } 497 pd = g_new0(PageDesc, V_L2_SIZE); 498 atomic_rcu_set(lp, pd); 499 } 500 501 return pd + (index & (V_L2_SIZE - 1)); 502 } 503 504 static inline PageDesc *page_find(tb_page_addr_t index) 505 { 506 return page_find_alloc(index, 0); 507 } 508 509 #if defined(CONFIG_USER_ONLY) 510 /* Currently it is not recommended to allocate big chunks of data in 511 user mode. It will change when a dedicated libc will be used. */ 512 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 513 region in which the guest needs to run. Revisit this. */ 514 #define USE_STATIC_CODE_GEN_BUFFER 515 #endif 516 517 /* Minimum size of the code gen buffer. This number is randomly chosen, 518 but not so small that we can't have a fair number of TB's live. */ 519 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 520 521 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 522 indicated, this is constrained by the range of direct branches on the 523 host cpu, as used by the TCG implementation of goto_tb. */ 524 #if defined(__x86_64__) 525 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 526 #elif defined(__sparc__) 527 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 528 #elif defined(__powerpc64__) 529 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 530 #elif defined(__powerpc__) 531 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 532 #elif defined(__aarch64__) 533 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 534 #elif defined(__s390x__) 535 /* We have a +- 4GB range on the branches; leave some slop. */ 536 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 537 #elif defined(__mips__) 538 /* We have a 256MB branch region, but leave room to make sure the 539 main executable is also within that region. */ 540 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 541 #else 542 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 543 #endif 544 545 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 546 547 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 548 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 549 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 550 551 static inline size_t size_code_gen_buffer(size_t tb_size) 552 { 553 /* Size the buffer. */ 554 if (tb_size == 0) { 555 #ifdef USE_STATIC_CODE_GEN_BUFFER 556 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 557 #else 558 /* ??? Needs adjustments. */ 559 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 560 static buffer, we could size this on RESERVED_VA, on the text 561 segment size of the executable, or continue to use the default. */ 562 tb_size = (unsigned long)(ram_size / 4); 563 #endif 564 } 565 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 566 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 567 } 568 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 569 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 570 } 571 return tb_size; 572 } 573 574 #ifdef __mips__ 575 /* In order to use J and JAL within the code_gen_buffer, we require 576 that the buffer not cross a 256MB boundary. */ 577 static inline bool cross_256mb(void *addr, size_t size) 578 { 579 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 580 } 581 582 /* We weren't able to allocate a buffer without crossing that boundary, 583 so make do with the larger portion of the buffer that doesn't cross. 584 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 585 static inline void *split_cross_256mb(void *buf1, size_t size1) 586 { 587 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 588 size_t size2 = buf1 + size1 - buf2; 589 590 size1 = buf2 - buf1; 591 if (size1 < size2) { 592 size1 = size2; 593 buf1 = buf2; 594 } 595 596 tcg_ctx->code_gen_buffer_size = size1; 597 return buf1; 598 } 599 #endif 600 601 #ifdef USE_STATIC_CODE_GEN_BUFFER 602 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 603 __attribute__((aligned(CODE_GEN_ALIGN))); 604 605 static inline void *alloc_code_gen_buffer(void) 606 { 607 void *buf = static_code_gen_buffer; 608 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 609 size_t size; 610 611 /* page-align the beginning and end of the buffer */ 612 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 613 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 614 615 size = end - buf; 616 617 /* Honor a command-line option limiting the size of the buffer. */ 618 if (size > tcg_ctx->code_gen_buffer_size) { 619 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 620 qemu_real_host_page_size); 621 } 622 tcg_ctx->code_gen_buffer_size = size; 623 624 #ifdef __mips__ 625 if (cross_256mb(buf, size)) { 626 buf = split_cross_256mb(buf, size); 627 size = tcg_ctx->code_gen_buffer_size; 628 } 629 #endif 630 631 if (qemu_mprotect_rwx(buf, size)) { 632 abort(); 633 } 634 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 635 636 return buf; 637 } 638 #elif defined(_WIN32) 639 static inline void *alloc_code_gen_buffer(void) 640 { 641 size_t size = tcg_ctx->code_gen_buffer_size; 642 void *buf; 643 644 buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 645 PAGE_EXECUTE_READWRITE); 646 return buf; 647 } 648 #else 649 static inline void *alloc_code_gen_buffer(void) 650 { 651 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 652 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 653 uintptr_t start = 0; 654 size_t size = tcg_ctx->code_gen_buffer_size; 655 void *buf; 656 657 /* Constrain the position of the buffer based on the host cpu. 658 Note that these addresses are chosen in concert with the 659 addresses assigned in the relevant linker script file. */ 660 # if defined(__PIE__) || defined(__PIC__) 661 /* Don't bother setting a preferred location if we're building 662 a position-independent executable. We're more likely to get 663 an address near the main executable if we let the kernel 664 choose the address. */ 665 # elif defined(__x86_64__) && defined(MAP_32BIT) 666 /* Force the memory down into low memory with the executable. 667 Leave the choice of exact location with the kernel. */ 668 flags |= MAP_32BIT; 669 /* Cannot expect to map more than 800MB in low memory. */ 670 if (size > 800u * 1024 * 1024) { 671 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 672 } 673 # elif defined(__sparc__) 674 start = 0x40000000ul; 675 # elif defined(__s390x__) 676 start = 0x90000000ul; 677 # elif defined(__mips__) 678 # if _MIPS_SIM == _ABI64 679 start = 0x128000000ul; 680 # else 681 start = 0x08000000ul; 682 # endif 683 # endif 684 685 buf = mmap((void *)start, size, prot, flags, -1, 0); 686 if (buf == MAP_FAILED) { 687 return NULL; 688 } 689 690 #ifdef __mips__ 691 if (cross_256mb(buf, size)) { 692 /* Try again, with the original still mapped, to avoid re-acquiring 693 that 256mb crossing. This time don't specify an address. */ 694 size_t size2; 695 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 696 switch ((int)(buf2 != MAP_FAILED)) { 697 case 1: 698 if (!cross_256mb(buf2, size)) { 699 /* Success! Use the new buffer. */ 700 munmap(buf, size); 701 break; 702 } 703 /* Failure. Work with what we had. */ 704 munmap(buf2, size); 705 /* fallthru */ 706 default: 707 /* Split the original buffer. Free the smaller half. */ 708 buf2 = split_cross_256mb(buf, size); 709 size2 = tcg_ctx->code_gen_buffer_size; 710 if (buf == buf2) { 711 munmap(buf + size2, size - size2); 712 } else { 713 munmap(buf, size - size2); 714 } 715 size = size2; 716 break; 717 } 718 buf = buf2; 719 } 720 #endif 721 722 /* Request large pages for the buffer. */ 723 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 724 725 return buf; 726 } 727 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 728 729 /* compare a pointer @ptr and a tb_tc @s */ 730 static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s) 731 { 732 if (ptr >= s->ptr + s->size) { 733 return 1; 734 } else if (ptr < s->ptr) { 735 return -1; 736 } 737 return 0; 738 } 739 740 static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp) 741 { 742 const struct tb_tc *a = ap; 743 const struct tb_tc *b = bp; 744 745 /* 746 * When both sizes are set, we know this isn't a lookup. 747 * This is the most likely case: every TB must be inserted; lookups 748 * are a lot less frequent. 749 */ 750 if (likely(a->size && b->size)) { 751 if (a->ptr > b->ptr) { 752 return 1; 753 } else if (a->ptr < b->ptr) { 754 return -1; 755 } 756 /* a->ptr == b->ptr should happen only on deletions */ 757 g_assert(a->size == b->size); 758 return 0; 759 } 760 /* 761 * All lookups have either .size field set to 0. 762 * From the glib sources we see that @ap is always the lookup key. However 763 * the docs provide no guarantee, so we just mark this case as likely. 764 */ 765 if (likely(a->size == 0)) { 766 return ptr_cmp_tb_tc(a->ptr, b); 767 } 768 return ptr_cmp_tb_tc(b->ptr, a); 769 } 770 771 static inline void code_gen_alloc(size_t tb_size) 772 { 773 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 774 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 775 if (tcg_ctx->code_gen_buffer == NULL) { 776 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 777 exit(1); 778 } 779 tb_ctx.tb_tree = g_tree_new(tb_tc_cmp); 780 qemu_mutex_init(&tb_ctx.tb_lock); 781 } 782 783 static void tb_htable_init(void) 784 { 785 unsigned int mode = QHT_MODE_AUTO_RESIZE; 786 787 qht_init(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode); 788 } 789 790 /* Must be called before using the QEMU cpus. 'tb_size' is the size 791 (in bytes) allocated to the translation buffer. Zero means default 792 size. */ 793 void tcg_exec_init(unsigned long tb_size) 794 { 795 tcg_allowed = true; 796 cpu_gen_init(); 797 page_init(); 798 tb_htable_init(); 799 code_gen_alloc(tb_size); 800 #if defined(CONFIG_SOFTMMU) 801 /* There's no guest base to take into account, so go ahead and 802 initialize the prologue now. */ 803 tcg_prologue_init(tcg_ctx); 804 #endif 805 } 806 807 /* 808 * Allocate a new translation block. Flush the translation buffer if 809 * too many translation blocks or too much generated code. 810 * 811 * Called with tb_lock held. 812 */ 813 static TranslationBlock *tb_alloc(target_ulong pc) 814 { 815 TranslationBlock *tb; 816 817 assert_tb_locked(); 818 819 tb = tcg_tb_alloc(tcg_ctx); 820 if (unlikely(tb == NULL)) { 821 return NULL; 822 } 823 return tb; 824 } 825 826 /* Called with tb_lock held. */ 827 void tb_remove(TranslationBlock *tb) 828 { 829 assert_tb_locked(); 830 831 g_tree_remove(tb_ctx.tb_tree, &tb->tc); 832 } 833 834 static inline void invalidate_page_bitmap(PageDesc *p) 835 { 836 #ifdef CONFIG_SOFTMMU 837 g_free(p->code_bitmap); 838 p->code_bitmap = NULL; 839 p->code_write_count = 0; 840 #endif 841 } 842 843 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 844 static void page_flush_tb_1(int level, void **lp) 845 { 846 int i; 847 848 if (*lp == NULL) { 849 return; 850 } 851 if (level == 0) { 852 PageDesc *pd = *lp; 853 854 for (i = 0; i < V_L2_SIZE; ++i) { 855 pd[i].first_tb = NULL; 856 invalidate_page_bitmap(pd + i); 857 } 858 } else { 859 void **pp = *lp; 860 861 for (i = 0; i < V_L2_SIZE; ++i) { 862 page_flush_tb_1(level - 1, pp + i); 863 } 864 } 865 } 866 867 static void page_flush_tb(void) 868 { 869 int i, l1_sz = v_l1_size; 870 871 for (i = 0; i < l1_sz; i++) { 872 page_flush_tb_1(v_l2_levels, l1_map + i); 873 } 874 } 875 876 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 877 { 878 const TranslationBlock *tb = value; 879 size_t *size = data; 880 881 *size += tb->tc.size; 882 return false; 883 } 884 885 /* flush all the translation blocks */ 886 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 887 { 888 tb_lock(); 889 890 /* If it is already been done on request of another CPU, 891 * just retry. 892 */ 893 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 894 goto done; 895 } 896 897 if (DEBUG_TB_FLUSH_GATE) { 898 size_t nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 899 size_t host_size = 0; 900 901 g_tree_foreach(tb_ctx.tb_tree, tb_host_size_iter, &host_size); 902 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 903 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 904 } 905 906 CPU_FOREACH(cpu) { 907 cpu_tb_jmp_cache_clear(cpu); 908 } 909 910 /* Increment the refcount first so that destroy acts as a reset */ 911 g_tree_ref(tb_ctx.tb_tree); 912 g_tree_destroy(tb_ctx.tb_tree); 913 914 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 915 page_flush_tb(); 916 917 tcg_region_reset_all(); 918 /* XXX: flush processor icache at this point if cache flush is 919 expensive */ 920 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 921 922 done: 923 tb_unlock(); 924 } 925 926 void tb_flush(CPUState *cpu) 927 { 928 if (tcg_enabled()) { 929 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 930 async_safe_run_on_cpu(cpu, do_tb_flush, 931 RUN_ON_CPU_HOST_INT(tb_flush_count)); 932 } 933 } 934 935 /* 936 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 937 * so in order to prevent bit rot we compile them unconditionally in user-mode, 938 * and let the optimizer get rid of them by wrapping their user-only callers 939 * with if (DEBUG_TB_CHECK_GATE). 940 */ 941 #ifdef CONFIG_USER_ONLY 942 943 static void 944 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 945 { 946 TranslationBlock *tb = p; 947 target_ulong addr = *(target_ulong *)userp; 948 949 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 950 printf("ERROR invalidate: address=" TARGET_FMT_lx 951 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 952 } 953 } 954 955 /* verify that all the pages have correct rights for code 956 * 957 * Called with tb_lock held. 958 */ 959 static void tb_invalidate_check(target_ulong address) 960 { 961 address &= TARGET_PAGE_MASK; 962 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 963 } 964 965 static void 966 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 967 { 968 TranslationBlock *tb = p; 969 int flags1, flags2; 970 971 flags1 = page_get_flags(tb->pc); 972 flags2 = page_get_flags(tb->pc + tb->size - 1); 973 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 974 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 975 (long)tb->pc, tb->size, flags1, flags2); 976 } 977 } 978 979 /* verify that all the pages have correct rights for code */ 980 static void tb_page_check(void) 981 { 982 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 983 } 984 985 #endif /* CONFIG_USER_ONLY */ 986 987 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) 988 { 989 TranslationBlock *tb1; 990 unsigned int n1; 991 992 for (;;) { 993 tb1 = *ptb; 994 n1 = (uintptr_t)tb1 & 3; 995 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); 996 if (tb1 == tb) { 997 *ptb = tb1->page_next[n1]; 998 break; 999 } 1000 ptb = &tb1->page_next[n1]; 1001 } 1002 } 1003 1004 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 1005 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 1006 { 1007 TranslationBlock *tb1; 1008 uintptr_t *ptb, ntb; 1009 unsigned int n1; 1010 1011 ptb = &tb->jmp_list_next[n]; 1012 if (*ptb) { 1013 /* find tb(n) in circular list */ 1014 for (;;) { 1015 ntb = *ptb; 1016 n1 = ntb & 3; 1017 tb1 = (TranslationBlock *)(ntb & ~3); 1018 if (n1 == n && tb1 == tb) { 1019 break; 1020 } 1021 if (n1 == 2) { 1022 ptb = &tb1->jmp_list_first; 1023 } else { 1024 ptb = &tb1->jmp_list_next[n1]; 1025 } 1026 } 1027 /* now we can suppress tb(n) from the list */ 1028 *ptb = tb->jmp_list_next[n]; 1029 1030 tb->jmp_list_next[n] = (uintptr_t)NULL; 1031 } 1032 } 1033 1034 /* reset the jump entry 'n' of a TB so that it is not chained to 1035 another TB */ 1036 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1037 { 1038 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1039 tb_set_jmp_target(tb, n, addr); 1040 } 1041 1042 /* remove any jumps to the TB */ 1043 static inline void tb_jmp_unlink(TranslationBlock *tb) 1044 { 1045 TranslationBlock *tb1; 1046 uintptr_t *ptb, ntb; 1047 unsigned int n1; 1048 1049 ptb = &tb->jmp_list_first; 1050 for (;;) { 1051 ntb = *ptb; 1052 n1 = ntb & 3; 1053 tb1 = (TranslationBlock *)(ntb & ~3); 1054 if (n1 == 2) { 1055 break; 1056 } 1057 tb_reset_jump(tb1, n1); 1058 *ptb = tb1->jmp_list_next[n1]; 1059 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1060 } 1061 } 1062 1063 /* invalidate one TB 1064 * 1065 * Called with tb_lock held. 1066 */ 1067 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1068 { 1069 CPUState *cpu; 1070 PageDesc *p; 1071 uint32_t h; 1072 tb_page_addr_t phys_pc; 1073 1074 assert_tb_locked(); 1075 1076 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1077 1078 /* remove the TB from the hash list */ 1079 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1080 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1081 tb->trace_vcpu_dstate); 1082 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1083 return; 1084 } 1085 1086 /* remove the TB from the page list */ 1087 if (tb->page_addr[0] != page_addr) { 1088 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1089 tb_page_remove(&p->first_tb, tb); 1090 invalidate_page_bitmap(p); 1091 } 1092 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1093 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1094 tb_page_remove(&p->first_tb, tb); 1095 invalidate_page_bitmap(p); 1096 } 1097 1098 /* remove the TB from the hash list */ 1099 h = tb_jmp_cache_hash_func(tb->pc); 1100 CPU_FOREACH(cpu) { 1101 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1102 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1103 } 1104 } 1105 1106 /* suppress this TB from the two jump lists */ 1107 tb_remove_from_jmp_list(tb, 0); 1108 tb_remove_from_jmp_list(tb, 1); 1109 1110 /* suppress any remaining jumps to this TB */ 1111 tb_jmp_unlink(tb); 1112 1113 tb_ctx.tb_phys_invalidate_count++; 1114 } 1115 1116 #ifdef CONFIG_SOFTMMU 1117 static void build_page_bitmap(PageDesc *p) 1118 { 1119 int n, tb_start, tb_end; 1120 TranslationBlock *tb; 1121 1122 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1123 1124 tb = p->first_tb; 1125 while (tb != NULL) { 1126 n = (uintptr_t)tb & 3; 1127 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1128 /* NOTE: this is subtle as a TB may span two physical pages */ 1129 if (n == 0) { 1130 /* NOTE: tb_end may be after the end of the page, but 1131 it is not a problem */ 1132 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1133 tb_end = tb_start + tb->size; 1134 if (tb_end > TARGET_PAGE_SIZE) { 1135 tb_end = TARGET_PAGE_SIZE; 1136 } 1137 } else { 1138 tb_start = 0; 1139 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1140 } 1141 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1142 tb = tb->page_next[n]; 1143 } 1144 } 1145 #endif 1146 1147 /* add the tb in the target page and protect it if necessary 1148 * 1149 * Called with mmap_lock held for user-mode emulation. 1150 */ 1151 static inline void tb_alloc_page(TranslationBlock *tb, 1152 unsigned int n, tb_page_addr_t page_addr) 1153 { 1154 PageDesc *p; 1155 #ifndef CONFIG_USER_ONLY 1156 bool page_already_protected; 1157 #endif 1158 1159 assert_memory_lock(); 1160 1161 tb->page_addr[n] = page_addr; 1162 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1163 tb->page_next[n] = p->first_tb; 1164 #ifndef CONFIG_USER_ONLY 1165 page_already_protected = p->first_tb != NULL; 1166 #endif 1167 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); 1168 invalidate_page_bitmap(p); 1169 1170 #if defined(CONFIG_USER_ONLY) 1171 if (p->flags & PAGE_WRITE) { 1172 target_ulong addr; 1173 PageDesc *p2; 1174 int prot; 1175 1176 /* force the host page as non writable (writes will have a 1177 page fault + mprotect overhead) */ 1178 page_addr &= qemu_host_page_mask; 1179 prot = 0; 1180 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1181 addr += TARGET_PAGE_SIZE) { 1182 1183 p2 = page_find(addr >> TARGET_PAGE_BITS); 1184 if (!p2) { 1185 continue; 1186 } 1187 prot |= p2->flags; 1188 p2->flags &= ~PAGE_WRITE; 1189 } 1190 mprotect(g2h(page_addr), qemu_host_page_size, 1191 (prot & PAGE_BITS) & ~PAGE_WRITE); 1192 if (DEBUG_TB_INVALIDATE_GATE) { 1193 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1194 } 1195 } 1196 #else 1197 /* if some code is already present, then the pages are already 1198 protected. So we handle the case where only the first TB is 1199 allocated in a physical page */ 1200 if (!page_already_protected) { 1201 tlb_protect_code(page_addr); 1202 } 1203 #endif 1204 } 1205 1206 /* add a new TB and link it to the physical page tables. phys_page2 is 1207 * (-1) to indicate that only one page contains the TB. 1208 * 1209 * Called with mmap_lock held for user-mode emulation. 1210 */ 1211 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1212 tb_page_addr_t phys_page2) 1213 { 1214 uint32_t h; 1215 1216 assert_memory_lock(); 1217 1218 /* add in the page list */ 1219 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1220 if (phys_page2 != -1) { 1221 tb_alloc_page(tb, 1, phys_page2); 1222 } else { 1223 tb->page_addr[1] = -1; 1224 } 1225 1226 /* add in the hash table */ 1227 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1228 tb->trace_vcpu_dstate); 1229 qht_insert(&tb_ctx.htable, tb, h); 1230 1231 #ifdef CONFIG_USER_ONLY 1232 if (DEBUG_TB_CHECK_GATE) { 1233 tb_page_check(); 1234 } 1235 #endif 1236 } 1237 1238 /* Called with mmap_lock held for user mode emulation. */ 1239 TranslationBlock *tb_gen_code(CPUState *cpu, 1240 target_ulong pc, target_ulong cs_base, 1241 uint32_t flags, int cflags) 1242 { 1243 CPUArchState *env = cpu->env_ptr; 1244 TranslationBlock *tb; 1245 tb_page_addr_t phys_pc, phys_page2; 1246 target_ulong virt_page2; 1247 tcg_insn_unit *gen_code_buf; 1248 int gen_code_size, search_size; 1249 #ifdef CONFIG_PROFILER 1250 TCGProfile *prof = &tcg_ctx->prof; 1251 int64_t ti; 1252 #endif 1253 assert_memory_lock(); 1254 1255 phys_pc = get_page_addr_code(env, pc); 1256 1257 buffer_overflow: 1258 tb = tb_alloc(pc); 1259 if (unlikely(!tb)) { 1260 /* flush must be done */ 1261 tb_flush(cpu); 1262 mmap_unlock(); 1263 /* Make the execution loop process the flush as soon as possible. */ 1264 cpu->exception_index = EXCP_INTERRUPT; 1265 cpu_loop_exit(cpu); 1266 } 1267 1268 gen_code_buf = tcg_ctx->code_gen_ptr; 1269 tb->tc.ptr = gen_code_buf; 1270 tb->pc = pc; 1271 tb->cs_base = cs_base; 1272 tb->flags = flags; 1273 tb->cflags = cflags; 1274 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1275 tcg_ctx->tb_cflags = cflags; 1276 1277 #ifdef CONFIG_PROFILER 1278 /* includes aborted translations because of exceptions */ 1279 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1280 ti = profile_getclock(); 1281 #endif 1282 1283 tcg_func_start(tcg_ctx); 1284 1285 tcg_ctx->cpu = ENV_GET_CPU(env); 1286 gen_intermediate_code(cpu, tb); 1287 tcg_ctx->cpu = NULL; 1288 1289 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1290 1291 /* generate machine code */ 1292 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1293 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1294 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1295 if (TCG_TARGET_HAS_direct_jump) { 1296 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1297 tcg_ctx->tb_jmp_target_addr = NULL; 1298 } else { 1299 tcg_ctx->tb_jmp_insn_offset = NULL; 1300 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1301 } 1302 1303 #ifdef CONFIG_PROFILER 1304 atomic_set(&prof->tb_count, prof->tb_count + 1); 1305 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1306 ti = profile_getclock(); 1307 #endif 1308 1309 /* ??? Overflow could be handled better here. In particular, we 1310 don't need to re-do gen_intermediate_code, nor should we re-do 1311 the tcg optimization currently hidden inside tcg_gen_code. All 1312 that should be required is to flush the TBs, allocate a new TB, 1313 re-initialize it per above, and re-do the actual code generation. */ 1314 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1315 if (unlikely(gen_code_size < 0)) { 1316 goto buffer_overflow; 1317 } 1318 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1319 if (unlikely(search_size < 0)) { 1320 goto buffer_overflow; 1321 } 1322 tb->tc.size = gen_code_size; 1323 1324 #ifdef CONFIG_PROFILER 1325 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1326 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1327 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1328 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1329 #endif 1330 1331 #ifdef DEBUG_DISAS 1332 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1333 qemu_log_in_addr_range(tb->pc)) { 1334 qemu_log_lock(); 1335 qemu_log("OUT: [size=%d]\n", gen_code_size); 1336 if (tcg_ctx->data_gen_ptr) { 1337 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1338 size_t data_size = gen_code_size - code_size; 1339 size_t i; 1340 1341 log_disas(tb->tc.ptr, code_size); 1342 1343 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1344 if (sizeof(tcg_target_ulong) == 8) { 1345 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1346 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1347 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1348 } else { 1349 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1350 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1351 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1352 } 1353 } 1354 } else { 1355 log_disas(tb->tc.ptr, gen_code_size); 1356 } 1357 qemu_log("\n"); 1358 qemu_log_flush(); 1359 qemu_log_unlock(); 1360 } 1361 #endif 1362 1363 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1364 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1365 CODE_GEN_ALIGN)); 1366 1367 /* init jump list */ 1368 assert(((uintptr_t)tb & 3) == 0); 1369 tb->jmp_list_first = (uintptr_t)tb | 2; 1370 tb->jmp_list_next[0] = (uintptr_t)NULL; 1371 tb->jmp_list_next[1] = (uintptr_t)NULL; 1372 1373 /* init original jump addresses wich has been set during tcg_gen_code() */ 1374 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1375 tb_reset_jump(tb, 0); 1376 } 1377 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1378 tb_reset_jump(tb, 1); 1379 } 1380 1381 /* check next page if needed */ 1382 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1383 phys_page2 = -1; 1384 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1385 phys_page2 = get_page_addr_code(env, virt_page2); 1386 } 1387 /* As long as consistency of the TB stuff is provided by tb_lock in user 1388 * mode and is implicit in single-threaded softmmu emulation, no explicit 1389 * memory barrier is required before tb_link_page() makes the TB visible 1390 * through the physical hash table and physical page list. 1391 */ 1392 tb_link_page(tb, phys_pc, phys_page2); 1393 g_tree_insert(tb_ctx.tb_tree, &tb->tc, tb); 1394 return tb; 1395 } 1396 1397 /* 1398 * Invalidate all TBs which intersect with the target physical address range 1399 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1400 * 'is_cpu_write_access' should be true if called from a real cpu write 1401 * access: the virtual CPU will exit the current TB if code is modified inside 1402 * this TB. 1403 * 1404 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1405 * Called with tb_lock held for system-mode emulation 1406 */ 1407 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1408 { 1409 while (start < end) { 1410 tb_invalidate_phys_page_range(start, end, 0); 1411 start &= TARGET_PAGE_MASK; 1412 start += TARGET_PAGE_SIZE; 1413 } 1414 } 1415 1416 #ifdef CONFIG_SOFTMMU 1417 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1418 { 1419 assert_tb_locked(); 1420 tb_invalidate_phys_range_1(start, end); 1421 } 1422 #else 1423 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1424 { 1425 assert_memory_lock(); 1426 tb_lock(); 1427 tb_invalidate_phys_range_1(start, end); 1428 tb_unlock(); 1429 } 1430 #endif 1431 /* 1432 * Invalidate all TBs which intersect with the target physical address range 1433 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1434 * 'is_cpu_write_access' should be true if called from a real cpu write 1435 * access: the virtual CPU will exit the current TB if code is modified inside 1436 * this TB. 1437 * 1438 * Called with tb_lock/mmap_lock held for user-mode emulation 1439 * Called with tb_lock held for system-mode emulation 1440 */ 1441 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1442 int is_cpu_write_access) 1443 { 1444 TranslationBlock *tb, *tb_next; 1445 tb_page_addr_t tb_start, tb_end; 1446 PageDesc *p; 1447 int n; 1448 #ifdef TARGET_HAS_PRECISE_SMC 1449 CPUState *cpu = current_cpu; 1450 CPUArchState *env = NULL; 1451 int current_tb_not_found = is_cpu_write_access; 1452 TranslationBlock *current_tb = NULL; 1453 int current_tb_modified = 0; 1454 target_ulong current_pc = 0; 1455 target_ulong current_cs_base = 0; 1456 uint32_t current_flags = 0; 1457 #endif /* TARGET_HAS_PRECISE_SMC */ 1458 1459 assert_memory_lock(); 1460 assert_tb_locked(); 1461 1462 p = page_find(start >> TARGET_PAGE_BITS); 1463 if (!p) { 1464 return; 1465 } 1466 #if defined(TARGET_HAS_PRECISE_SMC) 1467 if (cpu != NULL) { 1468 env = cpu->env_ptr; 1469 } 1470 #endif 1471 1472 /* we remove all the TBs in the range [start, end[ */ 1473 /* XXX: see if in some cases it could be faster to invalidate all 1474 the code */ 1475 tb = p->first_tb; 1476 while (tb != NULL) { 1477 n = (uintptr_t)tb & 3; 1478 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1479 tb_next = tb->page_next[n]; 1480 /* NOTE: this is subtle as a TB may span two physical pages */ 1481 if (n == 0) { 1482 /* NOTE: tb_end may be after the end of the page, but 1483 it is not a problem */ 1484 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1485 tb_end = tb_start + tb->size; 1486 } else { 1487 tb_start = tb->page_addr[1]; 1488 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1489 } 1490 if (!(tb_end <= start || tb_start >= end)) { 1491 #ifdef TARGET_HAS_PRECISE_SMC 1492 if (current_tb_not_found) { 1493 current_tb_not_found = 0; 1494 current_tb = NULL; 1495 if (cpu->mem_io_pc) { 1496 /* now we have a real cpu fault */ 1497 current_tb = tb_find_pc(cpu->mem_io_pc); 1498 } 1499 } 1500 if (current_tb == tb && 1501 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1502 /* If we are modifying the current TB, we must stop 1503 its execution. We could be more precise by checking 1504 that the modification is after the current PC, but it 1505 would require a specialized function to partially 1506 restore the CPU state */ 1507 1508 current_tb_modified = 1; 1509 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc); 1510 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1511 ¤t_flags); 1512 } 1513 #endif /* TARGET_HAS_PRECISE_SMC */ 1514 tb_phys_invalidate(tb, -1); 1515 } 1516 tb = tb_next; 1517 } 1518 #if !defined(CONFIG_USER_ONLY) 1519 /* if no code remaining, no need to continue to use slow writes */ 1520 if (!p->first_tb) { 1521 invalidate_page_bitmap(p); 1522 tlb_unprotect_code(start); 1523 } 1524 #endif 1525 #ifdef TARGET_HAS_PRECISE_SMC 1526 if (current_tb_modified) { 1527 /* Force execution of one insn next time. */ 1528 cpu->cflags_next_tb = 1 | curr_cflags(); 1529 cpu_loop_exit_noexc(cpu); 1530 } 1531 #endif 1532 } 1533 1534 #ifdef CONFIG_SOFTMMU 1535 /* len must be <= 8 and start must be a multiple of len. 1536 * Called via softmmu_template.h when code areas are written to with 1537 * iothread mutex not held. 1538 */ 1539 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1540 { 1541 PageDesc *p; 1542 1543 #if 0 1544 if (1) { 1545 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1546 cpu_single_env->mem_io_vaddr, len, 1547 cpu_single_env->eip, 1548 cpu_single_env->eip + 1549 (intptr_t)cpu_single_env->segs[R_CS].base); 1550 } 1551 #endif 1552 assert_memory_lock(); 1553 1554 p = page_find(start >> TARGET_PAGE_BITS); 1555 if (!p) { 1556 return; 1557 } 1558 if (!p->code_bitmap && 1559 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1560 /* build code bitmap. FIXME: writes should be protected by 1561 * tb_lock, reads by tb_lock or RCU. 1562 */ 1563 build_page_bitmap(p); 1564 } 1565 if (p->code_bitmap) { 1566 unsigned int nr; 1567 unsigned long b; 1568 1569 nr = start & ~TARGET_PAGE_MASK; 1570 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1571 if (b & ((1 << len) - 1)) { 1572 goto do_invalidate; 1573 } 1574 } else { 1575 do_invalidate: 1576 tb_invalidate_phys_page_range(start, start + len, 1); 1577 } 1578 } 1579 #else 1580 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1581 * host PC of the faulting store instruction that caused this invalidate. 1582 * Returns true if the caller needs to abort execution of the current 1583 * TB (because it was modified by this store and the guest CPU has 1584 * precise-SMC semantics). 1585 */ 1586 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1587 { 1588 TranslationBlock *tb; 1589 PageDesc *p; 1590 int n; 1591 #ifdef TARGET_HAS_PRECISE_SMC 1592 TranslationBlock *current_tb = NULL; 1593 CPUState *cpu = current_cpu; 1594 CPUArchState *env = NULL; 1595 int current_tb_modified = 0; 1596 target_ulong current_pc = 0; 1597 target_ulong current_cs_base = 0; 1598 uint32_t current_flags = 0; 1599 #endif 1600 1601 assert_memory_lock(); 1602 1603 addr &= TARGET_PAGE_MASK; 1604 p = page_find(addr >> TARGET_PAGE_BITS); 1605 if (!p) { 1606 return false; 1607 } 1608 1609 tb_lock(); 1610 tb = p->first_tb; 1611 #ifdef TARGET_HAS_PRECISE_SMC 1612 if (tb && pc != 0) { 1613 current_tb = tb_find_pc(pc); 1614 } 1615 if (cpu != NULL) { 1616 env = cpu->env_ptr; 1617 } 1618 #endif 1619 while (tb != NULL) { 1620 n = (uintptr_t)tb & 3; 1621 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1622 #ifdef TARGET_HAS_PRECISE_SMC 1623 if (current_tb == tb && 1624 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1625 /* If we are modifying the current TB, we must stop 1626 its execution. We could be more precise by checking 1627 that the modification is after the current PC, but it 1628 would require a specialized function to partially 1629 restore the CPU state */ 1630 1631 current_tb_modified = 1; 1632 cpu_restore_state_from_tb(cpu, current_tb, pc); 1633 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1634 ¤t_flags); 1635 } 1636 #endif /* TARGET_HAS_PRECISE_SMC */ 1637 tb_phys_invalidate(tb, addr); 1638 tb = tb->page_next[n]; 1639 } 1640 p->first_tb = NULL; 1641 #ifdef TARGET_HAS_PRECISE_SMC 1642 if (current_tb_modified) { 1643 /* Force execution of one insn next time. */ 1644 cpu->cflags_next_tb = 1 | curr_cflags(); 1645 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1646 * back into the cpu_exec loop. */ 1647 return true; 1648 } 1649 #endif 1650 tb_unlock(); 1651 1652 return false; 1653 } 1654 #endif 1655 1656 /* 1657 * Find the TB 'tb' such that 1658 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size 1659 * Return NULL if not found. 1660 */ 1661 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) 1662 { 1663 struct tb_tc s = { .ptr = (void *)tc_ptr }; 1664 1665 return g_tree_lookup(tb_ctx.tb_tree, &s); 1666 } 1667 1668 #if !defined(CONFIG_USER_ONLY) 1669 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr) 1670 { 1671 ram_addr_t ram_addr; 1672 MemoryRegion *mr; 1673 hwaddr l = 1; 1674 1675 rcu_read_lock(); 1676 mr = address_space_translate(as, addr, &addr, &l, false); 1677 if (!(memory_region_is_ram(mr) 1678 || memory_region_is_romd(mr))) { 1679 rcu_read_unlock(); 1680 return; 1681 } 1682 ram_addr = memory_region_get_ram_addr(mr) + addr; 1683 tb_lock(); 1684 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1685 tb_unlock(); 1686 rcu_read_unlock(); 1687 } 1688 #endif /* !defined(CONFIG_USER_ONLY) */ 1689 1690 /* Called with tb_lock held. */ 1691 void tb_check_watchpoint(CPUState *cpu) 1692 { 1693 TranslationBlock *tb; 1694 1695 tb = tb_find_pc(cpu->mem_io_pc); 1696 if (tb) { 1697 /* We can use retranslation to find the PC. */ 1698 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc); 1699 tb_phys_invalidate(tb, -1); 1700 } else { 1701 /* The exception probably happened in a helper. The CPU state should 1702 have been saved before calling it. Fetch the PC from there. */ 1703 CPUArchState *env = cpu->env_ptr; 1704 target_ulong pc, cs_base; 1705 tb_page_addr_t addr; 1706 uint32_t flags; 1707 1708 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1709 addr = get_page_addr_code(env, pc); 1710 tb_invalidate_phys_range(addr, addr + 1); 1711 } 1712 } 1713 1714 #ifndef CONFIG_USER_ONLY 1715 /* in deterministic execution mode, instructions doing device I/Os 1716 * must be at the end of the TB. 1717 * 1718 * Called by softmmu_template.h, with iothread mutex not held. 1719 */ 1720 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1721 { 1722 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1723 CPUArchState *env = cpu->env_ptr; 1724 #endif 1725 TranslationBlock *tb; 1726 uint32_t n; 1727 1728 tb_lock(); 1729 tb = tb_find_pc(retaddr); 1730 if (!tb) { 1731 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1732 (void *)retaddr); 1733 } 1734 n = cpu->icount_decr.u16.low + tb->icount; 1735 cpu_restore_state_from_tb(cpu, tb, retaddr); 1736 /* Calculate how many instructions had been executed before the fault 1737 occurred. */ 1738 n = n - cpu->icount_decr.u16.low; 1739 /* Generate a new TB ending on the I/O insn. */ 1740 n++; 1741 /* On MIPS and SH, delay slot instructions can only be restarted if 1742 they were already the first instruction in the TB. If this is not 1743 the first instruction in a TB then re-execute the preceding 1744 branch. */ 1745 #if defined(TARGET_MIPS) 1746 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) { 1747 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1748 cpu->icount_decr.u16.low++; 1749 env->hflags &= ~MIPS_HFLAG_BMASK; 1750 } 1751 #elif defined(TARGET_SH4) 1752 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1753 && n > 1) { 1754 env->pc -= 2; 1755 cpu->icount_decr.u16.low++; 1756 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1757 } 1758 #endif 1759 /* This should never happen. */ 1760 if (n > CF_COUNT_MASK) { 1761 cpu_abort(cpu, "TB too big during recompile"); 1762 } 1763 1764 /* Adjust the execution state of the next TB. */ 1765 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1766 1767 if (tb->cflags & CF_NOCACHE) { 1768 if (tb->orig_tb) { 1769 /* Invalidate original TB if this TB was generated in 1770 * cpu_exec_nocache() */ 1771 tb_phys_invalidate(tb->orig_tb, -1); 1772 } 1773 tb_remove(tb); 1774 } 1775 1776 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1777 * the first in the TB) then we end up generating a whole new TB and 1778 * repeating the fault, which is horribly inefficient. 1779 * Better would be to execute just this insn uncached, or generate a 1780 * second new TB. 1781 * 1782 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1783 * tb_lock gets reset. 1784 */ 1785 cpu_loop_exit_noexc(cpu); 1786 } 1787 1788 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1789 { 1790 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1791 1792 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1793 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1794 } 1795 } 1796 1797 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1798 { 1799 /* Discard jump cache entries for any tb which might potentially 1800 overlap the flushed page. */ 1801 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1802 tb_jmp_cache_clear_page(cpu, addr); 1803 } 1804 1805 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1806 struct qht_stats hst) 1807 { 1808 uint32_t hgram_opts; 1809 size_t hgram_bins; 1810 char *hgram; 1811 1812 if (!hst.head_buckets) { 1813 return; 1814 } 1815 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1816 hst.used_head_buckets, hst.head_buckets, 1817 (double)hst.used_head_buckets / hst.head_buckets * 100); 1818 1819 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1820 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1821 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1822 hgram_opts |= QDIST_PR_NODECIMAL; 1823 } 1824 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1825 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1826 qdist_avg(&hst.occupancy) * 100, hgram); 1827 g_free(hgram); 1828 1829 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1830 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1831 if (hgram_bins > 10) { 1832 hgram_bins = 10; 1833 } else { 1834 hgram_bins = 0; 1835 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1836 } 1837 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1838 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1839 qdist_avg(&hst.chain), hgram); 1840 g_free(hgram); 1841 } 1842 1843 struct tb_tree_stats { 1844 size_t host_size; 1845 size_t target_size; 1846 size_t max_target_size; 1847 size_t direct_jmp_count; 1848 size_t direct_jmp2_count; 1849 size_t cross_page; 1850 }; 1851 1852 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1853 { 1854 const TranslationBlock *tb = value; 1855 struct tb_tree_stats *tst = data; 1856 1857 tst->host_size += tb->tc.size; 1858 tst->target_size += tb->size; 1859 if (tb->size > tst->max_target_size) { 1860 tst->max_target_size = tb->size; 1861 } 1862 if (tb->page_addr[1] != -1) { 1863 tst->cross_page++; 1864 } 1865 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1866 tst->direct_jmp_count++; 1867 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1868 tst->direct_jmp2_count++; 1869 } 1870 } 1871 return false; 1872 } 1873 1874 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1875 { 1876 struct tb_tree_stats tst = {}; 1877 struct qht_stats hst; 1878 size_t nb_tbs; 1879 1880 tb_lock(); 1881 1882 nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 1883 g_tree_foreach(tb_ctx.tb_tree, tb_tree_stats_iter, &tst); 1884 /* XXX: avoid using doubles ? */ 1885 cpu_fprintf(f, "Translation buffer state:\n"); 1886 /* 1887 * Report total code size including the padding and TB structs; 1888 * otherwise users might think "-tb-size" is not honoured. 1889 * For avg host size we use the precise numbers from tb_tree_stats though. 1890 */ 1891 cpu_fprintf(f, "gen code size %zu/%zu\n", 1892 tcg_code_size(), tcg_code_capacity()); 1893 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1894 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1895 nb_tbs ? tst.target_size / nb_tbs : 0, 1896 tst.max_target_size); 1897 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1898 nb_tbs ? tst.host_size / nb_tbs : 0, 1899 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1900 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1901 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1902 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1903 tst.direct_jmp_count, 1904 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1905 tst.direct_jmp2_count, 1906 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1907 1908 qht_statistics_init(&tb_ctx.htable, &hst); 1909 print_qht_statistics(f, cpu_fprintf, hst); 1910 qht_statistics_destroy(&hst); 1911 1912 cpu_fprintf(f, "\nStatistics:\n"); 1913 cpu_fprintf(f, "TB flush count %u\n", 1914 atomic_read(&tb_ctx.tb_flush_count)); 1915 cpu_fprintf(f, "TB invalidate count %d\n", tb_ctx.tb_phys_invalidate_count); 1916 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1917 tcg_dump_info(f, cpu_fprintf); 1918 1919 tb_unlock(); 1920 } 1921 1922 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1923 { 1924 tcg_dump_op_count(f, cpu_fprintf); 1925 } 1926 1927 #else /* CONFIG_USER_ONLY */ 1928 1929 void cpu_interrupt(CPUState *cpu, int mask) 1930 { 1931 g_assert(qemu_mutex_iothread_locked()); 1932 cpu->interrupt_request |= mask; 1933 cpu->icount_decr.u16.high = -1; 1934 } 1935 1936 /* 1937 * Walks guest process memory "regions" one by one 1938 * and calls callback function 'fn' for each region. 1939 */ 1940 struct walk_memory_regions_data { 1941 walk_memory_regions_fn fn; 1942 void *priv; 1943 target_ulong start; 1944 int prot; 1945 }; 1946 1947 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1948 target_ulong end, int new_prot) 1949 { 1950 if (data->start != -1u) { 1951 int rc = data->fn(data->priv, data->start, end, data->prot); 1952 if (rc != 0) { 1953 return rc; 1954 } 1955 } 1956 1957 data->start = (new_prot ? end : -1u); 1958 data->prot = new_prot; 1959 1960 return 0; 1961 } 1962 1963 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1964 target_ulong base, int level, void **lp) 1965 { 1966 target_ulong pa; 1967 int i, rc; 1968 1969 if (*lp == NULL) { 1970 return walk_memory_regions_end(data, base, 0); 1971 } 1972 1973 if (level == 0) { 1974 PageDesc *pd = *lp; 1975 1976 for (i = 0; i < V_L2_SIZE; ++i) { 1977 int prot = pd[i].flags; 1978 1979 pa = base | (i << TARGET_PAGE_BITS); 1980 if (prot != data->prot) { 1981 rc = walk_memory_regions_end(data, pa, prot); 1982 if (rc != 0) { 1983 return rc; 1984 } 1985 } 1986 } 1987 } else { 1988 void **pp = *lp; 1989 1990 for (i = 0; i < V_L2_SIZE; ++i) { 1991 pa = base | ((target_ulong)i << 1992 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1993 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 1994 if (rc != 0) { 1995 return rc; 1996 } 1997 } 1998 } 1999 2000 return 0; 2001 } 2002 2003 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2004 { 2005 struct walk_memory_regions_data data; 2006 uintptr_t i, l1_sz = v_l1_size; 2007 2008 data.fn = fn; 2009 data.priv = priv; 2010 data.start = -1u; 2011 data.prot = 0; 2012 2013 for (i = 0; i < l1_sz; i++) { 2014 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2015 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2016 if (rc != 0) { 2017 return rc; 2018 } 2019 } 2020 2021 return walk_memory_regions_end(&data, 0, 0); 2022 } 2023 2024 static int dump_region(void *priv, target_ulong start, 2025 target_ulong end, unsigned long prot) 2026 { 2027 FILE *f = (FILE *)priv; 2028 2029 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2030 " "TARGET_FMT_lx" %c%c%c\n", 2031 start, end, end - start, 2032 ((prot & PAGE_READ) ? 'r' : '-'), 2033 ((prot & PAGE_WRITE) ? 'w' : '-'), 2034 ((prot & PAGE_EXEC) ? 'x' : '-')); 2035 2036 return 0; 2037 } 2038 2039 /* dump memory mappings */ 2040 void page_dump(FILE *f) 2041 { 2042 const int length = sizeof(target_ulong) * 2; 2043 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2044 length, "start", length, "end", length, "size", "prot"); 2045 walk_memory_regions(f, dump_region); 2046 } 2047 2048 int page_get_flags(target_ulong address) 2049 { 2050 PageDesc *p; 2051 2052 p = page_find(address >> TARGET_PAGE_BITS); 2053 if (!p) { 2054 return 0; 2055 } 2056 return p->flags; 2057 } 2058 2059 /* Modify the flags of a page and invalidate the code if necessary. 2060 The flag PAGE_WRITE_ORG is positioned automatically depending 2061 on PAGE_WRITE. The mmap_lock should already be held. */ 2062 void page_set_flags(target_ulong start, target_ulong end, int flags) 2063 { 2064 target_ulong addr, len; 2065 2066 /* This function should never be called with addresses outside the 2067 guest address space. If this assert fires, it probably indicates 2068 a missing call to h2g_valid. */ 2069 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2070 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2071 #endif 2072 assert(start < end); 2073 assert_memory_lock(); 2074 2075 start = start & TARGET_PAGE_MASK; 2076 end = TARGET_PAGE_ALIGN(end); 2077 2078 if (flags & PAGE_WRITE) { 2079 flags |= PAGE_WRITE_ORG; 2080 } 2081 2082 for (addr = start, len = end - start; 2083 len != 0; 2084 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2085 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2086 2087 /* If the write protection bit is set, then we invalidate 2088 the code inside. */ 2089 if (!(p->flags & PAGE_WRITE) && 2090 (flags & PAGE_WRITE) && 2091 p->first_tb) { 2092 tb_invalidate_phys_page(addr, 0); 2093 } 2094 p->flags = flags; 2095 } 2096 } 2097 2098 int page_check_range(target_ulong start, target_ulong len, int flags) 2099 { 2100 PageDesc *p; 2101 target_ulong end; 2102 target_ulong addr; 2103 2104 /* This function should never be called with addresses outside the 2105 guest address space. If this assert fires, it probably indicates 2106 a missing call to h2g_valid. */ 2107 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2108 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2109 #endif 2110 2111 if (len == 0) { 2112 return 0; 2113 } 2114 if (start + len - 1 < start) { 2115 /* We've wrapped around. */ 2116 return -1; 2117 } 2118 2119 /* must do before we loose bits in the next step */ 2120 end = TARGET_PAGE_ALIGN(start + len); 2121 start = start & TARGET_PAGE_MASK; 2122 2123 for (addr = start, len = end - start; 2124 len != 0; 2125 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2126 p = page_find(addr >> TARGET_PAGE_BITS); 2127 if (!p) { 2128 return -1; 2129 } 2130 if (!(p->flags & PAGE_VALID)) { 2131 return -1; 2132 } 2133 2134 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2135 return -1; 2136 } 2137 if (flags & PAGE_WRITE) { 2138 if (!(p->flags & PAGE_WRITE_ORG)) { 2139 return -1; 2140 } 2141 /* unprotect the page if it was put read-only because it 2142 contains translated code */ 2143 if (!(p->flags & PAGE_WRITE)) { 2144 if (!page_unprotect(addr, 0)) { 2145 return -1; 2146 } 2147 } 2148 } 2149 } 2150 return 0; 2151 } 2152 2153 /* called from signal handler: invalidate the code and unprotect the 2154 * page. Return 0 if the fault was not handled, 1 if it was handled, 2155 * and 2 if it was handled but the caller must cause the TB to be 2156 * immediately exited. (We can only return 2 if the 'pc' argument is 2157 * non-zero.) 2158 */ 2159 int page_unprotect(target_ulong address, uintptr_t pc) 2160 { 2161 unsigned int prot; 2162 bool current_tb_invalidated; 2163 PageDesc *p; 2164 target_ulong host_start, host_end, addr; 2165 2166 /* Technically this isn't safe inside a signal handler. However we 2167 know this only ever happens in a synchronous SEGV handler, so in 2168 practice it seems to be ok. */ 2169 mmap_lock(); 2170 2171 p = page_find(address >> TARGET_PAGE_BITS); 2172 if (!p) { 2173 mmap_unlock(); 2174 return 0; 2175 } 2176 2177 /* if the page was really writable, then we change its 2178 protection back to writable */ 2179 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) { 2180 host_start = address & qemu_host_page_mask; 2181 host_end = host_start + qemu_host_page_size; 2182 2183 prot = 0; 2184 current_tb_invalidated = false; 2185 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) { 2186 p = page_find(addr >> TARGET_PAGE_BITS); 2187 p->flags |= PAGE_WRITE; 2188 prot |= p->flags; 2189 2190 /* and since the content will be modified, we must invalidate 2191 the corresponding translated code. */ 2192 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2193 #ifdef CONFIG_USER_ONLY 2194 if (DEBUG_TB_CHECK_GATE) { 2195 tb_invalidate_check(addr); 2196 } 2197 #endif 2198 } 2199 mprotect((void *)g2h(host_start), qemu_host_page_size, 2200 prot & PAGE_BITS); 2201 2202 mmap_unlock(); 2203 /* If current TB was invalidated return to main loop */ 2204 return current_tb_invalidated ? 2 : 1; 2205 } 2206 mmap_unlock(); 2207 return 0; 2208 } 2209 #endif /* CONFIG_USER_ONLY */ 2210 2211 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2212 void tcg_flush_softmmu_tlb(CPUState *cs) 2213 { 2214 #ifdef CONFIG_SOFTMMU 2215 tlb_flush(cs); 2216 #endif 2217 } 2218