1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #include "exec/exec-all.h" 35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 36 #include <sys/param.h> 37 #if __FreeBSD_version >= 700104 38 #define HAVE_KINFO_GETVMMAP 39 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 40 #include <sys/proc.h> 41 #include <machine/profile.h> 42 #define _KERNEL 43 #include <sys/user.h> 44 #undef _KERNEL 45 #undef sigqueue 46 #include <libutil.h> 47 #endif 48 #endif 49 #else 50 #include "exec/address-spaces.h" 51 #endif 52 53 #include "exec/cputlb.h" 54 #include "exec/tb-hash.h" 55 #include "translate-all.h" 56 #include "qemu/bitmap.h" 57 #include "qemu/error-report.h" 58 #include "qemu/timer.h" 59 #include "qemu/main-loop.h" 60 #include "exec/log.h" 61 #include "sysemu/cpus.h" 62 63 /* #define DEBUG_TB_INVALIDATE */ 64 /* #define DEBUG_TB_FLUSH */ 65 /* make various TB consistency checks */ 66 /* #define DEBUG_TB_CHECK */ 67 68 #ifdef DEBUG_TB_INVALIDATE 69 #define DEBUG_TB_INVALIDATE_GATE 1 70 #else 71 #define DEBUG_TB_INVALIDATE_GATE 0 72 #endif 73 74 #ifdef DEBUG_TB_FLUSH 75 #define DEBUG_TB_FLUSH_GATE 1 76 #else 77 #define DEBUG_TB_FLUSH_GATE 0 78 #endif 79 80 #if !defined(CONFIG_USER_ONLY) 81 /* TB consistency checks only implemented for usermode emulation. */ 82 #undef DEBUG_TB_CHECK 83 #endif 84 85 #ifdef DEBUG_TB_CHECK 86 #define DEBUG_TB_CHECK_GATE 1 87 #else 88 #define DEBUG_TB_CHECK_GATE 0 89 #endif 90 91 /* Access to the various translations structures need to be serialised via locks 92 * for consistency. This is automatic for SoftMMU based system 93 * emulation due to its single threaded nature. In user-mode emulation 94 * access to the memory related structures are protected with the 95 * mmap_lock. 96 */ 97 #ifdef CONFIG_SOFTMMU 98 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 99 #else 100 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 101 #endif 102 103 #define SMC_BITMAP_USE_THRESHOLD 10 104 105 typedef struct PageDesc { 106 /* list of TBs intersecting this ram page */ 107 TranslationBlock *first_tb; 108 #ifdef CONFIG_SOFTMMU 109 /* in order to optimize self modifying code, we count the number 110 of lookups we do to a given page to use a bitmap */ 111 unsigned int code_write_count; 112 unsigned long *code_bitmap; 113 #else 114 unsigned long flags; 115 #endif 116 } PageDesc; 117 118 /* In system mode we want L1_MAP to be based on ram offsets, 119 while in user mode we want it to be based on virtual addresses. */ 120 #if !defined(CONFIG_USER_ONLY) 121 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 122 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 123 #else 124 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 125 #endif 126 #else 127 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 128 #endif 129 130 /* Size of the L2 (and L3, etc) page tables. */ 131 #define V_L2_BITS 10 132 #define V_L2_SIZE (1 << V_L2_BITS) 133 134 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 135 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 136 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 137 * BITS_PER_BYTE); 138 139 /* 140 * L1 Mapping properties 141 */ 142 static int v_l1_size; 143 static int v_l1_shift; 144 static int v_l2_levels; 145 146 /* The bottom level has pointers to PageDesc, and is indexed by 147 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 148 */ 149 #define V_L1_MIN_BITS 4 150 #define V_L1_MAX_BITS (V_L2_BITS + 3) 151 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 152 153 static void *l1_map[V_L1_MAX_SIZE]; 154 155 /* code generation context */ 156 TCGContext tcg_init_ctx; 157 __thread TCGContext *tcg_ctx; 158 TBContext tb_ctx; 159 bool parallel_cpus; 160 161 /* translation block context */ 162 static __thread int have_tb_lock; 163 164 static void page_table_config_init(void) 165 { 166 uint32_t v_l1_bits; 167 168 assert(TARGET_PAGE_BITS); 169 /* The bits remaining after N lower levels of page tables. */ 170 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 171 if (v_l1_bits < V_L1_MIN_BITS) { 172 v_l1_bits += V_L2_BITS; 173 } 174 175 v_l1_size = 1 << v_l1_bits; 176 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 177 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 178 179 assert(v_l1_bits <= V_L1_MAX_BITS); 180 assert(v_l1_shift % V_L2_BITS == 0); 181 assert(v_l2_levels >= 0); 182 } 183 184 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 185 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 186 187 void tb_lock(void) 188 { 189 assert_tb_unlocked(); 190 qemu_mutex_lock(&tb_ctx.tb_lock); 191 have_tb_lock++; 192 } 193 194 void tb_unlock(void) 195 { 196 assert_tb_locked(); 197 have_tb_lock--; 198 qemu_mutex_unlock(&tb_ctx.tb_lock); 199 } 200 201 void tb_lock_reset(void) 202 { 203 if (have_tb_lock) { 204 qemu_mutex_unlock(&tb_ctx.tb_lock); 205 have_tb_lock = 0; 206 } 207 } 208 209 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr); 210 211 void cpu_gen_init(void) 212 { 213 tcg_context_init(&tcg_init_ctx); 214 } 215 216 /* Encode VAL as a signed leb128 sequence at P. 217 Return P incremented past the encoded value. */ 218 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 219 { 220 int more, byte; 221 222 do { 223 byte = val & 0x7f; 224 val >>= 7; 225 more = !((val == 0 && (byte & 0x40) == 0) 226 || (val == -1 && (byte & 0x40) != 0)); 227 if (more) { 228 byte |= 0x80; 229 } 230 *p++ = byte; 231 } while (more); 232 233 return p; 234 } 235 236 /* Decode a signed leb128 sequence at *PP; increment *PP past the 237 decoded value. Return the decoded value. */ 238 static target_long decode_sleb128(uint8_t **pp) 239 { 240 uint8_t *p = *pp; 241 target_long val = 0; 242 int byte, shift = 0; 243 244 do { 245 byte = *p++; 246 val |= (target_ulong)(byte & 0x7f) << shift; 247 shift += 7; 248 } while (byte & 0x80); 249 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 250 val |= -(target_ulong)1 << shift; 251 } 252 253 *pp = p; 254 return val; 255 } 256 257 /* Encode the data collected about the instructions while compiling TB. 258 Place the data at BLOCK, and return the number of bytes consumed. 259 260 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's, 261 which come from the target's insn_start data, followed by a uintptr_t 262 which comes from the host pc of the end of the code implementing the insn. 263 264 Each line of the table is encoded as sleb128 deltas from the previous 265 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 266 That is, the first column is seeded with the guest pc, the last column 267 with the host pc, and the middle columns with zeros. */ 268 269 static int encode_search(TranslationBlock *tb, uint8_t *block) 270 { 271 uint8_t *highwater = tcg_ctx->code_gen_highwater; 272 uint8_t *p = block; 273 int i, j, n; 274 275 for (i = 0, n = tb->icount; i < n; ++i) { 276 target_ulong prev; 277 278 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 279 if (i == 0) { 280 prev = (j == 0 ? tb->pc : 0); 281 } else { 282 prev = tcg_ctx->gen_insn_data[i - 1][j]; 283 } 284 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 285 } 286 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 287 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 288 289 /* Test for (pending) buffer overflow. The assumption is that any 290 one row beginning below the high water mark cannot overrun 291 the buffer completely. Thus we can test for overflow after 292 encoding a row without having to check during encoding. */ 293 if (unlikely(p > highwater)) { 294 return -1; 295 } 296 } 297 298 return p - block; 299 } 300 301 /* The cpu state corresponding to 'searched_pc' is restored. 302 * Called with tb_lock held. 303 */ 304 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 305 uintptr_t searched_pc) 306 { 307 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 308 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 309 CPUArchState *env = cpu->env_ptr; 310 uint8_t *p = tb->tc.ptr + tb->tc.size; 311 int i, j, num_insns = tb->icount; 312 #ifdef CONFIG_PROFILER 313 TCGProfile *prof = &tcg_ctx->prof; 314 int64_t ti = profile_getclock(); 315 #endif 316 317 searched_pc -= GETPC_ADJ; 318 319 if (searched_pc < host_pc) { 320 return -1; 321 } 322 323 /* Reconstruct the stored insn data while looking for the point at 324 which the end of the insn exceeds the searched_pc. */ 325 for (i = 0; i < num_insns; ++i) { 326 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 327 data[j] += decode_sleb128(&p); 328 } 329 host_pc += decode_sleb128(&p); 330 if (host_pc > searched_pc) { 331 goto found; 332 } 333 } 334 return -1; 335 336 found: 337 if (tb->cflags & CF_USE_ICOUNT) { 338 assert(use_icount); 339 /* Reset the cycle counter to the start of the block. */ 340 cpu->icount_decr.u16.low += num_insns; 341 /* Clear the IO flag. */ 342 cpu->can_do_io = 0; 343 } 344 cpu->icount_decr.u16.low -= i; 345 restore_state_to_opc(env, tb, data); 346 347 #ifdef CONFIG_PROFILER 348 atomic_set(&prof->restore_time, 349 prof->restore_time + profile_getclock() - ti); 350 atomic_set(&prof->restore_count, prof->restore_count + 1); 351 #endif 352 return 0; 353 } 354 355 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr) 356 { 357 TranslationBlock *tb; 358 bool r = false; 359 360 /* A retaddr of zero is invalid so we really shouldn't have ended 361 * up here. The target code has likely forgotten to check retaddr 362 * != 0 before attempting to restore state. We return early to 363 * avoid blowing up on a recursive tb_lock(). The target must have 364 * previously survived a failed cpu_restore_state because 365 * tb_find_pc(0) would have failed anyway. It still should be 366 * fixed though. 367 */ 368 369 if (!retaddr) { 370 return r; 371 } 372 373 tb_lock(); 374 tb = tb_find_pc(retaddr); 375 if (tb) { 376 cpu_restore_state_from_tb(cpu, tb, retaddr); 377 if (tb->cflags & CF_NOCACHE) { 378 /* one-shot translation, invalidate it immediately */ 379 tb_phys_invalidate(tb, -1); 380 tb_remove(tb); 381 } 382 r = true; 383 } 384 tb_unlock(); 385 386 return r; 387 } 388 389 static void page_init(void) 390 { 391 page_size_init(); 392 page_table_config_init(); 393 394 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 395 { 396 #ifdef HAVE_KINFO_GETVMMAP 397 struct kinfo_vmentry *freep; 398 int i, cnt; 399 400 freep = kinfo_getvmmap(getpid(), &cnt); 401 if (freep) { 402 mmap_lock(); 403 for (i = 0; i < cnt; i++) { 404 unsigned long startaddr, endaddr; 405 406 startaddr = freep[i].kve_start; 407 endaddr = freep[i].kve_end; 408 if (h2g_valid(startaddr)) { 409 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 410 411 if (h2g_valid(endaddr)) { 412 endaddr = h2g(endaddr); 413 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 414 } else { 415 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 416 endaddr = ~0ul; 417 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 418 #endif 419 } 420 } 421 } 422 free(freep); 423 mmap_unlock(); 424 } 425 #else 426 FILE *f; 427 428 last_brk = (unsigned long)sbrk(0); 429 430 f = fopen("/compat/linux/proc/self/maps", "r"); 431 if (f) { 432 mmap_lock(); 433 434 do { 435 unsigned long startaddr, endaddr; 436 int n; 437 438 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 439 440 if (n == 2 && h2g_valid(startaddr)) { 441 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 442 443 if (h2g_valid(endaddr)) { 444 endaddr = h2g(endaddr); 445 } else { 446 endaddr = ~0ul; 447 } 448 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 449 } 450 } while (!feof(f)); 451 452 fclose(f); 453 mmap_unlock(); 454 } 455 #endif 456 } 457 #endif 458 } 459 460 /* If alloc=1: 461 * Called with tb_lock held for system emulation. 462 * Called with mmap_lock held for user-mode emulation. 463 */ 464 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 465 { 466 PageDesc *pd; 467 void **lp; 468 int i; 469 470 if (alloc) { 471 assert_memory_lock(); 472 } 473 474 /* Level 1. Always allocated. */ 475 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 476 477 /* Level 2..N-1. */ 478 for (i = v_l2_levels; i > 0; i--) { 479 void **p = atomic_rcu_read(lp); 480 481 if (p == NULL) { 482 if (!alloc) { 483 return NULL; 484 } 485 p = g_new0(void *, V_L2_SIZE); 486 atomic_rcu_set(lp, p); 487 } 488 489 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 490 } 491 492 pd = atomic_rcu_read(lp); 493 if (pd == NULL) { 494 if (!alloc) { 495 return NULL; 496 } 497 pd = g_new0(PageDesc, V_L2_SIZE); 498 atomic_rcu_set(lp, pd); 499 } 500 501 return pd + (index & (V_L2_SIZE - 1)); 502 } 503 504 static inline PageDesc *page_find(tb_page_addr_t index) 505 { 506 return page_find_alloc(index, 0); 507 } 508 509 #if defined(CONFIG_USER_ONLY) 510 /* Currently it is not recommended to allocate big chunks of data in 511 user mode. It will change when a dedicated libc will be used. */ 512 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 513 region in which the guest needs to run. Revisit this. */ 514 #define USE_STATIC_CODE_GEN_BUFFER 515 #endif 516 517 /* Minimum size of the code gen buffer. This number is randomly chosen, 518 but not so small that we can't have a fair number of TB's live. */ 519 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 520 521 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 522 indicated, this is constrained by the range of direct branches on the 523 host cpu, as used by the TCG implementation of goto_tb. */ 524 #if defined(__x86_64__) 525 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 526 #elif defined(__sparc__) 527 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 528 #elif defined(__powerpc64__) 529 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 530 #elif defined(__powerpc__) 531 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 532 #elif defined(__aarch64__) 533 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 534 #elif defined(__s390x__) 535 /* We have a +- 4GB range on the branches; leave some slop. */ 536 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 537 #elif defined(__mips__) 538 /* We have a 256MB branch region, but leave room to make sure the 539 main executable is also within that region. */ 540 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 541 #else 542 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 543 #endif 544 545 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 546 547 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 548 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 549 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 550 551 static inline size_t size_code_gen_buffer(size_t tb_size) 552 { 553 /* Size the buffer. */ 554 if (tb_size == 0) { 555 #ifdef USE_STATIC_CODE_GEN_BUFFER 556 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 557 #else 558 /* ??? Needs adjustments. */ 559 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 560 static buffer, we could size this on RESERVED_VA, on the text 561 segment size of the executable, or continue to use the default. */ 562 tb_size = (unsigned long)(ram_size / 4); 563 #endif 564 } 565 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 566 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 567 } 568 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 569 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 570 } 571 return tb_size; 572 } 573 574 #ifdef __mips__ 575 /* In order to use J and JAL within the code_gen_buffer, we require 576 that the buffer not cross a 256MB boundary. */ 577 static inline bool cross_256mb(void *addr, size_t size) 578 { 579 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 580 } 581 582 /* We weren't able to allocate a buffer without crossing that boundary, 583 so make do with the larger portion of the buffer that doesn't cross. 584 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 585 static inline void *split_cross_256mb(void *buf1, size_t size1) 586 { 587 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 588 size_t size2 = buf1 + size1 - buf2; 589 590 size1 = buf2 - buf1; 591 if (size1 < size2) { 592 size1 = size2; 593 buf1 = buf2; 594 } 595 596 tcg_ctx->code_gen_buffer_size = size1; 597 return buf1; 598 } 599 #endif 600 601 #ifdef USE_STATIC_CODE_GEN_BUFFER 602 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 603 __attribute__((aligned(CODE_GEN_ALIGN))); 604 605 static inline void *alloc_code_gen_buffer(void) 606 { 607 void *buf = static_code_gen_buffer; 608 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 609 size_t size; 610 611 /* page-align the beginning and end of the buffer */ 612 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 613 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 614 615 size = end - buf; 616 617 /* Honor a command-line option limiting the size of the buffer. */ 618 if (size > tcg_ctx->code_gen_buffer_size) { 619 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 620 qemu_real_host_page_size); 621 } 622 tcg_ctx->code_gen_buffer_size = size; 623 624 #ifdef __mips__ 625 if (cross_256mb(buf, size)) { 626 buf = split_cross_256mb(buf, size); 627 size = tcg_ctx->code_gen_buffer_size; 628 } 629 #endif 630 631 if (qemu_mprotect_rwx(buf, size)) { 632 abort(); 633 } 634 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 635 636 return buf; 637 } 638 #elif defined(_WIN32) 639 static inline void *alloc_code_gen_buffer(void) 640 { 641 size_t size = tcg_ctx->code_gen_buffer_size; 642 void *buf; 643 644 buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 645 PAGE_EXECUTE_READWRITE); 646 return buf; 647 } 648 #else 649 static inline void *alloc_code_gen_buffer(void) 650 { 651 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 652 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 653 uintptr_t start = 0; 654 size_t size = tcg_ctx->code_gen_buffer_size; 655 void *buf; 656 657 /* Constrain the position of the buffer based on the host cpu. 658 Note that these addresses are chosen in concert with the 659 addresses assigned in the relevant linker script file. */ 660 # if defined(__PIE__) || defined(__PIC__) 661 /* Don't bother setting a preferred location if we're building 662 a position-independent executable. We're more likely to get 663 an address near the main executable if we let the kernel 664 choose the address. */ 665 # elif defined(__x86_64__) && defined(MAP_32BIT) 666 /* Force the memory down into low memory with the executable. 667 Leave the choice of exact location with the kernel. */ 668 flags |= MAP_32BIT; 669 /* Cannot expect to map more than 800MB in low memory. */ 670 if (size > 800u * 1024 * 1024) { 671 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 672 } 673 # elif defined(__sparc__) 674 start = 0x40000000ul; 675 # elif defined(__s390x__) 676 start = 0x90000000ul; 677 # elif defined(__mips__) 678 # if _MIPS_SIM == _ABI64 679 start = 0x128000000ul; 680 # else 681 start = 0x08000000ul; 682 # endif 683 # endif 684 685 buf = mmap((void *)start, size, prot, flags, -1, 0); 686 if (buf == MAP_FAILED) { 687 return NULL; 688 } 689 690 #ifdef __mips__ 691 if (cross_256mb(buf, size)) { 692 /* Try again, with the original still mapped, to avoid re-acquiring 693 that 256mb crossing. This time don't specify an address. */ 694 size_t size2; 695 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 696 switch ((int)(buf2 != MAP_FAILED)) { 697 case 1: 698 if (!cross_256mb(buf2, size)) { 699 /* Success! Use the new buffer. */ 700 munmap(buf, size); 701 break; 702 } 703 /* Failure. Work with what we had. */ 704 munmap(buf2, size); 705 /* fallthru */ 706 default: 707 /* Split the original buffer. Free the smaller half. */ 708 buf2 = split_cross_256mb(buf, size); 709 size2 = tcg_ctx->code_gen_buffer_size; 710 if (buf == buf2) { 711 munmap(buf + size2, size - size2); 712 } else { 713 munmap(buf, size - size2); 714 } 715 size = size2; 716 break; 717 } 718 buf = buf2; 719 } 720 #endif 721 722 /* Request large pages for the buffer. */ 723 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 724 725 return buf; 726 } 727 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 728 729 /* compare a pointer @ptr and a tb_tc @s */ 730 static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s) 731 { 732 if (ptr >= s->ptr + s->size) { 733 return 1; 734 } else if (ptr < s->ptr) { 735 return -1; 736 } 737 return 0; 738 } 739 740 static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp) 741 { 742 const struct tb_tc *a = ap; 743 const struct tb_tc *b = bp; 744 745 /* 746 * When both sizes are set, we know this isn't a lookup. 747 * This is the most likely case: every TB must be inserted; lookups 748 * are a lot less frequent. 749 */ 750 if (likely(a->size && b->size)) { 751 if (a->ptr > b->ptr) { 752 return 1; 753 } else if (a->ptr < b->ptr) { 754 return -1; 755 } 756 /* a->ptr == b->ptr should happen only on deletions */ 757 g_assert(a->size == b->size); 758 return 0; 759 } 760 /* 761 * All lookups have either .size field set to 0. 762 * From the glib sources we see that @ap is always the lookup key. However 763 * the docs provide no guarantee, so we just mark this case as likely. 764 */ 765 if (likely(a->size == 0)) { 766 return ptr_cmp_tb_tc(a->ptr, b); 767 } 768 return ptr_cmp_tb_tc(b->ptr, a); 769 } 770 771 static inline void code_gen_alloc(size_t tb_size) 772 { 773 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 774 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 775 if (tcg_ctx->code_gen_buffer == NULL) { 776 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 777 exit(1); 778 } 779 tb_ctx.tb_tree = g_tree_new(tb_tc_cmp); 780 qemu_mutex_init(&tb_ctx.tb_lock); 781 } 782 783 static void tb_htable_init(void) 784 { 785 unsigned int mode = QHT_MODE_AUTO_RESIZE; 786 787 qht_init(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode); 788 } 789 790 /* Must be called before using the QEMU cpus. 'tb_size' is the size 791 (in bytes) allocated to the translation buffer. Zero means default 792 size. */ 793 void tcg_exec_init(unsigned long tb_size) 794 { 795 tcg_allowed = true; 796 cpu_gen_init(); 797 page_init(); 798 tb_htable_init(); 799 code_gen_alloc(tb_size); 800 #if defined(CONFIG_SOFTMMU) 801 /* There's no guest base to take into account, so go ahead and 802 initialize the prologue now. */ 803 tcg_prologue_init(tcg_ctx); 804 #endif 805 } 806 807 /* 808 * Allocate a new translation block. Flush the translation buffer if 809 * too many translation blocks or too much generated code. 810 * 811 * Called with tb_lock held. 812 */ 813 static TranslationBlock *tb_alloc(target_ulong pc) 814 { 815 TranslationBlock *tb; 816 817 assert_tb_locked(); 818 819 tb = tcg_tb_alloc(tcg_ctx); 820 if (unlikely(tb == NULL)) { 821 return NULL; 822 } 823 return tb; 824 } 825 826 /* Called with tb_lock held. */ 827 void tb_remove(TranslationBlock *tb) 828 { 829 assert_tb_locked(); 830 831 g_tree_remove(tb_ctx.tb_tree, &tb->tc); 832 } 833 834 static inline void invalidate_page_bitmap(PageDesc *p) 835 { 836 #ifdef CONFIG_SOFTMMU 837 g_free(p->code_bitmap); 838 p->code_bitmap = NULL; 839 p->code_write_count = 0; 840 #endif 841 } 842 843 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 844 static void page_flush_tb_1(int level, void **lp) 845 { 846 int i; 847 848 if (*lp == NULL) { 849 return; 850 } 851 if (level == 0) { 852 PageDesc *pd = *lp; 853 854 for (i = 0; i < V_L2_SIZE; ++i) { 855 pd[i].first_tb = NULL; 856 invalidate_page_bitmap(pd + i); 857 } 858 } else { 859 void **pp = *lp; 860 861 for (i = 0; i < V_L2_SIZE; ++i) { 862 page_flush_tb_1(level - 1, pp + i); 863 } 864 } 865 } 866 867 static void page_flush_tb(void) 868 { 869 int i, l1_sz = v_l1_size; 870 871 for (i = 0; i < l1_sz; i++) { 872 page_flush_tb_1(v_l2_levels, l1_map + i); 873 } 874 } 875 876 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 877 { 878 const TranslationBlock *tb = value; 879 size_t *size = data; 880 881 *size += tb->tc.size; 882 return false; 883 } 884 885 /* flush all the translation blocks */ 886 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 887 { 888 tb_lock(); 889 890 /* If it is already been done on request of another CPU, 891 * just retry. 892 */ 893 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 894 goto done; 895 } 896 897 if (DEBUG_TB_FLUSH_GATE) { 898 size_t nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 899 size_t host_size = 0; 900 901 g_tree_foreach(tb_ctx.tb_tree, tb_host_size_iter, &host_size); 902 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 903 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 904 } 905 906 CPU_FOREACH(cpu) { 907 cpu_tb_jmp_cache_clear(cpu); 908 } 909 910 /* Increment the refcount first so that destroy acts as a reset */ 911 g_tree_ref(tb_ctx.tb_tree); 912 g_tree_destroy(tb_ctx.tb_tree); 913 914 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 915 page_flush_tb(); 916 917 tcg_region_reset_all(); 918 /* XXX: flush processor icache at this point if cache flush is 919 expensive */ 920 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 921 922 done: 923 tb_unlock(); 924 } 925 926 void tb_flush(CPUState *cpu) 927 { 928 if (tcg_enabled()) { 929 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 930 async_safe_run_on_cpu(cpu, do_tb_flush, 931 RUN_ON_CPU_HOST_INT(tb_flush_count)); 932 } 933 } 934 935 /* 936 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 937 * so in order to prevent bit rot we compile them unconditionally in user-mode, 938 * and let the optimizer get rid of them by wrapping their user-only callers 939 * with if (DEBUG_TB_CHECK_GATE). 940 */ 941 #ifdef CONFIG_USER_ONLY 942 943 static void 944 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 945 { 946 TranslationBlock *tb = p; 947 target_ulong addr = *(target_ulong *)userp; 948 949 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 950 printf("ERROR invalidate: address=" TARGET_FMT_lx 951 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 952 } 953 } 954 955 /* verify that all the pages have correct rights for code 956 * 957 * Called with tb_lock held. 958 */ 959 static void tb_invalidate_check(target_ulong address) 960 { 961 address &= TARGET_PAGE_MASK; 962 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 963 } 964 965 static void 966 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 967 { 968 TranslationBlock *tb = p; 969 int flags1, flags2; 970 971 flags1 = page_get_flags(tb->pc); 972 flags2 = page_get_flags(tb->pc + tb->size - 1); 973 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 974 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 975 (long)tb->pc, tb->size, flags1, flags2); 976 } 977 } 978 979 /* verify that all the pages have correct rights for code */ 980 static void tb_page_check(void) 981 { 982 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 983 } 984 985 #endif /* CONFIG_USER_ONLY */ 986 987 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) 988 { 989 TranslationBlock *tb1; 990 unsigned int n1; 991 992 for (;;) { 993 tb1 = *ptb; 994 n1 = (uintptr_t)tb1 & 3; 995 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); 996 if (tb1 == tb) { 997 *ptb = tb1->page_next[n1]; 998 break; 999 } 1000 ptb = &tb1->page_next[n1]; 1001 } 1002 } 1003 1004 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 1005 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 1006 { 1007 TranslationBlock *tb1; 1008 uintptr_t *ptb, ntb; 1009 unsigned int n1; 1010 1011 ptb = &tb->jmp_list_next[n]; 1012 if (*ptb) { 1013 /* find tb(n) in circular list */ 1014 for (;;) { 1015 ntb = *ptb; 1016 n1 = ntb & 3; 1017 tb1 = (TranslationBlock *)(ntb & ~3); 1018 if (n1 == n && tb1 == tb) { 1019 break; 1020 } 1021 if (n1 == 2) { 1022 ptb = &tb1->jmp_list_first; 1023 } else { 1024 ptb = &tb1->jmp_list_next[n1]; 1025 } 1026 } 1027 /* now we can suppress tb(n) from the list */ 1028 *ptb = tb->jmp_list_next[n]; 1029 1030 tb->jmp_list_next[n] = (uintptr_t)NULL; 1031 } 1032 } 1033 1034 /* reset the jump entry 'n' of a TB so that it is not chained to 1035 another TB */ 1036 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1037 { 1038 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1039 tb_set_jmp_target(tb, n, addr); 1040 } 1041 1042 /* remove any jumps to the TB */ 1043 static inline void tb_jmp_unlink(TranslationBlock *tb) 1044 { 1045 TranslationBlock *tb1; 1046 uintptr_t *ptb, ntb; 1047 unsigned int n1; 1048 1049 ptb = &tb->jmp_list_first; 1050 for (;;) { 1051 ntb = *ptb; 1052 n1 = ntb & 3; 1053 tb1 = (TranslationBlock *)(ntb & ~3); 1054 if (n1 == 2) { 1055 break; 1056 } 1057 tb_reset_jump(tb1, n1); 1058 *ptb = tb1->jmp_list_next[n1]; 1059 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1060 } 1061 } 1062 1063 /* invalidate one TB 1064 * 1065 * Called with tb_lock held. 1066 */ 1067 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1068 { 1069 CPUState *cpu; 1070 PageDesc *p; 1071 uint32_t h; 1072 tb_page_addr_t phys_pc; 1073 1074 assert_tb_locked(); 1075 1076 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1077 1078 /* remove the TB from the hash list */ 1079 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1080 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1081 tb->trace_vcpu_dstate); 1082 qht_remove(&tb_ctx.htable, tb, h); 1083 1084 /* remove the TB from the page list */ 1085 if (tb->page_addr[0] != page_addr) { 1086 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1087 tb_page_remove(&p->first_tb, tb); 1088 invalidate_page_bitmap(p); 1089 } 1090 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1091 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1092 tb_page_remove(&p->first_tb, tb); 1093 invalidate_page_bitmap(p); 1094 } 1095 1096 /* remove the TB from the hash list */ 1097 h = tb_jmp_cache_hash_func(tb->pc); 1098 CPU_FOREACH(cpu) { 1099 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1100 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1101 } 1102 } 1103 1104 /* suppress this TB from the two jump lists */ 1105 tb_remove_from_jmp_list(tb, 0); 1106 tb_remove_from_jmp_list(tb, 1); 1107 1108 /* suppress any remaining jumps to this TB */ 1109 tb_jmp_unlink(tb); 1110 1111 tb_ctx.tb_phys_invalidate_count++; 1112 } 1113 1114 #ifdef CONFIG_SOFTMMU 1115 static void build_page_bitmap(PageDesc *p) 1116 { 1117 int n, tb_start, tb_end; 1118 TranslationBlock *tb; 1119 1120 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1121 1122 tb = p->first_tb; 1123 while (tb != NULL) { 1124 n = (uintptr_t)tb & 3; 1125 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1126 /* NOTE: this is subtle as a TB may span two physical pages */ 1127 if (n == 0) { 1128 /* NOTE: tb_end may be after the end of the page, but 1129 it is not a problem */ 1130 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1131 tb_end = tb_start + tb->size; 1132 if (tb_end > TARGET_PAGE_SIZE) { 1133 tb_end = TARGET_PAGE_SIZE; 1134 } 1135 } else { 1136 tb_start = 0; 1137 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1138 } 1139 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1140 tb = tb->page_next[n]; 1141 } 1142 } 1143 #endif 1144 1145 /* add the tb in the target page and protect it if necessary 1146 * 1147 * Called with mmap_lock held for user-mode emulation. 1148 */ 1149 static inline void tb_alloc_page(TranslationBlock *tb, 1150 unsigned int n, tb_page_addr_t page_addr) 1151 { 1152 PageDesc *p; 1153 #ifndef CONFIG_USER_ONLY 1154 bool page_already_protected; 1155 #endif 1156 1157 assert_memory_lock(); 1158 1159 tb->page_addr[n] = page_addr; 1160 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1161 tb->page_next[n] = p->first_tb; 1162 #ifndef CONFIG_USER_ONLY 1163 page_already_protected = p->first_tb != NULL; 1164 #endif 1165 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); 1166 invalidate_page_bitmap(p); 1167 1168 #if defined(CONFIG_USER_ONLY) 1169 if (p->flags & PAGE_WRITE) { 1170 target_ulong addr; 1171 PageDesc *p2; 1172 int prot; 1173 1174 /* force the host page as non writable (writes will have a 1175 page fault + mprotect overhead) */ 1176 page_addr &= qemu_host_page_mask; 1177 prot = 0; 1178 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1179 addr += TARGET_PAGE_SIZE) { 1180 1181 p2 = page_find(addr >> TARGET_PAGE_BITS); 1182 if (!p2) { 1183 continue; 1184 } 1185 prot |= p2->flags; 1186 p2->flags &= ~PAGE_WRITE; 1187 } 1188 mprotect(g2h(page_addr), qemu_host_page_size, 1189 (prot & PAGE_BITS) & ~PAGE_WRITE); 1190 if (DEBUG_TB_INVALIDATE_GATE) { 1191 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1192 } 1193 } 1194 #else 1195 /* if some code is already present, then the pages are already 1196 protected. So we handle the case where only the first TB is 1197 allocated in a physical page */ 1198 if (!page_already_protected) { 1199 tlb_protect_code(page_addr); 1200 } 1201 #endif 1202 } 1203 1204 /* add a new TB and link it to the physical page tables. phys_page2 is 1205 * (-1) to indicate that only one page contains the TB. 1206 * 1207 * Called with mmap_lock held for user-mode emulation. 1208 */ 1209 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1210 tb_page_addr_t phys_page2) 1211 { 1212 uint32_t h; 1213 1214 assert_memory_lock(); 1215 1216 /* add in the page list */ 1217 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1218 if (phys_page2 != -1) { 1219 tb_alloc_page(tb, 1, phys_page2); 1220 } else { 1221 tb->page_addr[1] = -1; 1222 } 1223 1224 /* add in the hash table */ 1225 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1226 tb->trace_vcpu_dstate); 1227 qht_insert(&tb_ctx.htable, tb, h); 1228 1229 #ifdef CONFIG_USER_ONLY 1230 if (DEBUG_TB_CHECK_GATE) { 1231 tb_page_check(); 1232 } 1233 #endif 1234 } 1235 1236 /* Called with mmap_lock held for user mode emulation. */ 1237 TranslationBlock *tb_gen_code(CPUState *cpu, 1238 target_ulong pc, target_ulong cs_base, 1239 uint32_t flags, int cflags) 1240 { 1241 CPUArchState *env = cpu->env_ptr; 1242 TranslationBlock *tb; 1243 tb_page_addr_t phys_pc, phys_page2; 1244 target_ulong virt_page2; 1245 tcg_insn_unit *gen_code_buf; 1246 int gen_code_size, search_size; 1247 #ifdef CONFIG_PROFILER 1248 TCGProfile *prof = &tcg_ctx->prof; 1249 int64_t ti; 1250 #endif 1251 assert_memory_lock(); 1252 1253 phys_pc = get_page_addr_code(env, pc); 1254 1255 buffer_overflow: 1256 tb = tb_alloc(pc); 1257 if (unlikely(!tb)) { 1258 /* flush must be done */ 1259 tb_flush(cpu); 1260 mmap_unlock(); 1261 /* Make the execution loop process the flush as soon as possible. */ 1262 cpu->exception_index = EXCP_INTERRUPT; 1263 cpu_loop_exit(cpu); 1264 } 1265 1266 gen_code_buf = tcg_ctx->code_gen_ptr; 1267 tb->tc.ptr = gen_code_buf; 1268 tb->pc = pc; 1269 tb->cs_base = cs_base; 1270 tb->flags = flags; 1271 tb->cflags = cflags; 1272 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1273 tcg_ctx->tb_cflags = cflags; 1274 1275 #ifdef CONFIG_PROFILER 1276 /* includes aborted translations because of exceptions */ 1277 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1278 ti = profile_getclock(); 1279 #endif 1280 1281 tcg_func_start(tcg_ctx); 1282 1283 tcg_ctx->cpu = ENV_GET_CPU(env); 1284 gen_intermediate_code(cpu, tb); 1285 tcg_ctx->cpu = NULL; 1286 1287 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1288 1289 /* generate machine code */ 1290 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1291 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1292 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1293 if (TCG_TARGET_HAS_direct_jump) { 1294 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1295 tcg_ctx->tb_jmp_target_addr = NULL; 1296 } else { 1297 tcg_ctx->tb_jmp_insn_offset = NULL; 1298 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1299 } 1300 1301 #ifdef CONFIG_PROFILER 1302 atomic_set(&prof->tb_count, prof->tb_count + 1); 1303 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1304 ti = profile_getclock(); 1305 #endif 1306 1307 /* ??? Overflow could be handled better here. In particular, we 1308 don't need to re-do gen_intermediate_code, nor should we re-do 1309 the tcg optimization currently hidden inside tcg_gen_code. All 1310 that should be required is to flush the TBs, allocate a new TB, 1311 re-initialize it per above, and re-do the actual code generation. */ 1312 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1313 if (unlikely(gen_code_size < 0)) { 1314 goto buffer_overflow; 1315 } 1316 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1317 if (unlikely(search_size < 0)) { 1318 goto buffer_overflow; 1319 } 1320 tb->tc.size = gen_code_size; 1321 1322 #ifdef CONFIG_PROFILER 1323 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1324 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1325 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1326 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1327 #endif 1328 1329 #ifdef DEBUG_DISAS 1330 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1331 qemu_log_in_addr_range(tb->pc)) { 1332 qemu_log_lock(); 1333 qemu_log("OUT: [size=%d]\n", gen_code_size); 1334 if (tcg_ctx->data_gen_ptr) { 1335 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1336 size_t data_size = gen_code_size - code_size; 1337 size_t i; 1338 1339 log_disas(tb->tc.ptr, code_size); 1340 1341 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1342 if (sizeof(tcg_target_ulong) == 8) { 1343 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1344 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1345 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1346 } else { 1347 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1348 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1349 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1350 } 1351 } 1352 } else { 1353 log_disas(tb->tc.ptr, gen_code_size); 1354 } 1355 qemu_log("\n"); 1356 qemu_log_flush(); 1357 qemu_log_unlock(); 1358 } 1359 #endif 1360 1361 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1362 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1363 CODE_GEN_ALIGN)); 1364 1365 /* init jump list */ 1366 assert(((uintptr_t)tb & 3) == 0); 1367 tb->jmp_list_first = (uintptr_t)tb | 2; 1368 tb->jmp_list_next[0] = (uintptr_t)NULL; 1369 tb->jmp_list_next[1] = (uintptr_t)NULL; 1370 1371 /* init original jump addresses wich has been set during tcg_gen_code() */ 1372 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1373 tb_reset_jump(tb, 0); 1374 } 1375 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1376 tb_reset_jump(tb, 1); 1377 } 1378 1379 /* check next page if needed */ 1380 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1381 phys_page2 = -1; 1382 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1383 phys_page2 = get_page_addr_code(env, virt_page2); 1384 } 1385 /* As long as consistency of the TB stuff is provided by tb_lock in user 1386 * mode and is implicit in single-threaded softmmu emulation, no explicit 1387 * memory barrier is required before tb_link_page() makes the TB visible 1388 * through the physical hash table and physical page list. 1389 */ 1390 tb_link_page(tb, phys_pc, phys_page2); 1391 g_tree_insert(tb_ctx.tb_tree, &tb->tc, tb); 1392 return tb; 1393 } 1394 1395 /* 1396 * Invalidate all TBs which intersect with the target physical address range 1397 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1398 * 'is_cpu_write_access' should be true if called from a real cpu write 1399 * access: the virtual CPU will exit the current TB if code is modified inside 1400 * this TB. 1401 * 1402 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1403 * Called with tb_lock held for system-mode emulation 1404 */ 1405 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1406 { 1407 while (start < end) { 1408 tb_invalidate_phys_page_range(start, end, 0); 1409 start &= TARGET_PAGE_MASK; 1410 start += TARGET_PAGE_SIZE; 1411 } 1412 } 1413 1414 #ifdef CONFIG_SOFTMMU 1415 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1416 { 1417 assert_tb_locked(); 1418 tb_invalidate_phys_range_1(start, end); 1419 } 1420 #else 1421 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1422 { 1423 assert_memory_lock(); 1424 tb_lock(); 1425 tb_invalidate_phys_range_1(start, end); 1426 tb_unlock(); 1427 } 1428 #endif 1429 /* 1430 * Invalidate all TBs which intersect with the target physical address range 1431 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1432 * 'is_cpu_write_access' should be true if called from a real cpu write 1433 * access: the virtual CPU will exit the current TB if code is modified inside 1434 * this TB. 1435 * 1436 * Called with tb_lock/mmap_lock held for user-mode emulation 1437 * Called with tb_lock held for system-mode emulation 1438 */ 1439 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1440 int is_cpu_write_access) 1441 { 1442 TranslationBlock *tb, *tb_next; 1443 tb_page_addr_t tb_start, tb_end; 1444 PageDesc *p; 1445 int n; 1446 #ifdef TARGET_HAS_PRECISE_SMC 1447 CPUState *cpu = current_cpu; 1448 CPUArchState *env = NULL; 1449 int current_tb_not_found = is_cpu_write_access; 1450 TranslationBlock *current_tb = NULL; 1451 int current_tb_modified = 0; 1452 target_ulong current_pc = 0; 1453 target_ulong current_cs_base = 0; 1454 uint32_t current_flags = 0; 1455 #endif /* TARGET_HAS_PRECISE_SMC */ 1456 1457 assert_memory_lock(); 1458 assert_tb_locked(); 1459 1460 p = page_find(start >> TARGET_PAGE_BITS); 1461 if (!p) { 1462 return; 1463 } 1464 #if defined(TARGET_HAS_PRECISE_SMC) 1465 if (cpu != NULL) { 1466 env = cpu->env_ptr; 1467 } 1468 #endif 1469 1470 /* we remove all the TBs in the range [start, end[ */ 1471 /* XXX: see if in some cases it could be faster to invalidate all 1472 the code */ 1473 tb = p->first_tb; 1474 while (tb != NULL) { 1475 n = (uintptr_t)tb & 3; 1476 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1477 tb_next = tb->page_next[n]; 1478 /* NOTE: this is subtle as a TB may span two physical pages */ 1479 if (n == 0) { 1480 /* NOTE: tb_end may be after the end of the page, but 1481 it is not a problem */ 1482 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1483 tb_end = tb_start + tb->size; 1484 } else { 1485 tb_start = tb->page_addr[1]; 1486 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1487 } 1488 if (!(tb_end <= start || tb_start >= end)) { 1489 #ifdef TARGET_HAS_PRECISE_SMC 1490 if (current_tb_not_found) { 1491 current_tb_not_found = 0; 1492 current_tb = NULL; 1493 if (cpu->mem_io_pc) { 1494 /* now we have a real cpu fault */ 1495 current_tb = tb_find_pc(cpu->mem_io_pc); 1496 } 1497 } 1498 if (current_tb == tb && 1499 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1500 /* If we are modifying the current TB, we must stop 1501 its execution. We could be more precise by checking 1502 that the modification is after the current PC, but it 1503 would require a specialized function to partially 1504 restore the CPU state */ 1505 1506 current_tb_modified = 1; 1507 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc); 1508 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1509 ¤t_flags); 1510 } 1511 #endif /* TARGET_HAS_PRECISE_SMC */ 1512 tb_phys_invalidate(tb, -1); 1513 } 1514 tb = tb_next; 1515 } 1516 #if !defined(CONFIG_USER_ONLY) 1517 /* if no code remaining, no need to continue to use slow writes */ 1518 if (!p->first_tb) { 1519 invalidate_page_bitmap(p); 1520 tlb_unprotect_code(start); 1521 } 1522 #endif 1523 #ifdef TARGET_HAS_PRECISE_SMC 1524 if (current_tb_modified) { 1525 /* Force execution of one insn next time. */ 1526 cpu->cflags_next_tb = 1 | curr_cflags(); 1527 cpu_loop_exit_noexc(cpu); 1528 } 1529 #endif 1530 } 1531 1532 #ifdef CONFIG_SOFTMMU 1533 /* len must be <= 8 and start must be a multiple of len. 1534 * Called via softmmu_template.h when code areas are written to with 1535 * iothread mutex not held. 1536 */ 1537 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1538 { 1539 PageDesc *p; 1540 1541 #if 0 1542 if (1) { 1543 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1544 cpu_single_env->mem_io_vaddr, len, 1545 cpu_single_env->eip, 1546 cpu_single_env->eip + 1547 (intptr_t)cpu_single_env->segs[R_CS].base); 1548 } 1549 #endif 1550 assert_memory_lock(); 1551 1552 p = page_find(start >> TARGET_PAGE_BITS); 1553 if (!p) { 1554 return; 1555 } 1556 if (!p->code_bitmap && 1557 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1558 /* build code bitmap. FIXME: writes should be protected by 1559 * tb_lock, reads by tb_lock or RCU. 1560 */ 1561 build_page_bitmap(p); 1562 } 1563 if (p->code_bitmap) { 1564 unsigned int nr; 1565 unsigned long b; 1566 1567 nr = start & ~TARGET_PAGE_MASK; 1568 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1569 if (b & ((1 << len) - 1)) { 1570 goto do_invalidate; 1571 } 1572 } else { 1573 do_invalidate: 1574 tb_invalidate_phys_page_range(start, start + len, 1); 1575 } 1576 } 1577 #else 1578 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1579 * host PC of the faulting store instruction that caused this invalidate. 1580 * Returns true if the caller needs to abort execution of the current 1581 * TB (because it was modified by this store and the guest CPU has 1582 * precise-SMC semantics). 1583 */ 1584 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1585 { 1586 TranslationBlock *tb; 1587 PageDesc *p; 1588 int n; 1589 #ifdef TARGET_HAS_PRECISE_SMC 1590 TranslationBlock *current_tb = NULL; 1591 CPUState *cpu = current_cpu; 1592 CPUArchState *env = NULL; 1593 int current_tb_modified = 0; 1594 target_ulong current_pc = 0; 1595 target_ulong current_cs_base = 0; 1596 uint32_t current_flags = 0; 1597 #endif 1598 1599 assert_memory_lock(); 1600 1601 addr &= TARGET_PAGE_MASK; 1602 p = page_find(addr >> TARGET_PAGE_BITS); 1603 if (!p) { 1604 return false; 1605 } 1606 1607 tb_lock(); 1608 tb = p->first_tb; 1609 #ifdef TARGET_HAS_PRECISE_SMC 1610 if (tb && pc != 0) { 1611 current_tb = tb_find_pc(pc); 1612 } 1613 if (cpu != NULL) { 1614 env = cpu->env_ptr; 1615 } 1616 #endif 1617 while (tb != NULL) { 1618 n = (uintptr_t)tb & 3; 1619 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1620 #ifdef TARGET_HAS_PRECISE_SMC 1621 if (current_tb == tb && 1622 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1623 /* If we are modifying the current TB, we must stop 1624 its execution. We could be more precise by checking 1625 that the modification is after the current PC, but it 1626 would require a specialized function to partially 1627 restore the CPU state */ 1628 1629 current_tb_modified = 1; 1630 cpu_restore_state_from_tb(cpu, current_tb, pc); 1631 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1632 ¤t_flags); 1633 } 1634 #endif /* TARGET_HAS_PRECISE_SMC */ 1635 tb_phys_invalidate(tb, addr); 1636 tb = tb->page_next[n]; 1637 } 1638 p->first_tb = NULL; 1639 #ifdef TARGET_HAS_PRECISE_SMC 1640 if (current_tb_modified) { 1641 /* Force execution of one insn next time. */ 1642 cpu->cflags_next_tb = 1 | curr_cflags(); 1643 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1644 * back into the cpu_exec loop. */ 1645 return true; 1646 } 1647 #endif 1648 tb_unlock(); 1649 1650 return false; 1651 } 1652 #endif 1653 1654 /* 1655 * Find the TB 'tb' such that 1656 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size 1657 * Return NULL if not found. 1658 */ 1659 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) 1660 { 1661 struct tb_tc s = { .ptr = (void *)tc_ptr }; 1662 1663 return g_tree_lookup(tb_ctx.tb_tree, &s); 1664 } 1665 1666 #if !defined(CONFIG_USER_ONLY) 1667 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr) 1668 { 1669 ram_addr_t ram_addr; 1670 MemoryRegion *mr; 1671 hwaddr l = 1; 1672 1673 rcu_read_lock(); 1674 mr = address_space_translate(as, addr, &addr, &l, false); 1675 if (!(memory_region_is_ram(mr) 1676 || memory_region_is_romd(mr))) { 1677 rcu_read_unlock(); 1678 return; 1679 } 1680 ram_addr = memory_region_get_ram_addr(mr) + addr; 1681 tb_lock(); 1682 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1683 tb_unlock(); 1684 rcu_read_unlock(); 1685 } 1686 #endif /* !defined(CONFIG_USER_ONLY) */ 1687 1688 /* Called with tb_lock held. */ 1689 void tb_check_watchpoint(CPUState *cpu) 1690 { 1691 TranslationBlock *tb; 1692 1693 tb = tb_find_pc(cpu->mem_io_pc); 1694 if (tb) { 1695 /* We can use retranslation to find the PC. */ 1696 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc); 1697 tb_phys_invalidate(tb, -1); 1698 } else { 1699 /* The exception probably happened in a helper. The CPU state should 1700 have been saved before calling it. Fetch the PC from there. */ 1701 CPUArchState *env = cpu->env_ptr; 1702 target_ulong pc, cs_base; 1703 tb_page_addr_t addr; 1704 uint32_t flags; 1705 1706 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1707 addr = get_page_addr_code(env, pc); 1708 tb_invalidate_phys_range(addr, addr + 1); 1709 } 1710 } 1711 1712 #ifndef CONFIG_USER_ONLY 1713 /* in deterministic execution mode, instructions doing device I/Os 1714 * must be at the end of the TB. 1715 * 1716 * Called by softmmu_template.h, with iothread mutex not held. 1717 */ 1718 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1719 { 1720 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1721 CPUArchState *env = cpu->env_ptr; 1722 #endif 1723 TranslationBlock *tb; 1724 uint32_t n; 1725 1726 tb_lock(); 1727 tb = tb_find_pc(retaddr); 1728 if (!tb) { 1729 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1730 (void *)retaddr); 1731 } 1732 n = cpu->icount_decr.u16.low + tb->icount; 1733 cpu_restore_state_from_tb(cpu, tb, retaddr); 1734 /* Calculate how many instructions had been executed before the fault 1735 occurred. */ 1736 n = n - cpu->icount_decr.u16.low; 1737 /* Generate a new TB ending on the I/O insn. */ 1738 n++; 1739 /* On MIPS and SH, delay slot instructions can only be restarted if 1740 they were already the first instruction in the TB. If this is not 1741 the first instruction in a TB then re-execute the preceding 1742 branch. */ 1743 #if defined(TARGET_MIPS) 1744 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) { 1745 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1746 cpu->icount_decr.u16.low++; 1747 env->hflags &= ~MIPS_HFLAG_BMASK; 1748 } 1749 #elif defined(TARGET_SH4) 1750 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1751 && n > 1) { 1752 env->pc -= 2; 1753 cpu->icount_decr.u16.low++; 1754 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1755 } 1756 #endif 1757 /* This should never happen. */ 1758 if (n > CF_COUNT_MASK) { 1759 cpu_abort(cpu, "TB too big during recompile"); 1760 } 1761 1762 /* Adjust the execution state of the next TB. */ 1763 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1764 1765 if (tb->cflags & CF_NOCACHE) { 1766 if (tb->orig_tb) { 1767 /* Invalidate original TB if this TB was generated in 1768 * cpu_exec_nocache() */ 1769 tb_phys_invalidate(tb->orig_tb, -1); 1770 } 1771 tb_remove(tb); 1772 } 1773 1774 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1775 * the first in the TB) then we end up generating a whole new TB and 1776 * repeating the fault, which is horribly inefficient. 1777 * Better would be to execute just this insn uncached, or generate a 1778 * second new TB. 1779 * 1780 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1781 * tb_lock gets reset. 1782 */ 1783 cpu_loop_exit_noexc(cpu); 1784 } 1785 1786 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1787 { 1788 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1789 1790 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1791 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1792 } 1793 } 1794 1795 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1796 { 1797 /* Discard jump cache entries for any tb which might potentially 1798 overlap the flushed page. */ 1799 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1800 tb_jmp_cache_clear_page(cpu, addr); 1801 } 1802 1803 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1804 struct qht_stats hst) 1805 { 1806 uint32_t hgram_opts; 1807 size_t hgram_bins; 1808 char *hgram; 1809 1810 if (!hst.head_buckets) { 1811 return; 1812 } 1813 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1814 hst.used_head_buckets, hst.head_buckets, 1815 (double)hst.used_head_buckets / hst.head_buckets * 100); 1816 1817 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1818 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1819 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1820 hgram_opts |= QDIST_PR_NODECIMAL; 1821 } 1822 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1823 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1824 qdist_avg(&hst.occupancy) * 100, hgram); 1825 g_free(hgram); 1826 1827 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1828 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1829 if (hgram_bins > 10) { 1830 hgram_bins = 10; 1831 } else { 1832 hgram_bins = 0; 1833 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1834 } 1835 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1836 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1837 qdist_avg(&hst.chain), hgram); 1838 g_free(hgram); 1839 } 1840 1841 struct tb_tree_stats { 1842 size_t host_size; 1843 size_t target_size; 1844 size_t max_target_size; 1845 size_t direct_jmp_count; 1846 size_t direct_jmp2_count; 1847 size_t cross_page; 1848 }; 1849 1850 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1851 { 1852 const TranslationBlock *tb = value; 1853 struct tb_tree_stats *tst = data; 1854 1855 tst->host_size += tb->tc.size; 1856 tst->target_size += tb->size; 1857 if (tb->size > tst->max_target_size) { 1858 tst->max_target_size = tb->size; 1859 } 1860 if (tb->page_addr[1] != -1) { 1861 tst->cross_page++; 1862 } 1863 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1864 tst->direct_jmp_count++; 1865 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1866 tst->direct_jmp2_count++; 1867 } 1868 } 1869 return false; 1870 } 1871 1872 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1873 { 1874 struct tb_tree_stats tst = {}; 1875 struct qht_stats hst; 1876 size_t nb_tbs; 1877 1878 tb_lock(); 1879 1880 nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 1881 g_tree_foreach(tb_ctx.tb_tree, tb_tree_stats_iter, &tst); 1882 /* XXX: avoid using doubles ? */ 1883 cpu_fprintf(f, "Translation buffer state:\n"); 1884 /* 1885 * Report total code size including the padding and TB structs; 1886 * otherwise users might think "-tb-size" is not honoured. 1887 * For avg host size we use the precise numbers from tb_tree_stats though. 1888 */ 1889 cpu_fprintf(f, "gen code size %zu/%zu\n", 1890 tcg_code_size(), tcg_code_capacity()); 1891 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1892 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1893 nb_tbs ? tst.target_size / nb_tbs : 0, 1894 tst.max_target_size); 1895 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1896 nb_tbs ? tst.host_size / nb_tbs : 0, 1897 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1898 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1899 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1900 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1901 tst.direct_jmp_count, 1902 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1903 tst.direct_jmp2_count, 1904 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1905 1906 qht_statistics_init(&tb_ctx.htable, &hst); 1907 print_qht_statistics(f, cpu_fprintf, hst); 1908 qht_statistics_destroy(&hst); 1909 1910 cpu_fprintf(f, "\nStatistics:\n"); 1911 cpu_fprintf(f, "TB flush count %u\n", 1912 atomic_read(&tb_ctx.tb_flush_count)); 1913 cpu_fprintf(f, "TB invalidate count %d\n", tb_ctx.tb_phys_invalidate_count); 1914 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1915 tcg_dump_info(f, cpu_fprintf); 1916 1917 tb_unlock(); 1918 } 1919 1920 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1921 { 1922 tcg_dump_op_count(f, cpu_fprintf); 1923 } 1924 1925 #else /* CONFIG_USER_ONLY */ 1926 1927 void cpu_interrupt(CPUState *cpu, int mask) 1928 { 1929 g_assert(qemu_mutex_iothread_locked()); 1930 cpu->interrupt_request |= mask; 1931 cpu->icount_decr.u16.high = -1; 1932 } 1933 1934 /* 1935 * Walks guest process memory "regions" one by one 1936 * and calls callback function 'fn' for each region. 1937 */ 1938 struct walk_memory_regions_data { 1939 walk_memory_regions_fn fn; 1940 void *priv; 1941 target_ulong start; 1942 int prot; 1943 }; 1944 1945 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1946 target_ulong end, int new_prot) 1947 { 1948 if (data->start != -1u) { 1949 int rc = data->fn(data->priv, data->start, end, data->prot); 1950 if (rc != 0) { 1951 return rc; 1952 } 1953 } 1954 1955 data->start = (new_prot ? end : -1u); 1956 data->prot = new_prot; 1957 1958 return 0; 1959 } 1960 1961 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1962 target_ulong base, int level, void **lp) 1963 { 1964 target_ulong pa; 1965 int i, rc; 1966 1967 if (*lp == NULL) { 1968 return walk_memory_regions_end(data, base, 0); 1969 } 1970 1971 if (level == 0) { 1972 PageDesc *pd = *lp; 1973 1974 for (i = 0; i < V_L2_SIZE; ++i) { 1975 int prot = pd[i].flags; 1976 1977 pa = base | (i << TARGET_PAGE_BITS); 1978 if (prot != data->prot) { 1979 rc = walk_memory_regions_end(data, pa, prot); 1980 if (rc != 0) { 1981 return rc; 1982 } 1983 } 1984 } 1985 } else { 1986 void **pp = *lp; 1987 1988 for (i = 0; i < V_L2_SIZE; ++i) { 1989 pa = base | ((target_ulong)i << 1990 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1991 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 1992 if (rc != 0) { 1993 return rc; 1994 } 1995 } 1996 } 1997 1998 return 0; 1999 } 2000 2001 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2002 { 2003 struct walk_memory_regions_data data; 2004 uintptr_t i, l1_sz = v_l1_size; 2005 2006 data.fn = fn; 2007 data.priv = priv; 2008 data.start = -1u; 2009 data.prot = 0; 2010 2011 for (i = 0; i < l1_sz; i++) { 2012 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2013 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2014 if (rc != 0) { 2015 return rc; 2016 } 2017 } 2018 2019 return walk_memory_regions_end(&data, 0, 0); 2020 } 2021 2022 static int dump_region(void *priv, target_ulong start, 2023 target_ulong end, unsigned long prot) 2024 { 2025 FILE *f = (FILE *)priv; 2026 2027 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2028 " "TARGET_FMT_lx" %c%c%c\n", 2029 start, end, end - start, 2030 ((prot & PAGE_READ) ? 'r' : '-'), 2031 ((prot & PAGE_WRITE) ? 'w' : '-'), 2032 ((prot & PAGE_EXEC) ? 'x' : '-')); 2033 2034 return 0; 2035 } 2036 2037 /* dump memory mappings */ 2038 void page_dump(FILE *f) 2039 { 2040 const int length = sizeof(target_ulong) * 2; 2041 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2042 length, "start", length, "end", length, "size", "prot"); 2043 walk_memory_regions(f, dump_region); 2044 } 2045 2046 int page_get_flags(target_ulong address) 2047 { 2048 PageDesc *p; 2049 2050 p = page_find(address >> TARGET_PAGE_BITS); 2051 if (!p) { 2052 return 0; 2053 } 2054 return p->flags; 2055 } 2056 2057 /* Modify the flags of a page and invalidate the code if necessary. 2058 The flag PAGE_WRITE_ORG is positioned automatically depending 2059 on PAGE_WRITE. The mmap_lock should already be held. */ 2060 void page_set_flags(target_ulong start, target_ulong end, int flags) 2061 { 2062 target_ulong addr, len; 2063 2064 /* This function should never be called with addresses outside the 2065 guest address space. If this assert fires, it probably indicates 2066 a missing call to h2g_valid. */ 2067 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2068 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2069 #endif 2070 assert(start < end); 2071 assert_memory_lock(); 2072 2073 start = start & TARGET_PAGE_MASK; 2074 end = TARGET_PAGE_ALIGN(end); 2075 2076 if (flags & PAGE_WRITE) { 2077 flags |= PAGE_WRITE_ORG; 2078 } 2079 2080 for (addr = start, len = end - start; 2081 len != 0; 2082 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2083 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2084 2085 /* If the write protection bit is set, then we invalidate 2086 the code inside. */ 2087 if (!(p->flags & PAGE_WRITE) && 2088 (flags & PAGE_WRITE) && 2089 p->first_tb) { 2090 tb_invalidate_phys_page(addr, 0); 2091 } 2092 p->flags = flags; 2093 } 2094 } 2095 2096 int page_check_range(target_ulong start, target_ulong len, int flags) 2097 { 2098 PageDesc *p; 2099 target_ulong end; 2100 target_ulong addr; 2101 2102 /* This function should never be called with addresses outside the 2103 guest address space. If this assert fires, it probably indicates 2104 a missing call to h2g_valid. */ 2105 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2106 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2107 #endif 2108 2109 if (len == 0) { 2110 return 0; 2111 } 2112 if (start + len - 1 < start) { 2113 /* We've wrapped around. */ 2114 return -1; 2115 } 2116 2117 /* must do before we loose bits in the next step */ 2118 end = TARGET_PAGE_ALIGN(start + len); 2119 start = start & TARGET_PAGE_MASK; 2120 2121 for (addr = start, len = end - start; 2122 len != 0; 2123 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2124 p = page_find(addr >> TARGET_PAGE_BITS); 2125 if (!p) { 2126 return -1; 2127 } 2128 if (!(p->flags & PAGE_VALID)) { 2129 return -1; 2130 } 2131 2132 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2133 return -1; 2134 } 2135 if (flags & PAGE_WRITE) { 2136 if (!(p->flags & PAGE_WRITE_ORG)) { 2137 return -1; 2138 } 2139 /* unprotect the page if it was put read-only because it 2140 contains translated code */ 2141 if (!(p->flags & PAGE_WRITE)) { 2142 if (!page_unprotect(addr, 0)) { 2143 return -1; 2144 } 2145 } 2146 } 2147 } 2148 return 0; 2149 } 2150 2151 /* called from signal handler: invalidate the code and unprotect the 2152 * page. Return 0 if the fault was not handled, 1 if it was handled, 2153 * and 2 if it was handled but the caller must cause the TB to be 2154 * immediately exited. (We can only return 2 if the 'pc' argument is 2155 * non-zero.) 2156 */ 2157 int page_unprotect(target_ulong address, uintptr_t pc) 2158 { 2159 unsigned int prot; 2160 bool current_tb_invalidated; 2161 PageDesc *p; 2162 target_ulong host_start, host_end, addr; 2163 2164 /* Technically this isn't safe inside a signal handler. However we 2165 know this only ever happens in a synchronous SEGV handler, so in 2166 practice it seems to be ok. */ 2167 mmap_lock(); 2168 2169 p = page_find(address >> TARGET_PAGE_BITS); 2170 if (!p) { 2171 mmap_unlock(); 2172 return 0; 2173 } 2174 2175 /* if the page was really writable, then we change its 2176 protection back to writable */ 2177 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) { 2178 host_start = address & qemu_host_page_mask; 2179 host_end = host_start + qemu_host_page_size; 2180 2181 prot = 0; 2182 current_tb_invalidated = false; 2183 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) { 2184 p = page_find(addr >> TARGET_PAGE_BITS); 2185 p->flags |= PAGE_WRITE; 2186 prot |= p->flags; 2187 2188 /* and since the content will be modified, we must invalidate 2189 the corresponding translated code. */ 2190 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2191 #ifdef CONFIG_USER_ONLY 2192 if (DEBUG_TB_CHECK_GATE) { 2193 tb_invalidate_check(addr); 2194 } 2195 #endif 2196 } 2197 mprotect((void *)g2h(host_start), qemu_host_page_size, 2198 prot & PAGE_BITS); 2199 2200 mmap_unlock(); 2201 /* If current TB was invalidated return to main loop */ 2202 return current_tb_invalidated ? 2 : 1; 2203 } 2204 mmap_unlock(); 2205 return 0; 2206 } 2207 #endif /* CONFIG_USER_ONLY */ 2208 2209 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2210 void tcg_flush_softmmu_tlb(CPUState *cs) 2211 { 2212 #ifdef CONFIG_SOFTMMU 2213 tlb_flush(cs); 2214 #endif 2215 } 2216