1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #include "exec/exec-all.h" 35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 36 #include <sys/param.h> 37 #if __FreeBSD_version >= 700104 38 #define HAVE_KINFO_GETVMMAP 39 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 40 #include <sys/proc.h> 41 #include <machine/profile.h> 42 #define _KERNEL 43 #include <sys/user.h> 44 #undef _KERNEL 45 #undef sigqueue 46 #include <libutil.h> 47 #endif 48 #endif 49 #else 50 #include "exec/address-spaces.h" 51 #endif 52 53 #include "exec/cputlb.h" 54 #include "exec/tb-hash.h" 55 #include "translate-all.h" 56 #include "qemu/bitmap.h" 57 #include "qemu/error-report.h" 58 #include "qemu/timer.h" 59 #include "qemu/main-loop.h" 60 #include "exec/log.h" 61 #include "sysemu/cpus.h" 62 63 /* #define DEBUG_TB_INVALIDATE */ 64 /* #define DEBUG_TB_FLUSH */ 65 /* make various TB consistency checks */ 66 /* #define DEBUG_TB_CHECK */ 67 68 #ifdef DEBUG_TB_INVALIDATE 69 #define DEBUG_TB_INVALIDATE_GATE 1 70 #else 71 #define DEBUG_TB_INVALIDATE_GATE 0 72 #endif 73 74 #ifdef DEBUG_TB_FLUSH 75 #define DEBUG_TB_FLUSH_GATE 1 76 #else 77 #define DEBUG_TB_FLUSH_GATE 0 78 #endif 79 80 #if !defined(CONFIG_USER_ONLY) 81 /* TB consistency checks only implemented for usermode emulation. */ 82 #undef DEBUG_TB_CHECK 83 #endif 84 85 #ifdef DEBUG_TB_CHECK 86 #define DEBUG_TB_CHECK_GATE 1 87 #else 88 #define DEBUG_TB_CHECK_GATE 0 89 #endif 90 91 /* Access to the various translations structures need to be serialised via locks 92 * for consistency. This is automatic for SoftMMU based system 93 * emulation due to its single threaded nature. In user-mode emulation 94 * access to the memory related structures are protected with the 95 * mmap_lock. 96 */ 97 #ifdef CONFIG_SOFTMMU 98 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 99 #else 100 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 101 #endif 102 103 #define SMC_BITMAP_USE_THRESHOLD 10 104 105 typedef struct PageDesc { 106 /* list of TBs intersecting this ram page */ 107 TranslationBlock *first_tb; 108 #ifdef CONFIG_SOFTMMU 109 /* in order to optimize self modifying code, we count the number 110 of lookups we do to a given page to use a bitmap */ 111 unsigned int code_write_count; 112 unsigned long *code_bitmap; 113 #else 114 unsigned long flags; 115 #endif 116 } PageDesc; 117 118 /* In system mode we want L1_MAP to be based on ram offsets, 119 while in user mode we want it to be based on virtual addresses. */ 120 #if !defined(CONFIG_USER_ONLY) 121 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 122 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 123 #else 124 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 125 #endif 126 #else 127 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 128 #endif 129 130 /* Size of the L2 (and L3, etc) page tables. */ 131 #define V_L2_BITS 10 132 #define V_L2_SIZE (1 << V_L2_BITS) 133 134 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 135 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 136 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 137 * BITS_PER_BYTE); 138 139 /* 140 * L1 Mapping properties 141 */ 142 static int v_l1_size; 143 static int v_l1_shift; 144 static int v_l2_levels; 145 146 /* The bottom level has pointers to PageDesc, and is indexed by 147 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 148 */ 149 #define V_L1_MIN_BITS 4 150 #define V_L1_MAX_BITS (V_L2_BITS + 3) 151 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 152 153 static void *l1_map[V_L1_MAX_SIZE]; 154 155 /* code generation context */ 156 TCGContext tcg_init_ctx; 157 __thread TCGContext *tcg_ctx; 158 TBContext tb_ctx; 159 bool parallel_cpus; 160 161 /* translation block context */ 162 static __thread int have_tb_lock; 163 164 static void page_table_config_init(void) 165 { 166 uint32_t v_l1_bits; 167 168 assert(TARGET_PAGE_BITS); 169 /* The bits remaining after N lower levels of page tables. */ 170 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 171 if (v_l1_bits < V_L1_MIN_BITS) { 172 v_l1_bits += V_L2_BITS; 173 } 174 175 v_l1_size = 1 << v_l1_bits; 176 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 177 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 178 179 assert(v_l1_bits <= V_L1_MAX_BITS); 180 assert(v_l1_shift % V_L2_BITS == 0); 181 assert(v_l2_levels >= 0); 182 } 183 184 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 185 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 186 187 void tb_lock(void) 188 { 189 assert_tb_unlocked(); 190 qemu_mutex_lock(&tb_ctx.tb_lock); 191 have_tb_lock++; 192 } 193 194 void tb_unlock(void) 195 { 196 assert_tb_locked(); 197 have_tb_lock--; 198 qemu_mutex_unlock(&tb_ctx.tb_lock); 199 } 200 201 void tb_lock_reset(void) 202 { 203 if (have_tb_lock) { 204 qemu_mutex_unlock(&tb_ctx.tb_lock); 205 have_tb_lock = 0; 206 } 207 } 208 209 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr); 210 211 void cpu_gen_init(void) 212 { 213 tcg_context_init(&tcg_init_ctx); 214 } 215 216 /* Encode VAL as a signed leb128 sequence at P. 217 Return P incremented past the encoded value. */ 218 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 219 { 220 int more, byte; 221 222 do { 223 byte = val & 0x7f; 224 val >>= 7; 225 more = !((val == 0 && (byte & 0x40) == 0) 226 || (val == -1 && (byte & 0x40) != 0)); 227 if (more) { 228 byte |= 0x80; 229 } 230 *p++ = byte; 231 } while (more); 232 233 return p; 234 } 235 236 /* Decode a signed leb128 sequence at *PP; increment *PP past the 237 decoded value. Return the decoded value. */ 238 static target_long decode_sleb128(uint8_t **pp) 239 { 240 uint8_t *p = *pp; 241 target_long val = 0; 242 int byte, shift = 0; 243 244 do { 245 byte = *p++; 246 val |= (target_ulong)(byte & 0x7f) << shift; 247 shift += 7; 248 } while (byte & 0x80); 249 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 250 val |= -(target_ulong)1 << shift; 251 } 252 253 *pp = p; 254 return val; 255 } 256 257 /* Encode the data collected about the instructions while compiling TB. 258 Place the data at BLOCK, and return the number of bytes consumed. 259 260 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's, 261 which come from the target's insn_start data, followed by a uintptr_t 262 which comes from the host pc of the end of the code implementing the insn. 263 264 Each line of the table is encoded as sleb128 deltas from the previous 265 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 266 That is, the first column is seeded with the guest pc, the last column 267 with the host pc, and the middle columns with zeros. */ 268 269 static int encode_search(TranslationBlock *tb, uint8_t *block) 270 { 271 uint8_t *highwater = tcg_ctx->code_gen_highwater; 272 uint8_t *p = block; 273 int i, j, n; 274 275 for (i = 0, n = tb->icount; i < n; ++i) { 276 target_ulong prev; 277 278 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 279 if (i == 0) { 280 prev = (j == 0 ? tb->pc : 0); 281 } else { 282 prev = tcg_ctx->gen_insn_data[i - 1][j]; 283 } 284 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 285 } 286 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 287 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 288 289 /* Test for (pending) buffer overflow. The assumption is that any 290 one row beginning below the high water mark cannot overrun 291 the buffer completely. Thus we can test for overflow after 292 encoding a row without having to check during encoding. */ 293 if (unlikely(p > highwater)) { 294 return -1; 295 } 296 } 297 298 return p - block; 299 } 300 301 /* The cpu state corresponding to 'searched_pc' is restored. 302 * Called with tb_lock held. 303 */ 304 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 305 uintptr_t searched_pc) 306 { 307 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 308 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 309 CPUArchState *env = cpu->env_ptr; 310 uint8_t *p = tb->tc.ptr + tb->tc.size; 311 int i, j, num_insns = tb->icount; 312 #ifdef CONFIG_PROFILER 313 TCGProfile *prof = &tcg_ctx->prof; 314 int64_t ti = profile_getclock(); 315 #endif 316 317 searched_pc -= GETPC_ADJ; 318 319 if (searched_pc < host_pc) { 320 return -1; 321 } 322 323 /* Reconstruct the stored insn data while looking for the point at 324 which the end of the insn exceeds the searched_pc. */ 325 for (i = 0; i < num_insns; ++i) { 326 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 327 data[j] += decode_sleb128(&p); 328 } 329 host_pc += decode_sleb128(&p); 330 if (host_pc > searched_pc) { 331 goto found; 332 } 333 } 334 return -1; 335 336 found: 337 if (tb->cflags & CF_USE_ICOUNT) { 338 assert(use_icount); 339 /* Reset the cycle counter to the start of the block. */ 340 cpu->icount_decr.u16.low += num_insns; 341 /* Clear the IO flag. */ 342 cpu->can_do_io = 0; 343 } 344 cpu->icount_decr.u16.low -= i; 345 restore_state_to_opc(env, tb, data); 346 347 #ifdef CONFIG_PROFILER 348 atomic_set(&prof->restore_time, 349 prof->restore_time + profile_getclock() - ti); 350 atomic_set(&prof->restore_count, prof->restore_count + 1); 351 #endif 352 return 0; 353 } 354 355 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc) 356 { 357 TranslationBlock *tb; 358 bool r = false; 359 uintptr_t check_offset; 360 361 /* The host_pc has to be in the region of current code buffer. If 362 * it is not we will not be able to resolve it here. The two cases 363 * where host_pc will not be correct are: 364 * 365 * - fault during translation (instruction fetch) 366 * - fault from helper (not using GETPC() macro) 367 * 368 * Either way we need return early to avoid blowing up on a 369 * recursive tb_lock() as we can't resolve it here. 370 * 371 * We are using unsigned arithmetic so if host_pc < 372 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way 373 * above the code_gen_buffer_size 374 */ 375 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; 376 377 if (check_offset < tcg_init_ctx.code_gen_buffer_size) { 378 tb_lock(); 379 tb = tb_find_pc(host_pc); 380 if (tb) { 381 cpu_restore_state_from_tb(cpu, tb, host_pc); 382 if (tb->cflags & CF_NOCACHE) { 383 /* one-shot translation, invalidate it immediately */ 384 tb_phys_invalidate(tb, -1); 385 tb_remove(tb); 386 } 387 r = true; 388 } 389 tb_unlock(); 390 } 391 392 return r; 393 } 394 395 static void page_init(void) 396 { 397 page_size_init(); 398 page_table_config_init(); 399 400 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 401 { 402 #ifdef HAVE_KINFO_GETVMMAP 403 struct kinfo_vmentry *freep; 404 int i, cnt; 405 406 freep = kinfo_getvmmap(getpid(), &cnt); 407 if (freep) { 408 mmap_lock(); 409 for (i = 0; i < cnt; i++) { 410 unsigned long startaddr, endaddr; 411 412 startaddr = freep[i].kve_start; 413 endaddr = freep[i].kve_end; 414 if (h2g_valid(startaddr)) { 415 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 416 417 if (h2g_valid(endaddr)) { 418 endaddr = h2g(endaddr); 419 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 420 } else { 421 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 422 endaddr = ~0ul; 423 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 424 #endif 425 } 426 } 427 } 428 free(freep); 429 mmap_unlock(); 430 } 431 #else 432 FILE *f; 433 434 last_brk = (unsigned long)sbrk(0); 435 436 f = fopen("/compat/linux/proc/self/maps", "r"); 437 if (f) { 438 mmap_lock(); 439 440 do { 441 unsigned long startaddr, endaddr; 442 int n; 443 444 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 445 446 if (n == 2 && h2g_valid(startaddr)) { 447 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 448 449 if (h2g_valid(endaddr)) { 450 endaddr = h2g(endaddr); 451 } else { 452 endaddr = ~0ul; 453 } 454 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 455 } 456 } while (!feof(f)); 457 458 fclose(f); 459 mmap_unlock(); 460 } 461 #endif 462 } 463 #endif 464 } 465 466 /* If alloc=1: 467 * Called with tb_lock held for system emulation. 468 * Called with mmap_lock held for user-mode emulation. 469 */ 470 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 471 { 472 PageDesc *pd; 473 void **lp; 474 int i; 475 476 if (alloc) { 477 assert_memory_lock(); 478 } 479 480 /* Level 1. Always allocated. */ 481 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 482 483 /* Level 2..N-1. */ 484 for (i = v_l2_levels; i > 0; i--) { 485 void **p = atomic_rcu_read(lp); 486 487 if (p == NULL) { 488 if (!alloc) { 489 return NULL; 490 } 491 p = g_new0(void *, V_L2_SIZE); 492 atomic_rcu_set(lp, p); 493 } 494 495 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 496 } 497 498 pd = atomic_rcu_read(lp); 499 if (pd == NULL) { 500 if (!alloc) { 501 return NULL; 502 } 503 pd = g_new0(PageDesc, V_L2_SIZE); 504 atomic_rcu_set(lp, pd); 505 } 506 507 return pd + (index & (V_L2_SIZE - 1)); 508 } 509 510 static inline PageDesc *page_find(tb_page_addr_t index) 511 { 512 return page_find_alloc(index, 0); 513 } 514 515 #if defined(CONFIG_USER_ONLY) 516 /* Currently it is not recommended to allocate big chunks of data in 517 user mode. It will change when a dedicated libc will be used. */ 518 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 519 region in which the guest needs to run. Revisit this. */ 520 #define USE_STATIC_CODE_GEN_BUFFER 521 #endif 522 523 /* Minimum size of the code gen buffer. This number is randomly chosen, 524 but not so small that we can't have a fair number of TB's live. */ 525 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 526 527 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 528 indicated, this is constrained by the range of direct branches on the 529 host cpu, as used by the TCG implementation of goto_tb. */ 530 #if defined(__x86_64__) 531 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 532 #elif defined(__sparc__) 533 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 534 #elif defined(__powerpc64__) 535 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 536 #elif defined(__powerpc__) 537 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 538 #elif defined(__aarch64__) 539 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 540 #elif defined(__s390x__) 541 /* We have a +- 4GB range on the branches; leave some slop. */ 542 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 543 #elif defined(__mips__) 544 /* We have a 256MB branch region, but leave room to make sure the 545 main executable is also within that region. */ 546 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 547 #else 548 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 549 #endif 550 551 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 552 553 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 554 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 555 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 556 557 static inline size_t size_code_gen_buffer(size_t tb_size) 558 { 559 /* Size the buffer. */ 560 if (tb_size == 0) { 561 #ifdef USE_STATIC_CODE_GEN_BUFFER 562 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 563 #else 564 /* ??? Needs adjustments. */ 565 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 566 static buffer, we could size this on RESERVED_VA, on the text 567 segment size of the executable, or continue to use the default. */ 568 tb_size = (unsigned long)(ram_size / 4); 569 #endif 570 } 571 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 572 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 573 } 574 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 575 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 576 } 577 return tb_size; 578 } 579 580 #ifdef __mips__ 581 /* In order to use J and JAL within the code_gen_buffer, we require 582 that the buffer not cross a 256MB boundary. */ 583 static inline bool cross_256mb(void *addr, size_t size) 584 { 585 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 586 } 587 588 /* We weren't able to allocate a buffer without crossing that boundary, 589 so make do with the larger portion of the buffer that doesn't cross. 590 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 591 static inline void *split_cross_256mb(void *buf1, size_t size1) 592 { 593 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 594 size_t size2 = buf1 + size1 - buf2; 595 596 size1 = buf2 - buf1; 597 if (size1 < size2) { 598 size1 = size2; 599 buf1 = buf2; 600 } 601 602 tcg_ctx->code_gen_buffer_size = size1; 603 return buf1; 604 } 605 #endif 606 607 #ifdef USE_STATIC_CODE_GEN_BUFFER 608 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 609 __attribute__((aligned(CODE_GEN_ALIGN))); 610 611 static inline void *alloc_code_gen_buffer(void) 612 { 613 void *buf = static_code_gen_buffer; 614 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 615 size_t size; 616 617 /* page-align the beginning and end of the buffer */ 618 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 619 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 620 621 size = end - buf; 622 623 /* Honor a command-line option limiting the size of the buffer. */ 624 if (size > tcg_ctx->code_gen_buffer_size) { 625 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 626 qemu_real_host_page_size); 627 } 628 tcg_ctx->code_gen_buffer_size = size; 629 630 #ifdef __mips__ 631 if (cross_256mb(buf, size)) { 632 buf = split_cross_256mb(buf, size); 633 size = tcg_ctx->code_gen_buffer_size; 634 } 635 #endif 636 637 if (qemu_mprotect_rwx(buf, size)) { 638 abort(); 639 } 640 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 641 642 return buf; 643 } 644 #elif defined(_WIN32) 645 static inline void *alloc_code_gen_buffer(void) 646 { 647 size_t size = tcg_ctx->code_gen_buffer_size; 648 void *buf; 649 650 buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 651 PAGE_EXECUTE_READWRITE); 652 return buf; 653 } 654 #else 655 static inline void *alloc_code_gen_buffer(void) 656 { 657 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 658 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 659 uintptr_t start = 0; 660 size_t size = tcg_ctx->code_gen_buffer_size; 661 void *buf; 662 663 /* Constrain the position of the buffer based on the host cpu. 664 Note that these addresses are chosen in concert with the 665 addresses assigned in the relevant linker script file. */ 666 # if defined(__PIE__) || defined(__PIC__) 667 /* Don't bother setting a preferred location if we're building 668 a position-independent executable. We're more likely to get 669 an address near the main executable if we let the kernel 670 choose the address. */ 671 # elif defined(__x86_64__) && defined(MAP_32BIT) 672 /* Force the memory down into low memory with the executable. 673 Leave the choice of exact location with the kernel. */ 674 flags |= MAP_32BIT; 675 /* Cannot expect to map more than 800MB in low memory. */ 676 if (size > 800u * 1024 * 1024) { 677 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 678 } 679 # elif defined(__sparc__) 680 start = 0x40000000ul; 681 # elif defined(__s390x__) 682 start = 0x90000000ul; 683 # elif defined(__mips__) 684 # if _MIPS_SIM == _ABI64 685 start = 0x128000000ul; 686 # else 687 start = 0x08000000ul; 688 # endif 689 # endif 690 691 buf = mmap((void *)start, size, prot, flags, -1, 0); 692 if (buf == MAP_FAILED) { 693 return NULL; 694 } 695 696 #ifdef __mips__ 697 if (cross_256mb(buf, size)) { 698 /* Try again, with the original still mapped, to avoid re-acquiring 699 that 256mb crossing. This time don't specify an address. */ 700 size_t size2; 701 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 702 switch ((int)(buf2 != MAP_FAILED)) { 703 case 1: 704 if (!cross_256mb(buf2, size)) { 705 /* Success! Use the new buffer. */ 706 munmap(buf, size); 707 break; 708 } 709 /* Failure. Work with what we had. */ 710 munmap(buf2, size); 711 /* fallthru */ 712 default: 713 /* Split the original buffer. Free the smaller half. */ 714 buf2 = split_cross_256mb(buf, size); 715 size2 = tcg_ctx->code_gen_buffer_size; 716 if (buf == buf2) { 717 munmap(buf + size2, size - size2); 718 } else { 719 munmap(buf, size - size2); 720 } 721 size = size2; 722 break; 723 } 724 buf = buf2; 725 } 726 #endif 727 728 /* Request large pages for the buffer. */ 729 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 730 731 return buf; 732 } 733 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 734 735 /* compare a pointer @ptr and a tb_tc @s */ 736 static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s) 737 { 738 if (ptr >= s->ptr + s->size) { 739 return 1; 740 } else if (ptr < s->ptr) { 741 return -1; 742 } 743 return 0; 744 } 745 746 static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp) 747 { 748 const struct tb_tc *a = ap; 749 const struct tb_tc *b = bp; 750 751 /* 752 * When both sizes are set, we know this isn't a lookup. 753 * This is the most likely case: every TB must be inserted; lookups 754 * are a lot less frequent. 755 */ 756 if (likely(a->size && b->size)) { 757 if (a->ptr > b->ptr) { 758 return 1; 759 } else if (a->ptr < b->ptr) { 760 return -1; 761 } 762 /* a->ptr == b->ptr should happen only on deletions */ 763 g_assert(a->size == b->size); 764 return 0; 765 } 766 /* 767 * All lookups have either .size field set to 0. 768 * From the glib sources we see that @ap is always the lookup key. However 769 * the docs provide no guarantee, so we just mark this case as likely. 770 */ 771 if (likely(a->size == 0)) { 772 return ptr_cmp_tb_tc(a->ptr, b); 773 } 774 return ptr_cmp_tb_tc(b->ptr, a); 775 } 776 777 static inline void code_gen_alloc(size_t tb_size) 778 { 779 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 780 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 781 if (tcg_ctx->code_gen_buffer == NULL) { 782 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 783 exit(1); 784 } 785 tb_ctx.tb_tree = g_tree_new(tb_tc_cmp); 786 qemu_mutex_init(&tb_ctx.tb_lock); 787 } 788 789 static void tb_htable_init(void) 790 { 791 unsigned int mode = QHT_MODE_AUTO_RESIZE; 792 793 qht_init(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode); 794 } 795 796 /* Must be called before using the QEMU cpus. 'tb_size' is the size 797 (in bytes) allocated to the translation buffer. Zero means default 798 size. */ 799 void tcg_exec_init(unsigned long tb_size) 800 { 801 tcg_allowed = true; 802 cpu_gen_init(); 803 page_init(); 804 tb_htable_init(); 805 code_gen_alloc(tb_size); 806 #if defined(CONFIG_SOFTMMU) 807 /* There's no guest base to take into account, so go ahead and 808 initialize the prologue now. */ 809 tcg_prologue_init(tcg_ctx); 810 #endif 811 } 812 813 /* 814 * Allocate a new translation block. Flush the translation buffer if 815 * too many translation blocks or too much generated code. 816 * 817 * Called with tb_lock held. 818 */ 819 static TranslationBlock *tb_alloc(target_ulong pc) 820 { 821 TranslationBlock *tb; 822 823 assert_tb_locked(); 824 825 tb = tcg_tb_alloc(tcg_ctx); 826 if (unlikely(tb == NULL)) { 827 return NULL; 828 } 829 return tb; 830 } 831 832 /* Called with tb_lock held. */ 833 void tb_remove(TranslationBlock *tb) 834 { 835 assert_tb_locked(); 836 837 g_tree_remove(tb_ctx.tb_tree, &tb->tc); 838 } 839 840 static inline void invalidate_page_bitmap(PageDesc *p) 841 { 842 #ifdef CONFIG_SOFTMMU 843 g_free(p->code_bitmap); 844 p->code_bitmap = NULL; 845 p->code_write_count = 0; 846 #endif 847 } 848 849 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 850 static void page_flush_tb_1(int level, void **lp) 851 { 852 int i; 853 854 if (*lp == NULL) { 855 return; 856 } 857 if (level == 0) { 858 PageDesc *pd = *lp; 859 860 for (i = 0; i < V_L2_SIZE; ++i) { 861 pd[i].first_tb = NULL; 862 invalidate_page_bitmap(pd + i); 863 } 864 } else { 865 void **pp = *lp; 866 867 for (i = 0; i < V_L2_SIZE; ++i) { 868 page_flush_tb_1(level - 1, pp + i); 869 } 870 } 871 } 872 873 static void page_flush_tb(void) 874 { 875 int i, l1_sz = v_l1_size; 876 877 for (i = 0; i < l1_sz; i++) { 878 page_flush_tb_1(v_l2_levels, l1_map + i); 879 } 880 } 881 882 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 883 { 884 const TranslationBlock *tb = value; 885 size_t *size = data; 886 887 *size += tb->tc.size; 888 return false; 889 } 890 891 /* flush all the translation blocks */ 892 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 893 { 894 tb_lock(); 895 896 /* If it is already been done on request of another CPU, 897 * just retry. 898 */ 899 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 900 goto done; 901 } 902 903 if (DEBUG_TB_FLUSH_GATE) { 904 size_t nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 905 size_t host_size = 0; 906 907 g_tree_foreach(tb_ctx.tb_tree, tb_host_size_iter, &host_size); 908 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 909 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 910 } 911 912 CPU_FOREACH(cpu) { 913 cpu_tb_jmp_cache_clear(cpu); 914 } 915 916 /* Increment the refcount first so that destroy acts as a reset */ 917 g_tree_ref(tb_ctx.tb_tree); 918 g_tree_destroy(tb_ctx.tb_tree); 919 920 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 921 page_flush_tb(); 922 923 tcg_region_reset_all(); 924 /* XXX: flush processor icache at this point if cache flush is 925 expensive */ 926 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 927 928 done: 929 tb_unlock(); 930 } 931 932 void tb_flush(CPUState *cpu) 933 { 934 if (tcg_enabled()) { 935 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 936 async_safe_run_on_cpu(cpu, do_tb_flush, 937 RUN_ON_CPU_HOST_INT(tb_flush_count)); 938 } 939 } 940 941 /* 942 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 943 * so in order to prevent bit rot we compile them unconditionally in user-mode, 944 * and let the optimizer get rid of them by wrapping their user-only callers 945 * with if (DEBUG_TB_CHECK_GATE). 946 */ 947 #ifdef CONFIG_USER_ONLY 948 949 static void 950 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 951 { 952 TranslationBlock *tb = p; 953 target_ulong addr = *(target_ulong *)userp; 954 955 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 956 printf("ERROR invalidate: address=" TARGET_FMT_lx 957 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 958 } 959 } 960 961 /* verify that all the pages have correct rights for code 962 * 963 * Called with tb_lock held. 964 */ 965 static void tb_invalidate_check(target_ulong address) 966 { 967 address &= TARGET_PAGE_MASK; 968 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 969 } 970 971 static void 972 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 973 { 974 TranslationBlock *tb = p; 975 int flags1, flags2; 976 977 flags1 = page_get_flags(tb->pc); 978 flags2 = page_get_flags(tb->pc + tb->size - 1); 979 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 980 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 981 (long)tb->pc, tb->size, flags1, flags2); 982 } 983 } 984 985 /* verify that all the pages have correct rights for code */ 986 static void tb_page_check(void) 987 { 988 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 989 } 990 991 #endif /* CONFIG_USER_ONLY */ 992 993 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) 994 { 995 TranslationBlock *tb1; 996 unsigned int n1; 997 998 for (;;) { 999 tb1 = *ptb; 1000 n1 = (uintptr_t)tb1 & 3; 1001 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); 1002 if (tb1 == tb) { 1003 *ptb = tb1->page_next[n1]; 1004 break; 1005 } 1006 ptb = &tb1->page_next[n1]; 1007 } 1008 } 1009 1010 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 1011 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 1012 { 1013 TranslationBlock *tb1; 1014 uintptr_t *ptb, ntb; 1015 unsigned int n1; 1016 1017 ptb = &tb->jmp_list_next[n]; 1018 if (*ptb) { 1019 /* find tb(n) in circular list */ 1020 for (;;) { 1021 ntb = *ptb; 1022 n1 = ntb & 3; 1023 tb1 = (TranslationBlock *)(ntb & ~3); 1024 if (n1 == n && tb1 == tb) { 1025 break; 1026 } 1027 if (n1 == 2) { 1028 ptb = &tb1->jmp_list_first; 1029 } else { 1030 ptb = &tb1->jmp_list_next[n1]; 1031 } 1032 } 1033 /* now we can suppress tb(n) from the list */ 1034 *ptb = tb->jmp_list_next[n]; 1035 1036 tb->jmp_list_next[n] = (uintptr_t)NULL; 1037 } 1038 } 1039 1040 /* reset the jump entry 'n' of a TB so that it is not chained to 1041 another TB */ 1042 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1043 { 1044 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1045 tb_set_jmp_target(tb, n, addr); 1046 } 1047 1048 /* remove any jumps to the TB */ 1049 static inline void tb_jmp_unlink(TranslationBlock *tb) 1050 { 1051 TranslationBlock *tb1; 1052 uintptr_t *ptb, ntb; 1053 unsigned int n1; 1054 1055 ptb = &tb->jmp_list_first; 1056 for (;;) { 1057 ntb = *ptb; 1058 n1 = ntb & 3; 1059 tb1 = (TranslationBlock *)(ntb & ~3); 1060 if (n1 == 2) { 1061 break; 1062 } 1063 tb_reset_jump(tb1, n1); 1064 *ptb = tb1->jmp_list_next[n1]; 1065 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1066 } 1067 } 1068 1069 /* invalidate one TB 1070 * 1071 * Called with tb_lock held. 1072 */ 1073 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1074 { 1075 CPUState *cpu; 1076 PageDesc *p; 1077 uint32_t h; 1078 tb_page_addr_t phys_pc; 1079 1080 assert_tb_locked(); 1081 1082 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1083 1084 /* remove the TB from the hash list */ 1085 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1086 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1087 tb->trace_vcpu_dstate); 1088 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1089 return; 1090 } 1091 1092 /* remove the TB from the page list */ 1093 if (tb->page_addr[0] != page_addr) { 1094 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1095 tb_page_remove(&p->first_tb, tb); 1096 invalidate_page_bitmap(p); 1097 } 1098 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1099 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1100 tb_page_remove(&p->first_tb, tb); 1101 invalidate_page_bitmap(p); 1102 } 1103 1104 /* remove the TB from the hash list */ 1105 h = tb_jmp_cache_hash_func(tb->pc); 1106 CPU_FOREACH(cpu) { 1107 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1108 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1109 } 1110 } 1111 1112 /* suppress this TB from the two jump lists */ 1113 tb_remove_from_jmp_list(tb, 0); 1114 tb_remove_from_jmp_list(tb, 1); 1115 1116 /* suppress any remaining jumps to this TB */ 1117 tb_jmp_unlink(tb); 1118 1119 tb_ctx.tb_phys_invalidate_count++; 1120 } 1121 1122 #ifdef CONFIG_SOFTMMU 1123 static void build_page_bitmap(PageDesc *p) 1124 { 1125 int n, tb_start, tb_end; 1126 TranslationBlock *tb; 1127 1128 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1129 1130 tb = p->first_tb; 1131 while (tb != NULL) { 1132 n = (uintptr_t)tb & 3; 1133 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1134 /* NOTE: this is subtle as a TB may span two physical pages */ 1135 if (n == 0) { 1136 /* NOTE: tb_end may be after the end of the page, but 1137 it is not a problem */ 1138 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1139 tb_end = tb_start + tb->size; 1140 if (tb_end > TARGET_PAGE_SIZE) { 1141 tb_end = TARGET_PAGE_SIZE; 1142 } 1143 } else { 1144 tb_start = 0; 1145 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1146 } 1147 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1148 tb = tb->page_next[n]; 1149 } 1150 } 1151 #endif 1152 1153 /* add the tb in the target page and protect it if necessary 1154 * 1155 * Called with mmap_lock held for user-mode emulation. 1156 */ 1157 static inline void tb_alloc_page(TranslationBlock *tb, 1158 unsigned int n, tb_page_addr_t page_addr) 1159 { 1160 PageDesc *p; 1161 #ifndef CONFIG_USER_ONLY 1162 bool page_already_protected; 1163 #endif 1164 1165 assert_memory_lock(); 1166 1167 tb->page_addr[n] = page_addr; 1168 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1169 tb->page_next[n] = p->first_tb; 1170 #ifndef CONFIG_USER_ONLY 1171 page_already_protected = p->first_tb != NULL; 1172 #endif 1173 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); 1174 invalidate_page_bitmap(p); 1175 1176 #if defined(CONFIG_USER_ONLY) 1177 if (p->flags & PAGE_WRITE) { 1178 target_ulong addr; 1179 PageDesc *p2; 1180 int prot; 1181 1182 /* force the host page as non writable (writes will have a 1183 page fault + mprotect overhead) */ 1184 page_addr &= qemu_host_page_mask; 1185 prot = 0; 1186 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1187 addr += TARGET_PAGE_SIZE) { 1188 1189 p2 = page_find(addr >> TARGET_PAGE_BITS); 1190 if (!p2) { 1191 continue; 1192 } 1193 prot |= p2->flags; 1194 p2->flags &= ~PAGE_WRITE; 1195 } 1196 mprotect(g2h(page_addr), qemu_host_page_size, 1197 (prot & PAGE_BITS) & ~PAGE_WRITE); 1198 if (DEBUG_TB_INVALIDATE_GATE) { 1199 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1200 } 1201 } 1202 #else 1203 /* if some code is already present, then the pages are already 1204 protected. So we handle the case where only the first TB is 1205 allocated in a physical page */ 1206 if (!page_already_protected) { 1207 tlb_protect_code(page_addr); 1208 } 1209 #endif 1210 } 1211 1212 /* add a new TB and link it to the physical page tables. phys_page2 is 1213 * (-1) to indicate that only one page contains the TB. 1214 * 1215 * Called with mmap_lock held for user-mode emulation. 1216 */ 1217 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1218 tb_page_addr_t phys_page2) 1219 { 1220 uint32_t h; 1221 1222 assert_memory_lock(); 1223 1224 /* add in the page list */ 1225 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1226 if (phys_page2 != -1) { 1227 tb_alloc_page(tb, 1, phys_page2); 1228 } else { 1229 tb->page_addr[1] = -1; 1230 } 1231 1232 /* add in the hash table */ 1233 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1234 tb->trace_vcpu_dstate); 1235 qht_insert(&tb_ctx.htable, tb, h); 1236 1237 #ifdef CONFIG_USER_ONLY 1238 if (DEBUG_TB_CHECK_GATE) { 1239 tb_page_check(); 1240 } 1241 #endif 1242 } 1243 1244 /* Called with mmap_lock held for user mode emulation. */ 1245 TranslationBlock *tb_gen_code(CPUState *cpu, 1246 target_ulong pc, target_ulong cs_base, 1247 uint32_t flags, int cflags) 1248 { 1249 CPUArchState *env = cpu->env_ptr; 1250 TranslationBlock *tb; 1251 tb_page_addr_t phys_pc, phys_page2; 1252 target_ulong virt_page2; 1253 tcg_insn_unit *gen_code_buf; 1254 int gen_code_size, search_size; 1255 #ifdef CONFIG_PROFILER 1256 TCGProfile *prof = &tcg_ctx->prof; 1257 int64_t ti; 1258 #endif 1259 assert_memory_lock(); 1260 1261 phys_pc = get_page_addr_code(env, pc); 1262 1263 buffer_overflow: 1264 tb = tb_alloc(pc); 1265 if (unlikely(!tb)) { 1266 /* flush must be done */ 1267 tb_flush(cpu); 1268 mmap_unlock(); 1269 /* Make the execution loop process the flush as soon as possible. */ 1270 cpu->exception_index = EXCP_INTERRUPT; 1271 cpu_loop_exit(cpu); 1272 } 1273 1274 gen_code_buf = tcg_ctx->code_gen_ptr; 1275 tb->tc.ptr = gen_code_buf; 1276 tb->pc = pc; 1277 tb->cs_base = cs_base; 1278 tb->flags = flags; 1279 tb->cflags = cflags; 1280 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1281 tcg_ctx->tb_cflags = cflags; 1282 1283 #ifdef CONFIG_PROFILER 1284 /* includes aborted translations because of exceptions */ 1285 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1286 ti = profile_getclock(); 1287 #endif 1288 1289 tcg_func_start(tcg_ctx); 1290 1291 tcg_ctx->cpu = ENV_GET_CPU(env); 1292 gen_intermediate_code(cpu, tb); 1293 tcg_ctx->cpu = NULL; 1294 1295 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1296 1297 /* generate machine code */ 1298 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1299 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1300 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1301 if (TCG_TARGET_HAS_direct_jump) { 1302 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1303 tcg_ctx->tb_jmp_target_addr = NULL; 1304 } else { 1305 tcg_ctx->tb_jmp_insn_offset = NULL; 1306 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1307 } 1308 1309 #ifdef CONFIG_PROFILER 1310 atomic_set(&prof->tb_count, prof->tb_count + 1); 1311 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1312 ti = profile_getclock(); 1313 #endif 1314 1315 /* ??? Overflow could be handled better here. In particular, we 1316 don't need to re-do gen_intermediate_code, nor should we re-do 1317 the tcg optimization currently hidden inside tcg_gen_code. All 1318 that should be required is to flush the TBs, allocate a new TB, 1319 re-initialize it per above, and re-do the actual code generation. */ 1320 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1321 if (unlikely(gen_code_size < 0)) { 1322 goto buffer_overflow; 1323 } 1324 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1325 if (unlikely(search_size < 0)) { 1326 goto buffer_overflow; 1327 } 1328 tb->tc.size = gen_code_size; 1329 1330 #ifdef CONFIG_PROFILER 1331 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1332 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1333 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1334 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1335 #endif 1336 1337 #ifdef DEBUG_DISAS 1338 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1339 qemu_log_in_addr_range(tb->pc)) { 1340 qemu_log_lock(); 1341 qemu_log("OUT: [size=%d]\n", gen_code_size); 1342 if (tcg_ctx->data_gen_ptr) { 1343 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1344 size_t data_size = gen_code_size - code_size; 1345 size_t i; 1346 1347 log_disas(tb->tc.ptr, code_size); 1348 1349 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1350 if (sizeof(tcg_target_ulong) == 8) { 1351 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1352 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1353 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1354 } else { 1355 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1356 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1357 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1358 } 1359 } 1360 } else { 1361 log_disas(tb->tc.ptr, gen_code_size); 1362 } 1363 qemu_log("\n"); 1364 qemu_log_flush(); 1365 qemu_log_unlock(); 1366 } 1367 #endif 1368 1369 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1370 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1371 CODE_GEN_ALIGN)); 1372 1373 /* init jump list */ 1374 assert(((uintptr_t)tb & 3) == 0); 1375 tb->jmp_list_first = (uintptr_t)tb | 2; 1376 tb->jmp_list_next[0] = (uintptr_t)NULL; 1377 tb->jmp_list_next[1] = (uintptr_t)NULL; 1378 1379 /* init original jump addresses wich has been set during tcg_gen_code() */ 1380 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1381 tb_reset_jump(tb, 0); 1382 } 1383 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1384 tb_reset_jump(tb, 1); 1385 } 1386 1387 /* check next page if needed */ 1388 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1389 phys_page2 = -1; 1390 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1391 phys_page2 = get_page_addr_code(env, virt_page2); 1392 } 1393 /* As long as consistency of the TB stuff is provided by tb_lock in user 1394 * mode and is implicit in single-threaded softmmu emulation, no explicit 1395 * memory barrier is required before tb_link_page() makes the TB visible 1396 * through the physical hash table and physical page list. 1397 */ 1398 tb_link_page(tb, phys_pc, phys_page2); 1399 g_tree_insert(tb_ctx.tb_tree, &tb->tc, tb); 1400 return tb; 1401 } 1402 1403 /* 1404 * Invalidate all TBs which intersect with the target physical address range 1405 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1406 * 'is_cpu_write_access' should be true if called from a real cpu write 1407 * access: the virtual CPU will exit the current TB if code is modified inside 1408 * this TB. 1409 * 1410 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1411 * Called with tb_lock held for system-mode emulation 1412 */ 1413 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1414 { 1415 while (start < end) { 1416 tb_invalidate_phys_page_range(start, end, 0); 1417 start &= TARGET_PAGE_MASK; 1418 start += TARGET_PAGE_SIZE; 1419 } 1420 } 1421 1422 #ifdef CONFIG_SOFTMMU 1423 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1424 { 1425 assert_tb_locked(); 1426 tb_invalidate_phys_range_1(start, end); 1427 } 1428 #else 1429 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1430 { 1431 assert_memory_lock(); 1432 tb_lock(); 1433 tb_invalidate_phys_range_1(start, end); 1434 tb_unlock(); 1435 } 1436 #endif 1437 /* 1438 * Invalidate all TBs which intersect with the target physical address range 1439 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1440 * 'is_cpu_write_access' should be true if called from a real cpu write 1441 * access: the virtual CPU will exit the current TB if code is modified inside 1442 * this TB. 1443 * 1444 * Called with tb_lock/mmap_lock held for user-mode emulation 1445 * Called with tb_lock held for system-mode emulation 1446 */ 1447 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1448 int is_cpu_write_access) 1449 { 1450 TranslationBlock *tb, *tb_next; 1451 tb_page_addr_t tb_start, tb_end; 1452 PageDesc *p; 1453 int n; 1454 #ifdef TARGET_HAS_PRECISE_SMC 1455 CPUState *cpu = current_cpu; 1456 CPUArchState *env = NULL; 1457 int current_tb_not_found = is_cpu_write_access; 1458 TranslationBlock *current_tb = NULL; 1459 int current_tb_modified = 0; 1460 target_ulong current_pc = 0; 1461 target_ulong current_cs_base = 0; 1462 uint32_t current_flags = 0; 1463 #endif /* TARGET_HAS_PRECISE_SMC */ 1464 1465 assert_memory_lock(); 1466 assert_tb_locked(); 1467 1468 p = page_find(start >> TARGET_PAGE_BITS); 1469 if (!p) { 1470 return; 1471 } 1472 #if defined(TARGET_HAS_PRECISE_SMC) 1473 if (cpu != NULL) { 1474 env = cpu->env_ptr; 1475 } 1476 #endif 1477 1478 /* we remove all the TBs in the range [start, end[ */ 1479 /* XXX: see if in some cases it could be faster to invalidate all 1480 the code */ 1481 tb = p->first_tb; 1482 while (tb != NULL) { 1483 n = (uintptr_t)tb & 3; 1484 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1485 tb_next = tb->page_next[n]; 1486 /* NOTE: this is subtle as a TB may span two physical pages */ 1487 if (n == 0) { 1488 /* NOTE: tb_end may be after the end of the page, but 1489 it is not a problem */ 1490 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1491 tb_end = tb_start + tb->size; 1492 } else { 1493 tb_start = tb->page_addr[1]; 1494 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1495 } 1496 if (!(tb_end <= start || tb_start >= end)) { 1497 #ifdef TARGET_HAS_PRECISE_SMC 1498 if (current_tb_not_found) { 1499 current_tb_not_found = 0; 1500 current_tb = NULL; 1501 if (cpu->mem_io_pc) { 1502 /* now we have a real cpu fault */ 1503 current_tb = tb_find_pc(cpu->mem_io_pc); 1504 } 1505 } 1506 if (current_tb == tb && 1507 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1508 /* If we are modifying the current TB, we must stop 1509 its execution. We could be more precise by checking 1510 that the modification is after the current PC, but it 1511 would require a specialized function to partially 1512 restore the CPU state */ 1513 1514 current_tb_modified = 1; 1515 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc); 1516 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1517 ¤t_flags); 1518 } 1519 #endif /* TARGET_HAS_PRECISE_SMC */ 1520 tb_phys_invalidate(tb, -1); 1521 } 1522 tb = tb_next; 1523 } 1524 #if !defined(CONFIG_USER_ONLY) 1525 /* if no code remaining, no need to continue to use slow writes */ 1526 if (!p->first_tb) { 1527 invalidate_page_bitmap(p); 1528 tlb_unprotect_code(start); 1529 } 1530 #endif 1531 #ifdef TARGET_HAS_PRECISE_SMC 1532 if (current_tb_modified) { 1533 /* Force execution of one insn next time. */ 1534 cpu->cflags_next_tb = 1 | curr_cflags(); 1535 cpu_loop_exit_noexc(cpu); 1536 } 1537 #endif 1538 } 1539 1540 #ifdef CONFIG_SOFTMMU 1541 /* len must be <= 8 and start must be a multiple of len. 1542 * Called via softmmu_template.h when code areas are written to with 1543 * iothread mutex not held. 1544 */ 1545 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1546 { 1547 PageDesc *p; 1548 1549 #if 0 1550 if (1) { 1551 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1552 cpu_single_env->mem_io_vaddr, len, 1553 cpu_single_env->eip, 1554 cpu_single_env->eip + 1555 (intptr_t)cpu_single_env->segs[R_CS].base); 1556 } 1557 #endif 1558 assert_memory_lock(); 1559 1560 p = page_find(start >> TARGET_PAGE_BITS); 1561 if (!p) { 1562 return; 1563 } 1564 if (!p->code_bitmap && 1565 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1566 /* build code bitmap. FIXME: writes should be protected by 1567 * tb_lock, reads by tb_lock or RCU. 1568 */ 1569 build_page_bitmap(p); 1570 } 1571 if (p->code_bitmap) { 1572 unsigned int nr; 1573 unsigned long b; 1574 1575 nr = start & ~TARGET_PAGE_MASK; 1576 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1577 if (b & ((1 << len) - 1)) { 1578 goto do_invalidate; 1579 } 1580 } else { 1581 do_invalidate: 1582 tb_invalidate_phys_page_range(start, start + len, 1); 1583 } 1584 } 1585 #else 1586 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1587 * host PC of the faulting store instruction that caused this invalidate. 1588 * Returns true if the caller needs to abort execution of the current 1589 * TB (because it was modified by this store and the guest CPU has 1590 * precise-SMC semantics). 1591 */ 1592 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1593 { 1594 TranslationBlock *tb; 1595 PageDesc *p; 1596 int n; 1597 #ifdef TARGET_HAS_PRECISE_SMC 1598 TranslationBlock *current_tb = NULL; 1599 CPUState *cpu = current_cpu; 1600 CPUArchState *env = NULL; 1601 int current_tb_modified = 0; 1602 target_ulong current_pc = 0; 1603 target_ulong current_cs_base = 0; 1604 uint32_t current_flags = 0; 1605 #endif 1606 1607 assert_memory_lock(); 1608 1609 addr &= TARGET_PAGE_MASK; 1610 p = page_find(addr >> TARGET_PAGE_BITS); 1611 if (!p) { 1612 return false; 1613 } 1614 1615 tb_lock(); 1616 tb = p->first_tb; 1617 #ifdef TARGET_HAS_PRECISE_SMC 1618 if (tb && pc != 0) { 1619 current_tb = tb_find_pc(pc); 1620 } 1621 if (cpu != NULL) { 1622 env = cpu->env_ptr; 1623 } 1624 #endif 1625 while (tb != NULL) { 1626 n = (uintptr_t)tb & 3; 1627 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1628 #ifdef TARGET_HAS_PRECISE_SMC 1629 if (current_tb == tb && 1630 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1631 /* If we are modifying the current TB, we must stop 1632 its execution. We could be more precise by checking 1633 that the modification is after the current PC, but it 1634 would require a specialized function to partially 1635 restore the CPU state */ 1636 1637 current_tb_modified = 1; 1638 cpu_restore_state_from_tb(cpu, current_tb, pc); 1639 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1640 ¤t_flags); 1641 } 1642 #endif /* TARGET_HAS_PRECISE_SMC */ 1643 tb_phys_invalidate(tb, addr); 1644 tb = tb->page_next[n]; 1645 } 1646 p->first_tb = NULL; 1647 #ifdef TARGET_HAS_PRECISE_SMC 1648 if (current_tb_modified) { 1649 /* Force execution of one insn next time. */ 1650 cpu->cflags_next_tb = 1 | curr_cflags(); 1651 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1652 * back into the cpu_exec loop. */ 1653 return true; 1654 } 1655 #endif 1656 tb_unlock(); 1657 1658 return false; 1659 } 1660 #endif 1661 1662 /* 1663 * Find the TB 'tb' such that 1664 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size 1665 * Return NULL if not found. 1666 */ 1667 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) 1668 { 1669 struct tb_tc s = { .ptr = (void *)tc_ptr }; 1670 1671 return g_tree_lookup(tb_ctx.tb_tree, &s); 1672 } 1673 1674 #if !defined(CONFIG_USER_ONLY) 1675 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr) 1676 { 1677 ram_addr_t ram_addr; 1678 MemoryRegion *mr; 1679 hwaddr l = 1; 1680 1681 rcu_read_lock(); 1682 mr = address_space_translate(as, addr, &addr, &l, false); 1683 if (!(memory_region_is_ram(mr) 1684 || memory_region_is_romd(mr))) { 1685 rcu_read_unlock(); 1686 return; 1687 } 1688 ram_addr = memory_region_get_ram_addr(mr) + addr; 1689 tb_lock(); 1690 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1691 tb_unlock(); 1692 rcu_read_unlock(); 1693 } 1694 #endif /* !defined(CONFIG_USER_ONLY) */ 1695 1696 /* Called with tb_lock held. */ 1697 void tb_check_watchpoint(CPUState *cpu) 1698 { 1699 TranslationBlock *tb; 1700 1701 tb = tb_find_pc(cpu->mem_io_pc); 1702 if (tb) { 1703 /* We can use retranslation to find the PC. */ 1704 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc); 1705 tb_phys_invalidate(tb, -1); 1706 } else { 1707 /* The exception probably happened in a helper. The CPU state should 1708 have been saved before calling it. Fetch the PC from there. */ 1709 CPUArchState *env = cpu->env_ptr; 1710 target_ulong pc, cs_base; 1711 tb_page_addr_t addr; 1712 uint32_t flags; 1713 1714 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1715 addr = get_page_addr_code(env, pc); 1716 tb_invalidate_phys_range(addr, addr + 1); 1717 } 1718 } 1719 1720 #ifndef CONFIG_USER_ONLY 1721 /* in deterministic execution mode, instructions doing device I/Os 1722 * must be at the end of the TB. 1723 * 1724 * Called by softmmu_template.h, with iothread mutex not held. 1725 */ 1726 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1727 { 1728 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1729 CPUArchState *env = cpu->env_ptr; 1730 #endif 1731 TranslationBlock *tb; 1732 uint32_t n; 1733 1734 tb_lock(); 1735 tb = tb_find_pc(retaddr); 1736 if (!tb) { 1737 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1738 (void *)retaddr); 1739 } 1740 n = cpu->icount_decr.u16.low + tb->icount; 1741 cpu_restore_state_from_tb(cpu, tb, retaddr); 1742 /* Calculate how many instructions had been executed before the fault 1743 occurred. */ 1744 n = n - cpu->icount_decr.u16.low; 1745 /* Generate a new TB ending on the I/O insn. */ 1746 n++; 1747 /* On MIPS and SH, delay slot instructions can only be restarted if 1748 they were already the first instruction in the TB. If this is not 1749 the first instruction in a TB then re-execute the preceding 1750 branch. */ 1751 #if defined(TARGET_MIPS) 1752 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) { 1753 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1754 cpu->icount_decr.u16.low++; 1755 env->hflags &= ~MIPS_HFLAG_BMASK; 1756 } 1757 #elif defined(TARGET_SH4) 1758 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1759 && n > 1) { 1760 env->pc -= 2; 1761 cpu->icount_decr.u16.low++; 1762 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1763 } 1764 #endif 1765 /* This should never happen. */ 1766 if (n > CF_COUNT_MASK) { 1767 cpu_abort(cpu, "TB too big during recompile"); 1768 } 1769 1770 /* Adjust the execution state of the next TB. */ 1771 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1772 1773 if (tb->cflags & CF_NOCACHE) { 1774 if (tb->orig_tb) { 1775 /* Invalidate original TB if this TB was generated in 1776 * cpu_exec_nocache() */ 1777 tb_phys_invalidate(tb->orig_tb, -1); 1778 } 1779 tb_remove(tb); 1780 } 1781 1782 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1783 * the first in the TB) then we end up generating a whole new TB and 1784 * repeating the fault, which is horribly inefficient. 1785 * Better would be to execute just this insn uncached, or generate a 1786 * second new TB. 1787 * 1788 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1789 * tb_lock gets reset. 1790 */ 1791 cpu_loop_exit_noexc(cpu); 1792 } 1793 1794 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1795 { 1796 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1797 1798 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1799 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1800 } 1801 } 1802 1803 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1804 { 1805 /* Discard jump cache entries for any tb which might potentially 1806 overlap the flushed page. */ 1807 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1808 tb_jmp_cache_clear_page(cpu, addr); 1809 } 1810 1811 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1812 struct qht_stats hst) 1813 { 1814 uint32_t hgram_opts; 1815 size_t hgram_bins; 1816 char *hgram; 1817 1818 if (!hst.head_buckets) { 1819 return; 1820 } 1821 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1822 hst.used_head_buckets, hst.head_buckets, 1823 (double)hst.used_head_buckets / hst.head_buckets * 100); 1824 1825 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1826 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1827 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1828 hgram_opts |= QDIST_PR_NODECIMAL; 1829 } 1830 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1831 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1832 qdist_avg(&hst.occupancy) * 100, hgram); 1833 g_free(hgram); 1834 1835 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1836 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1837 if (hgram_bins > 10) { 1838 hgram_bins = 10; 1839 } else { 1840 hgram_bins = 0; 1841 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1842 } 1843 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1844 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1845 qdist_avg(&hst.chain), hgram); 1846 g_free(hgram); 1847 } 1848 1849 struct tb_tree_stats { 1850 size_t host_size; 1851 size_t target_size; 1852 size_t max_target_size; 1853 size_t direct_jmp_count; 1854 size_t direct_jmp2_count; 1855 size_t cross_page; 1856 }; 1857 1858 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1859 { 1860 const TranslationBlock *tb = value; 1861 struct tb_tree_stats *tst = data; 1862 1863 tst->host_size += tb->tc.size; 1864 tst->target_size += tb->size; 1865 if (tb->size > tst->max_target_size) { 1866 tst->max_target_size = tb->size; 1867 } 1868 if (tb->page_addr[1] != -1) { 1869 tst->cross_page++; 1870 } 1871 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1872 tst->direct_jmp_count++; 1873 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1874 tst->direct_jmp2_count++; 1875 } 1876 } 1877 return false; 1878 } 1879 1880 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1881 { 1882 struct tb_tree_stats tst = {}; 1883 struct qht_stats hst; 1884 size_t nb_tbs; 1885 1886 tb_lock(); 1887 1888 nb_tbs = g_tree_nnodes(tb_ctx.tb_tree); 1889 g_tree_foreach(tb_ctx.tb_tree, tb_tree_stats_iter, &tst); 1890 /* XXX: avoid using doubles ? */ 1891 cpu_fprintf(f, "Translation buffer state:\n"); 1892 /* 1893 * Report total code size including the padding and TB structs; 1894 * otherwise users might think "-tb-size" is not honoured. 1895 * For avg host size we use the precise numbers from tb_tree_stats though. 1896 */ 1897 cpu_fprintf(f, "gen code size %zu/%zu\n", 1898 tcg_code_size(), tcg_code_capacity()); 1899 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1900 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1901 nb_tbs ? tst.target_size / nb_tbs : 0, 1902 tst.max_target_size); 1903 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1904 nb_tbs ? tst.host_size / nb_tbs : 0, 1905 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1906 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1907 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1908 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1909 tst.direct_jmp_count, 1910 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1911 tst.direct_jmp2_count, 1912 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1913 1914 qht_statistics_init(&tb_ctx.htable, &hst); 1915 print_qht_statistics(f, cpu_fprintf, hst); 1916 qht_statistics_destroy(&hst); 1917 1918 cpu_fprintf(f, "\nStatistics:\n"); 1919 cpu_fprintf(f, "TB flush count %u\n", 1920 atomic_read(&tb_ctx.tb_flush_count)); 1921 cpu_fprintf(f, "TB invalidate count %d\n", tb_ctx.tb_phys_invalidate_count); 1922 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1923 tcg_dump_info(f, cpu_fprintf); 1924 1925 tb_unlock(); 1926 } 1927 1928 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1929 { 1930 tcg_dump_op_count(f, cpu_fprintf); 1931 } 1932 1933 #else /* CONFIG_USER_ONLY */ 1934 1935 void cpu_interrupt(CPUState *cpu, int mask) 1936 { 1937 g_assert(qemu_mutex_iothread_locked()); 1938 cpu->interrupt_request |= mask; 1939 cpu->icount_decr.u16.high = -1; 1940 } 1941 1942 /* 1943 * Walks guest process memory "regions" one by one 1944 * and calls callback function 'fn' for each region. 1945 */ 1946 struct walk_memory_regions_data { 1947 walk_memory_regions_fn fn; 1948 void *priv; 1949 target_ulong start; 1950 int prot; 1951 }; 1952 1953 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1954 target_ulong end, int new_prot) 1955 { 1956 if (data->start != -1u) { 1957 int rc = data->fn(data->priv, data->start, end, data->prot); 1958 if (rc != 0) { 1959 return rc; 1960 } 1961 } 1962 1963 data->start = (new_prot ? end : -1u); 1964 data->prot = new_prot; 1965 1966 return 0; 1967 } 1968 1969 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1970 target_ulong base, int level, void **lp) 1971 { 1972 target_ulong pa; 1973 int i, rc; 1974 1975 if (*lp == NULL) { 1976 return walk_memory_regions_end(data, base, 0); 1977 } 1978 1979 if (level == 0) { 1980 PageDesc *pd = *lp; 1981 1982 for (i = 0; i < V_L2_SIZE; ++i) { 1983 int prot = pd[i].flags; 1984 1985 pa = base | (i << TARGET_PAGE_BITS); 1986 if (prot != data->prot) { 1987 rc = walk_memory_regions_end(data, pa, prot); 1988 if (rc != 0) { 1989 return rc; 1990 } 1991 } 1992 } 1993 } else { 1994 void **pp = *lp; 1995 1996 for (i = 0; i < V_L2_SIZE; ++i) { 1997 pa = base | ((target_ulong)i << 1998 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1999 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 2000 if (rc != 0) { 2001 return rc; 2002 } 2003 } 2004 } 2005 2006 return 0; 2007 } 2008 2009 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2010 { 2011 struct walk_memory_regions_data data; 2012 uintptr_t i, l1_sz = v_l1_size; 2013 2014 data.fn = fn; 2015 data.priv = priv; 2016 data.start = -1u; 2017 data.prot = 0; 2018 2019 for (i = 0; i < l1_sz; i++) { 2020 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2021 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2022 if (rc != 0) { 2023 return rc; 2024 } 2025 } 2026 2027 return walk_memory_regions_end(&data, 0, 0); 2028 } 2029 2030 static int dump_region(void *priv, target_ulong start, 2031 target_ulong end, unsigned long prot) 2032 { 2033 FILE *f = (FILE *)priv; 2034 2035 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2036 " "TARGET_FMT_lx" %c%c%c\n", 2037 start, end, end - start, 2038 ((prot & PAGE_READ) ? 'r' : '-'), 2039 ((prot & PAGE_WRITE) ? 'w' : '-'), 2040 ((prot & PAGE_EXEC) ? 'x' : '-')); 2041 2042 return 0; 2043 } 2044 2045 /* dump memory mappings */ 2046 void page_dump(FILE *f) 2047 { 2048 const int length = sizeof(target_ulong) * 2; 2049 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2050 length, "start", length, "end", length, "size", "prot"); 2051 walk_memory_regions(f, dump_region); 2052 } 2053 2054 int page_get_flags(target_ulong address) 2055 { 2056 PageDesc *p; 2057 2058 p = page_find(address >> TARGET_PAGE_BITS); 2059 if (!p) { 2060 return 0; 2061 } 2062 return p->flags; 2063 } 2064 2065 /* Modify the flags of a page and invalidate the code if necessary. 2066 The flag PAGE_WRITE_ORG is positioned automatically depending 2067 on PAGE_WRITE. The mmap_lock should already be held. */ 2068 void page_set_flags(target_ulong start, target_ulong end, int flags) 2069 { 2070 target_ulong addr, len; 2071 2072 /* This function should never be called with addresses outside the 2073 guest address space. If this assert fires, it probably indicates 2074 a missing call to h2g_valid. */ 2075 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2076 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2077 #endif 2078 assert(start < end); 2079 assert_memory_lock(); 2080 2081 start = start & TARGET_PAGE_MASK; 2082 end = TARGET_PAGE_ALIGN(end); 2083 2084 if (flags & PAGE_WRITE) { 2085 flags |= PAGE_WRITE_ORG; 2086 } 2087 2088 for (addr = start, len = end - start; 2089 len != 0; 2090 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2091 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2092 2093 /* If the write protection bit is set, then we invalidate 2094 the code inside. */ 2095 if (!(p->flags & PAGE_WRITE) && 2096 (flags & PAGE_WRITE) && 2097 p->first_tb) { 2098 tb_invalidate_phys_page(addr, 0); 2099 } 2100 p->flags = flags; 2101 } 2102 } 2103 2104 int page_check_range(target_ulong start, target_ulong len, int flags) 2105 { 2106 PageDesc *p; 2107 target_ulong end; 2108 target_ulong addr; 2109 2110 /* This function should never be called with addresses outside the 2111 guest address space. If this assert fires, it probably indicates 2112 a missing call to h2g_valid. */ 2113 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2114 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2115 #endif 2116 2117 if (len == 0) { 2118 return 0; 2119 } 2120 if (start + len - 1 < start) { 2121 /* We've wrapped around. */ 2122 return -1; 2123 } 2124 2125 /* must do before we loose bits in the next step */ 2126 end = TARGET_PAGE_ALIGN(start + len); 2127 start = start & TARGET_PAGE_MASK; 2128 2129 for (addr = start, len = end - start; 2130 len != 0; 2131 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2132 p = page_find(addr >> TARGET_PAGE_BITS); 2133 if (!p) { 2134 return -1; 2135 } 2136 if (!(p->flags & PAGE_VALID)) { 2137 return -1; 2138 } 2139 2140 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2141 return -1; 2142 } 2143 if (flags & PAGE_WRITE) { 2144 if (!(p->flags & PAGE_WRITE_ORG)) { 2145 return -1; 2146 } 2147 /* unprotect the page if it was put read-only because it 2148 contains translated code */ 2149 if (!(p->flags & PAGE_WRITE)) { 2150 if (!page_unprotect(addr, 0)) { 2151 return -1; 2152 } 2153 } 2154 } 2155 } 2156 return 0; 2157 } 2158 2159 /* called from signal handler: invalidate the code and unprotect the 2160 * page. Return 0 if the fault was not handled, 1 if it was handled, 2161 * and 2 if it was handled but the caller must cause the TB to be 2162 * immediately exited. (We can only return 2 if the 'pc' argument is 2163 * non-zero.) 2164 */ 2165 int page_unprotect(target_ulong address, uintptr_t pc) 2166 { 2167 unsigned int prot; 2168 bool current_tb_invalidated; 2169 PageDesc *p; 2170 target_ulong host_start, host_end, addr; 2171 2172 /* Technically this isn't safe inside a signal handler. However we 2173 know this only ever happens in a synchronous SEGV handler, so in 2174 practice it seems to be ok. */ 2175 mmap_lock(); 2176 2177 p = page_find(address >> TARGET_PAGE_BITS); 2178 if (!p) { 2179 mmap_unlock(); 2180 return 0; 2181 } 2182 2183 /* if the page was really writable, then we change its 2184 protection back to writable */ 2185 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) { 2186 host_start = address & qemu_host_page_mask; 2187 host_end = host_start + qemu_host_page_size; 2188 2189 prot = 0; 2190 current_tb_invalidated = false; 2191 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) { 2192 p = page_find(addr >> TARGET_PAGE_BITS); 2193 p->flags |= PAGE_WRITE; 2194 prot |= p->flags; 2195 2196 /* and since the content will be modified, we must invalidate 2197 the corresponding translated code. */ 2198 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2199 #ifdef CONFIG_USER_ONLY 2200 if (DEBUG_TB_CHECK_GATE) { 2201 tb_invalidate_check(addr); 2202 } 2203 #endif 2204 } 2205 mprotect((void *)g2h(host_start), qemu_host_page_size, 2206 prot & PAGE_BITS); 2207 2208 mmap_unlock(); 2209 /* If current TB was invalidated return to main loop */ 2210 return current_tb_invalidated ? 2 : 1; 2211 } 2212 mmap_unlock(); 2213 return 0; 2214 } 2215 #endif /* CONFIG_USER_ONLY */ 2216 2217 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2218 void tcg_flush_softmmu_tlb(CPUState *cs) 2219 { 2220 #ifdef CONFIG_SOFTMMU 2221 tlb_flush(cs); 2222 #endif 2223 } 2224