1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 35 #include <sys/param.h> 36 #if __FreeBSD_version >= 700104 37 #define HAVE_KINFO_GETVMMAP 38 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 39 #include <sys/proc.h> 40 #include <machine/profile.h> 41 #define _KERNEL 42 #include <sys/user.h> 43 #undef _KERNEL 44 #undef sigqueue 45 #include <libutil.h> 46 #endif 47 #endif 48 #else 49 #include "exec/address-spaces.h" 50 #endif 51 52 #include "exec/cputlb.h" 53 #include "exec/tb-hash.h" 54 #include "translate-all.h" 55 #include "qemu/bitmap.h" 56 #include "qemu/error-report.h" 57 #include "qemu/timer.h" 58 #include "qemu/main-loop.h" 59 #include "exec/log.h" 60 #include "sysemu/cpus.h" 61 62 /* #define DEBUG_TB_INVALIDATE */ 63 /* #define DEBUG_TB_FLUSH */ 64 /* make various TB consistency checks */ 65 /* #define DEBUG_TB_CHECK */ 66 67 #ifdef DEBUG_TB_INVALIDATE 68 #define DEBUG_TB_INVALIDATE_GATE 1 69 #else 70 #define DEBUG_TB_INVALIDATE_GATE 0 71 #endif 72 73 #ifdef DEBUG_TB_FLUSH 74 #define DEBUG_TB_FLUSH_GATE 1 75 #else 76 #define DEBUG_TB_FLUSH_GATE 0 77 #endif 78 79 #if !defined(CONFIG_USER_ONLY) 80 /* TB consistency checks only implemented for usermode emulation. */ 81 #undef DEBUG_TB_CHECK 82 #endif 83 84 #ifdef DEBUG_TB_CHECK 85 #define DEBUG_TB_CHECK_GATE 1 86 #else 87 #define DEBUG_TB_CHECK_GATE 0 88 #endif 89 90 /* Access to the various translations structures need to be serialised via locks 91 * for consistency. This is automatic for SoftMMU based system 92 * emulation due to its single threaded nature. In user-mode emulation 93 * access to the memory related structures are protected with the 94 * mmap_lock. 95 */ 96 #ifdef CONFIG_SOFTMMU 97 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 98 #else 99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 100 #endif 101 102 #define SMC_BITMAP_USE_THRESHOLD 10 103 104 typedef struct PageDesc { 105 /* list of TBs intersecting this ram page */ 106 TranslationBlock *first_tb; 107 #ifdef CONFIG_SOFTMMU 108 /* in order to optimize self modifying code, we count the number 109 of lookups we do to a given page to use a bitmap */ 110 unsigned int code_write_count; 111 unsigned long *code_bitmap; 112 #else 113 unsigned long flags; 114 #endif 115 } PageDesc; 116 117 /* In system mode we want L1_MAP to be based on ram offsets, 118 while in user mode we want it to be based on virtual addresses. */ 119 #if !defined(CONFIG_USER_ONLY) 120 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 121 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 122 #else 123 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 124 #endif 125 #else 126 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 127 #endif 128 129 /* Size of the L2 (and L3, etc) page tables. */ 130 #define V_L2_BITS 10 131 #define V_L2_SIZE (1 << V_L2_BITS) 132 133 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 134 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 135 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 136 * BITS_PER_BYTE); 137 138 /* 139 * L1 Mapping properties 140 */ 141 static int v_l1_size; 142 static int v_l1_shift; 143 static int v_l2_levels; 144 145 /* The bottom level has pointers to PageDesc, and is indexed by 146 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 147 */ 148 #define V_L1_MIN_BITS 4 149 #define V_L1_MAX_BITS (V_L2_BITS + 3) 150 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 151 152 static void *l1_map[V_L1_MAX_SIZE]; 153 154 /* code generation context */ 155 TCGContext tcg_init_ctx; 156 __thread TCGContext *tcg_ctx; 157 TBContext tb_ctx; 158 bool parallel_cpus; 159 160 /* translation block context */ 161 static __thread int have_tb_lock; 162 163 static void page_table_config_init(void) 164 { 165 uint32_t v_l1_bits; 166 167 assert(TARGET_PAGE_BITS); 168 /* The bits remaining after N lower levels of page tables. */ 169 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 170 if (v_l1_bits < V_L1_MIN_BITS) { 171 v_l1_bits += V_L2_BITS; 172 } 173 174 v_l1_size = 1 << v_l1_bits; 175 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 176 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 177 178 assert(v_l1_bits <= V_L1_MAX_BITS); 179 assert(v_l1_shift % V_L2_BITS == 0); 180 assert(v_l2_levels >= 0); 181 } 182 183 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 184 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 185 186 void tb_lock(void) 187 { 188 assert_tb_unlocked(); 189 qemu_mutex_lock(&tb_ctx.tb_lock); 190 have_tb_lock++; 191 } 192 193 void tb_unlock(void) 194 { 195 assert_tb_locked(); 196 have_tb_lock--; 197 qemu_mutex_unlock(&tb_ctx.tb_lock); 198 } 199 200 void tb_lock_reset(void) 201 { 202 if (have_tb_lock) { 203 qemu_mutex_unlock(&tb_ctx.tb_lock); 204 have_tb_lock = 0; 205 } 206 } 207 208 void cpu_gen_init(void) 209 { 210 tcg_context_init(&tcg_init_ctx); 211 } 212 213 /* Encode VAL as a signed leb128 sequence at P. 214 Return P incremented past the encoded value. */ 215 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 216 { 217 int more, byte; 218 219 do { 220 byte = val & 0x7f; 221 val >>= 7; 222 more = !((val == 0 && (byte & 0x40) == 0) 223 || (val == -1 && (byte & 0x40) != 0)); 224 if (more) { 225 byte |= 0x80; 226 } 227 *p++ = byte; 228 } while (more); 229 230 return p; 231 } 232 233 /* Decode a signed leb128 sequence at *PP; increment *PP past the 234 decoded value. Return the decoded value. */ 235 static target_long decode_sleb128(uint8_t **pp) 236 { 237 uint8_t *p = *pp; 238 target_long val = 0; 239 int byte, shift = 0; 240 241 do { 242 byte = *p++; 243 val |= (target_ulong)(byte & 0x7f) << shift; 244 shift += 7; 245 } while (byte & 0x80); 246 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 247 val |= -(target_ulong)1 << shift; 248 } 249 250 *pp = p; 251 return val; 252 } 253 254 /* Encode the data collected about the instructions while compiling TB. 255 Place the data at BLOCK, and return the number of bytes consumed. 256 257 The logical table consists of TARGET_INSN_START_WORDS target_ulong's, 258 which come from the target's insn_start data, followed by a uintptr_t 259 which comes from the host pc of the end of the code implementing the insn. 260 261 Each line of the table is encoded as sleb128 deltas from the previous 262 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 263 That is, the first column is seeded with the guest pc, the last column 264 with the host pc, and the middle columns with zeros. */ 265 266 static int encode_search(TranslationBlock *tb, uint8_t *block) 267 { 268 uint8_t *highwater = tcg_ctx->code_gen_highwater; 269 uint8_t *p = block; 270 int i, j, n; 271 272 for (i = 0, n = tb->icount; i < n; ++i) { 273 target_ulong prev; 274 275 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 276 if (i == 0) { 277 prev = (j == 0 ? tb->pc : 0); 278 } else { 279 prev = tcg_ctx->gen_insn_data[i - 1][j]; 280 } 281 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 282 } 283 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 284 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 285 286 /* Test for (pending) buffer overflow. The assumption is that any 287 one row beginning below the high water mark cannot overrun 288 the buffer completely. Thus we can test for overflow after 289 encoding a row without having to check during encoding. */ 290 if (unlikely(p > highwater)) { 291 return -1; 292 } 293 } 294 295 return p - block; 296 } 297 298 /* The cpu state corresponding to 'searched_pc' is restored. 299 * Called with tb_lock held. 300 * When reset_icount is true, current TB will be interrupted and 301 * icount should be recalculated. 302 */ 303 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 304 uintptr_t searched_pc, bool reset_icount) 305 { 306 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 307 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 308 CPUArchState *env = cpu->env_ptr; 309 uint8_t *p = tb->tc.ptr + tb->tc.size; 310 int i, j, num_insns = tb->icount; 311 #ifdef CONFIG_PROFILER 312 TCGProfile *prof = &tcg_ctx->prof; 313 int64_t ti = profile_getclock(); 314 #endif 315 316 searched_pc -= GETPC_ADJ; 317 318 if (searched_pc < host_pc) { 319 return -1; 320 } 321 322 /* Reconstruct the stored insn data while looking for the point at 323 which the end of the insn exceeds the searched_pc. */ 324 for (i = 0; i < num_insns; ++i) { 325 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 326 data[j] += decode_sleb128(&p); 327 } 328 host_pc += decode_sleb128(&p); 329 if (host_pc > searched_pc) { 330 goto found; 331 } 332 } 333 return -1; 334 335 found: 336 if (reset_icount && (tb->cflags & CF_USE_ICOUNT)) { 337 assert(use_icount); 338 /* Reset the cycle counter to the start of the block 339 and shift if to the number of actually executed instructions */ 340 cpu->icount_decr.u16.low += num_insns - i; 341 } 342 restore_state_to_opc(env, tb, data); 343 344 #ifdef CONFIG_PROFILER 345 atomic_set(&prof->restore_time, 346 prof->restore_time + profile_getclock() - ti); 347 atomic_set(&prof->restore_count, prof->restore_count + 1); 348 #endif 349 return 0; 350 } 351 352 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) 353 { 354 TranslationBlock *tb; 355 bool r = false; 356 uintptr_t check_offset; 357 358 /* The host_pc has to be in the region of current code buffer. If 359 * it is not we will not be able to resolve it here. The two cases 360 * where host_pc will not be correct are: 361 * 362 * - fault during translation (instruction fetch) 363 * - fault from helper (not using GETPC() macro) 364 * 365 * Either way we need return early to avoid blowing up on a 366 * recursive tb_lock() as we can't resolve it here. 367 * 368 * We are using unsigned arithmetic so if host_pc < 369 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way 370 * above the code_gen_buffer_size 371 */ 372 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; 373 374 if (check_offset < tcg_init_ctx.code_gen_buffer_size) { 375 tb_lock(); 376 tb = tcg_tb_lookup(host_pc); 377 if (tb) { 378 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); 379 if (tb->cflags & CF_NOCACHE) { 380 /* one-shot translation, invalidate it immediately */ 381 tb_phys_invalidate(tb, -1); 382 tcg_tb_remove(tb); 383 } 384 r = true; 385 } 386 tb_unlock(); 387 } 388 389 return r; 390 } 391 392 static void page_init(void) 393 { 394 page_size_init(); 395 page_table_config_init(); 396 397 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 398 { 399 #ifdef HAVE_KINFO_GETVMMAP 400 struct kinfo_vmentry *freep; 401 int i, cnt; 402 403 freep = kinfo_getvmmap(getpid(), &cnt); 404 if (freep) { 405 mmap_lock(); 406 for (i = 0; i < cnt; i++) { 407 unsigned long startaddr, endaddr; 408 409 startaddr = freep[i].kve_start; 410 endaddr = freep[i].kve_end; 411 if (h2g_valid(startaddr)) { 412 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 413 414 if (h2g_valid(endaddr)) { 415 endaddr = h2g(endaddr); 416 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 417 } else { 418 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 419 endaddr = ~0ul; 420 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 421 #endif 422 } 423 } 424 } 425 free(freep); 426 mmap_unlock(); 427 } 428 #else 429 FILE *f; 430 431 last_brk = (unsigned long)sbrk(0); 432 433 f = fopen("/compat/linux/proc/self/maps", "r"); 434 if (f) { 435 mmap_lock(); 436 437 do { 438 unsigned long startaddr, endaddr; 439 int n; 440 441 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 442 443 if (n == 2 && h2g_valid(startaddr)) { 444 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 445 446 if (h2g_valid(endaddr)) { 447 endaddr = h2g(endaddr); 448 } else { 449 endaddr = ~0ul; 450 } 451 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 452 } 453 } while (!feof(f)); 454 455 fclose(f); 456 mmap_unlock(); 457 } 458 #endif 459 } 460 #endif 461 } 462 463 /* If alloc=1: 464 * Called with tb_lock held for system emulation. 465 * Called with mmap_lock held for user-mode emulation. 466 */ 467 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 468 { 469 PageDesc *pd; 470 void **lp; 471 int i; 472 473 if (alloc) { 474 assert_memory_lock(); 475 } 476 477 /* Level 1. Always allocated. */ 478 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 479 480 /* Level 2..N-1. */ 481 for (i = v_l2_levels; i > 0; i--) { 482 void **p = atomic_rcu_read(lp); 483 484 if (p == NULL) { 485 if (!alloc) { 486 return NULL; 487 } 488 p = g_new0(void *, V_L2_SIZE); 489 atomic_rcu_set(lp, p); 490 } 491 492 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 493 } 494 495 pd = atomic_rcu_read(lp); 496 if (pd == NULL) { 497 if (!alloc) { 498 return NULL; 499 } 500 pd = g_new0(PageDesc, V_L2_SIZE); 501 atomic_rcu_set(lp, pd); 502 } 503 504 return pd + (index & (V_L2_SIZE - 1)); 505 } 506 507 static inline PageDesc *page_find(tb_page_addr_t index) 508 { 509 return page_find_alloc(index, 0); 510 } 511 512 #if defined(CONFIG_USER_ONLY) 513 /* Currently it is not recommended to allocate big chunks of data in 514 user mode. It will change when a dedicated libc will be used. */ 515 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 516 region in which the guest needs to run. Revisit this. */ 517 #define USE_STATIC_CODE_GEN_BUFFER 518 #endif 519 520 /* Minimum size of the code gen buffer. This number is randomly chosen, 521 but not so small that we can't have a fair number of TB's live. */ 522 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 523 524 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 525 indicated, this is constrained by the range of direct branches on the 526 host cpu, as used by the TCG implementation of goto_tb. */ 527 #if defined(__x86_64__) 528 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 529 #elif defined(__sparc__) 530 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 531 #elif defined(__powerpc64__) 532 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 533 #elif defined(__powerpc__) 534 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 535 #elif defined(__aarch64__) 536 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 537 #elif defined(__s390x__) 538 /* We have a +- 4GB range on the branches; leave some slop. */ 539 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 540 #elif defined(__mips__) 541 /* We have a 256MB branch region, but leave room to make sure the 542 main executable is also within that region. */ 543 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 544 #else 545 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 546 #endif 547 548 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 549 550 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 551 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 552 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 553 554 static inline size_t size_code_gen_buffer(size_t tb_size) 555 { 556 /* Size the buffer. */ 557 if (tb_size == 0) { 558 #ifdef USE_STATIC_CODE_GEN_BUFFER 559 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 560 #else 561 /* ??? Needs adjustments. */ 562 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 563 static buffer, we could size this on RESERVED_VA, on the text 564 segment size of the executable, or continue to use the default. */ 565 tb_size = (unsigned long)(ram_size / 4); 566 #endif 567 } 568 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 569 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 570 } 571 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 572 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 573 } 574 return tb_size; 575 } 576 577 #ifdef __mips__ 578 /* In order to use J and JAL within the code_gen_buffer, we require 579 that the buffer not cross a 256MB boundary. */ 580 static inline bool cross_256mb(void *addr, size_t size) 581 { 582 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 583 } 584 585 /* We weren't able to allocate a buffer without crossing that boundary, 586 so make do with the larger portion of the buffer that doesn't cross. 587 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 588 static inline void *split_cross_256mb(void *buf1, size_t size1) 589 { 590 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 591 size_t size2 = buf1 + size1 - buf2; 592 593 size1 = buf2 - buf1; 594 if (size1 < size2) { 595 size1 = size2; 596 buf1 = buf2; 597 } 598 599 tcg_ctx->code_gen_buffer_size = size1; 600 return buf1; 601 } 602 #endif 603 604 #ifdef USE_STATIC_CODE_GEN_BUFFER 605 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 606 __attribute__((aligned(CODE_GEN_ALIGN))); 607 608 static inline void *alloc_code_gen_buffer(void) 609 { 610 void *buf = static_code_gen_buffer; 611 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 612 size_t size; 613 614 /* page-align the beginning and end of the buffer */ 615 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 616 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 617 618 size = end - buf; 619 620 /* Honor a command-line option limiting the size of the buffer. */ 621 if (size > tcg_ctx->code_gen_buffer_size) { 622 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 623 qemu_real_host_page_size); 624 } 625 tcg_ctx->code_gen_buffer_size = size; 626 627 #ifdef __mips__ 628 if (cross_256mb(buf, size)) { 629 buf = split_cross_256mb(buf, size); 630 size = tcg_ctx->code_gen_buffer_size; 631 } 632 #endif 633 634 if (qemu_mprotect_rwx(buf, size)) { 635 abort(); 636 } 637 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 638 639 return buf; 640 } 641 #elif defined(_WIN32) 642 static inline void *alloc_code_gen_buffer(void) 643 { 644 size_t size = tcg_ctx->code_gen_buffer_size; 645 return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 646 PAGE_EXECUTE_READWRITE); 647 } 648 #else 649 static inline void *alloc_code_gen_buffer(void) 650 { 651 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 652 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 653 uintptr_t start = 0; 654 size_t size = tcg_ctx->code_gen_buffer_size; 655 void *buf; 656 657 /* Constrain the position of the buffer based on the host cpu. 658 Note that these addresses are chosen in concert with the 659 addresses assigned in the relevant linker script file. */ 660 # if defined(__PIE__) || defined(__PIC__) 661 /* Don't bother setting a preferred location if we're building 662 a position-independent executable. We're more likely to get 663 an address near the main executable if we let the kernel 664 choose the address. */ 665 # elif defined(__x86_64__) && defined(MAP_32BIT) 666 /* Force the memory down into low memory with the executable. 667 Leave the choice of exact location with the kernel. */ 668 flags |= MAP_32BIT; 669 /* Cannot expect to map more than 800MB in low memory. */ 670 if (size > 800u * 1024 * 1024) { 671 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 672 } 673 # elif defined(__sparc__) 674 start = 0x40000000ul; 675 # elif defined(__s390x__) 676 start = 0x90000000ul; 677 # elif defined(__mips__) 678 # if _MIPS_SIM == _ABI64 679 start = 0x128000000ul; 680 # else 681 start = 0x08000000ul; 682 # endif 683 # endif 684 685 buf = mmap((void *)start, size, prot, flags, -1, 0); 686 if (buf == MAP_FAILED) { 687 return NULL; 688 } 689 690 #ifdef __mips__ 691 if (cross_256mb(buf, size)) { 692 /* Try again, with the original still mapped, to avoid re-acquiring 693 that 256mb crossing. This time don't specify an address. */ 694 size_t size2; 695 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 696 switch ((int)(buf2 != MAP_FAILED)) { 697 case 1: 698 if (!cross_256mb(buf2, size)) { 699 /* Success! Use the new buffer. */ 700 munmap(buf, size); 701 break; 702 } 703 /* Failure. Work with what we had. */ 704 munmap(buf2, size); 705 /* fallthru */ 706 default: 707 /* Split the original buffer. Free the smaller half. */ 708 buf2 = split_cross_256mb(buf, size); 709 size2 = tcg_ctx->code_gen_buffer_size; 710 if (buf == buf2) { 711 munmap(buf + size2, size - size2); 712 } else { 713 munmap(buf, size - size2); 714 } 715 size = size2; 716 break; 717 } 718 buf = buf2; 719 } 720 #endif 721 722 /* Request large pages for the buffer. */ 723 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 724 725 return buf; 726 } 727 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 728 729 static inline void code_gen_alloc(size_t tb_size) 730 { 731 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 732 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 733 if (tcg_ctx->code_gen_buffer == NULL) { 734 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 735 exit(1); 736 } 737 qemu_mutex_init(&tb_ctx.tb_lock); 738 } 739 740 static bool tb_cmp(const void *ap, const void *bp) 741 { 742 const TranslationBlock *a = ap; 743 const TranslationBlock *b = bp; 744 745 return a->pc == b->pc && 746 a->cs_base == b->cs_base && 747 a->flags == b->flags && 748 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) && 749 a->trace_vcpu_dstate == b->trace_vcpu_dstate && 750 a->page_addr[0] == b->page_addr[0] && 751 a->page_addr[1] == b->page_addr[1]; 752 } 753 754 static void tb_htable_init(void) 755 { 756 unsigned int mode = QHT_MODE_AUTO_RESIZE; 757 758 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); 759 } 760 761 /* Must be called before using the QEMU cpus. 'tb_size' is the size 762 (in bytes) allocated to the translation buffer. Zero means default 763 size. */ 764 void tcg_exec_init(unsigned long tb_size) 765 { 766 tcg_allowed = true; 767 cpu_gen_init(); 768 page_init(); 769 tb_htable_init(); 770 code_gen_alloc(tb_size); 771 #if defined(CONFIG_SOFTMMU) 772 /* There's no guest base to take into account, so go ahead and 773 initialize the prologue now. */ 774 tcg_prologue_init(tcg_ctx); 775 #endif 776 } 777 778 /* 779 * Allocate a new translation block. Flush the translation buffer if 780 * too many translation blocks or too much generated code. 781 * 782 * Called with tb_lock held. 783 */ 784 static TranslationBlock *tb_alloc(target_ulong pc) 785 { 786 TranslationBlock *tb; 787 788 assert_tb_locked(); 789 790 tb = tcg_tb_alloc(tcg_ctx); 791 if (unlikely(tb == NULL)) { 792 return NULL; 793 } 794 return tb; 795 } 796 797 static inline void invalidate_page_bitmap(PageDesc *p) 798 { 799 #ifdef CONFIG_SOFTMMU 800 g_free(p->code_bitmap); 801 p->code_bitmap = NULL; 802 p->code_write_count = 0; 803 #endif 804 } 805 806 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 807 static void page_flush_tb_1(int level, void **lp) 808 { 809 int i; 810 811 if (*lp == NULL) { 812 return; 813 } 814 if (level == 0) { 815 PageDesc *pd = *lp; 816 817 for (i = 0; i < V_L2_SIZE; ++i) { 818 pd[i].first_tb = NULL; 819 invalidate_page_bitmap(pd + i); 820 } 821 } else { 822 void **pp = *lp; 823 824 for (i = 0; i < V_L2_SIZE; ++i) { 825 page_flush_tb_1(level - 1, pp + i); 826 } 827 } 828 } 829 830 static void page_flush_tb(void) 831 { 832 int i, l1_sz = v_l1_size; 833 834 for (i = 0; i < l1_sz; i++) { 835 page_flush_tb_1(v_l2_levels, l1_map + i); 836 } 837 } 838 839 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 840 { 841 const TranslationBlock *tb = value; 842 size_t *size = data; 843 844 *size += tb->tc.size; 845 return false; 846 } 847 848 /* flush all the translation blocks */ 849 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 850 { 851 tb_lock(); 852 853 /* If it is already been done on request of another CPU, 854 * just retry. 855 */ 856 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 857 goto done; 858 } 859 860 if (DEBUG_TB_FLUSH_GATE) { 861 size_t nb_tbs = tcg_nb_tbs(); 862 size_t host_size = 0; 863 864 tcg_tb_foreach(tb_host_size_iter, &host_size); 865 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 866 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 867 } 868 869 CPU_FOREACH(cpu) { 870 cpu_tb_jmp_cache_clear(cpu); 871 } 872 873 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 874 page_flush_tb(); 875 876 tcg_region_reset_all(); 877 /* XXX: flush processor icache at this point if cache flush is 878 expensive */ 879 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 880 881 done: 882 tb_unlock(); 883 } 884 885 void tb_flush(CPUState *cpu) 886 { 887 if (tcg_enabled()) { 888 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 889 async_safe_run_on_cpu(cpu, do_tb_flush, 890 RUN_ON_CPU_HOST_INT(tb_flush_count)); 891 } 892 } 893 894 /* 895 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 896 * so in order to prevent bit rot we compile them unconditionally in user-mode, 897 * and let the optimizer get rid of them by wrapping their user-only callers 898 * with if (DEBUG_TB_CHECK_GATE). 899 */ 900 #ifdef CONFIG_USER_ONLY 901 902 static void 903 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 904 { 905 TranslationBlock *tb = p; 906 target_ulong addr = *(target_ulong *)userp; 907 908 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 909 printf("ERROR invalidate: address=" TARGET_FMT_lx 910 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 911 } 912 } 913 914 /* verify that all the pages have correct rights for code 915 * 916 * Called with tb_lock held. 917 */ 918 static void tb_invalidate_check(target_ulong address) 919 { 920 address &= TARGET_PAGE_MASK; 921 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 922 } 923 924 static void 925 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 926 { 927 TranslationBlock *tb = p; 928 int flags1, flags2; 929 930 flags1 = page_get_flags(tb->pc); 931 flags2 = page_get_flags(tb->pc + tb->size - 1); 932 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 933 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 934 (long)tb->pc, tb->size, flags1, flags2); 935 } 936 } 937 938 /* verify that all the pages have correct rights for code */ 939 static void tb_page_check(void) 940 { 941 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 942 } 943 944 #endif /* CONFIG_USER_ONLY */ 945 946 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) 947 { 948 TranslationBlock *tb1; 949 unsigned int n1; 950 951 for (;;) { 952 tb1 = *ptb; 953 n1 = (uintptr_t)tb1 & 3; 954 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); 955 if (tb1 == tb) { 956 *ptb = tb1->page_next[n1]; 957 break; 958 } 959 ptb = &tb1->page_next[n1]; 960 } 961 } 962 963 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 964 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 965 { 966 TranslationBlock *tb1; 967 uintptr_t *ptb, ntb; 968 unsigned int n1; 969 970 ptb = &tb->jmp_list_next[n]; 971 if (*ptb) { 972 /* find tb(n) in circular list */ 973 for (;;) { 974 ntb = *ptb; 975 n1 = ntb & 3; 976 tb1 = (TranslationBlock *)(ntb & ~3); 977 if (n1 == n && tb1 == tb) { 978 break; 979 } 980 if (n1 == 2) { 981 ptb = &tb1->jmp_list_first; 982 } else { 983 ptb = &tb1->jmp_list_next[n1]; 984 } 985 } 986 /* now we can suppress tb(n) from the list */ 987 *ptb = tb->jmp_list_next[n]; 988 989 tb->jmp_list_next[n] = (uintptr_t)NULL; 990 } 991 } 992 993 /* reset the jump entry 'n' of a TB so that it is not chained to 994 another TB */ 995 static inline void tb_reset_jump(TranslationBlock *tb, int n) 996 { 997 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 998 tb_set_jmp_target(tb, n, addr); 999 } 1000 1001 /* remove any jumps to the TB */ 1002 static inline void tb_jmp_unlink(TranslationBlock *tb) 1003 { 1004 TranslationBlock *tb1; 1005 uintptr_t *ptb, ntb; 1006 unsigned int n1; 1007 1008 ptb = &tb->jmp_list_first; 1009 for (;;) { 1010 ntb = *ptb; 1011 n1 = ntb & 3; 1012 tb1 = (TranslationBlock *)(ntb & ~3); 1013 if (n1 == 2) { 1014 break; 1015 } 1016 tb_reset_jump(tb1, n1); 1017 *ptb = tb1->jmp_list_next[n1]; 1018 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1019 } 1020 } 1021 1022 /* invalidate one TB 1023 * 1024 * Called with tb_lock held. 1025 */ 1026 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1027 { 1028 CPUState *cpu; 1029 PageDesc *p; 1030 uint32_t h; 1031 tb_page_addr_t phys_pc; 1032 1033 assert_tb_locked(); 1034 1035 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1036 1037 /* remove the TB from the hash list */ 1038 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1039 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1040 tb->trace_vcpu_dstate); 1041 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1042 return; 1043 } 1044 1045 /* remove the TB from the page list */ 1046 if (tb->page_addr[0] != page_addr) { 1047 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1048 tb_page_remove(&p->first_tb, tb); 1049 invalidate_page_bitmap(p); 1050 } 1051 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1052 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1053 tb_page_remove(&p->first_tb, tb); 1054 invalidate_page_bitmap(p); 1055 } 1056 1057 /* remove the TB from the hash list */ 1058 h = tb_jmp_cache_hash_func(tb->pc); 1059 CPU_FOREACH(cpu) { 1060 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1061 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1062 } 1063 } 1064 1065 /* suppress this TB from the two jump lists */ 1066 tb_remove_from_jmp_list(tb, 0); 1067 tb_remove_from_jmp_list(tb, 1); 1068 1069 /* suppress any remaining jumps to this TB */ 1070 tb_jmp_unlink(tb); 1071 1072 tb_ctx.tb_phys_invalidate_count++; 1073 } 1074 1075 #ifdef CONFIG_SOFTMMU 1076 static void build_page_bitmap(PageDesc *p) 1077 { 1078 int n, tb_start, tb_end; 1079 TranslationBlock *tb; 1080 1081 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1082 1083 tb = p->first_tb; 1084 while (tb != NULL) { 1085 n = (uintptr_t)tb & 3; 1086 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1087 /* NOTE: this is subtle as a TB may span two physical pages */ 1088 if (n == 0) { 1089 /* NOTE: tb_end may be after the end of the page, but 1090 it is not a problem */ 1091 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1092 tb_end = tb_start + tb->size; 1093 if (tb_end > TARGET_PAGE_SIZE) { 1094 tb_end = TARGET_PAGE_SIZE; 1095 } 1096 } else { 1097 tb_start = 0; 1098 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1099 } 1100 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1101 tb = tb->page_next[n]; 1102 } 1103 } 1104 #endif 1105 1106 /* add the tb in the target page and protect it if necessary 1107 * 1108 * Called with mmap_lock held for user-mode emulation. 1109 */ 1110 static inline void tb_alloc_page(TranslationBlock *tb, 1111 unsigned int n, tb_page_addr_t page_addr) 1112 { 1113 PageDesc *p; 1114 #ifndef CONFIG_USER_ONLY 1115 bool page_already_protected; 1116 #endif 1117 1118 assert_memory_lock(); 1119 1120 tb->page_addr[n] = page_addr; 1121 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1122 tb->page_next[n] = p->first_tb; 1123 #ifndef CONFIG_USER_ONLY 1124 page_already_protected = p->first_tb != NULL; 1125 #endif 1126 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); 1127 invalidate_page_bitmap(p); 1128 1129 #if defined(CONFIG_USER_ONLY) 1130 if (p->flags & PAGE_WRITE) { 1131 target_ulong addr; 1132 PageDesc *p2; 1133 int prot; 1134 1135 /* force the host page as non writable (writes will have a 1136 page fault + mprotect overhead) */ 1137 page_addr &= qemu_host_page_mask; 1138 prot = 0; 1139 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1140 addr += TARGET_PAGE_SIZE) { 1141 1142 p2 = page_find(addr >> TARGET_PAGE_BITS); 1143 if (!p2) { 1144 continue; 1145 } 1146 prot |= p2->flags; 1147 p2->flags &= ~PAGE_WRITE; 1148 } 1149 mprotect(g2h(page_addr), qemu_host_page_size, 1150 (prot & PAGE_BITS) & ~PAGE_WRITE); 1151 if (DEBUG_TB_INVALIDATE_GATE) { 1152 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1153 } 1154 } 1155 #else 1156 /* if some code is already present, then the pages are already 1157 protected. So we handle the case where only the first TB is 1158 allocated in a physical page */ 1159 if (!page_already_protected) { 1160 tlb_protect_code(page_addr); 1161 } 1162 #endif 1163 } 1164 1165 /* add a new TB and link it to the physical page tables. phys_page2 is 1166 * (-1) to indicate that only one page contains the TB. 1167 * 1168 * Called with mmap_lock held for user-mode emulation. 1169 */ 1170 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1171 tb_page_addr_t phys_page2) 1172 { 1173 uint32_t h; 1174 1175 assert_memory_lock(); 1176 1177 /* add in the page list */ 1178 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1179 if (phys_page2 != -1) { 1180 tb_alloc_page(tb, 1, phys_page2); 1181 } else { 1182 tb->page_addr[1] = -1; 1183 } 1184 1185 /* add in the hash table */ 1186 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1187 tb->trace_vcpu_dstate); 1188 qht_insert(&tb_ctx.htable, tb, h, NULL); 1189 1190 #ifdef CONFIG_USER_ONLY 1191 if (DEBUG_TB_CHECK_GATE) { 1192 tb_page_check(); 1193 } 1194 #endif 1195 } 1196 1197 /* Called with mmap_lock held for user mode emulation. */ 1198 TranslationBlock *tb_gen_code(CPUState *cpu, 1199 target_ulong pc, target_ulong cs_base, 1200 uint32_t flags, int cflags) 1201 { 1202 CPUArchState *env = cpu->env_ptr; 1203 TranslationBlock *tb; 1204 tb_page_addr_t phys_pc, phys_page2; 1205 target_ulong virt_page2; 1206 tcg_insn_unit *gen_code_buf; 1207 int gen_code_size, search_size; 1208 #ifdef CONFIG_PROFILER 1209 TCGProfile *prof = &tcg_ctx->prof; 1210 int64_t ti; 1211 #endif 1212 assert_memory_lock(); 1213 1214 phys_pc = get_page_addr_code(env, pc); 1215 1216 buffer_overflow: 1217 tb = tb_alloc(pc); 1218 if (unlikely(!tb)) { 1219 /* flush must be done */ 1220 tb_flush(cpu); 1221 mmap_unlock(); 1222 /* Make the execution loop process the flush as soon as possible. */ 1223 cpu->exception_index = EXCP_INTERRUPT; 1224 cpu_loop_exit(cpu); 1225 } 1226 1227 gen_code_buf = tcg_ctx->code_gen_ptr; 1228 tb->tc.ptr = gen_code_buf; 1229 tb->pc = pc; 1230 tb->cs_base = cs_base; 1231 tb->flags = flags; 1232 tb->cflags = cflags; 1233 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1234 tcg_ctx->tb_cflags = cflags; 1235 1236 #ifdef CONFIG_PROFILER 1237 /* includes aborted translations because of exceptions */ 1238 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1239 ti = profile_getclock(); 1240 #endif 1241 1242 tcg_func_start(tcg_ctx); 1243 1244 tcg_ctx->cpu = ENV_GET_CPU(env); 1245 gen_intermediate_code(cpu, tb); 1246 tcg_ctx->cpu = NULL; 1247 1248 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1249 1250 /* generate machine code */ 1251 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1252 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1253 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1254 if (TCG_TARGET_HAS_direct_jump) { 1255 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1256 tcg_ctx->tb_jmp_target_addr = NULL; 1257 } else { 1258 tcg_ctx->tb_jmp_insn_offset = NULL; 1259 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1260 } 1261 1262 #ifdef CONFIG_PROFILER 1263 atomic_set(&prof->tb_count, prof->tb_count + 1); 1264 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1265 ti = profile_getclock(); 1266 #endif 1267 1268 /* ??? Overflow could be handled better here. In particular, we 1269 don't need to re-do gen_intermediate_code, nor should we re-do 1270 the tcg optimization currently hidden inside tcg_gen_code. All 1271 that should be required is to flush the TBs, allocate a new TB, 1272 re-initialize it per above, and re-do the actual code generation. */ 1273 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1274 if (unlikely(gen_code_size < 0)) { 1275 goto buffer_overflow; 1276 } 1277 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1278 if (unlikely(search_size < 0)) { 1279 goto buffer_overflow; 1280 } 1281 tb->tc.size = gen_code_size; 1282 1283 #ifdef CONFIG_PROFILER 1284 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1285 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1286 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1287 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1288 #endif 1289 1290 #ifdef DEBUG_DISAS 1291 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1292 qemu_log_in_addr_range(tb->pc)) { 1293 qemu_log_lock(); 1294 qemu_log("OUT: [size=%d]\n", gen_code_size); 1295 if (tcg_ctx->data_gen_ptr) { 1296 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1297 size_t data_size = gen_code_size - code_size; 1298 size_t i; 1299 1300 log_disas(tb->tc.ptr, code_size); 1301 1302 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1303 if (sizeof(tcg_target_ulong) == 8) { 1304 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1305 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1306 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1307 } else { 1308 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1309 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1310 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1311 } 1312 } 1313 } else { 1314 log_disas(tb->tc.ptr, gen_code_size); 1315 } 1316 qemu_log("\n"); 1317 qemu_log_flush(); 1318 qemu_log_unlock(); 1319 } 1320 #endif 1321 1322 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1323 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1324 CODE_GEN_ALIGN)); 1325 1326 /* init jump list */ 1327 assert(((uintptr_t)tb & 3) == 0); 1328 tb->jmp_list_first = (uintptr_t)tb | 2; 1329 tb->jmp_list_next[0] = (uintptr_t)NULL; 1330 tb->jmp_list_next[1] = (uintptr_t)NULL; 1331 1332 /* init original jump addresses wich has been set during tcg_gen_code() */ 1333 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1334 tb_reset_jump(tb, 0); 1335 } 1336 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1337 tb_reset_jump(tb, 1); 1338 } 1339 1340 /* check next page if needed */ 1341 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1342 phys_page2 = -1; 1343 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1344 phys_page2 = get_page_addr_code(env, virt_page2); 1345 } 1346 /* As long as consistency of the TB stuff is provided by tb_lock in user 1347 * mode and is implicit in single-threaded softmmu emulation, no explicit 1348 * memory barrier is required before tb_link_page() makes the TB visible 1349 * through the physical hash table and physical page list. 1350 */ 1351 tb_link_page(tb, phys_pc, phys_page2); 1352 tcg_tb_insert(tb); 1353 return tb; 1354 } 1355 1356 /* 1357 * Invalidate all TBs which intersect with the target physical address range 1358 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1359 * 'is_cpu_write_access' should be true if called from a real cpu write 1360 * access: the virtual CPU will exit the current TB if code is modified inside 1361 * this TB. 1362 * 1363 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1364 * Called with tb_lock held for system-mode emulation 1365 */ 1366 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1367 { 1368 while (start < end) { 1369 tb_invalidate_phys_page_range(start, end, 0); 1370 start &= TARGET_PAGE_MASK; 1371 start += TARGET_PAGE_SIZE; 1372 } 1373 } 1374 1375 #ifdef CONFIG_SOFTMMU 1376 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1377 { 1378 assert_tb_locked(); 1379 tb_invalidate_phys_range_1(start, end); 1380 } 1381 #else 1382 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1383 { 1384 assert_memory_lock(); 1385 tb_lock(); 1386 tb_invalidate_phys_range_1(start, end); 1387 tb_unlock(); 1388 } 1389 #endif 1390 /* 1391 * Invalidate all TBs which intersect with the target physical address range 1392 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1393 * 'is_cpu_write_access' should be true if called from a real cpu write 1394 * access: the virtual CPU will exit the current TB if code is modified inside 1395 * this TB. 1396 * 1397 * Called with tb_lock/mmap_lock held for user-mode emulation 1398 * Called with tb_lock held for system-mode emulation 1399 */ 1400 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1401 int is_cpu_write_access) 1402 { 1403 TranslationBlock *tb, *tb_next; 1404 tb_page_addr_t tb_start, tb_end; 1405 PageDesc *p; 1406 int n; 1407 #ifdef TARGET_HAS_PRECISE_SMC 1408 CPUState *cpu = current_cpu; 1409 CPUArchState *env = NULL; 1410 int current_tb_not_found = is_cpu_write_access; 1411 TranslationBlock *current_tb = NULL; 1412 int current_tb_modified = 0; 1413 target_ulong current_pc = 0; 1414 target_ulong current_cs_base = 0; 1415 uint32_t current_flags = 0; 1416 #endif /* TARGET_HAS_PRECISE_SMC */ 1417 1418 assert_memory_lock(); 1419 assert_tb_locked(); 1420 1421 p = page_find(start >> TARGET_PAGE_BITS); 1422 if (!p) { 1423 return; 1424 } 1425 #if defined(TARGET_HAS_PRECISE_SMC) 1426 if (cpu != NULL) { 1427 env = cpu->env_ptr; 1428 } 1429 #endif 1430 1431 /* we remove all the TBs in the range [start, end[ */ 1432 /* XXX: see if in some cases it could be faster to invalidate all 1433 the code */ 1434 tb = p->first_tb; 1435 while (tb != NULL) { 1436 n = (uintptr_t)tb & 3; 1437 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1438 tb_next = tb->page_next[n]; 1439 /* NOTE: this is subtle as a TB may span two physical pages */ 1440 if (n == 0) { 1441 /* NOTE: tb_end may be after the end of the page, but 1442 it is not a problem */ 1443 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1444 tb_end = tb_start + tb->size; 1445 } else { 1446 tb_start = tb->page_addr[1]; 1447 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1448 } 1449 if (!(tb_end <= start || tb_start >= end)) { 1450 #ifdef TARGET_HAS_PRECISE_SMC 1451 if (current_tb_not_found) { 1452 current_tb_not_found = 0; 1453 current_tb = NULL; 1454 if (cpu->mem_io_pc) { 1455 /* now we have a real cpu fault */ 1456 current_tb = tcg_tb_lookup(cpu->mem_io_pc); 1457 } 1458 } 1459 if (current_tb == tb && 1460 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1461 /* If we are modifying the current TB, we must stop 1462 its execution. We could be more precise by checking 1463 that the modification is after the current PC, but it 1464 would require a specialized function to partially 1465 restore the CPU state */ 1466 1467 current_tb_modified = 1; 1468 cpu_restore_state_from_tb(cpu, current_tb, 1469 cpu->mem_io_pc, true); 1470 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1471 ¤t_flags); 1472 } 1473 #endif /* TARGET_HAS_PRECISE_SMC */ 1474 tb_phys_invalidate(tb, -1); 1475 } 1476 tb = tb_next; 1477 } 1478 #if !defined(CONFIG_USER_ONLY) 1479 /* if no code remaining, no need to continue to use slow writes */ 1480 if (!p->first_tb) { 1481 invalidate_page_bitmap(p); 1482 tlb_unprotect_code(start); 1483 } 1484 #endif 1485 #ifdef TARGET_HAS_PRECISE_SMC 1486 if (current_tb_modified) { 1487 /* Force execution of one insn next time. */ 1488 cpu->cflags_next_tb = 1 | curr_cflags(); 1489 cpu_loop_exit_noexc(cpu); 1490 } 1491 #endif 1492 } 1493 1494 #ifdef CONFIG_SOFTMMU 1495 /* len must be <= 8 and start must be a multiple of len. 1496 * Called via softmmu_template.h when code areas are written to with 1497 * iothread mutex not held. 1498 */ 1499 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1500 { 1501 PageDesc *p; 1502 1503 #if 0 1504 if (1) { 1505 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1506 cpu_single_env->mem_io_vaddr, len, 1507 cpu_single_env->eip, 1508 cpu_single_env->eip + 1509 (intptr_t)cpu_single_env->segs[R_CS].base); 1510 } 1511 #endif 1512 assert_memory_lock(); 1513 1514 p = page_find(start >> TARGET_PAGE_BITS); 1515 if (!p) { 1516 return; 1517 } 1518 if (!p->code_bitmap && 1519 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1520 /* build code bitmap. FIXME: writes should be protected by 1521 * tb_lock, reads by tb_lock or RCU. 1522 */ 1523 build_page_bitmap(p); 1524 } 1525 if (p->code_bitmap) { 1526 unsigned int nr; 1527 unsigned long b; 1528 1529 nr = start & ~TARGET_PAGE_MASK; 1530 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1531 if (b & ((1 << len) - 1)) { 1532 goto do_invalidate; 1533 } 1534 } else { 1535 do_invalidate: 1536 tb_invalidate_phys_page_range(start, start + len, 1); 1537 } 1538 } 1539 #else 1540 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1541 * host PC of the faulting store instruction that caused this invalidate. 1542 * Returns true if the caller needs to abort execution of the current 1543 * TB (because it was modified by this store and the guest CPU has 1544 * precise-SMC semantics). 1545 */ 1546 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1547 { 1548 TranslationBlock *tb; 1549 PageDesc *p; 1550 int n; 1551 #ifdef TARGET_HAS_PRECISE_SMC 1552 TranslationBlock *current_tb = NULL; 1553 CPUState *cpu = current_cpu; 1554 CPUArchState *env = NULL; 1555 int current_tb_modified = 0; 1556 target_ulong current_pc = 0; 1557 target_ulong current_cs_base = 0; 1558 uint32_t current_flags = 0; 1559 #endif 1560 1561 assert_memory_lock(); 1562 1563 addr &= TARGET_PAGE_MASK; 1564 p = page_find(addr >> TARGET_PAGE_BITS); 1565 if (!p) { 1566 return false; 1567 } 1568 1569 tb_lock(); 1570 tb = p->first_tb; 1571 #ifdef TARGET_HAS_PRECISE_SMC 1572 if (tb && pc != 0) { 1573 current_tb = tcg_tb_lookup(pc); 1574 } 1575 if (cpu != NULL) { 1576 env = cpu->env_ptr; 1577 } 1578 #endif 1579 while (tb != NULL) { 1580 n = (uintptr_t)tb & 3; 1581 tb = (TranslationBlock *)((uintptr_t)tb & ~3); 1582 #ifdef TARGET_HAS_PRECISE_SMC 1583 if (current_tb == tb && 1584 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1585 /* If we are modifying the current TB, we must stop 1586 its execution. We could be more precise by checking 1587 that the modification is after the current PC, but it 1588 would require a specialized function to partially 1589 restore the CPU state */ 1590 1591 current_tb_modified = 1; 1592 cpu_restore_state_from_tb(cpu, current_tb, pc, true); 1593 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1594 ¤t_flags); 1595 } 1596 #endif /* TARGET_HAS_PRECISE_SMC */ 1597 tb_phys_invalidate(tb, addr); 1598 tb = tb->page_next[n]; 1599 } 1600 p->first_tb = NULL; 1601 #ifdef TARGET_HAS_PRECISE_SMC 1602 if (current_tb_modified) { 1603 /* Force execution of one insn next time. */ 1604 cpu->cflags_next_tb = 1 | curr_cflags(); 1605 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1606 * back into the cpu_exec loop. */ 1607 return true; 1608 } 1609 #endif 1610 tb_unlock(); 1611 1612 return false; 1613 } 1614 #endif 1615 1616 #if !defined(CONFIG_USER_ONLY) 1617 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs) 1618 { 1619 ram_addr_t ram_addr; 1620 MemoryRegion *mr; 1621 hwaddr l = 1; 1622 1623 rcu_read_lock(); 1624 mr = address_space_translate(as, addr, &addr, &l, false, attrs); 1625 if (!(memory_region_is_ram(mr) 1626 || memory_region_is_romd(mr))) { 1627 rcu_read_unlock(); 1628 return; 1629 } 1630 ram_addr = memory_region_get_ram_addr(mr) + addr; 1631 tb_lock(); 1632 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1633 tb_unlock(); 1634 rcu_read_unlock(); 1635 } 1636 #endif /* !defined(CONFIG_USER_ONLY) */ 1637 1638 /* Called with tb_lock held. */ 1639 void tb_check_watchpoint(CPUState *cpu) 1640 { 1641 TranslationBlock *tb; 1642 1643 tb = tcg_tb_lookup(cpu->mem_io_pc); 1644 if (tb) { 1645 /* We can use retranslation to find the PC. */ 1646 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc, true); 1647 tb_phys_invalidate(tb, -1); 1648 } else { 1649 /* The exception probably happened in a helper. The CPU state should 1650 have been saved before calling it. Fetch the PC from there. */ 1651 CPUArchState *env = cpu->env_ptr; 1652 target_ulong pc, cs_base; 1653 tb_page_addr_t addr; 1654 uint32_t flags; 1655 1656 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1657 addr = get_page_addr_code(env, pc); 1658 tb_invalidate_phys_range(addr, addr + 1); 1659 } 1660 } 1661 1662 #ifndef CONFIG_USER_ONLY 1663 /* in deterministic execution mode, instructions doing device I/Os 1664 * must be at the end of the TB. 1665 * 1666 * Called by softmmu_template.h, with iothread mutex not held. 1667 */ 1668 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1669 { 1670 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1671 CPUArchState *env = cpu->env_ptr; 1672 #endif 1673 TranslationBlock *tb; 1674 uint32_t n; 1675 1676 tb_lock(); 1677 tb = tcg_tb_lookup(retaddr); 1678 if (!tb) { 1679 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1680 (void *)retaddr); 1681 } 1682 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 1683 1684 /* On MIPS and SH, delay slot instructions can only be restarted if 1685 they were already the first instruction in the TB. If this is not 1686 the first instruction in a TB then re-execute the preceding 1687 branch. */ 1688 n = 1; 1689 #if defined(TARGET_MIPS) 1690 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 1691 && env->active_tc.PC != tb->pc) { 1692 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1693 cpu->icount_decr.u16.low++; 1694 env->hflags &= ~MIPS_HFLAG_BMASK; 1695 n = 2; 1696 } 1697 #elif defined(TARGET_SH4) 1698 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1699 && env->pc != tb->pc) { 1700 env->pc -= 2; 1701 cpu->icount_decr.u16.low++; 1702 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1703 n = 2; 1704 } 1705 #endif 1706 1707 /* Generate a new TB executing the I/O insn. */ 1708 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1709 1710 if (tb->cflags & CF_NOCACHE) { 1711 if (tb->orig_tb) { 1712 /* Invalidate original TB if this TB was generated in 1713 * cpu_exec_nocache() */ 1714 tb_phys_invalidate(tb->orig_tb, -1); 1715 } 1716 tcg_tb_remove(tb); 1717 } 1718 1719 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1720 * the first in the TB) then we end up generating a whole new TB and 1721 * repeating the fault, which is horribly inefficient. 1722 * Better would be to execute just this insn uncached, or generate a 1723 * second new TB. 1724 * 1725 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1726 * tb_lock gets reset. 1727 */ 1728 cpu_loop_exit_noexc(cpu); 1729 } 1730 1731 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1732 { 1733 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1734 1735 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1736 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1737 } 1738 } 1739 1740 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1741 { 1742 /* Discard jump cache entries for any tb which might potentially 1743 overlap the flushed page. */ 1744 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1745 tb_jmp_cache_clear_page(cpu, addr); 1746 } 1747 1748 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1749 struct qht_stats hst) 1750 { 1751 uint32_t hgram_opts; 1752 size_t hgram_bins; 1753 char *hgram; 1754 1755 if (!hst.head_buckets) { 1756 return; 1757 } 1758 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1759 hst.used_head_buckets, hst.head_buckets, 1760 (double)hst.used_head_buckets / hst.head_buckets * 100); 1761 1762 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1763 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1764 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1765 hgram_opts |= QDIST_PR_NODECIMAL; 1766 } 1767 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1768 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1769 qdist_avg(&hst.occupancy) * 100, hgram); 1770 g_free(hgram); 1771 1772 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1773 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1774 if (hgram_bins > 10) { 1775 hgram_bins = 10; 1776 } else { 1777 hgram_bins = 0; 1778 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1779 } 1780 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1781 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1782 qdist_avg(&hst.chain), hgram); 1783 g_free(hgram); 1784 } 1785 1786 struct tb_tree_stats { 1787 size_t nb_tbs; 1788 size_t host_size; 1789 size_t target_size; 1790 size_t max_target_size; 1791 size_t direct_jmp_count; 1792 size_t direct_jmp2_count; 1793 size_t cross_page; 1794 }; 1795 1796 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1797 { 1798 const TranslationBlock *tb = value; 1799 struct tb_tree_stats *tst = data; 1800 1801 tst->nb_tbs++; 1802 tst->host_size += tb->tc.size; 1803 tst->target_size += tb->size; 1804 if (tb->size > tst->max_target_size) { 1805 tst->max_target_size = tb->size; 1806 } 1807 if (tb->page_addr[1] != -1) { 1808 tst->cross_page++; 1809 } 1810 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1811 tst->direct_jmp_count++; 1812 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1813 tst->direct_jmp2_count++; 1814 } 1815 } 1816 return false; 1817 } 1818 1819 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1820 { 1821 struct tb_tree_stats tst = {}; 1822 struct qht_stats hst; 1823 size_t nb_tbs; 1824 1825 tcg_tb_foreach(tb_tree_stats_iter, &tst); 1826 nb_tbs = tst.nb_tbs; 1827 /* XXX: avoid using doubles ? */ 1828 cpu_fprintf(f, "Translation buffer state:\n"); 1829 /* 1830 * Report total code size including the padding and TB structs; 1831 * otherwise users might think "-tb-size" is not honoured. 1832 * For avg host size we use the precise numbers from tb_tree_stats though. 1833 */ 1834 cpu_fprintf(f, "gen code size %zu/%zu\n", 1835 tcg_code_size(), tcg_code_capacity()); 1836 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1837 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1838 nb_tbs ? tst.target_size / nb_tbs : 0, 1839 tst.max_target_size); 1840 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1841 nb_tbs ? tst.host_size / nb_tbs : 0, 1842 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1843 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1844 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1845 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1846 tst.direct_jmp_count, 1847 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1848 tst.direct_jmp2_count, 1849 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1850 1851 qht_statistics_init(&tb_ctx.htable, &hst); 1852 print_qht_statistics(f, cpu_fprintf, hst); 1853 qht_statistics_destroy(&hst); 1854 1855 cpu_fprintf(f, "\nStatistics:\n"); 1856 cpu_fprintf(f, "TB flush count %u\n", 1857 atomic_read(&tb_ctx.tb_flush_count)); 1858 cpu_fprintf(f, "TB invalidate count %d\n", tb_ctx.tb_phys_invalidate_count); 1859 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1860 tcg_dump_info(f, cpu_fprintf); 1861 } 1862 1863 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1864 { 1865 tcg_dump_op_count(f, cpu_fprintf); 1866 } 1867 1868 #else /* CONFIG_USER_ONLY */ 1869 1870 void cpu_interrupt(CPUState *cpu, int mask) 1871 { 1872 g_assert(qemu_mutex_iothread_locked()); 1873 cpu->interrupt_request |= mask; 1874 cpu->icount_decr.u16.high = -1; 1875 } 1876 1877 /* 1878 * Walks guest process memory "regions" one by one 1879 * and calls callback function 'fn' for each region. 1880 */ 1881 struct walk_memory_regions_data { 1882 walk_memory_regions_fn fn; 1883 void *priv; 1884 target_ulong start; 1885 int prot; 1886 }; 1887 1888 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1889 target_ulong end, int new_prot) 1890 { 1891 if (data->start != -1u) { 1892 int rc = data->fn(data->priv, data->start, end, data->prot); 1893 if (rc != 0) { 1894 return rc; 1895 } 1896 } 1897 1898 data->start = (new_prot ? end : -1u); 1899 data->prot = new_prot; 1900 1901 return 0; 1902 } 1903 1904 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1905 target_ulong base, int level, void **lp) 1906 { 1907 target_ulong pa; 1908 int i, rc; 1909 1910 if (*lp == NULL) { 1911 return walk_memory_regions_end(data, base, 0); 1912 } 1913 1914 if (level == 0) { 1915 PageDesc *pd = *lp; 1916 1917 for (i = 0; i < V_L2_SIZE; ++i) { 1918 int prot = pd[i].flags; 1919 1920 pa = base | (i << TARGET_PAGE_BITS); 1921 if (prot != data->prot) { 1922 rc = walk_memory_regions_end(data, pa, prot); 1923 if (rc != 0) { 1924 return rc; 1925 } 1926 } 1927 } 1928 } else { 1929 void **pp = *lp; 1930 1931 for (i = 0; i < V_L2_SIZE; ++i) { 1932 pa = base | ((target_ulong)i << 1933 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1934 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 1935 if (rc != 0) { 1936 return rc; 1937 } 1938 } 1939 } 1940 1941 return 0; 1942 } 1943 1944 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 1945 { 1946 struct walk_memory_regions_data data; 1947 uintptr_t i, l1_sz = v_l1_size; 1948 1949 data.fn = fn; 1950 data.priv = priv; 1951 data.start = -1u; 1952 data.prot = 0; 1953 1954 for (i = 0; i < l1_sz; i++) { 1955 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 1956 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 1957 if (rc != 0) { 1958 return rc; 1959 } 1960 } 1961 1962 return walk_memory_regions_end(&data, 0, 0); 1963 } 1964 1965 static int dump_region(void *priv, target_ulong start, 1966 target_ulong end, unsigned long prot) 1967 { 1968 FILE *f = (FILE *)priv; 1969 1970 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 1971 " "TARGET_FMT_lx" %c%c%c\n", 1972 start, end, end - start, 1973 ((prot & PAGE_READ) ? 'r' : '-'), 1974 ((prot & PAGE_WRITE) ? 'w' : '-'), 1975 ((prot & PAGE_EXEC) ? 'x' : '-')); 1976 1977 return 0; 1978 } 1979 1980 /* dump memory mappings */ 1981 void page_dump(FILE *f) 1982 { 1983 const int length = sizeof(target_ulong) * 2; 1984 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 1985 length, "start", length, "end", length, "size", "prot"); 1986 walk_memory_regions(f, dump_region); 1987 } 1988 1989 int page_get_flags(target_ulong address) 1990 { 1991 PageDesc *p; 1992 1993 p = page_find(address >> TARGET_PAGE_BITS); 1994 if (!p) { 1995 return 0; 1996 } 1997 return p->flags; 1998 } 1999 2000 /* Modify the flags of a page and invalidate the code if necessary. 2001 The flag PAGE_WRITE_ORG is positioned automatically depending 2002 on PAGE_WRITE. The mmap_lock should already be held. */ 2003 void page_set_flags(target_ulong start, target_ulong end, int flags) 2004 { 2005 target_ulong addr, len; 2006 2007 /* This function should never be called with addresses outside the 2008 guest address space. If this assert fires, it probably indicates 2009 a missing call to h2g_valid. */ 2010 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2011 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2012 #endif 2013 assert(start < end); 2014 assert_memory_lock(); 2015 2016 start = start & TARGET_PAGE_MASK; 2017 end = TARGET_PAGE_ALIGN(end); 2018 2019 if (flags & PAGE_WRITE) { 2020 flags |= PAGE_WRITE_ORG; 2021 } 2022 2023 for (addr = start, len = end - start; 2024 len != 0; 2025 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2026 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2027 2028 /* If the write protection bit is set, then we invalidate 2029 the code inside. */ 2030 if (!(p->flags & PAGE_WRITE) && 2031 (flags & PAGE_WRITE) && 2032 p->first_tb) { 2033 tb_invalidate_phys_page(addr, 0); 2034 } 2035 p->flags = flags; 2036 } 2037 } 2038 2039 int page_check_range(target_ulong start, target_ulong len, int flags) 2040 { 2041 PageDesc *p; 2042 target_ulong end; 2043 target_ulong addr; 2044 2045 /* This function should never be called with addresses outside the 2046 guest address space. If this assert fires, it probably indicates 2047 a missing call to h2g_valid. */ 2048 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2049 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2050 #endif 2051 2052 if (len == 0) { 2053 return 0; 2054 } 2055 if (start + len - 1 < start) { 2056 /* We've wrapped around. */ 2057 return -1; 2058 } 2059 2060 /* must do before we loose bits in the next step */ 2061 end = TARGET_PAGE_ALIGN(start + len); 2062 start = start & TARGET_PAGE_MASK; 2063 2064 for (addr = start, len = end - start; 2065 len != 0; 2066 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2067 p = page_find(addr >> TARGET_PAGE_BITS); 2068 if (!p) { 2069 return -1; 2070 } 2071 if (!(p->flags & PAGE_VALID)) { 2072 return -1; 2073 } 2074 2075 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2076 return -1; 2077 } 2078 if (flags & PAGE_WRITE) { 2079 if (!(p->flags & PAGE_WRITE_ORG)) { 2080 return -1; 2081 } 2082 /* unprotect the page if it was put read-only because it 2083 contains translated code */ 2084 if (!(p->flags & PAGE_WRITE)) { 2085 if (!page_unprotect(addr, 0)) { 2086 return -1; 2087 } 2088 } 2089 } 2090 } 2091 return 0; 2092 } 2093 2094 /* called from signal handler: invalidate the code and unprotect the 2095 * page. Return 0 if the fault was not handled, 1 if it was handled, 2096 * and 2 if it was handled but the caller must cause the TB to be 2097 * immediately exited. (We can only return 2 if the 'pc' argument is 2098 * non-zero.) 2099 */ 2100 int page_unprotect(target_ulong address, uintptr_t pc) 2101 { 2102 unsigned int prot; 2103 bool current_tb_invalidated; 2104 PageDesc *p; 2105 target_ulong host_start, host_end, addr; 2106 2107 /* Technically this isn't safe inside a signal handler. However we 2108 know this only ever happens in a synchronous SEGV handler, so in 2109 practice it seems to be ok. */ 2110 mmap_lock(); 2111 2112 p = page_find(address >> TARGET_PAGE_BITS); 2113 if (!p) { 2114 mmap_unlock(); 2115 return 0; 2116 } 2117 2118 /* if the page was really writable, then we change its 2119 protection back to writable */ 2120 if (p->flags & PAGE_WRITE_ORG) { 2121 current_tb_invalidated = false; 2122 if (p->flags & PAGE_WRITE) { 2123 /* If the page is actually marked WRITE then assume this is because 2124 * this thread raced with another one which got here first and 2125 * set the page to PAGE_WRITE and did the TB invalidate for us. 2126 */ 2127 #ifdef TARGET_HAS_PRECISE_SMC 2128 TranslationBlock *current_tb = tcg_tb_lookup(pc); 2129 if (current_tb) { 2130 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; 2131 } 2132 #endif 2133 } else { 2134 host_start = address & qemu_host_page_mask; 2135 host_end = host_start + qemu_host_page_size; 2136 2137 prot = 0; 2138 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) { 2139 p = page_find(addr >> TARGET_PAGE_BITS); 2140 p->flags |= PAGE_WRITE; 2141 prot |= p->flags; 2142 2143 /* and since the content will be modified, we must invalidate 2144 the corresponding translated code. */ 2145 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2146 #ifdef CONFIG_USER_ONLY 2147 if (DEBUG_TB_CHECK_GATE) { 2148 tb_invalidate_check(addr); 2149 } 2150 #endif 2151 } 2152 mprotect((void *)g2h(host_start), qemu_host_page_size, 2153 prot & PAGE_BITS); 2154 } 2155 mmap_unlock(); 2156 /* If current TB was invalidated return to main loop */ 2157 return current_tb_invalidated ? 2 : 1; 2158 } 2159 mmap_unlock(); 2160 return 0; 2161 } 2162 #endif /* CONFIG_USER_ONLY */ 2163 2164 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2165 void tcg_flush_softmmu_tlb(CPUState *cs) 2166 { 2167 #ifdef CONFIG_SOFTMMU 2168 tlb_flush(cs); 2169 #endif 2170 } 2171