1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 35 #include <sys/param.h> 36 #if __FreeBSD_version >= 700104 37 #define HAVE_KINFO_GETVMMAP 38 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 39 #include <sys/proc.h> 40 #include <machine/profile.h> 41 #define _KERNEL 42 #include <sys/user.h> 43 #undef _KERNEL 44 #undef sigqueue 45 #include <libutil.h> 46 #endif 47 #endif 48 #else 49 #include "exec/address-spaces.h" 50 #endif 51 52 #include "exec/cputlb.h" 53 #include "exec/tb-hash.h" 54 #include "translate-all.h" 55 #include "qemu/bitmap.h" 56 #include "qemu/error-report.h" 57 #include "qemu/timer.h" 58 #include "qemu/main-loop.h" 59 #include "exec/log.h" 60 #include "sysemu/cpus.h" 61 62 /* #define DEBUG_TB_INVALIDATE */ 63 /* #define DEBUG_TB_FLUSH */ 64 /* make various TB consistency checks */ 65 /* #define DEBUG_TB_CHECK */ 66 67 #ifdef DEBUG_TB_INVALIDATE 68 #define DEBUG_TB_INVALIDATE_GATE 1 69 #else 70 #define DEBUG_TB_INVALIDATE_GATE 0 71 #endif 72 73 #ifdef DEBUG_TB_FLUSH 74 #define DEBUG_TB_FLUSH_GATE 1 75 #else 76 #define DEBUG_TB_FLUSH_GATE 0 77 #endif 78 79 #if !defined(CONFIG_USER_ONLY) 80 /* TB consistency checks only implemented for usermode emulation. */ 81 #undef DEBUG_TB_CHECK 82 #endif 83 84 #ifdef DEBUG_TB_CHECK 85 #define DEBUG_TB_CHECK_GATE 1 86 #else 87 #define DEBUG_TB_CHECK_GATE 0 88 #endif 89 90 /* Access to the various translations structures need to be serialised via locks 91 * for consistency. This is automatic for SoftMMU based system 92 * emulation due to its single threaded nature. In user-mode emulation 93 * access to the memory related structures are protected with the 94 * mmap_lock. 95 */ 96 #ifdef CONFIG_SOFTMMU 97 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 98 #else 99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 100 #endif 101 102 #define SMC_BITMAP_USE_THRESHOLD 10 103 104 typedef struct PageDesc { 105 /* list of TBs intersecting this ram page */ 106 uintptr_t first_tb; 107 #ifdef CONFIG_SOFTMMU 108 /* in order to optimize self modifying code, we count the number 109 of lookups we do to a given page to use a bitmap */ 110 unsigned long *code_bitmap; 111 unsigned int code_write_count; 112 #else 113 unsigned long flags; 114 #endif 115 } PageDesc; 116 117 /* list iterators for lists of tagged pointers in TranslationBlock */ 118 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \ 119 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \ 120 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \ 121 tb = (TranslationBlock *)((uintptr_t)tb & ~1)) 122 123 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \ 124 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next) 125 126 /* In system mode we want L1_MAP to be based on ram offsets, 127 while in user mode we want it to be based on virtual addresses. */ 128 #if !defined(CONFIG_USER_ONLY) 129 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 130 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 131 #else 132 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 133 #endif 134 #else 135 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 136 #endif 137 138 /* Size of the L2 (and L3, etc) page tables. */ 139 #define V_L2_BITS 10 140 #define V_L2_SIZE (1 << V_L2_BITS) 141 142 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 143 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 144 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 145 * BITS_PER_BYTE); 146 147 /* 148 * L1 Mapping properties 149 */ 150 static int v_l1_size; 151 static int v_l1_shift; 152 static int v_l2_levels; 153 154 /* The bottom level has pointers to PageDesc, and is indexed by 155 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 156 */ 157 #define V_L1_MIN_BITS 4 158 #define V_L1_MAX_BITS (V_L2_BITS + 3) 159 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 160 161 static void *l1_map[V_L1_MAX_SIZE]; 162 163 /* code generation context */ 164 TCGContext tcg_init_ctx; 165 __thread TCGContext *tcg_ctx; 166 TBContext tb_ctx; 167 bool parallel_cpus; 168 169 /* translation block context */ 170 static __thread int have_tb_lock; 171 172 static void page_table_config_init(void) 173 { 174 uint32_t v_l1_bits; 175 176 assert(TARGET_PAGE_BITS); 177 /* The bits remaining after N lower levels of page tables. */ 178 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 179 if (v_l1_bits < V_L1_MIN_BITS) { 180 v_l1_bits += V_L2_BITS; 181 } 182 183 v_l1_size = 1 << v_l1_bits; 184 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 185 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 186 187 assert(v_l1_bits <= V_L1_MAX_BITS); 188 assert(v_l1_shift % V_L2_BITS == 0); 189 assert(v_l2_levels >= 0); 190 } 191 192 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 193 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 194 195 void tb_lock(void) 196 { 197 assert_tb_unlocked(); 198 qemu_mutex_lock(&tb_ctx.tb_lock); 199 have_tb_lock++; 200 } 201 202 void tb_unlock(void) 203 { 204 assert_tb_locked(); 205 have_tb_lock--; 206 qemu_mutex_unlock(&tb_ctx.tb_lock); 207 } 208 209 void tb_lock_reset(void) 210 { 211 if (have_tb_lock) { 212 qemu_mutex_unlock(&tb_ctx.tb_lock); 213 have_tb_lock = 0; 214 } 215 } 216 217 void cpu_gen_init(void) 218 { 219 tcg_context_init(&tcg_init_ctx); 220 } 221 222 /* Encode VAL as a signed leb128 sequence at P. 223 Return P incremented past the encoded value. */ 224 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 225 { 226 int more, byte; 227 228 do { 229 byte = val & 0x7f; 230 val >>= 7; 231 more = !((val == 0 && (byte & 0x40) == 0) 232 || (val == -1 && (byte & 0x40) != 0)); 233 if (more) { 234 byte |= 0x80; 235 } 236 *p++ = byte; 237 } while (more); 238 239 return p; 240 } 241 242 /* Decode a signed leb128 sequence at *PP; increment *PP past the 243 decoded value. Return the decoded value. */ 244 static target_long decode_sleb128(uint8_t **pp) 245 { 246 uint8_t *p = *pp; 247 target_long val = 0; 248 int byte, shift = 0; 249 250 do { 251 byte = *p++; 252 val |= (target_ulong)(byte & 0x7f) << shift; 253 shift += 7; 254 } while (byte & 0x80); 255 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 256 val |= -(target_ulong)1 << shift; 257 } 258 259 *pp = p; 260 return val; 261 } 262 263 /* Encode the data collected about the instructions while compiling TB. 264 Place the data at BLOCK, and return the number of bytes consumed. 265 266 The logical table consists of TARGET_INSN_START_WORDS target_ulong's, 267 which come from the target's insn_start data, followed by a uintptr_t 268 which comes from the host pc of the end of the code implementing the insn. 269 270 Each line of the table is encoded as sleb128 deltas from the previous 271 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 272 That is, the first column is seeded with the guest pc, the last column 273 with the host pc, and the middle columns with zeros. */ 274 275 static int encode_search(TranslationBlock *tb, uint8_t *block) 276 { 277 uint8_t *highwater = tcg_ctx->code_gen_highwater; 278 uint8_t *p = block; 279 int i, j, n; 280 281 for (i = 0, n = tb->icount; i < n; ++i) { 282 target_ulong prev; 283 284 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 285 if (i == 0) { 286 prev = (j == 0 ? tb->pc : 0); 287 } else { 288 prev = tcg_ctx->gen_insn_data[i - 1][j]; 289 } 290 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 291 } 292 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 293 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 294 295 /* Test for (pending) buffer overflow. The assumption is that any 296 one row beginning below the high water mark cannot overrun 297 the buffer completely. Thus we can test for overflow after 298 encoding a row without having to check during encoding. */ 299 if (unlikely(p > highwater)) { 300 return -1; 301 } 302 } 303 304 return p - block; 305 } 306 307 /* The cpu state corresponding to 'searched_pc' is restored. 308 * Called with tb_lock held. 309 * When reset_icount is true, current TB will be interrupted and 310 * icount should be recalculated. 311 */ 312 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 313 uintptr_t searched_pc, bool reset_icount) 314 { 315 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 316 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 317 CPUArchState *env = cpu->env_ptr; 318 uint8_t *p = tb->tc.ptr + tb->tc.size; 319 int i, j, num_insns = tb->icount; 320 #ifdef CONFIG_PROFILER 321 TCGProfile *prof = &tcg_ctx->prof; 322 int64_t ti = profile_getclock(); 323 #endif 324 325 searched_pc -= GETPC_ADJ; 326 327 if (searched_pc < host_pc) { 328 return -1; 329 } 330 331 /* Reconstruct the stored insn data while looking for the point at 332 which the end of the insn exceeds the searched_pc. */ 333 for (i = 0; i < num_insns; ++i) { 334 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 335 data[j] += decode_sleb128(&p); 336 } 337 host_pc += decode_sleb128(&p); 338 if (host_pc > searched_pc) { 339 goto found; 340 } 341 } 342 return -1; 343 344 found: 345 if (reset_icount && (tb->cflags & CF_USE_ICOUNT)) { 346 assert(use_icount); 347 /* Reset the cycle counter to the start of the block 348 and shift if to the number of actually executed instructions */ 349 cpu->icount_decr.u16.low += num_insns - i; 350 } 351 restore_state_to_opc(env, tb, data); 352 353 #ifdef CONFIG_PROFILER 354 atomic_set(&prof->restore_time, 355 prof->restore_time + profile_getclock() - ti); 356 atomic_set(&prof->restore_count, prof->restore_count + 1); 357 #endif 358 return 0; 359 } 360 361 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) 362 { 363 TranslationBlock *tb; 364 bool r = false; 365 uintptr_t check_offset; 366 367 /* The host_pc has to be in the region of current code buffer. If 368 * it is not we will not be able to resolve it here. The two cases 369 * where host_pc will not be correct are: 370 * 371 * - fault during translation (instruction fetch) 372 * - fault from helper (not using GETPC() macro) 373 * 374 * Either way we need return early to avoid blowing up on a 375 * recursive tb_lock() as we can't resolve it here. 376 * 377 * We are using unsigned arithmetic so if host_pc < 378 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way 379 * above the code_gen_buffer_size 380 */ 381 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; 382 383 if (check_offset < tcg_init_ctx.code_gen_buffer_size) { 384 tb_lock(); 385 tb = tcg_tb_lookup(host_pc); 386 if (tb) { 387 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); 388 if (tb->cflags & CF_NOCACHE) { 389 /* one-shot translation, invalidate it immediately */ 390 tb_phys_invalidate(tb, -1); 391 tcg_tb_remove(tb); 392 } 393 r = true; 394 } 395 tb_unlock(); 396 } 397 398 return r; 399 } 400 401 static void page_init(void) 402 { 403 page_size_init(); 404 page_table_config_init(); 405 406 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 407 { 408 #ifdef HAVE_KINFO_GETVMMAP 409 struct kinfo_vmentry *freep; 410 int i, cnt; 411 412 freep = kinfo_getvmmap(getpid(), &cnt); 413 if (freep) { 414 mmap_lock(); 415 for (i = 0; i < cnt; i++) { 416 unsigned long startaddr, endaddr; 417 418 startaddr = freep[i].kve_start; 419 endaddr = freep[i].kve_end; 420 if (h2g_valid(startaddr)) { 421 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 422 423 if (h2g_valid(endaddr)) { 424 endaddr = h2g(endaddr); 425 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 426 } else { 427 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 428 endaddr = ~0ul; 429 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 430 #endif 431 } 432 } 433 } 434 free(freep); 435 mmap_unlock(); 436 } 437 #else 438 FILE *f; 439 440 last_brk = (unsigned long)sbrk(0); 441 442 f = fopen("/compat/linux/proc/self/maps", "r"); 443 if (f) { 444 mmap_lock(); 445 446 do { 447 unsigned long startaddr, endaddr; 448 int n; 449 450 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 451 452 if (n == 2 && h2g_valid(startaddr)) { 453 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 454 455 if (h2g_valid(endaddr)) { 456 endaddr = h2g(endaddr); 457 } else { 458 endaddr = ~0ul; 459 } 460 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 461 } 462 } while (!feof(f)); 463 464 fclose(f); 465 mmap_unlock(); 466 } 467 #endif 468 } 469 #endif 470 } 471 472 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 473 { 474 PageDesc *pd; 475 void **lp; 476 int i; 477 478 /* Level 1. Always allocated. */ 479 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 480 481 /* Level 2..N-1. */ 482 for (i = v_l2_levels; i > 0; i--) { 483 void **p = atomic_rcu_read(lp); 484 485 if (p == NULL) { 486 void *existing; 487 488 if (!alloc) { 489 return NULL; 490 } 491 p = g_new0(void *, V_L2_SIZE); 492 existing = atomic_cmpxchg(lp, NULL, p); 493 if (unlikely(existing)) { 494 g_free(p); 495 p = existing; 496 } 497 } 498 499 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 500 } 501 502 pd = atomic_rcu_read(lp); 503 if (pd == NULL) { 504 void *existing; 505 506 if (!alloc) { 507 return NULL; 508 } 509 pd = g_new0(PageDesc, V_L2_SIZE); 510 existing = atomic_cmpxchg(lp, NULL, pd); 511 if (unlikely(existing)) { 512 g_free(pd); 513 pd = existing; 514 } 515 } 516 517 return pd + (index & (V_L2_SIZE - 1)); 518 } 519 520 static inline PageDesc *page_find(tb_page_addr_t index) 521 { 522 return page_find_alloc(index, 0); 523 } 524 525 #if defined(CONFIG_USER_ONLY) 526 /* Currently it is not recommended to allocate big chunks of data in 527 user mode. It will change when a dedicated libc will be used. */ 528 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 529 region in which the guest needs to run. Revisit this. */ 530 #define USE_STATIC_CODE_GEN_BUFFER 531 #endif 532 533 /* Minimum size of the code gen buffer. This number is randomly chosen, 534 but not so small that we can't have a fair number of TB's live. */ 535 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 536 537 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 538 indicated, this is constrained by the range of direct branches on the 539 host cpu, as used by the TCG implementation of goto_tb. */ 540 #if defined(__x86_64__) 541 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 542 #elif defined(__sparc__) 543 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 544 #elif defined(__powerpc64__) 545 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 546 #elif defined(__powerpc__) 547 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 548 #elif defined(__aarch64__) 549 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 550 #elif defined(__s390x__) 551 /* We have a +- 4GB range on the branches; leave some slop. */ 552 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 553 #elif defined(__mips__) 554 /* We have a 256MB branch region, but leave room to make sure the 555 main executable is also within that region. */ 556 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 557 #else 558 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 559 #endif 560 561 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 562 563 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 564 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 565 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 566 567 static inline size_t size_code_gen_buffer(size_t tb_size) 568 { 569 /* Size the buffer. */ 570 if (tb_size == 0) { 571 #ifdef USE_STATIC_CODE_GEN_BUFFER 572 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 573 #else 574 /* ??? Needs adjustments. */ 575 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 576 static buffer, we could size this on RESERVED_VA, on the text 577 segment size of the executable, or continue to use the default. */ 578 tb_size = (unsigned long)(ram_size / 4); 579 #endif 580 } 581 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 582 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 583 } 584 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 585 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 586 } 587 return tb_size; 588 } 589 590 #ifdef __mips__ 591 /* In order to use J and JAL within the code_gen_buffer, we require 592 that the buffer not cross a 256MB boundary. */ 593 static inline bool cross_256mb(void *addr, size_t size) 594 { 595 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 596 } 597 598 /* We weren't able to allocate a buffer without crossing that boundary, 599 so make do with the larger portion of the buffer that doesn't cross. 600 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 601 static inline void *split_cross_256mb(void *buf1, size_t size1) 602 { 603 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 604 size_t size2 = buf1 + size1 - buf2; 605 606 size1 = buf2 - buf1; 607 if (size1 < size2) { 608 size1 = size2; 609 buf1 = buf2; 610 } 611 612 tcg_ctx->code_gen_buffer_size = size1; 613 return buf1; 614 } 615 #endif 616 617 #ifdef USE_STATIC_CODE_GEN_BUFFER 618 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 619 __attribute__((aligned(CODE_GEN_ALIGN))); 620 621 static inline void *alloc_code_gen_buffer(void) 622 { 623 void *buf = static_code_gen_buffer; 624 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 625 size_t size; 626 627 /* page-align the beginning and end of the buffer */ 628 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 629 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 630 631 size = end - buf; 632 633 /* Honor a command-line option limiting the size of the buffer. */ 634 if (size > tcg_ctx->code_gen_buffer_size) { 635 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 636 qemu_real_host_page_size); 637 } 638 tcg_ctx->code_gen_buffer_size = size; 639 640 #ifdef __mips__ 641 if (cross_256mb(buf, size)) { 642 buf = split_cross_256mb(buf, size); 643 size = tcg_ctx->code_gen_buffer_size; 644 } 645 #endif 646 647 if (qemu_mprotect_rwx(buf, size)) { 648 abort(); 649 } 650 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 651 652 return buf; 653 } 654 #elif defined(_WIN32) 655 static inline void *alloc_code_gen_buffer(void) 656 { 657 size_t size = tcg_ctx->code_gen_buffer_size; 658 return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 659 PAGE_EXECUTE_READWRITE); 660 } 661 #else 662 static inline void *alloc_code_gen_buffer(void) 663 { 664 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 665 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 666 uintptr_t start = 0; 667 size_t size = tcg_ctx->code_gen_buffer_size; 668 void *buf; 669 670 /* Constrain the position of the buffer based on the host cpu. 671 Note that these addresses are chosen in concert with the 672 addresses assigned in the relevant linker script file. */ 673 # if defined(__PIE__) || defined(__PIC__) 674 /* Don't bother setting a preferred location if we're building 675 a position-independent executable. We're more likely to get 676 an address near the main executable if we let the kernel 677 choose the address. */ 678 # elif defined(__x86_64__) && defined(MAP_32BIT) 679 /* Force the memory down into low memory with the executable. 680 Leave the choice of exact location with the kernel. */ 681 flags |= MAP_32BIT; 682 /* Cannot expect to map more than 800MB in low memory. */ 683 if (size > 800u * 1024 * 1024) { 684 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 685 } 686 # elif defined(__sparc__) 687 start = 0x40000000ul; 688 # elif defined(__s390x__) 689 start = 0x90000000ul; 690 # elif defined(__mips__) 691 # if _MIPS_SIM == _ABI64 692 start = 0x128000000ul; 693 # else 694 start = 0x08000000ul; 695 # endif 696 # endif 697 698 buf = mmap((void *)start, size, prot, flags, -1, 0); 699 if (buf == MAP_FAILED) { 700 return NULL; 701 } 702 703 #ifdef __mips__ 704 if (cross_256mb(buf, size)) { 705 /* Try again, with the original still mapped, to avoid re-acquiring 706 that 256mb crossing. This time don't specify an address. */ 707 size_t size2; 708 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 709 switch ((int)(buf2 != MAP_FAILED)) { 710 case 1: 711 if (!cross_256mb(buf2, size)) { 712 /* Success! Use the new buffer. */ 713 munmap(buf, size); 714 break; 715 } 716 /* Failure. Work with what we had. */ 717 munmap(buf2, size); 718 /* fallthru */ 719 default: 720 /* Split the original buffer. Free the smaller half. */ 721 buf2 = split_cross_256mb(buf, size); 722 size2 = tcg_ctx->code_gen_buffer_size; 723 if (buf == buf2) { 724 munmap(buf + size2, size - size2); 725 } else { 726 munmap(buf, size - size2); 727 } 728 size = size2; 729 break; 730 } 731 buf = buf2; 732 } 733 #endif 734 735 /* Request large pages for the buffer. */ 736 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 737 738 return buf; 739 } 740 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 741 742 static inline void code_gen_alloc(size_t tb_size) 743 { 744 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 745 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 746 if (tcg_ctx->code_gen_buffer == NULL) { 747 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 748 exit(1); 749 } 750 qemu_mutex_init(&tb_ctx.tb_lock); 751 } 752 753 static bool tb_cmp(const void *ap, const void *bp) 754 { 755 const TranslationBlock *a = ap; 756 const TranslationBlock *b = bp; 757 758 return a->pc == b->pc && 759 a->cs_base == b->cs_base && 760 a->flags == b->flags && 761 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) && 762 a->trace_vcpu_dstate == b->trace_vcpu_dstate && 763 a->page_addr[0] == b->page_addr[0] && 764 a->page_addr[1] == b->page_addr[1]; 765 } 766 767 static void tb_htable_init(void) 768 { 769 unsigned int mode = QHT_MODE_AUTO_RESIZE; 770 771 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); 772 } 773 774 /* Must be called before using the QEMU cpus. 'tb_size' is the size 775 (in bytes) allocated to the translation buffer. Zero means default 776 size. */ 777 void tcg_exec_init(unsigned long tb_size) 778 { 779 tcg_allowed = true; 780 cpu_gen_init(); 781 page_init(); 782 tb_htable_init(); 783 code_gen_alloc(tb_size); 784 #if defined(CONFIG_SOFTMMU) 785 /* There's no guest base to take into account, so go ahead and 786 initialize the prologue now. */ 787 tcg_prologue_init(tcg_ctx); 788 #endif 789 } 790 791 /* 792 * Allocate a new translation block. Flush the translation buffer if 793 * too many translation blocks or too much generated code. 794 * 795 * Called with tb_lock held. 796 */ 797 static TranslationBlock *tb_alloc(target_ulong pc) 798 { 799 TranslationBlock *tb; 800 801 assert_tb_locked(); 802 803 tb = tcg_tb_alloc(tcg_ctx); 804 if (unlikely(tb == NULL)) { 805 return NULL; 806 } 807 return tb; 808 } 809 810 static inline void invalidate_page_bitmap(PageDesc *p) 811 { 812 #ifdef CONFIG_SOFTMMU 813 g_free(p->code_bitmap); 814 p->code_bitmap = NULL; 815 p->code_write_count = 0; 816 #endif 817 } 818 819 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 820 static void page_flush_tb_1(int level, void **lp) 821 { 822 int i; 823 824 if (*lp == NULL) { 825 return; 826 } 827 if (level == 0) { 828 PageDesc *pd = *lp; 829 830 for (i = 0; i < V_L2_SIZE; ++i) { 831 pd[i].first_tb = (uintptr_t)NULL; 832 invalidate_page_bitmap(pd + i); 833 } 834 } else { 835 void **pp = *lp; 836 837 for (i = 0; i < V_L2_SIZE; ++i) { 838 page_flush_tb_1(level - 1, pp + i); 839 } 840 } 841 } 842 843 static void page_flush_tb(void) 844 { 845 int i, l1_sz = v_l1_size; 846 847 for (i = 0; i < l1_sz; i++) { 848 page_flush_tb_1(v_l2_levels, l1_map + i); 849 } 850 } 851 852 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 853 { 854 const TranslationBlock *tb = value; 855 size_t *size = data; 856 857 *size += tb->tc.size; 858 return false; 859 } 860 861 /* flush all the translation blocks */ 862 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 863 { 864 tb_lock(); 865 866 /* If it is already been done on request of another CPU, 867 * just retry. 868 */ 869 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 870 goto done; 871 } 872 873 if (DEBUG_TB_FLUSH_GATE) { 874 size_t nb_tbs = tcg_nb_tbs(); 875 size_t host_size = 0; 876 877 tcg_tb_foreach(tb_host_size_iter, &host_size); 878 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 879 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 880 } 881 882 CPU_FOREACH(cpu) { 883 cpu_tb_jmp_cache_clear(cpu); 884 } 885 886 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 887 page_flush_tb(); 888 889 tcg_region_reset_all(); 890 /* XXX: flush processor icache at this point if cache flush is 891 expensive */ 892 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 893 894 done: 895 tb_unlock(); 896 } 897 898 void tb_flush(CPUState *cpu) 899 { 900 if (tcg_enabled()) { 901 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 902 async_safe_run_on_cpu(cpu, do_tb_flush, 903 RUN_ON_CPU_HOST_INT(tb_flush_count)); 904 } 905 } 906 907 /* 908 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 909 * so in order to prevent bit rot we compile them unconditionally in user-mode, 910 * and let the optimizer get rid of them by wrapping their user-only callers 911 * with if (DEBUG_TB_CHECK_GATE). 912 */ 913 #ifdef CONFIG_USER_ONLY 914 915 static void 916 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 917 { 918 TranslationBlock *tb = p; 919 target_ulong addr = *(target_ulong *)userp; 920 921 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 922 printf("ERROR invalidate: address=" TARGET_FMT_lx 923 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 924 } 925 } 926 927 /* verify that all the pages have correct rights for code 928 * 929 * Called with tb_lock held. 930 */ 931 static void tb_invalidate_check(target_ulong address) 932 { 933 address &= TARGET_PAGE_MASK; 934 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 935 } 936 937 static void 938 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 939 { 940 TranslationBlock *tb = p; 941 int flags1, flags2; 942 943 flags1 = page_get_flags(tb->pc); 944 flags2 = page_get_flags(tb->pc + tb->size - 1); 945 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 946 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 947 (long)tb->pc, tb->size, flags1, flags2); 948 } 949 } 950 951 /* verify that all the pages have correct rights for code */ 952 static void tb_page_check(void) 953 { 954 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 955 } 956 957 #endif /* CONFIG_USER_ONLY */ 958 959 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb) 960 { 961 TranslationBlock *tb1; 962 uintptr_t *pprev; 963 unsigned int n1; 964 965 pprev = &pd->first_tb; 966 PAGE_FOR_EACH_TB(pd, tb1, n1) { 967 if (tb1 == tb) { 968 *pprev = tb1->page_next[n1]; 969 return; 970 } 971 pprev = &tb1->page_next[n1]; 972 } 973 g_assert_not_reached(); 974 } 975 976 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 977 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 978 { 979 TranslationBlock *tb1; 980 uintptr_t *ptb, ntb; 981 unsigned int n1; 982 983 ptb = &tb->jmp_list_next[n]; 984 if (*ptb) { 985 /* find tb(n) in circular list */ 986 for (;;) { 987 ntb = *ptb; 988 n1 = ntb & 3; 989 tb1 = (TranslationBlock *)(ntb & ~3); 990 if (n1 == n && tb1 == tb) { 991 break; 992 } 993 if (n1 == 2) { 994 ptb = &tb1->jmp_list_first; 995 } else { 996 ptb = &tb1->jmp_list_next[n1]; 997 } 998 } 999 /* now we can suppress tb(n) from the list */ 1000 *ptb = tb->jmp_list_next[n]; 1001 1002 tb->jmp_list_next[n] = (uintptr_t)NULL; 1003 } 1004 } 1005 1006 /* reset the jump entry 'n' of a TB so that it is not chained to 1007 another TB */ 1008 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1009 { 1010 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1011 tb_set_jmp_target(tb, n, addr); 1012 } 1013 1014 /* remove any jumps to the TB */ 1015 static inline void tb_jmp_unlink(TranslationBlock *tb) 1016 { 1017 TranslationBlock *tb1; 1018 uintptr_t *ptb, ntb; 1019 unsigned int n1; 1020 1021 ptb = &tb->jmp_list_first; 1022 for (;;) { 1023 ntb = *ptb; 1024 n1 = ntb & 3; 1025 tb1 = (TranslationBlock *)(ntb & ~3); 1026 if (n1 == 2) { 1027 break; 1028 } 1029 tb_reset_jump(tb1, n1); 1030 *ptb = tb1->jmp_list_next[n1]; 1031 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1032 } 1033 } 1034 1035 /* invalidate one TB 1036 * 1037 * Called with tb_lock held. 1038 */ 1039 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1040 { 1041 CPUState *cpu; 1042 PageDesc *p; 1043 uint32_t h; 1044 tb_page_addr_t phys_pc; 1045 1046 assert_tb_locked(); 1047 1048 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1049 1050 /* remove the TB from the hash list */ 1051 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1052 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1053 tb->trace_vcpu_dstate); 1054 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1055 return; 1056 } 1057 1058 /* remove the TB from the page list */ 1059 if (tb->page_addr[0] != page_addr) { 1060 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1061 tb_page_remove(p, tb); 1062 invalidate_page_bitmap(p); 1063 } 1064 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1065 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1066 tb_page_remove(p, tb); 1067 invalidate_page_bitmap(p); 1068 } 1069 1070 /* remove the TB from the hash list */ 1071 h = tb_jmp_cache_hash_func(tb->pc); 1072 CPU_FOREACH(cpu) { 1073 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1074 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1075 } 1076 } 1077 1078 /* suppress this TB from the two jump lists */ 1079 tb_remove_from_jmp_list(tb, 0); 1080 tb_remove_from_jmp_list(tb, 1); 1081 1082 /* suppress any remaining jumps to this TB */ 1083 tb_jmp_unlink(tb); 1084 1085 atomic_set(&tcg_ctx->tb_phys_invalidate_count, 1086 tcg_ctx->tb_phys_invalidate_count + 1); 1087 } 1088 1089 #ifdef CONFIG_SOFTMMU 1090 static void build_page_bitmap(PageDesc *p) 1091 { 1092 int n, tb_start, tb_end; 1093 TranslationBlock *tb; 1094 1095 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1096 1097 PAGE_FOR_EACH_TB(p, tb, n) { 1098 /* NOTE: this is subtle as a TB may span two physical pages */ 1099 if (n == 0) { 1100 /* NOTE: tb_end may be after the end of the page, but 1101 it is not a problem */ 1102 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1103 tb_end = tb_start + tb->size; 1104 if (tb_end > TARGET_PAGE_SIZE) { 1105 tb_end = TARGET_PAGE_SIZE; 1106 } 1107 } else { 1108 tb_start = 0; 1109 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1110 } 1111 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1112 } 1113 } 1114 #endif 1115 1116 /* add the tb in the target page and protect it if necessary 1117 * 1118 * Called with mmap_lock held for user-mode emulation. 1119 */ 1120 static inline void tb_alloc_page(TranslationBlock *tb, 1121 unsigned int n, tb_page_addr_t page_addr) 1122 { 1123 PageDesc *p; 1124 #ifndef CONFIG_USER_ONLY 1125 bool page_already_protected; 1126 #endif 1127 1128 assert_memory_lock(); 1129 1130 tb->page_addr[n] = page_addr; 1131 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1132 tb->page_next[n] = p->first_tb; 1133 #ifndef CONFIG_USER_ONLY 1134 page_already_protected = p->first_tb != (uintptr_t)NULL; 1135 #endif 1136 p->first_tb = (uintptr_t)tb | n; 1137 invalidate_page_bitmap(p); 1138 1139 #if defined(CONFIG_USER_ONLY) 1140 if (p->flags & PAGE_WRITE) { 1141 target_ulong addr; 1142 PageDesc *p2; 1143 int prot; 1144 1145 /* force the host page as non writable (writes will have a 1146 page fault + mprotect overhead) */ 1147 page_addr &= qemu_host_page_mask; 1148 prot = 0; 1149 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1150 addr += TARGET_PAGE_SIZE) { 1151 1152 p2 = page_find(addr >> TARGET_PAGE_BITS); 1153 if (!p2) { 1154 continue; 1155 } 1156 prot |= p2->flags; 1157 p2->flags &= ~PAGE_WRITE; 1158 } 1159 mprotect(g2h(page_addr), qemu_host_page_size, 1160 (prot & PAGE_BITS) & ~PAGE_WRITE); 1161 if (DEBUG_TB_INVALIDATE_GATE) { 1162 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1163 } 1164 } 1165 #else 1166 /* if some code is already present, then the pages are already 1167 protected. So we handle the case where only the first TB is 1168 allocated in a physical page */ 1169 if (!page_already_protected) { 1170 tlb_protect_code(page_addr); 1171 } 1172 #endif 1173 } 1174 1175 /* add a new TB and link it to the physical page tables. phys_page2 is 1176 * (-1) to indicate that only one page contains the TB. 1177 * 1178 * Called with mmap_lock held for user-mode emulation. 1179 */ 1180 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1181 tb_page_addr_t phys_page2) 1182 { 1183 uint32_t h; 1184 1185 assert_memory_lock(); 1186 1187 /* add in the page list */ 1188 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1189 if (phys_page2 != -1) { 1190 tb_alloc_page(tb, 1, phys_page2); 1191 } else { 1192 tb->page_addr[1] = -1; 1193 } 1194 1195 /* add in the hash table */ 1196 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1197 tb->trace_vcpu_dstate); 1198 qht_insert(&tb_ctx.htable, tb, h, NULL); 1199 1200 #ifdef CONFIG_USER_ONLY 1201 if (DEBUG_TB_CHECK_GATE) { 1202 tb_page_check(); 1203 } 1204 #endif 1205 } 1206 1207 /* Called with mmap_lock held for user mode emulation. */ 1208 TranslationBlock *tb_gen_code(CPUState *cpu, 1209 target_ulong pc, target_ulong cs_base, 1210 uint32_t flags, int cflags) 1211 { 1212 CPUArchState *env = cpu->env_ptr; 1213 TranslationBlock *tb; 1214 tb_page_addr_t phys_pc, phys_page2; 1215 target_ulong virt_page2; 1216 tcg_insn_unit *gen_code_buf; 1217 int gen_code_size, search_size; 1218 #ifdef CONFIG_PROFILER 1219 TCGProfile *prof = &tcg_ctx->prof; 1220 int64_t ti; 1221 #endif 1222 assert_memory_lock(); 1223 1224 phys_pc = get_page_addr_code(env, pc); 1225 1226 buffer_overflow: 1227 tb = tb_alloc(pc); 1228 if (unlikely(!tb)) { 1229 /* flush must be done */ 1230 tb_flush(cpu); 1231 mmap_unlock(); 1232 /* Make the execution loop process the flush as soon as possible. */ 1233 cpu->exception_index = EXCP_INTERRUPT; 1234 cpu_loop_exit(cpu); 1235 } 1236 1237 gen_code_buf = tcg_ctx->code_gen_ptr; 1238 tb->tc.ptr = gen_code_buf; 1239 tb->pc = pc; 1240 tb->cs_base = cs_base; 1241 tb->flags = flags; 1242 tb->cflags = cflags; 1243 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1244 tcg_ctx->tb_cflags = cflags; 1245 1246 #ifdef CONFIG_PROFILER 1247 /* includes aborted translations because of exceptions */ 1248 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1249 ti = profile_getclock(); 1250 #endif 1251 1252 tcg_func_start(tcg_ctx); 1253 1254 tcg_ctx->cpu = ENV_GET_CPU(env); 1255 gen_intermediate_code(cpu, tb); 1256 tcg_ctx->cpu = NULL; 1257 1258 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1259 1260 /* generate machine code */ 1261 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1262 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1263 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1264 if (TCG_TARGET_HAS_direct_jump) { 1265 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1266 tcg_ctx->tb_jmp_target_addr = NULL; 1267 } else { 1268 tcg_ctx->tb_jmp_insn_offset = NULL; 1269 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1270 } 1271 1272 #ifdef CONFIG_PROFILER 1273 atomic_set(&prof->tb_count, prof->tb_count + 1); 1274 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1275 ti = profile_getclock(); 1276 #endif 1277 1278 /* ??? Overflow could be handled better here. In particular, we 1279 don't need to re-do gen_intermediate_code, nor should we re-do 1280 the tcg optimization currently hidden inside tcg_gen_code. All 1281 that should be required is to flush the TBs, allocate a new TB, 1282 re-initialize it per above, and re-do the actual code generation. */ 1283 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1284 if (unlikely(gen_code_size < 0)) { 1285 goto buffer_overflow; 1286 } 1287 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1288 if (unlikely(search_size < 0)) { 1289 goto buffer_overflow; 1290 } 1291 tb->tc.size = gen_code_size; 1292 1293 #ifdef CONFIG_PROFILER 1294 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1295 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1296 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1297 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1298 #endif 1299 1300 #ifdef DEBUG_DISAS 1301 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1302 qemu_log_in_addr_range(tb->pc)) { 1303 qemu_log_lock(); 1304 qemu_log("OUT: [size=%d]\n", gen_code_size); 1305 if (tcg_ctx->data_gen_ptr) { 1306 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1307 size_t data_size = gen_code_size - code_size; 1308 size_t i; 1309 1310 log_disas(tb->tc.ptr, code_size); 1311 1312 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1313 if (sizeof(tcg_target_ulong) == 8) { 1314 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1315 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1316 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1317 } else { 1318 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1319 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1320 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1321 } 1322 } 1323 } else { 1324 log_disas(tb->tc.ptr, gen_code_size); 1325 } 1326 qemu_log("\n"); 1327 qemu_log_flush(); 1328 qemu_log_unlock(); 1329 } 1330 #endif 1331 1332 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1333 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1334 CODE_GEN_ALIGN)); 1335 1336 /* init jump list */ 1337 assert(((uintptr_t)tb & 3) == 0); 1338 tb->jmp_list_first = (uintptr_t)tb | 2; 1339 tb->jmp_list_next[0] = (uintptr_t)NULL; 1340 tb->jmp_list_next[1] = (uintptr_t)NULL; 1341 1342 /* init original jump addresses wich has been set during tcg_gen_code() */ 1343 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1344 tb_reset_jump(tb, 0); 1345 } 1346 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1347 tb_reset_jump(tb, 1); 1348 } 1349 1350 /* check next page if needed */ 1351 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1352 phys_page2 = -1; 1353 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1354 phys_page2 = get_page_addr_code(env, virt_page2); 1355 } 1356 /* As long as consistency of the TB stuff is provided by tb_lock in user 1357 * mode and is implicit in single-threaded softmmu emulation, no explicit 1358 * memory barrier is required before tb_link_page() makes the TB visible 1359 * through the physical hash table and physical page list. 1360 */ 1361 tb_link_page(tb, phys_pc, phys_page2); 1362 tcg_tb_insert(tb); 1363 return tb; 1364 } 1365 1366 /* 1367 * Invalidate all TBs which intersect with the target physical address range 1368 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1369 * 'is_cpu_write_access' should be true if called from a real cpu write 1370 * access: the virtual CPU will exit the current TB if code is modified inside 1371 * this TB. 1372 * 1373 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1374 * Called with tb_lock held for system-mode emulation 1375 */ 1376 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1377 { 1378 while (start < end) { 1379 tb_invalidate_phys_page_range(start, end, 0); 1380 start &= TARGET_PAGE_MASK; 1381 start += TARGET_PAGE_SIZE; 1382 } 1383 } 1384 1385 #ifdef CONFIG_SOFTMMU 1386 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1387 { 1388 assert_tb_locked(); 1389 tb_invalidate_phys_range_1(start, end); 1390 } 1391 #else 1392 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1393 { 1394 assert_memory_lock(); 1395 tb_lock(); 1396 tb_invalidate_phys_range_1(start, end); 1397 tb_unlock(); 1398 } 1399 #endif 1400 /* 1401 * Invalidate all TBs which intersect with the target physical address range 1402 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1403 * 'is_cpu_write_access' should be true if called from a real cpu write 1404 * access: the virtual CPU will exit the current TB if code is modified inside 1405 * this TB. 1406 * 1407 * Called with tb_lock/mmap_lock held for user-mode emulation 1408 * Called with tb_lock held for system-mode emulation 1409 */ 1410 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1411 int is_cpu_write_access) 1412 { 1413 TranslationBlock *tb; 1414 tb_page_addr_t tb_start, tb_end; 1415 PageDesc *p; 1416 int n; 1417 #ifdef TARGET_HAS_PRECISE_SMC 1418 CPUState *cpu = current_cpu; 1419 CPUArchState *env = NULL; 1420 int current_tb_not_found = is_cpu_write_access; 1421 TranslationBlock *current_tb = NULL; 1422 int current_tb_modified = 0; 1423 target_ulong current_pc = 0; 1424 target_ulong current_cs_base = 0; 1425 uint32_t current_flags = 0; 1426 #endif /* TARGET_HAS_PRECISE_SMC */ 1427 1428 assert_memory_lock(); 1429 assert_tb_locked(); 1430 1431 p = page_find(start >> TARGET_PAGE_BITS); 1432 if (!p) { 1433 return; 1434 } 1435 #if defined(TARGET_HAS_PRECISE_SMC) 1436 if (cpu != NULL) { 1437 env = cpu->env_ptr; 1438 } 1439 #endif 1440 1441 /* we remove all the TBs in the range [start, end[ */ 1442 /* XXX: see if in some cases it could be faster to invalidate all 1443 the code */ 1444 PAGE_FOR_EACH_TB(p, tb, n) { 1445 /* NOTE: this is subtle as a TB may span two physical pages */ 1446 if (n == 0) { 1447 /* NOTE: tb_end may be after the end of the page, but 1448 it is not a problem */ 1449 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1450 tb_end = tb_start + tb->size; 1451 } else { 1452 tb_start = tb->page_addr[1]; 1453 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1454 } 1455 if (!(tb_end <= start || tb_start >= end)) { 1456 #ifdef TARGET_HAS_PRECISE_SMC 1457 if (current_tb_not_found) { 1458 current_tb_not_found = 0; 1459 current_tb = NULL; 1460 if (cpu->mem_io_pc) { 1461 /* now we have a real cpu fault */ 1462 current_tb = tcg_tb_lookup(cpu->mem_io_pc); 1463 } 1464 } 1465 if (current_tb == tb && 1466 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1467 /* If we are modifying the current TB, we must stop 1468 its execution. We could be more precise by checking 1469 that the modification is after the current PC, but it 1470 would require a specialized function to partially 1471 restore the CPU state */ 1472 1473 current_tb_modified = 1; 1474 cpu_restore_state_from_tb(cpu, current_tb, 1475 cpu->mem_io_pc, true); 1476 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1477 ¤t_flags); 1478 } 1479 #endif /* TARGET_HAS_PRECISE_SMC */ 1480 tb_phys_invalidate(tb, -1); 1481 } 1482 } 1483 #if !defined(CONFIG_USER_ONLY) 1484 /* if no code remaining, no need to continue to use slow writes */ 1485 if (!p->first_tb) { 1486 invalidate_page_bitmap(p); 1487 tlb_unprotect_code(start); 1488 } 1489 #endif 1490 #ifdef TARGET_HAS_PRECISE_SMC 1491 if (current_tb_modified) { 1492 /* Force execution of one insn next time. */ 1493 cpu->cflags_next_tb = 1 | curr_cflags(); 1494 cpu_loop_exit_noexc(cpu); 1495 } 1496 #endif 1497 } 1498 1499 #ifdef CONFIG_SOFTMMU 1500 /* len must be <= 8 and start must be a multiple of len. 1501 * Called via softmmu_template.h when code areas are written to with 1502 * iothread mutex not held. 1503 */ 1504 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1505 { 1506 PageDesc *p; 1507 1508 #if 0 1509 if (1) { 1510 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1511 cpu_single_env->mem_io_vaddr, len, 1512 cpu_single_env->eip, 1513 cpu_single_env->eip + 1514 (intptr_t)cpu_single_env->segs[R_CS].base); 1515 } 1516 #endif 1517 assert_memory_lock(); 1518 1519 p = page_find(start >> TARGET_PAGE_BITS); 1520 if (!p) { 1521 return; 1522 } 1523 if (!p->code_bitmap && 1524 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1525 /* build code bitmap. FIXME: writes should be protected by 1526 * tb_lock, reads by tb_lock or RCU. 1527 */ 1528 build_page_bitmap(p); 1529 } 1530 if (p->code_bitmap) { 1531 unsigned int nr; 1532 unsigned long b; 1533 1534 nr = start & ~TARGET_PAGE_MASK; 1535 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1536 if (b & ((1 << len) - 1)) { 1537 goto do_invalidate; 1538 } 1539 } else { 1540 do_invalidate: 1541 tb_invalidate_phys_page_range(start, start + len, 1); 1542 } 1543 } 1544 #else 1545 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1546 * host PC of the faulting store instruction that caused this invalidate. 1547 * Returns true if the caller needs to abort execution of the current 1548 * TB (because it was modified by this store and the guest CPU has 1549 * precise-SMC semantics). 1550 */ 1551 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1552 { 1553 TranslationBlock *tb; 1554 PageDesc *p; 1555 int n; 1556 #ifdef TARGET_HAS_PRECISE_SMC 1557 TranslationBlock *current_tb = NULL; 1558 CPUState *cpu = current_cpu; 1559 CPUArchState *env = NULL; 1560 int current_tb_modified = 0; 1561 target_ulong current_pc = 0; 1562 target_ulong current_cs_base = 0; 1563 uint32_t current_flags = 0; 1564 #endif 1565 1566 assert_memory_lock(); 1567 1568 addr &= TARGET_PAGE_MASK; 1569 p = page_find(addr >> TARGET_PAGE_BITS); 1570 if (!p) { 1571 return false; 1572 } 1573 1574 tb_lock(); 1575 #ifdef TARGET_HAS_PRECISE_SMC 1576 if (p->first_tb && pc != 0) { 1577 current_tb = tcg_tb_lookup(pc); 1578 } 1579 if (cpu != NULL) { 1580 env = cpu->env_ptr; 1581 } 1582 #endif 1583 PAGE_FOR_EACH_TB(p, tb, n) { 1584 #ifdef TARGET_HAS_PRECISE_SMC 1585 if (current_tb == tb && 1586 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1587 /* If we are modifying the current TB, we must stop 1588 its execution. We could be more precise by checking 1589 that the modification is after the current PC, but it 1590 would require a specialized function to partially 1591 restore the CPU state */ 1592 1593 current_tb_modified = 1; 1594 cpu_restore_state_from_tb(cpu, current_tb, pc, true); 1595 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1596 ¤t_flags); 1597 } 1598 #endif /* TARGET_HAS_PRECISE_SMC */ 1599 tb_phys_invalidate(tb, addr); 1600 } 1601 p->first_tb = (uintptr_t)NULL; 1602 #ifdef TARGET_HAS_PRECISE_SMC 1603 if (current_tb_modified) { 1604 /* Force execution of one insn next time. */ 1605 cpu->cflags_next_tb = 1 | curr_cflags(); 1606 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1607 * back into the cpu_exec loop. */ 1608 return true; 1609 } 1610 #endif 1611 tb_unlock(); 1612 1613 return false; 1614 } 1615 #endif 1616 1617 #if !defined(CONFIG_USER_ONLY) 1618 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs) 1619 { 1620 ram_addr_t ram_addr; 1621 MemoryRegion *mr; 1622 hwaddr l = 1; 1623 1624 rcu_read_lock(); 1625 mr = address_space_translate(as, addr, &addr, &l, false, attrs); 1626 if (!(memory_region_is_ram(mr) 1627 || memory_region_is_romd(mr))) { 1628 rcu_read_unlock(); 1629 return; 1630 } 1631 ram_addr = memory_region_get_ram_addr(mr) + addr; 1632 tb_lock(); 1633 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1634 tb_unlock(); 1635 rcu_read_unlock(); 1636 } 1637 #endif /* !defined(CONFIG_USER_ONLY) */ 1638 1639 /* Called with tb_lock held. */ 1640 void tb_check_watchpoint(CPUState *cpu) 1641 { 1642 TranslationBlock *tb; 1643 1644 tb = tcg_tb_lookup(cpu->mem_io_pc); 1645 if (tb) { 1646 /* We can use retranslation to find the PC. */ 1647 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc, true); 1648 tb_phys_invalidate(tb, -1); 1649 } else { 1650 /* The exception probably happened in a helper. The CPU state should 1651 have been saved before calling it. Fetch the PC from there. */ 1652 CPUArchState *env = cpu->env_ptr; 1653 target_ulong pc, cs_base; 1654 tb_page_addr_t addr; 1655 uint32_t flags; 1656 1657 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1658 addr = get_page_addr_code(env, pc); 1659 tb_invalidate_phys_range(addr, addr + 1); 1660 } 1661 } 1662 1663 #ifndef CONFIG_USER_ONLY 1664 /* in deterministic execution mode, instructions doing device I/Os 1665 * must be at the end of the TB. 1666 * 1667 * Called by softmmu_template.h, with iothread mutex not held. 1668 */ 1669 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1670 { 1671 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1672 CPUArchState *env = cpu->env_ptr; 1673 #endif 1674 TranslationBlock *tb; 1675 uint32_t n; 1676 1677 tb_lock(); 1678 tb = tcg_tb_lookup(retaddr); 1679 if (!tb) { 1680 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1681 (void *)retaddr); 1682 } 1683 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 1684 1685 /* On MIPS and SH, delay slot instructions can only be restarted if 1686 they were already the first instruction in the TB. If this is not 1687 the first instruction in a TB then re-execute the preceding 1688 branch. */ 1689 n = 1; 1690 #if defined(TARGET_MIPS) 1691 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 1692 && env->active_tc.PC != tb->pc) { 1693 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1694 cpu->icount_decr.u16.low++; 1695 env->hflags &= ~MIPS_HFLAG_BMASK; 1696 n = 2; 1697 } 1698 #elif defined(TARGET_SH4) 1699 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1700 && env->pc != tb->pc) { 1701 env->pc -= 2; 1702 cpu->icount_decr.u16.low++; 1703 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1704 n = 2; 1705 } 1706 #endif 1707 1708 /* Generate a new TB executing the I/O insn. */ 1709 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1710 1711 if (tb->cflags & CF_NOCACHE) { 1712 if (tb->orig_tb) { 1713 /* Invalidate original TB if this TB was generated in 1714 * cpu_exec_nocache() */ 1715 tb_phys_invalidate(tb->orig_tb, -1); 1716 } 1717 tcg_tb_remove(tb); 1718 } 1719 1720 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1721 * the first in the TB) then we end up generating a whole new TB and 1722 * repeating the fault, which is horribly inefficient. 1723 * Better would be to execute just this insn uncached, or generate a 1724 * second new TB. 1725 * 1726 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1727 * tb_lock gets reset. 1728 */ 1729 cpu_loop_exit_noexc(cpu); 1730 } 1731 1732 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1733 { 1734 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1735 1736 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1737 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1738 } 1739 } 1740 1741 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1742 { 1743 /* Discard jump cache entries for any tb which might potentially 1744 overlap the flushed page. */ 1745 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1746 tb_jmp_cache_clear_page(cpu, addr); 1747 } 1748 1749 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1750 struct qht_stats hst) 1751 { 1752 uint32_t hgram_opts; 1753 size_t hgram_bins; 1754 char *hgram; 1755 1756 if (!hst.head_buckets) { 1757 return; 1758 } 1759 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1760 hst.used_head_buckets, hst.head_buckets, 1761 (double)hst.used_head_buckets / hst.head_buckets * 100); 1762 1763 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1764 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1765 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1766 hgram_opts |= QDIST_PR_NODECIMAL; 1767 } 1768 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1769 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1770 qdist_avg(&hst.occupancy) * 100, hgram); 1771 g_free(hgram); 1772 1773 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1774 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1775 if (hgram_bins > 10) { 1776 hgram_bins = 10; 1777 } else { 1778 hgram_bins = 0; 1779 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1780 } 1781 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1782 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1783 qdist_avg(&hst.chain), hgram); 1784 g_free(hgram); 1785 } 1786 1787 struct tb_tree_stats { 1788 size_t nb_tbs; 1789 size_t host_size; 1790 size_t target_size; 1791 size_t max_target_size; 1792 size_t direct_jmp_count; 1793 size_t direct_jmp2_count; 1794 size_t cross_page; 1795 }; 1796 1797 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1798 { 1799 const TranslationBlock *tb = value; 1800 struct tb_tree_stats *tst = data; 1801 1802 tst->nb_tbs++; 1803 tst->host_size += tb->tc.size; 1804 tst->target_size += tb->size; 1805 if (tb->size > tst->max_target_size) { 1806 tst->max_target_size = tb->size; 1807 } 1808 if (tb->page_addr[1] != -1) { 1809 tst->cross_page++; 1810 } 1811 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1812 tst->direct_jmp_count++; 1813 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1814 tst->direct_jmp2_count++; 1815 } 1816 } 1817 return false; 1818 } 1819 1820 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1821 { 1822 struct tb_tree_stats tst = {}; 1823 struct qht_stats hst; 1824 size_t nb_tbs; 1825 1826 tcg_tb_foreach(tb_tree_stats_iter, &tst); 1827 nb_tbs = tst.nb_tbs; 1828 /* XXX: avoid using doubles ? */ 1829 cpu_fprintf(f, "Translation buffer state:\n"); 1830 /* 1831 * Report total code size including the padding and TB structs; 1832 * otherwise users might think "-tb-size" is not honoured. 1833 * For avg host size we use the precise numbers from tb_tree_stats though. 1834 */ 1835 cpu_fprintf(f, "gen code size %zu/%zu\n", 1836 tcg_code_size(), tcg_code_capacity()); 1837 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1838 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1839 nb_tbs ? tst.target_size / nb_tbs : 0, 1840 tst.max_target_size); 1841 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1842 nb_tbs ? tst.host_size / nb_tbs : 0, 1843 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1844 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1845 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1846 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1847 tst.direct_jmp_count, 1848 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1849 tst.direct_jmp2_count, 1850 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1851 1852 qht_statistics_init(&tb_ctx.htable, &hst); 1853 print_qht_statistics(f, cpu_fprintf, hst); 1854 qht_statistics_destroy(&hst); 1855 1856 cpu_fprintf(f, "\nStatistics:\n"); 1857 cpu_fprintf(f, "TB flush count %u\n", 1858 atomic_read(&tb_ctx.tb_flush_count)); 1859 cpu_fprintf(f, "TB invalidate count %zu\n", tcg_tb_phys_invalidate_count()); 1860 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1861 tcg_dump_info(f, cpu_fprintf); 1862 } 1863 1864 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1865 { 1866 tcg_dump_op_count(f, cpu_fprintf); 1867 } 1868 1869 #else /* CONFIG_USER_ONLY */ 1870 1871 void cpu_interrupt(CPUState *cpu, int mask) 1872 { 1873 g_assert(qemu_mutex_iothread_locked()); 1874 cpu->interrupt_request |= mask; 1875 cpu->icount_decr.u16.high = -1; 1876 } 1877 1878 /* 1879 * Walks guest process memory "regions" one by one 1880 * and calls callback function 'fn' for each region. 1881 */ 1882 struct walk_memory_regions_data { 1883 walk_memory_regions_fn fn; 1884 void *priv; 1885 target_ulong start; 1886 int prot; 1887 }; 1888 1889 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1890 target_ulong end, int new_prot) 1891 { 1892 if (data->start != -1u) { 1893 int rc = data->fn(data->priv, data->start, end, data->prot); 1894 if (rc != 0) { 1895 return rc; 1896 } 1897 } 1898 1899 data->start = (new_prot ? end : -1u); 1900 data->prot = new_prot; 1901 1902 return 0; 1903 } 1904 1905 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1906 target_ulong base, int level, void **lp) 1907 { 1908 target_ulong pa; 1909 int i, rc; 1910 1911 if (*lp == NULL) { 1912 return walk_memory_regions_end(data, base, 0); 1913 } 1914 1915 if (level == 0) { 1916 PageDesc *pd = *lp; 1917 1918 for (i = 0; i < V_L2_SIZE; ++i) { 1919 int prot = pd[i].flags; 1920 1921 pa = base | (i << TARGET_PAGE_BITS); 1922 if (prot != data->prot) { 1923 rc = walk_memory_regions_end(data, pa, prot); 1924 if (rc != 0) { 1925 return rc; 1926 } 1927 } 1928 } 1929 } else { 1930 void **pp = *lp; 1931 1932 for (i = 0; i < V_L2_SIZE; ++i) { 1933 pa = base | ((target_ulong)i << 1934 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1935 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 1936 if (rc != 0) { 1937 return rc; 1938 } 1939 } 1940 } 1941 1942 return 0; 1943 } 1944 1945 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 1946 { 1947 struct walk_memory_regions_data data; 1948 uintptr_t i, l1_sz = v_l1_size; 1949 1950 data.fn = fn; 1951 data.priv = priv; 1952 data.start = -1u; 1953 data.prot = 0; 1954 1955 for (i = 0; i < l1_sz; i++) { 1956 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 1957 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 1958 if (rc != 0) { 1959 return rc; 1960 } 1961 } 1962 1963 return walk_memory_regions_end(&data, 0, 0); 1964 } 1965 1966 static int dump_region(void *priv, target_ulong start, 1967 target_ulong end, unsigned long prot) 1968 { 1969 FILE *f = (FILE *)priv; 1970 1971 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 1972 " "TARGET_FMT_lx" %c%c%c\n", 1973 start, end, end - start, 1974 ((prot & PAGE_READ) ? 'r' : '-'), 1975 ((prot & PAGE_WRITE) ? 'w' : '-'), 1976 ((prot & PAGE_EXEC) ? 'x' : '-')); 1977 1978 return 0; 1979 } 1980 1981 /* dump memory mappings */ 1982 void page_dump(FILE *f) 1983 { 1984 const int length = sizeof(target_ulong) * 2; 1985 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 1986 length, "start", length, "end", length, "size", "prot"); 1987 walk_memory_regions(f, dump_region); 1988 } 1989 1990 int page_get_flags(target_ulong address) 1991 { 1992 PageDesc *p; 1993 1994 p = page_find(address >> TARGET_PAGE_BITS); 1995 if (!p) { 1996 return 0; 1997 } 1998 return p->flags; 1999 } 2000 2001 /* Modify the flags of a page and invalidate the code if necessary. 2002 The flag PAGE_WRITE_ORG is positioned automatically depending 2003 on PAGE_WRITE. The mmap_lock should already be held. */ 2004 void page_set_flags(target_ulong start, target_ulong end, int flags) 2005 { 2006 target_ulong addr, len; 2007 2008 /* This function should never be called with addresses outside the 2009 guest address space. If this assert fires, it probably indicates 2010 a missing call to h2g_valid. */ 2011 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2012 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2013 #endif 2014 assert(start < end); 2015 assert_memory_lock(); 2016 2017 start = start & TARGET_PAGE_MASK; 2018 end = TARGET_PAGE_ALIGN(end); 2019 2020 if (flags & PAGE_WRITE) { 2021 flags |= PAGE_WRITE_ORG; 2022 } 2023 2024 for (addr = start, len = end - start; 2025 len != 0; 2026 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2027 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2028 2029 /* If the write protection bit is set, then we invalidate 2030 the code inside. */ 2031 if (!(p->flags & PAGE_WRITE) && 2032 (flags & PAGE_WRITE) && 2033 p->first_tb) { 2034 tb_invalidate_phys_page(addr, 0); 2035 } 2036 p->flags = flags; 2037 } 2038 } 2039 2040 int page_check_range(target_ulong start, target_ulong len, int flags) 2041 { 2042 PageDesc *p; 2043 target_ulong end; 2044 target_ulong addr; 2045 2046 /* This function should never be called with addresses outside the 2047 guest address space. If this assert fires, it probably indicates 2048 a missing call to h2g_valid. */ 2049 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2050 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2051 #endif 2052 2053 if (len == 0) { 2054 return 0; 2055 } 2056 if (start + len - 1 < start) { 2057 /* We've wrapped around. */ 2058 return -1; 2059 } 2060 2061 /* must do before we loose bits in the next step */ 2062 end = TARGET_PAGE_ALIGN(start + len); 2063 start = start & TARGET_PAGE_MASK; 2064 2065 for (addr = start, len = end - start; 2066 len != 0; 2067 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2068 p = page_find(addr >> TARGET_PAGE_BITS); 2069 if (!p) { 2070 return -1; 2071 } 2072 if (!(p->flags & PAGE_VALID)) { 2073 return -1; 2074 } 2075 2076 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2077 return -1; 2078 } 2079 if (flags & PAGE_WRITE) { 2080 if (!(p->flags & PAGE_WRITE_ORG)) { 2081 return -1; 2082 } 2083 /* unprotect the page if it was put read-only because it 2084 contains translated code */ 2085 if (!(p->flags & PAGE_WRITE)) { 2086 if (!page_unprotect(addr, 0)) { 2087 return -1; 2088 } 2089 } 2090 } 2091 } 2092 return 0; 2093 } 2094 2095 /* called from signal handler: invalidate the code and unprotect the 2096 * page. Return 0 if the fault was not handled, 1 if it was handled, 2097 * and 2 if it was handled but the caller must cause the TB to be 2098 * immediately exited. (We can only return 2 if the 'pc' argument is 2099 * non-zero.) 2100 */ 2101 int page_unprotect(target_ulong address, uintptr_t pc) 2102 { 2103 unsigned int prot; 2104 bool current_tb_invalidated; 2105 PageDesc *p; 2106 target_ulong host_start, host_end, addr; 2107 2108 /* Technically this isn't safe inside a signal handler. However we 2109 know this only ever happens in a synchronous SEGV handler, so in 2110 practice it seems to be ok. */ 2111 mmap_lock(); 2112 2113 p = page_find(address >> TARGET_PAGE_BITS); 2114 if (!p) { 2115 mmap_unlock(); 2116 return 0; 2117 } 2118 2119 /* if the page was really writable, then we change its 2120 protection back to writable */ 2121 if (p->flags & PAGE_WRITE_ORG) { 2122 current_tb_invalidated = false; 2123 if (p->flags & PAGE_WRITE) { 2124 /* If the page is actually marked WRITE then assume this is because 2125 * this thread raced with another one which got here first and 2126 * set the page to PAGE_WRITE and did the TB invalidate for us. 2127 */ 2128 #ifdef TARGET_HAS_PRECISE_SMC 2129 TranslationBlock *current_tb = tcg_tb_lookup(pc); 2130 if (current_tb) { 2131 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; 2132 } 2133 #endif 2134 } else { 2135 host_start = address & qemu_host_page_mask; 2136 host_end = host_start + qemu_host_page_size; 2137 2138 prot = 0; 2139 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) { 2140 p = page_find(addr >> TARGET_PAGE_BITS); 2141 p->flags |= PAGE_WRITE; 2142 prot |= p->flags; 2143 2144 /* and since the content will be modified, we must invalidate 2145 the corresponding translated code. */ 2146 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2147 #ifdef CONFIG_USER_ONLY 2148 if (DEBUG_TB_CHECK_GATE) { 2149 tb_invalidate_check(addr); 2150 } 2151 #endif 2152 } 2153 mprotect((void *)g2h(host_start), qemu_host_page_size, 2154 prot & PAGE_BITS); 2155 } 2156 mmap_unlock(); 2157 /* If current TB was invalidated return to main loop */ 2158 return current_tb_invalidated ? 2 : 1; 2159 } 2160 mmap_unlock(); 2161 return 0; 2162 } 2163 #endif /* CONFIG_USER_ONLY */ 2164 2165 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2166 void tcg_flush_softmmu_tlb(CPUState *cs) 2167 { 2168 #ifdef CONFIG_SOFTMMU 2169 tlb_flush(cs); 2170 #endif 2171 } 2172