1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 35 #include <sys/param.h> 36 #if __FreeBSD_version >= 700104 37 #define HAVE_KINFO_GETVMMAP 38 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 39 #include <sys/proc.h> 40 #include <machine/profile.h> 41 #define _KERNEL 42 #include <sys/user.h> 43 #undef _KERNEL 44 #undef sigqueue 45 #include <libutil.h> 46 #endif 47 #endif 48 #else 49 #include "exec/address-spaces.h" 50 #endif 51 52 #include "exec/cputlb.h" 53 #include "exec/tb-hash.h" 54 #include "translate-all.h" 55 #include "qemu/bitmap.h" 56 #include "qemu/error-report.h" 57 #include "qemu/timer.h" 58 #include "qemu/main-loop.h" 59 #include "exec/log.h" 60 #include "sysemu/cpus.h" 61 62 /* #define DEBUG_TB_INVALIDATE */ 63 /* #define DEBUG_TB_FLUSH */ 64 /* make various TB consistency checks */ 65 /* #define DEBUG_TB_CHECK */ 66 67 #ifdef DEBUG_TB_INVALIDATE 68 #define DEBUG_TB_INVALIDATE_GATE 1 69 #else 70 #define DEBUG_TB_INVALIDATE_GATE 0 71 #endif 72 73 #ifdef DEBUG_TB_FLUSH 74 #define DEBUG_TB_FLUSH_GATE 1 75 #else 76 #define DEBUG_TB_FLUSH_GATE 0 77 #endif 78 79 #if !defined(CONFIG_USER_ONLY) 80 /* TB consistency checks only implemented for usermode emulation. */ 81 #undef DEBUG_TB_CHECK 82 #endif 83 84 #ifdef DEBUG_TB_CHECK 85 #define DEBUG_TB_CHECK_GATE 1 86 #else 87 #define DEBUG_TB_CHECK_GATE 0 88 #endif 89 90 /* Access to the various translations structures need to be serialised via locks 91 * for consistency. This is automatic for SoftMMU based system 92 * emulation due to its single threaded nature. In user-mode emulation 93 * access to the memory related structures are protected with the 94 * mmap_lock. 95 */ 96 #ifdef CONFIG_SOFTMMU 97 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 98 #else 99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 100 #endif 101 102 #define SMC_BITMAP_USE_THRESHOLD 10 103 104 typedef struct PageDesc { 105 /* list of TBs intersecting this ram page */ 106 uintptr_t first_tb; 107 #ifdef CONFIG_SOFTMMU 108 /* in order to optimize self modifying code, we count the number 109 of lookups we do to a given page to use a bitmap */ 110 unsigned long *code_bitmap; 111 unsigned int code_write_count; 112 #else 113 unsigned long flags; 114 #endif 115 } PageDesc; 116 117 /* list iterators for lists of tagged pointers in TranslationBlock */ 118 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \ 119 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \ 120 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \ 121 tb = (TranslationBlock *)((uintptr_t)tb & ~1)) 122 123 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \ 124 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next) 125 126 /* In system mode we want L1_MAP to be based on ram offsets, 127 while in user mode we want it to be based on virtual addresses. */ 128 #if !defined(CONFIG_USER_ONLY) 129 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 130 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 131 #else 132 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 133 #endif 134 #else 135 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 136 #endif 137 138 /* Size of the L2 (and L3, etc) page tables. */ 139 #define V_L2_BITS 10 140 #define V_L2_SIZE (1 << V_L2_BITS) 141 142 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 143 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 144 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 145 * BITS_PER_BYTE); 146 147 /* 148 * L1 Mapping properties 149 */ 150 static int v_l1_size; 151 static int v_l1_shift; 152 static int v_l2_levels; 153 154 /* The bottom level has pointers to PageDesc, and is indexed by 155 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 156 */ 157 #define V_L1_MIN_BITS 4 158 #define V_L1_MAX_BITS (V_L2_BITS + 3) 159 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 160 161 static void *l1_map[V_L1_MAX_SIZE]; 162 163 /* code generation context */ 164 TCGContext tcg_init_ctx; 165 __thread TCGContext *tcg_ctx; 166 TBContext tb_ctx; 167 bool parallel_cpus; 168 169 /* translation block context */ 170 static __thread int have_tb_lock; 171 172 static void page_table_config_init(void) 173 { 174 uint32_t v_l1_bits; 175 176 assert(TARGET_PAGE_BITS); 177 /* The bits remaining after N lower levels of page tables. */ 178 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 179 if (v_l1_bits < V_L1_MIN_BITS) { 180 v_l1_bits += V_L2_BITS; 181 } 182 183 v_l1_size = 1 << v_l1_bits; 184 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 185 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 186 187 assert(v_l1_bits <= V_L1_MAX_BITS); 188 assert(v_l1_shift % V_L2_BITS == 0); 189 assert(v_l2_levels >= 0); 190 } 191 192 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 193 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 194 195 void tb_lock(void) 196 { 197 assert_tb_unlocked(); 198 qemu_mutex_lock(&tb_ctx.tb_lock); 199 have_tb_lock++; 200 } 201 202 void tb_unlock(void) 203 { 204 assert_tb_locked(); 205 have_tb_lock--; 206 qemu_mutex_unlock(&tb_ctx.tb_lock); 207 } 208 209 void tb_lock_reset(void) 210 { 211 if (have_tb_lock) { 212 qemu_mutex_unlock(&tb_ctx.tb_lock); 213 have_tb_lock = 0; 214 } 215 } 216 217 void cpu_gen_init(void) 218 { 219 tcg_context_init(&tcg_init_ctx); 220 } 221 222 /* Encode VAL as a signed leb128 sequence at P. 223 Return P incremented past the encoded value. */ 224 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 225 { 226 int more, byte; 227 228 do { 229 byte = val & 0x7f; 230 val >>= 7; 231 more = !((val == 0 && (byte & 0x40) == 0) 232 || (val == -1 && (byte & 0x40) != 0)); 233 if (more) { 234 byte |= 0x80; 235 } 236 *p++ = byte; 237 } while (more); 238 239 return p; 240 } 241 242 /* Decode a signed leb128 sequence at *PP; increment *PP past the 243 decoded value. Return the decoded value. */ 244 static target_long decode_sleb128(uint8_t **pp) 245 { 246 uint8_t *p = *pp; 247 target_long val = 0; 248 int byte, shift = 0; 249 250 do { 251 byte = *p++; 252 val |= (target_ulong)(byte & 0x7f) << shift; 253 shift += 7; 254 } while (byte & 0x80); 255 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 256 val |= -(target_ulong)1 << shift; 257 } 258 259 *pp = p; 260 return val; 261 } 262 263 /* Encode the data collected about the instructions while compiling TB. 264 Place the data at BLOCK, and return the number of bytes consumed. 265 266 The logical table consists of TARGET_INSN_START_WORDS target_ulong's, 267 which come from the target's insn_start data, followed by a uintptr_t 268 which comes from the host pc of the end of the code implementing the insn. 269 270 Each line of the table is encoded as sleb128 deltas from the previous 271 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 272 That is, the first column is seeded with the guest pc, the last column 273 with the host pc, and the middle columns with zeros. */ 274 275 static int encode_search(TranslationBlock *tb, uint8_t *block) 276 { 277 uint8_t *highwater = tcg_ctx->code_gen_highwater; 278 uint8_t *p = block; 279 int i, j, n; 280 281 for (i = 0, n = tb->icount; i < n; ++i) { 282 target_ulong prev; 283 284 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 285 if (i == 0) { 286 prev = (j == 0 ? tb->pc : 0); 287 } else { 288 prev = tcg_ctx->gen_insn_data[i - 1][j]; 289 } 290 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 291 } 292 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 293 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 294 295 /* Test for (pending) buffer overflow. The assumption is that any 296 one row beginning below the high water mark cannot overrun 297 the buffer completely. Thus we can test for overflow after 298 encoding a row without having to check during encoding. */ 299 if (unlikely(p > highwater)) { 300 return -1; 301 } 302 } 303 304 return p - block; 305 } 306 307 /* The cpu state corresponding to 'searched_pc' is restored. 308 * Called with tb_lock held. 309 * When reset_icount is true, current TB will be interrupted and 310 * icount should be recalculated. 311 */ 312 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 313 uintptr_t searched_pc, bool reset_icount) 314 { 315 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 316 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 317 CPUArchState *env = cpu->env_ptr; 318 uint8_t *p = tb->tc.ptr + tb->tc.size; 319 int i, j, num_insns = tb->icount; 320 #ifdef CONFIG_PROFILER 321 TCGProfile *prof = &tcg_ctx->prof; 322 int64_t ti = profile_getclock(); 323 #endif 324 325 searched_pc -= GETPC_ADJ; 326 327 if (searched_pc < host_pc) { 328 return -1; 329 } 330 331 /* Reconstruct the stored insn data while looking for the point at 332 which the end of the insn exceeds the searched_pc. */ 333 for (i = 0; i < num_insns; ++i) { 334 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 335 data[j] += decode_sleb128(&p); 336 } 337 host_pc += decode_sleb128(&p); 338 if (host_pc > searched_pc) { 339 goto found; 340 } 341 } 342 return -1; 343 344 found: 345 if (reset_icount && (tb->cflags & CF_USE_ICOUNT)) { 346 assert(use_icount); 347 /* Reset the cycle counter to the start of the block 348 and shift if to the number of actually executed instructions */ 349 cpu->icount_decr.u16.low += num_insns - i; 350 } 351 restore_state_to_opc(env, tb, data); 352 353 #ifdef CONFIG_PROFILER 354 atomic_set(&prof->restore_time, 355 prof->restore_time + profile_getclock() - ti); 356 atomic_set(&prof->restore_count, prof->restore_count + 1); 357 #endif 358 return 0; 359 } 360 361 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) 362 { 363 TranslationBlock *tb; 364 bool r = false; 365 uintptr_t check_offset; 366 367 /* The host_pc has to be in the region of current code buffer. If 368 * it is not we will not be able to resolve it here. The two cases 369 * where host_pc will not be correct are: 370 * 371 * - fault during translation (instruction fetch) 372 * - fault from helper (not using GETPC() macro) 373 * 374 * Either way we need return early to avoid blowing up on a 375 * recursive tb_lock() as we can't resolve it here. 376 * 377 * We are using unsigned arithmetic so if host_pc < 378 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way 379 * above the code_gen_buffer_size 380 */ 381 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; 382 383 if (check_offset < tcg_init_ctx.code_gen_buffer_size) { 384 tb_lock(); 385 tb = tcg_tb_lookup(host_pc); 386 if (tb) { 387 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); 388 if (tb->cflags & CF_NOCACHE) { 389 /* one-shot translation, invalidate it immediately */ 390 tb_phys_invalidate(tb, -1); 391 tcg_tb_remove(tb); 392 } 393 r = true; 394 } 395 tb_unlock(); 396 } 397 398 return r; 399 } 400 401 static void page_init(void) 402 { 403 page_size_init(); 404 page_table_config_init(); 405 406 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 407 { 408 #ifdef HAVE_KINFO_GETVMMAP 409 struct kinfo_vmentry *freep; 410 int i, cnt; 411 412 freep = kinfo_getvmmap(getpid(), &cnt); 413 if (freep) { 414 mmap_lock(); 415 for (i = 0; i < cnt; i++) { 416 unsigned long startaddr, endaddr; 417 418 startaddr = freep[i].kve_start; 419 endaddr = freep[i].kve_end; 420 if (h2g_valid(startaddr)) { 421 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 422 423 if (h2g_valid(endaddr)) { 424 endaddr = h2g(endaddr); 425 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 426 } else { 427 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 428 endaddr = ~0ul; 429 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 430 #endif 431 } 432 } 433 } 434 free(freep); 435 mmap_unlock(); 436 } 437 #else 438 FILE *f; 439 440 last_brk = (unsigned long)sbrk(0); 441 442 f = fopen("/compat/linux/proc/self/maps", "r"); 443 if (f) { 444 mmap_lock(); 445 446 do { 447 unsigned long startaddr, endaddr; 448 int n; 449 450 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 451 452 if (n == 2 && h2g_valid(startaddr)) { 453 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 454 455 if (h2g_valid(endaddr)) { 456 endaddr = h2g(endaddr); 457 } else { 458 endaddr = ~0ul; 459 } 460 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 461 } 462 } while (!feof(f)); 463 464 fclose(f); 465 mmap_unlock(); 466 } 467 #endif 468 } 469 #endif 470 } 471 472 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 473 { 474 PageDesc *pd; 475 void **lp; 476 int i; 477 478 /* Level 1. Always allocated. */ 479 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 480 481 /* Level 2..N-1. */ 482 for (i = v_l2_levels; i > 0; i--) { 483 void **p = atomic_rcu_read(lp); 484 485 if (p == NULL) { 486 void *existing; 487 488 if (!alloc) { 489 return NULL; 490 } 491 p = g_new0(void *, V_L2_SIZE); 492 existing = atomic_cmpxchg(lp, NULL, p); 493 if (unlikely(existing)) { 494 g_free(p); 495 p = existing; 496 } 497 } 498 499 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 500 } 501 502 pd = atomic_rcu_read(lp); 503 if (pd == NULL) { 504 void *existing; 505 506 if (!alloc) { 507 return NULL; 508 } 509 pd = g_new0(PageDesc, V_L2_SIZE); 510 existing = atomic_cmpxchg(lp, NULL, pd); 511 if (unlikely(existing)) { 512 g_free(pd); 513 pd = existing; 514 } 515 } 516 517 return pd + (index & (V_L2_SIZE - 1)); 518 } 519 520 static inline PageDesc *page_find(tb_page_addr_t index) 521 { 522 return page_find_alloc(index, 0); 523 } 524 525 #if defined(CONFIG_USER_ONLY) 526 /* Currently it is not recommended to allocate big chunks of data in 527 user mode. It will change when a dedicated libc will be used. */ 528 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 529 region in which the guest needs to run. Revisit this. */ 530 #define USE_STATIC_CODE_GEN_BUFFER 531 #endif 532 533 /* Minimum size of the code gen buffer. This number is randomly chosen, 534 but not so small that we can't have a fair number of TB's live. */ 535 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 536 537 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 538 indicated, this is constrained by the range of direct branches on the 539 host cpu, as used by the TCG implementation of goto_tb. */ 540 #if defined(__x86_64__) 541 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 542 #elif defined(__sparc__) 543 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 544 #elif defined(__powerpc64__) 545 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 546 #elif defined(__powerpc__) 547 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 548 #elif defined(__aarch64__) 549 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 550 #elif defined(__s390x__) 551 /* We have a +- 4GB range on the branches; leave some slop. */ 552 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 553 #elif defined(__mips__) 554 /* We have a 256MB branch region, but leave room to make sure the 555 main executable is also within that region. */ 556 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 557 #else 558 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 559 #endif 560 561 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 562 563 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 564 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 565 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 566 567 static inline size_t size_code_gen_buffer(size_t tb_size) 568 { 569 /* Size the buffer. */ 570 if (tb_size == 0) { 571 #ifdef USE_STATIC_CODE_GEN_BUFFER 572 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 573 #else 574 /* ??? Needs adjustments. */ 575 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 576 static buffer, we could size this on RESERVED_VA, on the text 577 segment size of the executable, or continue to use the default. */ 578 tb_size = (unsigned long)(ram_size / 4); 579 #endif 580 } 581 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 582 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 583 } 584 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 585 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 586 } 587 return tb_size; 588 } 589 590 #ifdef __mips__ 591 /* In order to use J and JAL within the code_gen_buffer, we require 592 that the buffer not cross a 256MB boundary. */ 593 static inline bool cross_256mb(void *addr, size_t size) 594 { 595 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 596 } 597 598 /* We weren't able to allocate a buffer without crossing that boundary, 599 so make do with the larger portion of the buffer that doesn't cross. 600 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 601 static inline void *split_cross_256mb(void *buf1, size_t size1) 602 { 603 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 604 size_t size2 = buf1 + size1 - buf2; 605 606 size1 = buf2 - buf1; 607 if (size1 < size2) { 608 size1 = size2; 609 buf1 = buf2; 610 } 611 612 tcg_ctx->code_gen_buffer_size = size1; 613 return buf1; 614 } 615 #endif 616 617 #ifdef USE_STATIC_CODE_GEN_BUFFER 618 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 619 __attribute__((aligned(CODE_GEN_ALIGN))); 620 621 static inline void *alloc_code_gen_buffer(void) 622 { 623 void *buf = static_code_gen_buffer; 624 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 625 size_t size; 626 627 /* page-align the beginning and end of the buffer */ 628 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 629 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 630 631 size = end - buf; 632 633 /* Honor a command-line option limiting the size of the buffer. */ 634 if (size > tcg_ctx->code_gen_buffer_size) { 635 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 636 qemu_real_host_page_size); 637 } 638 tcg_ctx->code_gen_buffer_size = size; 639 640 #ifdef __mips__ 641 if (cross_256mb(buf, size)) { 642 buf = split_cross_256mb(buf, size); 643 size = tcg_ctx->code_gen_buffer_size; 644 } 645 #endif 646 647 if (qemu_mprotect_rwx(buf, size)) { 648 abort(); 649 } 650 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 651 652 return buf; 653 } 654 #elif defined(_WIN32) 655 static inline void *alloc_code_gen_buffer(void) 656 { 657 size_t size = tcg_ctx->code_gen_buffer_size; 658 return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 659 PAGE_EXECUTE_READWRITE); 660 } 661 #else 662 static inline void *alloc_code_gen_buffer(void) 663 { 664 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 665 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 666 uintptr_t start = 0; 667 size_t size = tcg_ctx->code_gen_buffer_size; 668 void *buf; 669 670 /* Constrain the position of the buffer based on the host cpu. 671 Note that these addresses are chosen in concert with the 672 addresses assigned in the relevant linker script file. */ 673 # if defined(__PIE__) || defined(__PIC__) 674 /* Don't bother setting a preferred location if we're building 675 a position-independent executable. We're more likely to get 676 an address near the main executable if we let the kernel 677 choose the address. */ 678 # elif defined(__x86_64__) && defined(MAP_32BIT) 679 /* Force the memory down into low memory with the executable. 680 Leave the choice of exact location with the kernel. */ 681 flags |= MAP_32BIT; 682 /* Cannot expect to map more than 800MB in low memory. */ 683 if (size > 800u * 1024 * 1024) { 684 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 685 } 686 # elif defined(__sparc__) 687 start = 0x40000000ul; 688 # elif defined(__s390x__) 689 start = 0x90000000ul; 690 # elif defined(__mips__) 691 # if _MIPS_SIM == _ABI64 692 start = 0x128000000ul; 693 # else 694 start = 0x08000000ul; 695 # endif 696 # endif 697 698 buf = mmap((void *)start, size, prot, flags, -1, 0); 699 if (buf == MAP_FAILED) { 700 return NULL; 701 } 702 703 #ifdef __mips__ 704 if (cross_256mb(buf, size)) { 705 /* Try again, with the original still mapped, to avoid re-acquiring 706 that 256mb crossing. This time don't specify an address. */ 707 size_t size2; 708 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 709 switch ((int)(buf2 != MAP_FAILED)) { 710 case 1: 711 if (!cross_256mb(buf2, size)) { 712 /* Success! Use the new buffer. */ 713 munmap(buf, size); 714 break; 715 } 716 /* Failure. Work with what we had. */ 717 munmap(buf2, size); 718 /* fallthru */ 719 default: 720 /* Split the original buffer. Free the smaller half. */ 721 buf2 = split_cross_256mb(buf, size); 722 size2 = tcg_ctx->code_gen_buffer_size; 723 if (buf == buf2) { 724 munmap(buf + size2, size - size2); 725 } else { 726 munmap(buf, size - size2); 727 } 728 size = size2; 729 break; 730 } 731 buf = buf2; 732 } 733 #endif 734 735 /* Request large pages for the buffer. */ 736 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 737 738 return buf; 739 } 740 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 741 742 static inline void code_gen_alloc(size_t tb_size) 743 { 744 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 745 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 746 if (tcg_ctx->code_gen_buffer == NULL) { 747 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 748 exit(1); 749 } 750 qemu_mutex_init(&tb_ctx.tb_lock); 751 } 752 753 static bool tb_cmp(const void *ap, const void *bp) 754 { 755 const TranslationBlock *a = ap; 756 const TranslationBlock *b = bp; 757 758 return a->pc == b->pc && 759 a->cs_base == b->cs_base && 760 a->flags == b->flags && 761 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) && 762 a->trace_vcpu_dstate == b->trace_vcpu_dstate && 763 a->page_addr[0] == b->page_addr[0] && 764 a->page_addr[1] == b->page_addr[1]; 765 } 766 767 static void tb_htable_init(void) 768 { 769 unsigned int mode = QHT_MODE_AUTO_RESIZE; 770 771 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); 772 } 773 774 /* Must be called before using the QEMU cpus. 'tb_size' is the size 775 (in bytes) allocated to the translation buffer. Zero means default 776 size. */ 777 void tcg_exec_init(unsigned long tb_size) 778 { 779 tcg_allowed = true; 780 cpu_gen_init(); 781 page_init(); 782 tb_htable_init(); 783 code_gen_alloc(tb_size); 784 #if defined(CONFIG_SOFTMMU) 785 /* There's no guest base to take into account, so go ahead and 786 initialize the prologue now. */ 787 tcg_prologue_init(tcg_ctx); 788 #endif 789 } 790 791 /* 792 * Allocate a new translation block. Flush the translation buffer if 793 * too many translation blocks or too much generated code. 794 * 795 * Called with tb_lock held. 796 */ 797 static TranslationBlock *tb_alloc(target_ulong pc) 798 { 799 TranslationBlock *tb; 800 801 assert_tb_locked(); 802 803 tb = tcg_tb_alloc(tcg_ctx); 804 if (unlikely(tb == NULL)) { 805 return NULL; 806 } 807 return tb; 808 } 809 810 static inline void invalidate_page_bitmap(PageDesc *p) 811 { 812 #ifdef CONFIG_SOFTMMU 813 g_free(p->code_bitmap); 814 p->code_bitmap = NULL; 815 p->code_write_count = 0; 816 #endif 817 } 818 819 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 820 static void page_flush_tb_1(int level, void **lp) 821 { 822 int i; 823 824 if (*lp == NULL) { 825 return; 826 } 827 if (level == 0) { 828 PageDesc *pd = *lp; 829 830 for (i = 0; i < V_L2_SIZE; ++i) { 831 pd[i].first_tb = (uintptr_t)NULL; 832 invalidate_page_bitmap(pd + i); 833 } 834 } else { 835 void **pp = *lp; 836 837 for (i = 0; i < V_L2_SIZE; ++i) { 838 page_flush_tb_1(level - 1, pp + i); 839 } 840 } 841 } 842 843 static void page_flush_tb(void) 844 { 845 int i, l1_sz = v_l1_size; 846 847 for (i = 0; i < l1_sz; i++) { 848 page_flush_tb_1(v_l2_levels, l1_map + i); 849 } 850 } 851 852 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 853 { 854 const TranslationBlock *tb = value; 855 size_t *size = data; 856 857 *size += tb->tc.size; 858 return false; 859 } 860 861 /* flush all the translation blocks */ 862 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 863 { 864 tb_lock(); 865 866 /* If it is already been done on request of another CPU, 867 * just retry. 868 */ 869 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 870 goto done; 871 } 872 873 if (DEBUG_TB_FLUSH_GATE) { 874 size_t nb_tbs = tcg_nb_tbs(); 875 size_t host_size = 0; 876 877 tcg_tb_foreach(tb_host_size_iter, &host_size); 878 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 879 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 880 } 881 882 CPU_FOREACH(cpu) { 883 cpu_tb_jmp_cache_clear(cpu); 884 } 885 886 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 887 page_flush_tb(); 888 889 tcg_region_reset_all(); 890 /* XXX: flush processor icache at this point if cache flush is 891 expensive */ 892 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 893 894 done: 895 tb_unlock(); 896 } 897 898 void tb_flush(CPUState *cpu) 899 { 900 if (tcg_enabled()) { 901 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 902 async_safe_run_on_cpu(cpu, do_tb_flush, 903 RUN_ON_CPU_HOST_INT(tb_flush_count)); 904 } 905 } 906 907 /* 908 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 909 * so in order to prevent bit rot we compile them unconditionally in user-mode, 910 * and let the optimizer get rid of them by wrapping their user-only callers 911 * with if (DEBUG_TB_CHECK_GATE). 912 */ 913 #ifdef CONFIG_USER_ONLY 914 915 static void 916 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 917 { 918 TranslationBlock *tb = p; 919 target_ulong addr = *(target_ulong *)userp; 920 921 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 922 printf("ERROR invalidate: address=" TARGET_FMT_lx 923 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 924 } 925 } 926 927 /* verify that all the pages have correct rights for code 928 * 929 * Called with tb_lock held. 930 */ 931 static void tb_invalidate_check(target_ulong address) 932 { 933 address &= TARGET_PAGE_MASK; 934 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 935 } 936 937 static void 938 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 939 { 940 TranslationBlock *tb = p; 941 int flags1, flags2; 942 943 flags1 = page_get_flags(tb->pc); 944 flags2 = page_get_flags(tb->pc + tb->size - 1); 945 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 946 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 947 (long)tb->pc, tb->size, flags1, flags2); 948 } 949 } 950 951 /* verify that all the pages have correct rights for code */ 952 static void tb_page_check(void) 953 { 954 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 955 } 956 957 #endif /* CONFIG_USER_ONLY */ 958 959 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb) 960 { 961 TranslationBlock *tb1; 962 uintptr_t *pprev; 963 unsigned int n1; 964 965 pprev = &pd->first_tb; 966 PAGE_FOR_EACH_TB(pd, tb1, n1) { 967 if (tb1 == tb) { 968 *pprev = tb1->page_next[n1]; 969 return; 970 } 971 pprev = &tb1->page_next[n1]; 972 } 973 g_assert_not_reached(); 974 } 975 976 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 977 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 978 { 979 TranslationBlock *tb1; 980 uintptr_t *ptb, ntb; 981 unsigned int n1; 982 983 ptb = &tb->jmp_list_next[n]; 984 if (*ptb) { 985 /* find tb(n) in circular list */ 986 for (;;) { 987 ntb = *ptb; 988 n1 = ntb & 3; 989 tb1 = (TranslationBlock *)(ntb & ~3); 990 if (n1 == n && tb1 == tb) { 991 break; 992 } 993 if (n1 == 2) { 994 ptb = &tb1->jmp_list_first; 995 } else { 996 ptb = &tb1->jmp_list_next[n1]; 997 } 998 } 999 /* now we can suppress tb(n) from the list */ 1000 *ptb = tb->jmp_list_next[n]; 1001 1002 tb->jmp_list_next[n] = (uintptr_t)NULL; 1003 } 1004 } 1005 1006 /* reset the jump entry 'n' of a TB so that it is not chained to 1007 another TB */ 1008 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1009 { 1010 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1011 tb_set_jmp_target(tb, n, addr); 1012 } 1013 1014 /* remove any jumps to the TB */ 1015 static inline void tb_jmp_unlink(TranslationBlock *tb) 1016 { 1017 TranslationBlock *tb1; 1018 uintptr_t *ptb, ntb; 1019 unsigned int n1; 1020 1021 ptb = &tb->jmp_list_first; 1022 for (;;) { 1023 ntb = *ptb; 1024 n1 = ntb & 3; 1025 tb1 = (TranslationBlock *)(ntb & ~3); 1026 if (n1 == 2) { 1027 break; 1028 } 1029 tb_reset_jump(tb1, n1); 1030 *ptb = tb1->jmp_list_next[n1]; 1031 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1032 } 1033 } 1034 1035 /* invalidate one TB 1036 * 1037 * Called with tb_lock held. 1038 */ 1039 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1040 { 1041 CPUState *cpu; 1042 PageDesc *p; 1043 uint32_t h; 1044 tb_page_addr_t phys_pc; 1045 1046 assert_tb_locked(); 1047 1048 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1049 1050 /* remove the TB from the hash list */ 1051 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1052 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1053 tb->trace_vcpu_dstate); 1054 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1055 return; 1056 } 1057 1058 /* remove the TB from the page list */ 1059 if (tb->page_addr[0] != page_addr) { 1060 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1061 tb_page_remove(p, tb); 1062 invalidate_page_bitmap(p); 1063 } 1064 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1065 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1066 tb_page_remove(p, tb); 1067 invalidate_page_bitmap(p); 1068 } 1069 1070 /* remove the TB from the hash list */ 1071 h = tb_jmp_cache_hash_func(tb->pc); 1072 CPU_FOREACH(cpu) { 1073 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1074 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1075 } 1076 } 1077 1078 /* suppress this TB from the two jump lists */ 1079 tb_remove_from_jmp_list(tb, 0); 1080 tb_remove_from_jmp_list(tb, 1); 1081 1082 /* suppress any remaining jumps to this TB */ 1083 tb_jmp_unlink(tb); 1084 1085 atomic_set(&tcg_ctx->tb_phys_invalidate_count, 1086 tcg_ctx->tb_phys_invalidate_count + 1); 1087 } 1088 1089 #ifdef CONFIG_SOFTMMU 1090 static void build_page_bitmap(PageDesc *p) 1091 { 1092 int n, tb_start, tb_end; 1093 TranslationBlock *tb; 1094 1095 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1096 1097 PAGE_FOR_EACH_TB(p, tb, n) { 1098 /* NOTE: this is subtle as a TB may span two physical pages */ 1099 if (n == 0) { 1100 /* NOTE: tb_end may be after the end of the page, but 1101 it is not a problem */ 1102 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1103 tb_end = tb_start + tb->size; 1104 if (tb_end > TARGET_PAGE_SIZE) { 1105 tb_end = TARGET_PAGE_SIZE; 1106 } 1107 } else { 1108 tb_start = 0; 1109 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1110 } 1111 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1112 } 1113 } 1114 #endif 1115 1116 /* add the tb in the target page and protect it if necessary 1117 * 1118 * Called with mmap_lock held for user-mode emulation. 1119 */ 1120 static inline void tb_alloc_page(TranslationBlock *tb, 1121 unsigned int n, tb_page_addr_t page_addr) 1122 { 1123 PageDesc *p; 1124 #ifndef CONFIG_USER_ONLY 1125 bool page_already_protected; 1126 #endif 1127 1128 assert_memory_lock(); 1129 1130 tb->page_addr[n] = page_addr; 1131 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1132 tb->page_next[n] = p->first_tb; 1133 #ifndef CONFIG_USER_ONLY 1134 page_already_protected = p->first_tb != (uintptr_t)NULL; 1135 #endif 1136 p->first_tb = (uintptr_t)tb | n; 1137 invalidate_page_bitmap(p); 1138 1139 #if defined(CONFIG_USER_ONLY) 1140 if (p->flags & PAGE_WRITE) { 1141 target_ulong addr; 1142 PageDesc *p2; 1143 int prot; 1144 1145 /* force the host page as non writable (writes will have a 1146 page fault + mprotect overhead) */ 1147 page_addr &= qemu_host_page_mask; 1148 prot = 0; 1149 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1150 addr += TARGET_PAGE_SIZE) { 1151 1152 p2 = page_find(addr >> TARGET_PAGE_BITS); 1153 if (!p2) { 1154 continue; 1155 } 1156 prot |= p2->flags; 1157 p2->flags &= ~PAGE_WRITE; 1158 } 1159 mprotect(g2h(page_addr), qemu_host_page_size, 1160 (prot & PAGE_BITS) & ~PAGE_WRITE); 1161 if (DEBUG_TB_INVALIDATE_GATE) { 1162 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1163 } 1164 } 1165 #else 1166 /* if some code is already present, then the pages are already 1167 protected. So we handle the case where only the first TB is 1168 allocated in a physical page */ 1169 if (!page_already_protected) { 1170 tlb_protect_code(page_addr); 1171 } 1172 #endif 1173 } 1174 1175 /* add a new TB and link it to the physical page tables. phys_page2 is 1176 * (-1) to indicate that only one page contains the TB. 1177 * 1178 * Called with mmap_lock held for user-mode emulation. 1179 */ 1180 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1181 tb_page_addr_t phys_page2) 1182 { 1183 uint32_t h; 1184 1185 assert_memory_lock(); 1186 1187 /* add in the page list */ 1188 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1189 if (phys_page2 != -1) { 1190 tb_alloc_page(tb, 1, phys_page2); 1191 } else { 1192 tb->page_addr[1] = -1; 1193 } 1194 1195 /* add in the hash table */ 1196 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1197 tb->trace_vcpu_dstate); 1198 qht_insert(&tb_ctx.htable, tb, h, NULL); 1199 1200 #ifdef CONFIG_USER_ONLY 1201 if (DEBUG_TB_CHECK_GATE) { 1202 tb_page_check(); 1203 } 1204 #endif 1205 } 1206 1207 /* Called with mmap_lock held for user mode emulation. */ 1208 TranslationBlock *tb_gen_code(CPUState *cpu, 1209 target_ulong pc, target_ulong cs_base, 1210 uint32_t flags, int cflags) 1211 { 1212 CPUArchState *env = cpu->env_ptr; 1213 TranslationBlock *tb; 1214 tb_page_addr_t phys_pc, phys_page2; 1215 target_ulong virt_page2; 1216 tcg_insn_unit *gen_code_buf; 1217 int gen_code_size, search_size; 1218 #ifdef CONFIG_PROFILER 1219 TCGProfile *prof = &tcg_ctx->prof; 1220 int64_t ti; 1221 #endif 1222 assert_memory_lock(); 1223 1224 phys_pc = get_page_addr_code(env, pc); 1225 1226 buffer_overflow: 1227 tb = tb_alloc(pc); 1228 if (unlikely(!tb)) { 1229 /* flush must be done */ 1230 tb_flush(cpu); 1231 mmap_unlock(); 1232 /* Make the execution loop process the flush as soon as possible. */ 1233 cpu->exception_index = EXCP_INTERRUPT; 1234 cpu_loop_exit(cpu); 1235 } 1236 1237 gen_code_buf = tcg_ctx->code_gen_ptr; 1238 tb->tc.ptr = gen_code_buf; 1239 tb->pc = pc; 1240 tb->cs_base = cs_base; 1241 tb->flags = flags; 1242 tb->cflags = cflags; 1243 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1244 tcg_ctx->tb_cflags = cflags; 1245 1246 #ifdef CONFIG_PROFILER 1247 /* includes aborted translations because of exceptions */ 1248 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1249 ti = profile_getclock(); 1250 #endif 1251 1252 tcg_func_start(tcg_ctx); 1253 1254 tcg_ctx->cpu = ENV_GET_CPU(env); 1255 gen_intermediate_code(cpu, tb); 1256 tcg_ctx->cpu = NULL; 1257 1258 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1259 1260 /* generate machine code */ 1261 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1262 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1263 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1264 if (TCG_TARGET_HAS_direct_jump) { 1265 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1266 tcg_ctx->tb_jmp_target_addr = NULL; 1267 } else { 1268 tcg_ctx->tb_jmp_insn_offset = NULL; 1269 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1270 } 1271 1272 #ifdef CONFIG_PROFILER 1273 atomic_set(&prof->tb_count, prof->tb_count + 1); 1274 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1275 ti = profile_getclock(); 1276 #endif 1277 1278 /* ??? Overflow could be handled better here. In particular, we 1279 don't need to re-do gen_intermediate_code, nor should we re-do 1280 the tcg optimization currently hidden inside tcg_gen_code. All 1281 that should be required is to flush the TBs, allocate a new TB, 1282 re-initialize it per above, and re-do the actual code generation. */ 1283 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1284 if (unlikely(gen_code_size < 0)) { 1285 goto buffer_overflow; 1286 } 1287 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1288 if (unlikely(search_size < 0)) { 1289 goto buffer_overflow; 1290 } 1291 tb->tc.size = gen_code_size; 1292 1293 #ifdef CONFIG_PROFILER 1294 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1295 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1296 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1297 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1298 #endif 1299 1300 #ifdef DEBUG_DISAS 1301 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1302 qemu_log_in_addr_range(tb->pc)) { 1303 qemu_log_lock(); 1304 qemu_log("OUT: [size=%d]\n", gen_code_size); 1305 if (tcg_ctx->data_gen_ptr) { 1306 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1307 size_t data_size = gen_code_size - code_size; 1308 size_t i; 1309 1310 log_disas(tb->tc.ptr, code_size); 1311 1312 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1313 if (sizeof(tcg_target_ulong) == 8) { 1314 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1315 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1316 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1317 } else { 1318 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1319 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1320 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1321 } 1322 } 1323 } else { 1324 log_disas(tb->tc.ptr, gen_code_size); 1325 } 1326 qemu_log("\n"); 1327 qemu_log_flush(); 1328 qemu_log_unlock(); 1329 } 1330 #endif 1331 1332 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1333 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1334 CODE_GEN_ALIGN)); 1335 1336 /* init jump list */ 1337 assert(((uintptr_t)tb & 3) == 0); 1338 tb->jmp_list_first = (uintptr_t)tb | 2; 1339 tb->jmp_list_next[0] = (uintptr_t)NULL; 1340 tb->jmp_list_next[1] = (uintptr_t)NULL; 1341 1342 /* init original jump addresses wich has been set during tcg_gen_code() */ 1343 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1344 tb_reset_jump(tb, 0); 1345 } 1346 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1347 tb_reset_jump(tb, 1); 1348 } 1349 1350 /* check next page if needed */ 1351 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1352 phys_page2 = -1; 1353 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1354 phys_page2 = get_page_addr_code(env, virt_page2); 1355 } 1356 /* As long as consistency of the TB stuff is provided by tb_lock in user 1357 * mode and is implicit in single-threaded softmmu emulation, no explicit 1358 * memory barrier is required before tb_link_page() makes the TB visible 1359 * through the physical hash table and physical page list. 1360 */ 1361 tb_link_page(tb, phys_pc, phys_page2); 1362 tcg_tb_insert(tb); 1363 return tb; 1364 } 1365 1366 /* 1367 * Invalidate all TBs which intersect with the target physical address range 1368 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1369 * 'is_cpu_write_access' should be true if called from a real cpu write 1370 * access: the virtual CPU will exit the current TB if code is modified inside 1371 * this TB. 1372 * 1373 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1374 * Called with tb_lock held for system-mode emulation 1375 */ 1376 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1377 { 1378 tb_page_addr_t next; 1379 1380 for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; 1381 start < end; 1382 start = next, next += TARGET_PAGE_SIZE) { 1383 tb_page_addr_t bound = MIN(next, end); 1384 1385 tb_invalidate_phys_page_range(start, bound, 0); 1386 } 1387 } 1388 1389 #ifdef CONFIG_SOFTMMU 1390 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1391 { 1392 assert_tb_locked(); 1393 tb_invalidate_phys_range_1(start, end); 1394 } 1395 #else 1396 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1397 { 1398 assert_memory_lock(); 1399 tb_lock(); 1400 tb_invalidate_phys_range_1(start, end); 1401 tb_unlock(); 1402 } 1403 #endif 1404 /* 1405 * Invalidate all TBs which intersect with the target physical address range 1406 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1407 * 'is_cpu_write_access' should be true if called from a real cpu write 1408 * access: the virtual CPU will exit the current TB if code is modified inside 1409 * this TB. 1410 * 1411 * Called with tb_lock/mmap_lock held for user-mode emulation 1412 * Called with tb_lock held for system-mode emulation 1413 */ 1414 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1415 int is_cpu_write_access) 1416 { 1417 TranslationBlock *tb; 1418 tb_page_addr_t tb_start, tb_end; 1419 PageDesc *p; 1420 int n; 1421 #ifdef TARGET_HAS_PRECISE_SMC 1422 CPUState *cpu = current_cpu; 1423 CPUArchState *env = NULL; 1424 int current_tb_not_found = is_cpu_write_access; 1425 TranslationBlock *current_tb = NULL; 1426 int current_tb_modified = 0; 1427 target_ulong current_pc = 0; 1428 target_ulong current_cs_base = 0; 1429 uint32_t current_flags = 0; 1430 #endif /* TARGET_HAS_PRECISE_SMC */ 1431 1432 assert_memory_lock(); 1433 assert_tb_locked(); 1434 1435 p = page_find(start >> TARGET_PAGE_BITS); 1436 if (!p) { 1437 return; 1438 } 1439 #if defined(TARGET_HAS_PRECISE_SMC) 1440 if (cpu != NULL) { 1441 env = cpu->env_ptr; 1442 } 1443 #endif 1444 1445 /* we remove all the TBs in the range [start, end[ */ 1446 /* XXX: see if in some cases it could be faster to invalidate all 1447 the code */ 1448 PAGE_FOR_EACH_TB(p, tb, n) { 1449 /* NOTE: this is subtle as a TB may span two physical pages */ 1450 if (n == 0) { 1451 /* NOTE: tb_end may be after the end of the page, but 1452 it is not a problem */ 1453 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1454 tb_end = tb_start + tb->size; 1455 } else { 1456 tb_start = tb->page_addr[1]; 1457 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1458 } 1459 if (!(tb_end <= start || tb_start >= end)) { 1460 #ifdef TARGET_HAS_PRECISE_SMC 1461 if (current_tb_not_found) { 1462 current_tb_not_found = 0; 1463 current_tb = NULL; 1464 if (cpu->mem_io_pc) { 1465 /* now we have a real cpu fault */ 1466 current_tb = tcg_tb_lookup(cpu->mem_io_pc); 1467 } 1468 } 1469 if (current_tb == tb && 1470 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1471 /* If we are modifying the current TB, we must stop 1472 its execution. We could be more precise by checking 1473 that the modification is after the current PC, but it 1474 would require a specialized function to partially 1475 restore the CPU state */ 1476 1477 current_tb_modified = 1; 1478 cpu_restore_state_from_tb(cpu, current_tb, 1479 cpu->mem_io_pc, true); 1480 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1481 ¤t_flags); 1482 } 1483 #endif /* TARGET_HAS_PRECISE_SMC */ 1484 tb_phys_invalidate(tb, -1); 1485 } 1486 } 1487 #if !defined(CONFIG_USER_ONLY) 1488 /* if no code remaining, no need to continue to use slow writes */ 1489 if (!p->first_tb) { 1490 invalidate_page_bitmap(p); 1491 tlb_unprotect_code(start); 1492 } 1493 #endif 1494 #ifdef TARGET_HAS_PRECISE_SMC 1495 if (current_tb_modified) { 1496 /* Force execution of one insn next time. */ 1497 cpu->cflags_next_tb = 1 | curr_cflags(); 1498 cpu_loop_exit_noexc(cpu); 1499 } 1500 #endif 1501 } 1502 1503 #ifdef CONFIG_SOFTMMU 1504 /* len must be <= 8 and start must be a multiple of len. 1505 * Called via softmmu_template.h when code areas are written to with 1506 * iothread mutex not held. 1507 */ 1508 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1509 { 1510 PageDesc *p; 1511 1512 #if 0 1513 if (1) { 1514 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1515 cpu_single_env->mem_io_vaddr, len, 1516 cpu_single_env->eip, 1517 cpu_single_env->eip + 1518 (intptr_t)cpu_single_env->segs[R_CS].base); 1519 } 1520 #endif 1521 assert_memory_lock(); 1522 1523 p = page_find(start >> TARGET_PAGE_BITS); 1524 if (!p) { 1525 return; 1526 } 1527 if (!p->code_bitmap && 1528 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1529 /* build code bitmap. FIXME: writes should be protected by 1530 * tb_lock, reads by tb_lock or RCU. 1531 */ 1532 build_page_bitmap(p); 1533 } 1534 if (p->code_bitmap) { 1535 unsigned int nr; 1536 unsigned long b; 1537 1538 nr = start & ~TARGET_PAGE_MASK; 1539 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1540 if (b & ((1 << len) - 1)) { 1541 goto do_invalidate; 1542 } 1543 } else { 1544 do_invalidate: 1545 tb_invalidate_phys_page_range(start, start + len, 1); 1546 } 1547 } 1548 #else 1549 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1550 * host PC of the faulting store instruction that caused this invalidate. 1551 * Returns true if the caller needs to abort execution of the current 1552 * TB (because it was modified by this store and the guest CPU has 1553 * precise-SMC semantics). 1554 */ 1555 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1556 { 1557 TranslationBlock *tb; 1558 PageDesc *p; 1559 int n; 1560 #ifdef TARGET_HAS_PRECISE_SMC 1561 TranslationBlock *current_tb = NULL; 1562 CPUState *cpu = current_cpu; 1563 CPUArchState *env = NULL; 1564 int current_tb_modified = 0; 1565 target_ulong current_pc = 0; 1566 target_ulong current_cs_base = 0; 1567 uint32_t current_flags = 0; 1568 #endif 1569 1570 assert_memory_lock(); 1571 1572 addr &= TARGET_PAGE_MASK; 1573 p = page_find(addr >> TARGET_PAGE_BITS); 1574 if (!p) { 1575 return false; 1576 } 1577 1578 tb_lock(); 1579 #ifdef TARGET_HAS_PRECISE_SMC 1580 if (p->first_tb && pc != 0) { 1581 current_tb = tcg_tb_lookup(pc); 1582 } 1583 if (cpu != NULL) { 1584 env = cpu->env_ptr; 1585 } 1586 #endif 1587 PAGE_FOR_EACH_TB(p, tb, n) { 1588 #ifdef TARGET_HAS_PRECISE_SMC 1589 if (current_tb == tb && 1590 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1591 /* If we are modifying the current TB, we must stop 1592 its execution. We could be more precise by checking 1593 that the modification is after the current PC, but it 1594 would require a specialized function to partially 1595 restore the CPU state */ 1596 1597 current_tb_modified = 1; 1598 cpu_restore_state_from_tb(cpu, current_tb, pc, true); 1599 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1600 ¤t_flags); 1601 } 1602 #endif /* TARGET_HAS_PRECISE_SMC */ 1603 tb_phys_invalidate(tb, addr); 1604 } 1605 p->first_tb = (uintptr_t)NULL; 1606 #ifdef TARGET_HAS_PRECISE_SMC 1607 if (current_tb_modified) { 1608 /* Force execution of one insn next time. */ 1609 cpu->cflags_next_tb = 1 | curr_cflags(); 1610 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1611 * back into the cpu_exec loop. */ 1612 return true; 1613 } 1614 #endif 1615 tb_unlock(); 1616 1617 return false; 1618 } 1619 #endif 1620 1621 #if !defined(CONFIG_USER_ONLY) 1622 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs) 1623 { 1624 ram_addr_t ram_addr; 1625 MemoryRegion *mr; 1626 hwaddr l = 1; 1627 1628 rcu_read_lock(); 1629 mr = address_space_translate(as, addr, &addr, &l, false, attrs); 1630 if (!(memory_region_is_ram(mr) 1631 || memory_region_is_romd(mr))) { 1632 rcu_read_unlock(); 1633 return; 1634 } 1635 ram_addr = memory_region_get_ram_addr(mr) + addr; 1636 tb_lock(); 1637 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1638 tb_unlock(); 1639 rcu_read_unlock(); 1640 } 1641 #endif /* !defined(CONFIG_USER_ONLY) */ 1642 1643 /* Called with tb_lock held. */ 1644 void tb_check_watchpoint(CPUState *cpu) 1645 { 1646 TranslationBlock *tb; 1647 1648 tb = tcg_tb_lookup(cpu->mem_io_pc); 1649 if (tb) { 1650 /* We can use retranslation to find the PC. */ 1651 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc, true); 1652 tb_phys_invalidate(tb, -1); 1653 } else { 1654 /* The exception probably happened in a helper. The CPU state should 1655 have been saved before calling it. Fetch the PC from there. */ 1656 CPUArchState *env = cpu->env_ptr; 1657 target_ulong pc, cs_base; 1658 tb_page_addr_t addr; 1659 uint32_t flags; 1660 1661 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1662 addr = get_page_addr_code(env, pc); 1663 tb_invalidate_phys_range(addr, addr + 1); 1664 } 1665 } 1666 1667 #ifndef CONFIG_USER_ONLY 1668 /* in deterministic execution mode, instructions doing device I/Os 1669 * must be at the end of the TB. 1670 * 1671 * Called by softmmu_template.h, with iothread mutex not held. 1672 */ 1673 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1674 { 1675 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1676 CPUArchState *env = cpu->env_ptr; 1677 #endif 1678 TranslationBlock *tb; 1679 uint32_t n; 1680 1681 tb_lock(); 1682 tb = tcg_tb_lookup(retaddr); 1683 if (!tb) { 1684 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1685 (void *)retaddr); 1686 } 1687 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 1688 1689 /* On MIPS and SH, delay slot instructions can only be restarted if 1690 they were already the first instruction in the TB. If this is not 1691 the first instruction in a TB then re-execute the preceding 1692 branch. */ 1693 n = 1; 1694 #if defined(TARGET_MIPS) 1695 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 1696 && env->active_tc.PC != tb->pc) { 1697 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1698 cpu->icount_decr.u16.low++; 1699 env->hflags &= ~MIPS_HFLAG_BMASK; 1700 n = 2; 1701 } 1702 #elif defined(TARGET_SH4) 1703 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1704 && env->pc != tb->pc) { 1705 env->pc -= 2; 1706 cpu->icount_decr.u16.low++; 1707 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1708 n = 2; 1709 } 1710 #endif 1711 1712 /* Generate a new TB executing the I/O insn. */ 1713 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1714 1715 if (tb->cflags & CF_NOCACHE) { 1716 if (tb->orig_tb) { 1717 /* Invalidate original TB if this TB was generated in 1718 * cpu_exec_nocache() */ 1719 tb_phys_invalidate(tb->orig_tb, -1); 1720 } 1721 tcg_tb_remove(tb); 1722 } 1723 1724 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1725 * the first in the TB) then we end up generating a whole new TB and 1726 * repeating the fault, which is horribly inefficient. 1727 * Better would be to execute just this insn uncached, or generate a 1728 * second new TB. 1729 * 1730 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1731 * tb_lock gets reset. 1732 */ 1733 cpu_loop_exit_noexc(cpu); 1734 } 1735 1736 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1737 { 1738 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1739 1740 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1741 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1742 } 1743 } 1744 1745 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1746 { 1747 /* Discard jump cache entries for any tb which might potentially 1748 overlap the flushed page. */ 1749 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1750 tb_jmp_cache_clear_page(cpu, addr); 1751 } 1752 1753 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1754 struct qht_stats hst) 1755 { 1756 uint32_t hgram_opts; 1757 size_t hgram_bins; 1758 char *hgram; 1759 1760 if (!hst.head_buckets) { 1761 return; 1762 } 1763 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1764 hst.used_head_buckets, hst.head_buckets, 1765 (double)hst.used_head_buckets / hst.head_buckets * 100); 1766 1767 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1768 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1769 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1770 hgram_opts |= QDIST_PR_NODECIMAL; 1771 } 1772 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1773 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1774 qdist_avg(&hst.occupancy) * 100, hgram); 1775 g_free(hgram); 1776 1777 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1778 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1779 if (hgram_bins > 10) { 1780 hgram_bins = 10; 1781 } else { 1782 hgram_bins = 0; 1783 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1784 } 1785 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1786 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1787 qdist_avg(&hst.chain), hgram); 1788 g_free(hgram); 1789 } 1790 1791 struct tb_tree_stats { 1792 size_t nb_tbs; 1793 size_t host_size; 1794 size_t target_size; 1795 size_t max_target_size; 1796 size_t direct_jmp_count; 1797 size_t direct_jmp2_count; 1798 size_t cross_page; 1799 }; 1800 1801 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1802 { 1803 const TranslationBlock *tb = value; 1804 struct tb_tree_stats *tst = data; 1805 1806 tst->nb_tbs++; 1807 tst->host_size += tb->tc.size; 1808 tst->target_size += tb->size; 1809 if (tb->size > tst->max_target_size) { 1810 tst->max_target_size = tb->size; 1811 } 1812 if (tb->page_addr[1] != -1) { 1813 tst->cross_page++; 1814 } 1815 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1816 tst->direct_jmp_count++; 1817 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1818 tst->direct_jmp2_count++; 1819 } 1820 } 1821 return false; 1822 } 1823 1824 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1825 { 1826 struct tb_tree_stats tst = {}; 1827 struct qht_stats hst; 1828 size_t nb_tbs; 1829 1830 tcg_tb_foreach(tb_tree_stats_iter, &tst); 1831 nb_tbs = tst.nb_tbs; 1832 /* XXX: avoid using doubles ? */ 1833 cpu_fprintf(f, "Translation buffer state:\n"); 1834 /* 1835 * Report total code size including the padding and TB structs; 1836 * otherwise users might think "-tb-size" is not honoured. 1837 * For avg host size we use the precise numbers from tb_tree_stats though. 1838 */ 1839 cpu_fprintf(f, "gen code size %zu/%zu\n", 1840 tcg_code_size(), tcg_code_capacity()); 1841 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1842 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1843 nb_tbs ? tst.target_size / nb_tbs : 0, 1844 tst.max_target_size); 1845 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1846 nb_tbs ? tst.host_size / nb_tbs : 0, 1847 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1848 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1849 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1850 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1851 tst.direct_jmp_count, 1852 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1853 tst.direct_jmp2_count, 1854 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1855 1856 qht_statistics_init(&tb_ctx.htable, &hst); 1857 print_qht_statistics(f, cpu_fprintf, hst); 1858 qht_statistics_destroy(&hst); 1859 1860 cpu_fprintf(f, "\nStatistics:\n"); 1861 cpu_fprintf(f, "TB flush count %u\n", 1862 atomic_read(&tb_ctx.tb_flush_count)); 1863 cpu_fprintf(f, "TB invalidate count %zu\n", tcg_tb_phys_invalidate_count()); 1864 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1865 tcg_dump_info(f, cpu_fprintf); 1866 } 1867 1868 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1869 { 1870 tcg_dump_op_count(f, cpu_fprintf); 1871 } 1872 1873 #else /* CONFIG_USER_ONLY */ 1874 1875 void cpu_interrupt(CPUState *cpu, int mask) 1876 { 1877 g_assert(qemu_mutex_iothread_locked()); 1878 cpu->interrupt_request |= mask; 1879 cpu->icount_decr.u16.high = -1; 1880 } 1881 1882 /* 1883 * Walks guest process memory "regions" one by one 1884 * and calls callback function 'fn' for each region. 1885 */ 1886 struct walk_memory_regions_data { 1887 walk_memory_regions_fn fn; 1888 void *priv; 1889 target_ulong start; 1890 int prot; 1891 }; 1892 1893 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1894 target_ulong end, int new_prot) 1895 { 1896 if (data->start != -1u) { 1897 int rc = data->fn(data->priv, data->start, end, data->prot); 1898 if (rc != 0) { 1899 return rc; 1900 } 1901 } 1902 1903 data->start = (new_prot ? end : -1u); 1904 data->prot = new_prot; 1905 1906 return 0; 1907 } 1908 1909 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1910 target_ulong base, int level, void **lp) 1911 { 1912 target_ulong pa; 1913 int i, rc; 1914 1915 if (*lp == NULL) { 1916 return walk_memory_regions_end(data, base, 0); 1917 } 1918 1919 if (level == 0) { 1920 PageDesc *pd = *lp; 1921 1922 for (i = 0; i < V_L2_SIZE; ++i) { 1923 int prot = pd[i].flags; 1924 1925 pa = base | (i << TARGET_PAGE_BITS); 1926 if (prot != data->prot) { 1927 rc = walk_memory_regions_end(data, pa, prot); 1928 if (rc != 0) { 1929 return rc; 1930 } 1931 } 1932 } 1933 } else { 1934 void **pp = *lp; 1935 1936 for (i = 0; i < V_L2_SIZE; ++i) { 1937 pa = base | ((target_ulong)i << 1938 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1939 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 1940 if (rc != 0) { 1941 return rc; 1942 } 1943 } 1944 } 1945 1946 return 0; 1947 } 1948 1949 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 1950 { 1951 struct walk_memory_regions_data data; 1952 uintptr_t i, l1_sz = v_l1_size; 1953 1954 data.fn = fn; 1955 data.priv = priv; 1956 data.start = -1u; 1957 data.prot = 0; 1958 1959 for (i = 0; i < l1_sz; i++) { 1960 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 1961 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 1962 if (rc != 0) { 1963 return rc; 1964 } 1965 } 1966 1967 return walk_memory_regions_end(&data, 0, 0); 1968 } 1969 1970 static int dump_region(void *priv, target_ulong start, 1971 target_ulong end, unsigned long prot) 1972 { 1973 FILE *f = (FILE *)priv; 1974 1975 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 1976 " "TARGET_FMT_lx" %c%c%c\n", 1977 start, end, end - start, 1978 ((prot & PAGE_READ) ? 'r' : '-'), 1979 ((prot & PAGE_WRITE) ? 'w' : '-'), 1980 ((prot & PAGE_EXEC) ? 'x' : '-')); 1981 1982 return 0; 1983 } 1984 1985 /* dump memory mappings */ 1986 void page_dump(FILE *f) 1987 { 1988 const int length = sizeof(target_ulong) * 2; 1989 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 1990 length, "start", length, "end", length, "size", "prot"); 1991 walk_memory_regions(f, dump_region); 1992 } 1993 1994 int page_get_flags(target_ulong address) 1995 { 1996 PageDesc *p; 1997 1998 p = page_find(address >> TARGET_PAGE_BITS); 1999 if (!p) { 2000 return 0; 2001 } 2002 return p->flags; 2003 } 2004 2005 /* Modify the flags of a page and invalidate the code if necessary. 2006 The flag PAGE_WRITE_ORG is positioned automatically depending 2007 on PAGE_WRITE. The mmap_lock should already be held. */ 2008 void page_set_flags(target_ulong start, target_ulong end, int flags) 2009 { 2010 target_ulong addr, len; 2011 2012 /* This function should never be called with addresses outside the 2013 guest address space. If this assert fires, it probably indicates 2014 a missing call to h2g_valid. */ 2015 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2016 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2017 #endif 2018 assert(start < end); 2019 assert_memory_lock(); 2020 2021 start = start & TARGET_PAGE_MASK; 2022 end = TARGET_PAGE_ALIGN(end); 2023 2024 if (flags & PAGE_WRITE) { 2025 flags |= PAGE_WRITE_ORG; 2026 } 2027 2028 for (addr = start, len = end - start; 2029 len != 0; 2030 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2031 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2032 2033 /* If the write protection bit is set, then we invalidate 2034 the code inside. */ 2035 if (!(p->flags & PAGE_WRITE) && 2036 (flags & PAGE_WRITE) && 2037 p->first_tb) { 2038 tb_invalidate_phys_page(addr, 0); 2039 } 2040 p->flags = flags; 2041 } 2042 } 2043 2044 int page_check_range(target_ulong start, target_ulong len, int flags) 2045 { 2046 PageDesc *p; 2047 target_ulong end; 2048 target_ulong addr; 2049 2050 /* This function should never be called with addresses outside the 2051 guest address space. If this assert fires, it probably indicates 2052 a missing call to h2g_valid. */ 2053 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2054 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2055 #endif 2056 2057 if (len == 0) { 2058 return 0; 2059 } 2060 if (start + len - 1 < start) { 2061 /* We've wrapped around. */ 2062 return -1; 2063 } 2064 2065 /* must do before we loose bits in the next step */ 2066 end = TARGET_PAGE_ALIGN(start + len); 2067 start = start & TARGET_PAGE_MASK; 2068 2069 for (addr = start, len = end - start; 2070 len != 0; 2071 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2072 p = page_find(addr >> TARGET_PAGE_BITS); 2073 if (!p) { 2074 return -1; 2075 } 2076 if (!(p->flags & PAGE_VALID)) { 2077 return -1; 2078 } 2079 2080 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2081 return -1; 2082 } 2083 if (flags & PAGE_WRITE) { 2084 if (!(p->flags & PAGE_WRITE_ORG)) { 2085 return -1; 2086 } 2087 /* unprotect the page if it was put read-only because it 2088 contains translated code */ 2089 if (!(p->flags & PAGE_WRITE)) { 2090 if (!page_unprotect(addr, 0)) { 2091 return -1; 2092 } 2093 } 2094 } 2095 } 2096 return 0; 2097 } 2098 2099 /* called from signal handler: invalidate the code and unprotect the 2100 * page. Return 0 if the fault was not handled, 1 if it was handled, 2101 * and 2 if it was handled but the caller must cause the TB to be 2102 * immediately exited. (We can only return 2 if the 'pc' argument is 2103 * non-zero.) 2104 */ 2105 int page_unprotect(target_ulong address, uintptr_t pc) 2106 { 2107 unsigned int prot; 2108 bool current_tb_invalidated; 2109 PageDesc *p; 2110 target_ulong host_start, host_end, addr; 2111 2112 /* Technically this isn't safe inside a signal handler. However we 2113 know this only ever happens in a synchronous SEGV handler, so in 2114 practice it seems to be ok. */ 2115 mmap_lock(); 2116 2117 p = page_find(address >> TARGET_PAGE_BITS); 2118 if (!p) { 2119 mmap_unlock(); 2120 return 0; 2121 } 2122 2123 /* if the page was really writable, then we change its 2124 protection back to writable */ 2125 if (p->flags & PAGE_WRITE_ORG) { 2126 current_tb_invalidated = false; 2127 if (p->flags & PAGE_WRITE) { 2128 /* If the page is actually marked WRITE then assume this is because 2129 * this thread raced with another one which got here first and 2130 * set the page to PAGE_WRITE and did the TB invalidate for us. 2131 */ 2132 #ifdef TARGET_HAS_PRECISE_SMC 2133 TranslationBlock *current_tb = tcg_tb_lookup(pc); 2134 if (current_tb) { 2135 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; 2136 } 2137 #endif 2138 } else { 2139 host_start = address & qemu_host_page_mask; 2140 host_end = host_start + qemu_host_page_size; 2141 2142 prot = 0; 2143 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) { 2144 p = page_find(addr >> TARGET_PAGE_BITS); 2145 p->flags |= PAGE_WRITE; 2146 prot |= p->flags; 2147 2148 /* and since the content will be modified, we must invalidate 2149 the corresponding translated code. */ 2150 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2151 #ifdef CONFIG_USER_ONLY 2152 if (DEBUG_TB_CHECK_GATE) { 2153 tb_invalidate_check(addr); 2154 } 2155 #endif 2156 } 2157 mprotect((void *)g2h(host_start), qemu_host_page_size, 2158 prot & PAGE_BITS); 2159 } 2160 mmap_unlock(); 2161 /* If current TB was invalidated return to main loop */ 2162 return current_tb_invalidated ? 2 : 1; 2163 } 2164 mmap_unlock(); 2165 return 0; 2166 } 2167 #endif /* CONFIG_USER_ONLY */ 2168 2169 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2170 void tcg_flush_softmmu_tlb(CPUState *cs) 2171 { 2172 #ifdef CONFIG_SOFTMMU 2173 tlb_flush(cs); 2174 #endif 2175 } 2176