1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #ifdef _WIN32 20 #include <windows.h> 21 #endif 22 #include "qemu/osdep.h" 23 24 25 #include "qemu-common.h" 26 #define NO_CPU_IO_DEFS 27 #include "cpu.h" 28 #include "trace.h" 29 #include "disas/disas.h" 30 #include "exec/exec-all.h" 31 #include "tcg.h" 32 #if defined(CONFIG_USER_ONLY) 33 #include "qemu.h" 34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 35 #include <sys/param.h> 36 #if __FreeBSD_version >= 700104 37 #define HAVE_KINFO_GETVMMAP 38 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 39 #include <sys/proc.h> 40 #include <machine/profile.h> 41 #define _KERNEL 42 #include <sys/user.h> 43 #undef _KERNEL 44 #undef sigqueue 45 #include <libutil.h> 46 #endif 47 #endif 48 #else 49 #include "exec/address-spaces.h" 50 #endif 51 52 #include "exec/cputlb.h" 53 #include "exec/tb-hash.h" 54 #include "translate-all.h" 55 #include "qemu/bitmap.h" 56 #include "qemu/error-report.h" 57 #include "qemu/timer.h" 58 #include "qemu/main-loop.h" 59 #include "exec/log.h" 60 #include "sysemu/cpus.h" 61 62 /* #define DEBUG_TB_INVALIDATE */ 63 /* #define DEBUG_TB_FLUSH */ 64 /* make various TB consistency checks */ 65 /* #define DEBUG_TB_CHECK */ 66 67 #ifdef DEBUG_TB_INVALIDATE 68 #define DEBUG_TB_INVALIDATE_GATE 1 69 #else 70 #define DEBUG_TB_INVALIDATE_GATE 0 71 #endif 72 73 #ifdef DEBUG_TB_FLUSH 74 #define DEBUG_TB_FLUSH_GATE 1 75 #else 76 #define DEBUG_TB_FLUSH_GATE 0 77 #endif 78 79 #if !defined(CONFIG_USER_ONLY) 80 /* TB consistency checks only implemented for usermode emulation. */ 81 #undef DEBUG_TB_CHECK 82 #endif 83 84 #ifdef DEBUG_TB_CHECK 85 #define DEBUG_TB_CHECK_GATE 1 86 #else 87 #define DEBUG_TB_CHECK_GATE 0 88 #endif 89 90 /* Access to the various translations structures need to be serialised via locks 91 * for consistency. This is automatic for SoftMMU based system 92 * emulation due to its single threaded nature. In user-mode emulation 93 * access to the memory related structures are protected with the 94 * mmap_lock. 95 */ 96 #ifdef CONFIG_SOFTMMU 97 #define assert_memory_lock() tcg_debug_assert(have_tb_lock) 98 #else 99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 100 #endif 101 102 #define SMC_BITMAP_USE_THRESHOLD 10 103 104 typedef struct PageDesc { 105 /* list of TBs intersecting this ram page */ 106 uintptr_t first_tb; 107 #ifdef CONFIG_SOFTMMU 108 /* in order to optimize self modifying code, we count the number 109 of lookups we do to a given page to use a bitmap */ 110 unsigned long *code_bitmap; 111 unsigned int code_write_count; 112 #else 113 unsigned long flags; 114 #endif 115 } PageDesc; 116 117 /* list iterators for lists of tagged pointers in TranslationBlock */ 118 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \ 119 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \ 120 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \ 121 tb = (TranslationBlock *)((uintptr_t)tb & ~1)) 122 123 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \ 124 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next) 125 126 /* In system mode we want L1_MAP to be based on ram offsets, 127 while in user mode we want it to be based on virtual addresses. */ 128 #if !defined(CONFIG_USER_ONLY) 129 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 130 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 131 #else 132 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 133 #endif 134 #else 135 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS 136 #endif 137 138 /* Size of the L2 (and L3, etc) page tables. */ 139 #define V_L2_BITS 10 140 #define V_L2_SIZE (1 << V_L2_BITS) 141 142 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 143 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 144 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate) 145 * BITS_PER_BYTE); 146 147 /* 148 * L1 Mapping properties 149 */ 150 static int v_l1_size; 151 static int v_l1_shift; 152 static int v_l2_levels; 153 154 /* The bottom level has pointers to PageDesc, and is indexed by 155 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 156 */ 157 #define V_L1_MIN_BITS 4 158 #define V_L1_MAX_BITS (V_L2_BITS + 3) 159 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 160 161 static void *l1_map[V_L1_MAX_SIZE]; 162 163 /* code generation context */ 164 TCGContext tcg_init_ctx; 165 __thread TCGContext *tcg_ctx; 166 TBContext tb_ctx; 167 bool parallel_cpus; 168 169 /* translation block context */ 170 static __thread int have_tb_lock; 171 172 static void page_table_config_init(void) 173 { 174 uint32_t v_l1_bits; 175 176 assert(TARGET_PAGE_BITS); 177 /* The bits remaining after N lower levels of page tables. */ 178 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 179 if (v_l1_bits < V_L1_MIN_BITS) { 180 v_l1_bits += V_L2_BITS; 181 } 182 183 v_l1_size = 1 << v_l1_bits; 184 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 185 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 186 187 assert(v_l1_bits <= V_L1_MAX_BITS); 188 assert(v_l1_shift % V_L2_BITS == 0); 189 assert(v_l2_levels >= 0); 190 } 191 192 #define assert_tb_locked() tcg_debug_assert(have_tb_lock) 193 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock) 194 195 void tb_lock(void) 196 { 197 assert_tb_unlocked(); 198 qemu_mutex_lock(&tb_ctx.tb_lock); 199 have_tb_lock++; 200 } 201 202 void tb_unlock(void) 203 { 204 assert_tb_locked(); 205 have_tb_lock--; 206 qemu_mutex_unlock(&tb_ctx.tb_lock); 207 } 208 209 void tb_lock_reset(void) 210 { 211 if (have_tb_lock) { 212 qemu_mutex_unlock(&tb_ctx.tb_lock); 213 have_tb_lock = 0; 214 } 215 } 216 217 void cpu_gen_init(void) 218 { 219 tcg_context_init(&tcg_init_ctx); 220 } 221 222 /* Encode VAL as a signed leb128 sequence at P. 223 Return P incremented past the encoded value. */ 224 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 225 { 226 int more, byte; 227 228 do { 229 byte = val & 0x7f; 230 val >>= 7; 231 more = !((val == 0 && (byte & 0x40) == 0) 232 || (val == -1 && (byte & 0x40) != 0)); 233 if (more) { 234 byte |= 0x80; 235 } 236 *p++ = byte; 237 } while (more); 238 239 return p; 240 } 241 242 /* Decode a signed leb128 sequence at *PP; increment *PP past the 243 decoded value. Return the decoded value. */ 244 static target_long decode_sleb128(uint8_t **pp) 245 { 246 uint8_t *p = *pp; 247 target_long val = 0; 248 int byte, shift = 0; 249 250 do { 251 byte = *p++; 252 val |= (target_ulong)(byte & 0x7f) << shift; 253 shift += 7; 254 } while (byte & 0x80); 255 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 256 val |= -(target_ulong)1 << shift; 257 } 258 259 *pp = p; 260 return val; 261 } 262 263 /* Encode the data collected about the instructions while compiling TB. 264 Place the data at BLOCK, and return the number of bytes consumed. 265 266 The logical table consists of TARGET_INSN_START_WORDS target_ulong's, 267 which come from the target's insn_start data, followed by a uintptr_t 268 which comes from the host pc of the end of the code implementing the insn. 269 270 Each line of the table is encoded as sleb128 deltas from the previous 271 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 272 That is, the first column is seeded with the guest pc, the last column 273 with the host pc, and the middle columns with zeros. */ 274 275 static int encode_search(TranslationBlock *tb, uint8_t *block) 276 { 277 uint8_t *highwater = tcg_ctx->code_gen_highwater; 278 uint8_t *p = block; 279 int i, j, n; 280 281 for (i = 0, n = tb->icount; i < n; ++i) { 282 target_ulong prev; 283 284 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 285 if (i == 0) { 286 prev = (j == 0 ? tb->pc : 0); 287 } else { 288 prev = tcg_ctx->gen_insn_data[i - 1][j]; 289 } 290 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 291 } 292 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 293 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 294 295 /* Test for (pending) buffer overflow. The assumption is that any 296 one row beginning below the high water mark cannot overrun 297 the buffer completely. Thus we can test for overflow after 298 encoding a row without having to check during encoding. */ 299 if (unlikely(p > highwater)) { 300 return -1; 301 } 302 } 303 304 return p - block; 305 } 306 307 /* The cpu state corresponding to 'searched_pc' is restored. 308 * Called with tb_lock held. 309 * When reset_icount is true, current TB will be interrupted and 310 * icount should be recalculated. 311 */ 312 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 313 uintptr_t searched_pc, bool reset_icount) 314 { 315 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 316 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 317 CPUArchState *env = cpu->env_ptr; 318 uint8_t *p = tb->tc.ptr + tb->tc.size; 319 int i, j, num_insns = tb->icount; 320 #ifdef CONFIG_PROFILER 321 TCGProfile *prof = &tcg_ctx->prof; 322 int64_t ti = profile_getclock(); 323 #endif 324 325 searched_pc -= GETPC_ADJ; 326 327 if (searched_pc < host_pc) { 328 return -1; 329 } 330 331 /* Reconstruct the stored insn data while looking for the point at 332 which the end of the insn exceeds the searched_pc. */ 333 for (i = 0; i < num_insns; ++i) { 334 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 335 data[j] += decode_sleb128(&p); 336 } 337 host_pc += decode_sleb128(&p); 338 if (host_pc > searched_pc) { 339 goto found; 340 } 341 } 342 return -1; 343 344 found: 345 if (reset_icount && (tb->cflags & CF_USE_ICOUNT)) { 346 assert(use_icount); 347 /* Reset the cycle counter to the start of the block 348 and shift if to the number of actually executed instructions */ 349 cpu->icount_decr.u16.low += num_insns - i; 350 } 351 restore_state_to_opc(env, tb, data); 352 353 #ifdef CONFIG_PROFILER 354 atomic_set(&prof->restore_time, 355 prof->restore_time + profile_getclock() - ti); 356 atomic_set(&prof->restore_count, prof->restore_count + 1); 357 #endif 358 return 0; 359 } 360 361 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) 362 { 363 TranslationBlock *tb; 364 bool r = false; 365 uintptr_t check_offset; 366 367 /* The host_pc has to be in the region of current code buffer. If 368 * it is not we will not be able to resolve it here. The two cases 369 * where host_pc will not be correct are: 370 * 371 * - fault during translation (instruction fetch) 372 * - fault from helper (not using GETPC() macro) 373 * 374 * Either way we need return early to avoid blowing up on a 375 * recursive tb_lock() as we can't resolve it here. 376 * 377 * We are using unsigned arithmetic so if host_pc < 378 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way 379 * above the code_gen_buffer_size 380 */ 381 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer; 382 383 if (check_offset < tcg_init_ctx.code_gen_buffer_size) { 384 tb_lock(); 385 tb = tcg_tb_lookup(host_pc); 386 if (tb) { 387 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); 388 if (tb->cflags & CF_NOCACHE) { 389 /* one-shot translation, invalidate it immediately */ 390 tb_phys_invalidate(tb, -1); 391 tcg_tb_remove(tb); 392 } 393 r = true; 394 } 395 tb_unlock(); 396 } 397 398 return r; 399 } 400 401 static void page_init(void) 402 { 403 page_size_init(); 404 page_table_config_init(); 405 406 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 407 { 408 #ifdef HAVE_KINFO_GETVMMAP 409 struct kinfo_vmentry *freep; 410 int i, cnt; 411 412 freep = kinfo_getvmmap(getpid(), &cnt); 413 if (freep) { 414 mmap_lock(); 415 for (i = 0; i < cnt; i++) { 416 unsigned long startaddr, endaddr; 417 418 startaddr = freep[i].kve_start; 419 endaddr = freep[i].kve_end; 420 if (h2g_valid(startaddr)) { 421 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 422 423 if (h2g_valid(endaddr)) { 424 endaddr = h2g(endaddr); 425 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 426 } else { 427 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 428 endaddr = ~0ul; 429 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 430 #endif 431 } 432 } 433 } 434 free(freep); 435 mmap_unlock(); 436 } 437 #else 438 FILE *f; 439 440 last_brk = (unsigned long)sbrk(0); 441 442 f = fopen("/compat/linux/proc/self/maps", "r"); 443 if (f) { 444 mmap_lock(); 445 446 do { 447 unsigned long startaddr, endaddr; 448 int n; 449 450 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 451 452 if (n == 2 && h2g_valid(startaddr)) { 453 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 454 455 if (h2g_valid(endaddr)) { 456 endaddr = h2g(endaddr); 457 } else { 458 endaddr = ~0ul; 459 } 460 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 461 } 462 } while (!feof(f)); 463 464 fclose(f); 465 mmap_unlock(); 466 } 467 #endif 468 } 469 #endif 470 } 471 472 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 473 { 474 PageDesc *pd; 475 void **lp; 476 int i; 477 478 /* Level 1. Always allocated. */ 479 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 480 481 /* Level 2..N-1. */ 482 for (i = v_l2_levels; i > 0; i--) { 483 void **p = atomic_rcu_read(lp); 484 485 if (p == NULL) { 486 void *existing; 487 488 if (!alloc) { 489 return NULL; 490 } 491 p = g_new0(void *, V_L2_SIZE); 492 existing = atomic_cmpxchg(lp, NULL, p); 493 if (unlikely(existing)) { 494 g_free(p); 495 p = existing; 496 } 497 } 498 499 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 500 } 501 502 pd = atomic_rcu_read(lp); 503 if (pd == NULL) { 504 void *existing; 505 506 if (!alloc) { 507 return NULL; 508 } 509 pd = g_new0(PageDesc, V_L2_SIZE); 510 existing = atomic_cmpxchg(lp, NULL, pd); 511 if (unlikely(existing)) { 512 g_free(pd); 513 pd = existing; 514 } 515 } 516 517 return pd + (index & (V_L2_SIZE - 1)); 518 } 519 520 static inline PageDesc *page_find(tb_page_addr_t index) 521 { 522 return page_find_alloc(index, 0); 523 } 524 525 #if defined(CONFIG_USER_ONLY) 526 /* Currently it is not recommended to allocate big chunks of data in 527 user mode. It will change when a dedicated libc will be used. */ 528 /* ??? 64-bit hosts ought to have no problem mmaping data outside the 529 region in which the guest needs to run. Revisit this. */ 530 #define USE_STATIC_CODE_GEN_BUFFER 531 #endif 532 533 /* Minimum size of the code gen buffer. This number is randomly chosen, 534 but not so small that we can't have a fair number of TB's live. */ 535 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024) 536 537 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 538 indicated, this is constrained by the range of direct branches on the 539 host cpu, as used by the TCG implementation of goto_tb. */ 540 #if defined(__x86_64__) 541 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 542 #elif defined(__sparc__) 543 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 544 #elif defined(__powerpc64__) 545 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 546 #elif defined(__powerpc__) 547 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) 548 #elif defined(__aarch64__) 549 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) 550 #elif defined(__s390x__) 551 /* We have a +- 4GB range on the branches; leave some slop. */ 552 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024) 553 #elif defined(__mips__) 554 /* We have a 256MB branch region, but leave room to make sure the 555 main executable is also within that region. */ 556 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) 557 #else 558 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 559 #endif 560 561 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024) 562 563 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 564 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 565 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 566 567 static inline size_t size_code_gen_buffer(size_t tb_size) 568 { 569 /* Size the buffer. */ 570 if (tb_size == 0) { 571 #ifdef USE_STATIC_CODE_GEN_BUFFER 572 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 573 #else 574 /* ??? Needs adjustments. */ 575 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the 576 static buffer, we could size this on RESERVED_VA, on the text 577 segment size of the executable, or continue to use the default. */ 578 tb_size = (unsigned long)(ram_size / 4); 579 #endif 580 } 581 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 582 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 583 } 584 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 585 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 586 } 587 return tb_size; 588 } 589 590 #ifdef __mips__ 591 /* In order to use J and JAL within the code_gen_buffer, we require 592 that the buffer not cross a 256MB boundary. */ 593 static inline bool cross_256mb(void *addr, size_t size) 594 { 595 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 596 } 597 598 /* We weren't able to allocate a buffer without crossing that boundary, 599 so make do with the larger portion of the buffer that doesn't cross. 600 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 601 static inline void *split_cross_256mb(void *buf1, size_t size1) 602 { 603 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 604 size_t size2 = buf1 + size1 - buf2; 605 606 size1 = buf2 - buf1; 607 if (size1 < size2) { 608 size1 = size2; 609 buf1 = buf2; 610 } 611 612 tcg_ctx->code_gen_buffer_size = size1; 613 return buf1; 614 } 615 #endif 616 617 #ifdef USE_STATIC_CODE_GEN_BUFFER 618 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 619 __attribute__((aligned(CODE_GEN_ALIGN))); 620 621 static inline void *alloc_code_gen_buffer(void) 622 { 623 void *buf = static_code_gen_buffer; 624 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 625 size_t size; 626 627 /* page-align the beginning and end of the buffer */ 628 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 629 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 630 631 size = end - buf; 632 633 /* Honor a command-line option limiting the size of the buffer. */ 634 if (size > tcg_ctx->code_gen_buffer_size) { 635 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size, 636 qemu_real_host_page_size); 637 } 638 tcg_ctx->code_gen_buffer_size = size; 639 640 #ifdef __mips__ 641 if (cross_256mb(buf, size)) { 642 buf = split_cross_256mb(buf, size); 643 size = tcg_ctx->code_gen_buffer_size; 644 } 645 #endif 646 647 if (qemu_mprotect_rwx(buf, size)) { 648 abort(); 649 } 650 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 651 652 return buf; 653 } 654 #elif defined(_WIN32) 655 static inline void *alloc_code_gen_buffer(void) 656 { 657 size_t size = tcg_ctx->code_gen_buffer_size; 658 return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 659 PAGE_EXECUTE_READWRITE); 660 } 661 #else 662 static inline void *alloc_code_gen_buffer(void) 663 { 664 int prot = PROT_WRITE | PROT_READ | PROT_EXEC; 665 int flags = MAP_PRIVATE | MAP_ANONYMOUS; 666 uintptr_t start = 0; 667 size_t size = tcg_ctx->code_gen_buffer_size; 668 void *buf; 669 670 /* Constrain the position of the buffer based on the host cpu. 671 Note that these addresses are chosen in concert with the 672 addresses assigned in the relevant linker script file. */ 673 # if defined(__PIE__) || defined(__PIC__) 674 /* Don't bother setting a preferred location if we're building 675 a position-independent executable. We're more likely to get 676 an address near the main executable if we let the kernel 677 choose the address. */ 678 # elif defined(__x86_64__) && defined(MAP_32BIT) 679 /* Force the memory down into low memory with the executable. 680 Leave the choice of exact location with the kernel. */ 681 flags |= MAP_32BIT; 682 /* Cannot expect to map more than 800MB in low memory. */ 683 if (size > 800u * 1024 * 1024) { 684 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024; 685 } 686 # elif defined(__sparc__) 687 start = 0x40000000ul; 688 # elif defined(__s390x__) 689 start = 0x90000000ul; 690 # elif defined(__mips__) 691 # if _MIPS_SIM == _ABI64 692 start = 0x128000000ul; 693 # else 694 start = 0x08000000ul; 695 # endif 696 # endif 697 698 buf = mmap((void *)start, size, prot, flags, -1, 0); 699 if (buf == MAP_FAILED) { 700 return NULL; 701 } 702 703 #ifdef __mips__ 704 if (cross_256mb(buf, size)) { 705 /* Try again, with the original still mapped, to avoid re-acquiring 706 that 256mb crossing. This time don't specify an address. */ 707 size_t size2; 708 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 709 switch ((int)(buf2 != MAP_FAILED)) { 710 case 1: 711 if (!cross_256mb(buf2, size)) { 712 /* Success! Use the new buffer. */ 713 munmap(buf, size); 714 break; 715 } 716 /* Failure. Work with what we had. */ 717 munmap(buf2, size); 718 /* fallthru */ 719 default: 720 /* Split the original buffer. Free the smaller half. */ 721 buf2 = split_cross_256mb(buf, size); 722 size2 = tcg_ctx->code_gen_buffer_size; 723 if (buf == buf2) { 724 munmap(buf + size2, size - size2); 725 } else { 726 munmap(buf, size - size2); 727 } 728 size = size2; 729 break; 730 } 731 buf = buf2; 732 } 733 #endif 734 735 /* Request large pages for the buffer. */ 736 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 737 738 return buf; 739 } 740 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 741 742 static inline void code_gen_alloc(size_t tb_size) 743 { 744 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size); 745 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(); 746 if (tcg_ctx->code_gen_buffer == NULL) { 747 fprintf(stderr, "Could not allocate dynamic translator buffer\n"); 748 exit(1); 749 } 750 qemu_mutex_init(&tb_ctx.tb_lock); 751 } 752 753 static bool tb_cmp(const void *ap, const void *bp) 754 { 755 const TranslationBlock *a = ap; 756 const TranslationBlock *b = bp; 757 758 return a->pc == b->pc && 759 a->cs_base == b->cs_base && 760 a->flags == b->flags && 761 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) && 762 a->trace_vcpu_dstate == b->trace_vcpu_dstate && 763 a->page_addr[0] == b->page_addr[0] && 764 a->page_addr[1] == b->page_addr[1]; 765 } 766 767 static void tb_htable_init(void) 768 { 769 unsigned int mode = QHT_MODE_AUTO_RESIZE; 770 771 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); 772 } 773 774 /* Must be called before using the QEMU cpus. 'tb_size' is the size 775 (in bytes) allocated to the translation buffer. Zero means default 776 size. */ 777 void tcg_exec_init(unsigned long tb_size) 778 { 779 tcg_allowed = true; 780 cpu_gen_init(); 781 page_init(); 782 tb_htable_init(); 783 code_gen_alloc(tb_size); 784 #if defined(CONFIG_SOFTMMU) 785 /* There's no guest base to take into account, so go ahead and 786 initialize the prologue now. */ 787 tcg_prologue_init(tcg_ctx); 788 #endif 789 } 790 791 /* 792 * Allocate a new translation block. Flush the translation buffer if 793 * too many translation blocks or too much generated code. 794 * 795 * Called with tb_lock held. 796 */ 797 static TranslationBlock *tb_alloc(target_ulong pc) 798 { 799 TranslationBlock *tb; 800 801 assert_tb_locked(); 802 803 tb = tcg_tb_alloc(tcg_ctx); 804 if (unlikely(tb == NULL)) { 805 return NULL; 806 } 807 return tb; 808 } 809 810 static inline void invalidate_page_bitmap(PageDesc *p) 811 { 812 #ifdef CONFIG_SOFTMMU 813 g_free(p->code_bitmap); 814 p->code_bitmap = NULL; 815 p->code_write_count = 0; 816 #endif 817 } 818 819 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 820 static void page_flush_tb_1(int level, void **lp) 821 { 822 int i; 823 824 if (*lp == NULL) { 825 return; 826 } 827 if (level == 0) { 828 PageDesc *pd = *lp; 829 830 for (i = 0; i < V_L2_SIZE; ++i) { 831 pd[i].first_tb = (uintptr_t)NULL; 832 invalidate_page_bitmap(pd + i); 833 } 834 } else { 835 void **pp = *lp; 836 837 for (i = 0; i < V_L2_SIZE; ++i) { 838 page_flush_tb_1(level - 1, pp + i); 839 } 840 } 841 } 842 843 static void page_flush_tb(void) 844 { 845 int i, l1_sz = v_l1_size; 846 847 for (i = 0; i < l1_sz; i++) { 848 page_flush_tb_1(v_l2_levels, l1_map + i); 849 } 850 } 851 852 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 853 { 854 const TranslationBlock *tb = value; 855 size_t *size = data; 856 857 *size += tb->tc.size; 858 return false; 859 } 860 861 /* flush all the translation blocks */ 862 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 863 { 864 tb_lock(); 865 866 /* If it is already been done on request of another CPU, 867 * just retry. 868 */ 869 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 870 goto done; 871 } 872 873 if (DEBUG_TB_FLUSH_GATE) { 874 size_t nb_tbs = tcg_nb_tbs(); 875 size_t host_size = 0; 876 877 tcg_tb_foreach(tb_host_size_iter, &host_size); 878 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 879 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 880 } 881 882 CPU_FOREACH(cpu) { 883 cpu_tb_jmp_cache_clear(cpu); 884 } 885 886 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 887 page_flush_tb(); 888 889 tcg_region_reset_all(); 890 /* XXX: flush processor icache at this point if cache flush is 891 expensive */ 892 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 893 894 done: 895 tb_unlock(); 896 } 897 898 void tb_flush(CPUState *cpu) 899 { 900 if (tcg_enabled()) { 901 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count); 902 async_safe_run_on_cpu(cpu, do_tb_flush, 903 RUN_ON_CPU_HOST_INT(tb_flush_count)); 904 } 905 } 906 907 /* 908 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 909 * so in order to prevent bit rot we compile them unconditionally in user-mode, 910 * and let the optimizer get rid of them by wrapping their user-only callers 911 * with if (DEBUG_TB_CHECK_GATE). 912 */ 913 #ifdef CONFIG_USER_ONLY 914 915 static void 916 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp) 917 { 918 TranslationBlock *tb = p; 919 target_ulong addr = *(target_ulong *)userp; 920 921 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 922 printf("ERROR invalidate: address=" TARGET_FMT_lx 923 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 924 } 925 } 926 927 /* verify that all the pages have correct rights for code 928 * 929 * Called with tb_lock held. 930 */ 931 static void tb_invalidate_check(target_ulong address) 932 { 933 address &= TARGET_PAGE_MASK; 934 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 935 } 936 937 static void 938 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp) 939 { 940 TranslationBlock *tb = p; 941 int flags1, flags2; 942 943 flags1 = page_get_flags(tb->pc); 944 flags2 = page_get_flags(tb->pc + tb->size - 1); 945 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 946 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 947 (long)tb->pc, tb->size, flags1, flags2); 948 } 949 } 950 951 /* verify that all the pages have correct rights for code */ 952 static void tb_page_check(void) 953 { 954 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 955 } 956 957 #endif /* CONFIG_USER_ONLY */ 958 959 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb) 960 { 961 TranslationBlock *tb1; 962 uintptr_t *pprev; 963 unsigned int n1; 964 965 pprev = &pd->first_tb; 966 PAGE_FOR_EACH_TB(pd, tb1, n1) { 967 if (tb1 == tb) { 968 *pprev = tb1->page_next[n1]; 969 return; 970 } 971 pprev = &tb1->page_next[n1]; 972 } 973 g_assert_not_reached(); 974 } 975 976 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ 977 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) 978 { 979 TranslationBlock *tb1; 980 uintptr_t *ptb, ntb; 981 unsigned int n1; 982 983 ptb = &tb->jmp_list_next[n]; 984 if (*ptb) { 985 /* find tb(n) in circular list */ 986 for (;;) { 987 ntb = *ptb; 988 n1 = ntb & 3; 989 tb1 = (TranslationBlock *)(ntb & ~3); 990 if (n1 == n && tb1 == tb) { 991 break; 992 } 993 if (n1 == 2) { 994 ptb = &tb1->jmp_list_first; 995 } else { 996 ptb = &tb1->jmp_list_next[n1]; 997 } 998 } 999 /* now we can suppress tb(n) from the list */ 1000 *ptb = tb->jmp_list_next[n]; 1001 1002 tb->jmp_list_next[n] = (uintptr_t)NULL; 1003 } 1004 } 1005 1006 /* reset the jump entry 'n' of a TB so that it is not chained to 1007 another TB */ 1008 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1009 { 1010 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1011 tb_set_jmp_target(tb, n, addr); 1012 } 1013 1014 /* remove any jumps to the TB */ 1015 static inline void tb_jmp_unlink(TranslationBlock *tb) 1016 { 1017 TranslationBlock *tb1; 1018 uintptr_t *ptb, ntb; 1019 unsigned int n1; 1020 1021 ptb = &tb->jmp_list_first; 1022 for (;;) { 1023 ntb = *ptb; 1024 n1 = ntb & 3; 1025 tb1 = (TranslationBlock *)(ntb & ~3); 1026 if (n1 == 2) { 1027 break; 1028 } 1029 tb_reset_jump(tb1, n1); 1030 *ptb = tb1->jmp_list_next[n1]; 1031 tb1->jmp_list_next[n1] = (uintptr_t)NULL; 1032 } 1033 } 1034 1035 /* invalidate one TB 1036 * 1037 * Called with tb_lock held. 1038 */ 1039 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1040 { 1041 CPUState *cpu; 1042 PageDesc *p; 1043 uint32_t h; 1044 tb_page_addr_t phys_pc; 1045 1046 assert_tb_locked(); 1047 1048 atomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1049 1050 /* remove the TB from the hash list */ 1051 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1052 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1053 tb->trace_vcpu_dstate); 1054 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1055 return; 1056 } 1057 1058 /* remove the TB from the page list */ 1059 if (tb->page_addr[0] != page_addr) { 1060 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1061 tb_page_remove(p, tb); 1062 invalidate_page_bitmap(p); 1063 } 1064 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) { 1065 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1066 tb_page_remove(p, tb); 1067 invalidate_page_bitmap(p); 1068 } 1069 1070 /* remove the TB from the hash list */ 1071 h = tb_jmp_cache_hash_func(tb->pc); 1072 CPU_FOREACH(cpu) { 1073 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1074 atomic_set(&cpu->tb_jmp_cache[h], NULL); 1075 } 1076 } 1077 1078 /* suppress this TB from the two jump lists */ 1079 tb_remove_from_jmp_list(tb, 0); 1080 tb_remove_from_jmp_list(tb, 1); 1081 1082 /* suppress any remaining jumps to this TB */ 1083 tb_jmp_unlink(tb); 1084 1085 atomic_set(&tcg_ctx->tb_phys_invalidate_count, 1086 tcg_ctx->tb_phys_invalidate_count + 1); 1087 } 1088 1089 #ifdef CONFIG_SOFTMMU 1090 static void build_page_bitmap(PageDesc *p) 1091 { 1092 int n, tb_start, tb_end; 1093 TranslationBlock *tb; 1094 1095 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1096 1097 PAGE_FOR_EACH_TB(p, tb, n) { 1098 /* NOTE: this is subtle as a TB may span two physical pages */ 1099 if (n == 0) { 1100 /* NOTE: tb_end may be after the end of the page, but 1101 it is not a problem */ 1102 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1103 tb_end = tb_start + tb->size; 1104 if (tb_end > TARGET_PAGE_SIZE) { 1105 tb_end = TARGET_PAGE_SIZE; 1106 } 1107 } else { 1108 tb_start = 0; 1109 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1110 } 1111 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1112 } 1113 } 1114 #endif 1115 1116 /* add the tb in the target page and protect it if necessary 1117 * 1118 * Called with mmap_lock held for user-mode emulation. 1119 */ 1120 static inline void tb_alloc_page(TranslationBlock *tb, 1121 unsigned int n, tb_page_addr_t page_addr) 1122 { 1123 PageDesc *p; 1124 #ifndef CONFIG_USER_ONLY 1125 bool page_already_protected; 1126 #endif 1127 1128 assert_memory_lock(); 1129 1130 tb->page_addr[n] = page_addr; 1131 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); 1132 tb->page_next[n] = p->first_tb; 1133 #ifndef CONFIG_USER_ONLY 1134 page_already_protected = p->first_tb != (uintptr_t)NULL; 1135 #endif 1136 p->first_tb = (uintptr_t)tb | n; 1137 invalidate_page_bitmap(p); 1138 1139 #if defined(CONFIG_USER_ONLY) 1140 if (p->flags & PAGE_WRITE) { 1141 target_ulong addr; 1142 PageDesc *p2; 1143 int prot; 1144 1145 /* force the host page as non writable (writes will have a 1146 page fault + mprotect overhead) */ 1147 page_addr &= qemu_host_page_mask; 1148 prot = 0; 1149 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1150 addr += TARGET_PAGE_SIZE) { 1151 1152 p2 = page_find(addr >> TARGET_PAGE_BITS); 1153 if (!p2) { 1154 continue; 1155 } 1156 prot |= p2->flags; 1157 p2->flags &= ~PAGE_WRITE; 1158 } 1159 mprotect(g2h(page_addr), qemu_host_page_size, 1160 (prot & PAGE_BITS) & ~PAGE_WRITE); 1161 if (DEBUG_TB_INVALIDATE_GATE) { 1162 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1163 } 1164 } 1165 #else 1166 /* if some code is already present, then the pages are already 1167 protected. So we handle the case where only the first TB is 1168 allocated in a physical page */ 1169 if (!page_already_protected) { 1170 tlb_protect_code(page_addr); 1171 } 1172 #endif 1173 } 1174 1175 /* add a new TB and link it to the physical page tables. phys_page2 is 1176 * (-1) to indicate that only one page contains the TB. 1177 * 1178 * Called with mmap_lock held for user-mode emulation. 1179 */ 1180 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1181 tb_page_addr_t phys_page2) 1182 { 1183 uint32_t h; 1184 1185 assert_memory_lock(); 1186 1187 /* add in the page list */ 1188 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); 1189 if (phys_page2 != -1) { 1190 tb_alloc_page(tb, 1, phys_page2); 1191 } else { 1192 tb->page_addr[1] = -1; 1193 } 1194 1195 /* add in the hash table */ 1196 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1197 tb->trace_vcpu_dstate); 1198 qht_insert(&tb_ctx.htable, tb, h, NULL); 1199 1200 #ifdef CONFIG_USER_ONLY 1201 if (DEBUG_TB_CHECK_GATE) { 1202 tb_page_check(); 1203 } 1204 #endif 1205 } 1206 1207 /* Called with mmap_lock held for user mode emulation. */ 1208 TranslationBlock *tb_gen_code(CPUState *cpu, 1209 target_ulong pc, target_ulong cs_base, 1210 uint32_t flags, int cflags) 1211 { 1212 CPUArchState *env = cpu->env_ptr; 1213 TranslationBlock *tb; 1214 tb_page_addr_t phys_pc, phys_page2; 1215 target_ulong virt_page2; 1216 tcg_insn_unit *gen_code_buf; 1217 int gen_code_size, search_size; 1218 #ifdef CONFIG_PROFILER 1219 TCGProfile *prof = &tcg_ctx->prof; 1220 int64_t ti; 1221 #endif 1222 assert_memory_lock(); 1223 1224 phys_pc = get_page_addr_code(env, pc); 1225 1226 buffer_overflow: 1227 tb = tb_alloc(pc); 1228 if (unlikely(!tb)) { 1229 /* flush must be done */ 1230 tb_flush(cpu); 1231 mmap_unlock(); 1232 /* Make the execution loop process the flush as soon as possible. */ 1233 cpu->exception_index = EXCP_INTERRUPT; 1234 cpu_loop_exit(cpu); 1235 } 1236 1237 gen_code_buf = tcg_ctx->code_gen_ptr; 1238 tb->tc.ptr = gen_code_buf; 1239 tb->pc = pc; 1240 tb->cs_base = cs_base; 1241 tb->flags = flags; 1242 tb->cflags = cflags; 1243 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1244 tcg_ctx->tb_cflags = cflags; 1245 1246 #ifdef CONFIG_PROFILER 1247 /* includes aborted translations because of exceptions */ 1248 atomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1249 ti = profile_getclock(); 1250 #endif 1251 1252 tcg_func_start(tcg_ctx); 1253 1254 tcg_ctx->cpu = ENV_GET_CPU(env); 1255 gen_intermediate_code(cpu, tb); 1256 tcg_ctx->cpu = NULL; 1257 1258 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1259 1260 /* generate machine code */ 1261 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1262 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1263 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1264 if (TCG_TARGET_HAS_direct_jump) { 1265 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1266 tcg_ctx->tb_jmp_target_addr = NULL; 1267 } else { 1268 tcg_ctx->tb_jmp_insn_offset = NULL; 1269 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1270 } 1271 1272 #ifdef CONFIG_PROFILER 1273 atomic_set(&prof->tb_count, prof->tb_count + 1); 1274 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti); 1275 ti = profile_getclock(); 1276 #endif 1277 1278 /* ??? Overflow could be handled better here. In particular, we 1279 don't need to re-do gen_intermediate_code, nor should we re-do 1280 the tcg optimization currently hidden inside tcg_gen_code. All 1281 that should be required is to flush the TBs, allocate a new TB, 1282 re-initialize it per above, and re-do the actual code generation. */ 1283 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1284 if (unlikely(gen_code_size < 0)) { 1285 goto buffer_overflow; 1286 } 1287 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1288 if (unlikely(search_size < 0)) { 1289 goto buffer_overflow; 1290 } 1291 tb->tc.size = gen_code_size; 1292 1293 #ifdef CONFIG_PROFILER 1294 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1295 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1296 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1297 atomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1298 #endif 1299 1300 #ifdef DEBUG_DISAS 1301 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1302 qemu_log_in_addr_range(tb->pc)) { 1303 qemu_log_lock(); 1304 qemu_log("OUT: [size=%d]\n", gen_code_size); 1305 if (tcg_ctx->data_gen_ptr) { 1306 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr; 1307 size_t data_size = gen_code_size - code_size; 1308 size_t i; 1309 1310 log_disas(tb->tc.ptr, code_size); 1311 1312 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1313 if (sizeof(tcg_target_ulong) == 8) { 1314 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1315 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1316 *(uint64_t *)(tcg_ctx->data_gen_ptr + i)); 1317 } else { 1318 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 1319 (uintptr_t)tcg_ctx->data_gen_ptr + i, 1320 *(uint32_t *)(tcg_ctx->data_gen_ptr + i)); 1321 } 1322 } 1323 } else { 1324 log_disas(tb->tc.ptr, gen_code_size); 1325 } 1326 qemu_log("\n"); 1327 qemu_log_flush(); 1328 qemu_log_unlock(); 1329 } 1330 #endif 1331 1332 atomic_set(&tcg_ctx->code_gen_ptr, (void *) 1333 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 1334 CODE_GEN_ALIGN)); 1335 1336 /* init jump list */ 1337 assert(((uintptr_t)tb & 3) == 0); 1338 tb->jmp_list_first = (uintptr_t)tb | 2; 1339 tb->jmp_list_next[0] = (uintptr_t)NULL; 1340 tb->jmp_list_next[1] = (uintptr_t)NULL; 1341 1342 /* init original jump addresses wich has been set during tcg_gen_code() */ 1343 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1344 tb_reset_jump(tb, 0); 1345 } 1346 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1347 tb_reset_jump(tb, 1); 1348 } 1349 1350 /* check next page if needed */ 1351 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 1352 phys_page2 = -1; 1353 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 1354 phys_page2 = get_page_addr_code(env, virt_page2); 1355 } 1356 /* As long as consistency of the TB stuff is provided by tb_lock in user 1357 * mode and is implicit in single-threaded softmmu emulation, no explicit 1358 * memory barrier is required before tb_link_page() makes the TB visible 1359 * through the physical hash table and physical page list. 1360 */ 1361 tb_link_page(tb, phys_pc, phys_page2); 1362 tcg_tb_insert(tb); 1363 return tb; 1364 } 1365 1366 /* 1367 * Invalidate all TBs which intersect with the target physical address range 1368 * [start;end[. NOTE: start and end must refer to the *same* physical page. 1369 * 'is_cpu_write_access' should be true if called from a real cpu write 1370 * access: the virtual CPU will exit the current TB if code is modified inside 1371 * this TB. 1372 * 1373 * Called with tb_lock/mmap_lock held for user-mode emulation 1374 * Called with tb_lock held for system-mode emulation 1375 */ 1376 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, 1377 int is_cpu_write_access) 1378 { 1379 TranslationBlock *tb; 1380 tb_page_addr_t tb_start, tb_end; 1381 PageDesc *p; 1382 int n; 1383 #ifdef TARGET_HAS_PRECISE_SMC 1384 CPUState *cpu = current_cpu; 1385 CPUArchState *env = NULL; 1386 int current_tb_not_found = is_cpu_write_access; 1387 TranslationBlock *current_tb = NULL; 1388 int current_tb_modified = 0; 1389 target_ulong current_pc = 0; 1390 target_ulong current_cs_base = 0; 1391 uint32_t current_flags = 0; 1392 #endif /* TARGET_HAS_PRECISE_SMC */ 1393 1394 assert_memory_lock(); 1395 assert_tb_locked(); 1396 1397 p = page_find(start >> TARGET_PAGE_BITS); 1398 if (!p) { 1399 return; 1400 } 1401 #if defined(TARGET_HAS_PRECISE_SMC) 1402 if (cpu != NULL) { 1403 env = cpu->env_ptr; 1404 } 1405 #endif 1406 1407 /* we remove all the TBs in the range [start, end[ */ 1408 /* XXX: see if in some cases it could be faster to invalidate all 1409 the code */ 1410 PAGE_FOR_EACH_TB(p, tb, n) { 1411 /* NOTE: this is subtle as a TB may span two physical pages */ 1412 if (n == 0) { 1413 /* NOTE: tb_end may be after the end of the page, but 1414 it is not a problem */ 1415 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1416 tb_end = tb_start + tb->size; 1417 } else { 1418 tb_start = tb->page_addr[1]; 1419 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1420 } 1421 if (!(tb_end <= start || tb_start >= end)) { 1422 #ifdef TARGET_HAS_PRECISE_SMC 1423 if (current_tb_not_found) { 1424 current_tb_not_found = 0; 1425 current_tb = NULL; 1426 if (cpu->mem_io_pc) { 1427 /* now we have a real cpu fault */ 1428 current_tb = tcg_tb_lookup(cpu->mem_io_pc); 1429 } 1430 } 1431 if (current_tb == tb && 1432 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1433 /* If we are modifying the current TB, we must stop 1434 its execution. We could be more precise by checking 1435 that the modification is after the current PC, but it 1436 would require a specialized function to partially 1437 restore the CPU state */ 1438 1439 current_tb_modified = 1; 1440 cpu_restore_state_from_tb(cpu, current_tb, 1441 cpu->mem_io_pc, true); 1442 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1443 ¤t_flags); 1444 } 1445 #endif /* TARGET_HAS_PRECISE_SMC */ 1446 tb_phys_invalidate(tb, -1); 1447 } 1448 } 1449 #if !defined(CONFIG_USER_ONLY) 1450 /* if no code remaining, no need to continue to use slow writes */ 1451 if (!p->first_tb) { 1452 invalidate_page_bitmap(p); 1453 tlb_unprotect_code(start); 1454 } 1455 #endif 1456 #ifdef TARGET_HAS_PRECISE_SMC 1457 if (current_tb_modified) { 1458 /* Force execution of one insn next time. */ 1459 cpu->cflags_next_tb = 1 | curr_cflags(); 1460 cpu_loop_exit_noexc(cpu); 1461 } 1462 #endif 1463 } 1464 1465 /* 1466 * Invalidate all TBs which intersect with the target physical address range 1467 * [start;end[. NOTE: start and end may refer to *different* physical pages. 1468 * 'is_cpu_write_access' should be true if called from a real cpu write 1469 * access: the virtual CPU will exit the current TB if code is modified inside 1470 * this TB. 1471 * 1472 * Called with mmap_lock held for user-mode emulation, grabs tb_lock 1473 * Called with tb_lock held for system-mode emulation 1474 */ 1475 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end) 1476 { 1477 tb_page_addr_t next; 1478 1479 for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; 1480 start < end; 1481 start = next, next += TARGET_PAGE_SIZE) { 1482 tb_page_addr_t bound = MIN(next, end); 1483 1484 tb_invalidate_phys_page_range(start, bound, 0); 1485 } 1486 } 1487 1488 #ifdef CONFIG_SOFTMMU 1489 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1490 { 1491 assert_tb_locked(); 1492 tb_invalidate_phys_range_1(start, end); 1493 } 1494 #else 1495 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) 1496 { 1497 assert_memory_lock(); 1498 tb_lock(); 1499 tb_invalidate_phys_range_1(start, end); 1500 tb_unlock(); 1501 } 1502 #endif 1503 1504 #ifdef CONFIG_SOFTMMU 1505 /* len must be <= 8 and start must be a multiple of len. 1506 * Called via softmmu_template.h when code areas are written to with 1507 * iothread mutex not held. 1508 */ 1509 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) 1510 { 1511 PageDesc *p; 1512 1513 #if 0 1514 if (1) { 1515 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 1516 cpu_single_env->mem_io_vaddr, len, 1517 cpu_single_env->eip, 1518 cpu_single_env->eip + 1519 (intptr_t)cpu_single_env->segs[R_CS].base); 1520 } 1521 #endif 1522 assert_memory_lock(); 1523 1524 p = page_find(start >> TARGET_PAGE_BITS); 1525 if (!p) { 1526 return; 1527 } 1528 if (!p->code_bitmap && 1529 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 1530 /* build code bitmap. FIXME: writes should be protected by 1531 * tb_lock, reads by tb_lock or RCU. 1532 */ 1533 build_page_bitmap(p); 1534 } 1535 if (p->code_bitmap) { 1536 unsigned int nr; 1537 unsigned long b; 1538 1539 nr = start & ~TARGET_PAGE_MASK; 1540 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 1541 if (b & ((1 << len) - 1)) { 1542 goto do_invalidate; 1543 } 1544 } else { 1545 do_invalidate: 1546 tb_invalidate_phys_page_range(start, start + len, 1); 1547 } 1548 } 1549 #else 1550 /* Called with mmap_lock held. If pc is not 0 then it indicates the 1551 * host PC of the faulting store instruction that caused this invalidate. 1552 * Returns true if the caller needs to abort execution of the current 1553 * TB (because it was modified by this store and the guest CPU has 1554 * precise-SMC semantics). 1555 */ 1556 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 1557 { 1558 TranslationBlock *tb; 1559 PageDesc *p; 1560 int n; 1561 #ifdef TARGET_HAS_PRECISE_SMC 1562 TranslationBlock *current_tb = NULL; 1563 CPUState *cpu = current_cpu; 1564 CPUArchState *env = NULL; 1565 int current_tb_modified = 0; 1566 target_ulong current_pc = 0; 1567 target_ulong current_cs_base = 0; 1568 uint32_t current_flags = 0; 1569 #endif 1570 1571 assert_memory_lock(); 1572 1573 addr &= TARGET_PAGE_MASK; 1574 p = page_find(addr >> TARGET_PAGE_BITS); 1575 if (!p) { 1576 return false; 1577 } 1578 1579 tb_lock(); 1580 #ifdef TARGET_HAS_PRECISE_SMC 1581 if (p->first_tb && pc != 0) { 1582 current_tb = tcg_tb_lookup(pc); 1583 } 1584 if (cpu != NULL) { 1585 env = cpu->env_ptr; 1586 } 1587 #endif 1588 PAGE_FOR_EACH_TB(p, tb, n) { 1589 #ifdef TARGET_HAS_PRECISE_SMC 1590 if (current_tb == tb && 1591 (current_tb->cflags & CF_COUNT_MASK) != 1) { 1592 /* If we are modifying the current TB, we must stop 1593 its execution. We could be more precise by checking 1594 that the modification is after the current PC, but it 1595 would require a specialized function to partially 1596 restore the CPU state */ 1597 1598 current_tb_modified = 1; 1599 cpu_restore_state_from_tb(cpu, current_tb, pc, true); 1600 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 1601 ¤t_flags); 1602 } 1603 #endif /* TARGET_HAS_PRECISE_SMC */ 1604 tb_phys_invalidate(tb, addr); 1605 } 1606 p->first_tb = (uintptr_t)NULL; 1607 #ifdef TARGET_HAS_PRECISE_SMC 1608 if (current_tb_modified) { 1609 /* Force execution of one insn next time. */ 1610 cpu->cflags_next_tb = 1 | curr_cflags(); 1611 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps 1612 * back into the cpu_exec loop. */ 1613 return true; 1614 } 1615 #endif 1616 tb_unlock(); 1617 1618 return false; 1619 } 1620 #endif 1621 1622 #if !defined(CONFIG_USER_ONLY) 1623 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs) 1624 { 1625 ram_addr_t ram_addr; 1626 MemoryRegion *mr; 1627 hwaddr l = 1; 1628 1629 rcu_read_lock(); 1630 mr = address_space_translate(as, addr, &addr, &l, false, attrs); 1631 if (!(memory_region_is_ram(mr) 1632 || memory_region_is_romd(mr))) { 1633 rcu_read_unlock(); 1634 return; 1635 } 1636 ram_addr = memory_region_get_ram_addr(mr) + addr; 1637 tb_lock(); 1638 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); 1639 tb_unlock(); 1640 rcu_read_unlock(); 1641 } 1642 #endif /* !defined(CONFIG_USER_ONLY) */ 1643 1644 /* Called with tb_lock held. */ 1645 void tb_check_watchpoint(CPUState *cpu) 1646 { 1647 TranslationBlock *tb; 1648 1649 tb = tcg_tb_lookup(cpu->mem_io_pc); 1650 if (tb) { 1651 /* We can use retranslation to find the PC. */ 1652 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc, true); 1653 tb_phys_invalidate(tb, -1); 1654 } else { 1655 /* The exception probably happened in a helper. The CPU state should 1656 have been saved before calling it. Fetch the PC from there. */ 1657 CPUArchState *env = cpu->env_ptr; 1658 target_ulong pc, cs_base; 1659 tb_page_addr_t addr; 1660 uint32_t flags; 1661 1662 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 1663 addr = get_page_addr_code(env, pc); 1664 tb_invalidate_phys_range(addr, addr + 1); 1665 } 1666 } 1667 1668 #ifndef CONFIG_USER_ONLY 1669 /* in deterministic execution mode, instructions doing device I/Os 1670 * must be at the end of the TB. 1671 * 1672 * Called by softmmu_template.h, with iothread mutex not held. 1673 */ 1674 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 1675 { 1676 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 1677 CPUArchState *env = cpu->env_ptr; 1678 #endif 1679 TranslationBlock *tb; 1680 uint32_t n; 1681 1682 tb_lock(); 1683 tb = tcg_tb_lookup(retaddr); 1684 if (!tb) { 1685 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 1686 (void *)retaddr); 1687 } 1688 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 1689 1690 /* On MIPS and SH, delay slot instructions can only be restarted if 1691 they were already the first instruction in the TB. If this is not 1692 the first instruction in a TB then re-execute the preceding 1693 branch. */ 1694 n = 1; 1695 #if defined(TARGET_MIPS) 1696 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 1697 && env->active_tc.PC != tb->pc) { 1698 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 1699 cpu->icount_decr.u16.low++; 1700 env->hflags &= ~MIPS_HFLAG_BMASK; 1701 n = 2; 1702 } 1703 #elif defined(TARGET_SH4) 1704 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 1705 && env->pc != tb->pc) { 1706 env->pc -= 2; 1707 cpu->icount_decr.u16.low++; 1708 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 1709 n = 2; 1710 } 1711 #endif 1712 1713 /* Generate a new TB executing the I/O insn. */ 1714 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 1715 1716 if (tb->cflags & CF_NOCACHE) { 1717 if (tb->orig_tb) { 1718 /* Invalidate original TB if this TB was generated in 1719 * cpu_exec_nocache() */ 1720 tb_phys_invalidate(tb->orig_tb, -1); 1721 } 1722 tcg_tb_remove(tb); 1723 } 1724 1725 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 1726 * the first in the TB) then we end up generating a whole new TB and 1727 * repeating the fault, which is horribly inefficient. 1728 * Better would be to execute just this insn uncached, or generate a 1729 * second new TB. 1730 * 1731 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the 1732 * tb_lock gets reset. 1733 */ 1734 cpu_loop_exit_noexc(cpu); 1735 } 1736 1737 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 1738 { 1739 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 1740 1741 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 1742 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 1743 } 1744 } 1745 1746 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 1747 { 1748 /* Discard jump cache entries for any tb which might potentially 1749 overlap the flushed page. */ 1750 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 1751 tb_jmp_cache_clear_page(cpu, addr); 1752 } 1753 1754 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf, 1755 struct qht_stats hst) 1756 { 1757 uint32_t hgram_opts; 1758 size_t hgram_bins; 1759 char *hgram; 1760 1761 if (!hst.head_buckets) { 1762 return; 1763 } 1764 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 1765 hst.used_head_buckets, hst.head_buckets, 1766 (double)hst.used_head_buckets / hst.head_buckets * 100); 1767 1768 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1769 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 1770 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 1771 hgram_opts |= QDIST_PR_NODECIMAL; 1772 } 1773 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 1774 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 1775 qdist_avg(&hst.occupancy) * 100, hgram); 1776 g_free(hgram); 1777 1778 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 1779 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 1780 if (hgram_bins > 10) { 1781 hgram_bins = 10; 1782 } else { 1783 hgram_bins = 0; 1784 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 1785 } 1786 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 1787 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n", 1788 qdist_avg(&hst.chain), hgram); 1789 g_free(hgram); 1790 } 1791 1792 struct tb_tree_stats { 1793 size_t nb_tbs; 1794 size_t host_size; 1795 size_t target_size; 1796 size_t max_target_size; 1797 size_t direct_jmp_count; 1798 size_t direct_jmp2_count; 1799 size_t cross_page; 1800 }; 1801 1802 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 1803 { 1804 const TranslationBlock *tb = value; 1805 struct tb_tree_stats *tst = data; 1806 1807 tst->nb_tbs++; 1808 tst->host_size += tb->tc.size; 1809 tst->target_size += tb->size; 1810 if (tb->size > tst->max_target_size) { 1811 tst->max_target_size = tb->size; 1812 } 1813 if (tb->page_addr[1] != -1) { 1814 tst->cross_page++; 1815 } 1816 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 1817 tst->direct_jmp_count++; 1818 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 1819 tst->direct_jmp2_count++; 1820 } 1821 } 1822 return false; 1823 } 1824 1825 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) 1826 { 1827 struct tb_tree_stats tst = {}; 1828 struct qht_stats hst; 1829 size_t nb_tbs; 1830 1831 tcg_tb_foreach(tb_tree_stats_iter, &tst); 1832 nb_tbs = tst.nb_tbs; 1833 /* XXX: avoid using doubles ? */ 1834 cpu_fprintf(f, "Translation buffer state:\n"); 1835 /* 1836 * Report total code size including the padding and TB structs; 1837 * otherwise users might think "-tb-size" is not honoured. 1838 * For avg host size we use the precise numbers from tb_tree_stats though. 1839 */ 1840 cpu_fprintf(f, "gen code size %zu/%zu\n", 1841 tcg_code_size(), tcg_code_capacity()); 1842 cpu_fprintf(f, "TB count %zu\n", nb_tbs); 1843 cpu_fprintf(f, "TB avg target size %zu max=%zu bytes\n", 1844 nb_tbs ? tst.target_size / nb_tbs : 0, 1845 tst.max_target_size); 1846 cpu_fprintf(f, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 1847 nb_tbs ? tst.host_size / nb_tbs : 0, 1848 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 1849 cpu_fprintf(f, "cross page TB count %zu (%zu%%)\n", tst.cross_page, 1850 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 1851 cpu_fprintf(f, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 1852 tst.direct_jmp_count, 1853 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 1854 tst.direct_jmp2_count, 1855 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 1856 1857 qht_statistics_init(&tb_ctx.htable, &hst); 1858 print_qht_statistics(f, cpu_fprintf, hst); 1859 qht_statistics_destroy(&hst); 1860 1861 cpu_fprintf(f, "\nStatistics:\n"); 1862 cpu_fprintf(f, "TB flush count %u\n", 1863 atomic_read(&tb_ctx.tb_flush_count)); 1864 cpu_fprintf(f, "TB invalidate count %zu\n", tcg_tb_phys_invalidate_count()); 1865 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count()); 1866 tcg_dump_info(f, cpu_fprintf); 1867 } 1868 1869 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf) 1870 { 1871 tcg_dump_op_count(f, cpu_fprintf); 1872 } 1873 1874 #else /* CONFIG_USER_ONLY */ 1875 1876 void cpu_interrupt(CPUState *cpu, int mask) 1877 { 1878 g_assert(qemu_mutex_iothread_locked()); 1879 cpu->interrupt_request |= mask; 1880 cpu->icount_decr.u16.high = -1; 1881 } 1882 1883 /* 1884 * Walks guest process memory "regions" one by one 1885 * and calls callback function 'fn' for each region. 1886 */ 1887 struct walk_memory_regions_data { 1888 walk_memory_regions_fn fn; 1889 void *priv; 1890 target_ulong start; 1891 int prot; 1892 }; 1893 1894 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 1895 target_ulong end, int new_prot) 1896 { 1897 if (data->start != -1u) { 1898 int rc = data->fn(data->priv, data->start, end, data->prot); 1899 if (rc != 0) { 1900 return rc; 1901 } 1902 } 1903 1904 data->start = (new_prot ? end : -1u); 1905 data->prot = new_prot; 1906 1907 return 0; 1908 } 1909 1910 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 1911 target_ulong base, int level, void **lp) 1912 { 1913 target_ulong pa; 1914 int i, rc; 1915 1916 if (*lp == NULL) { 1917 return walk_memory_regions_end(data, base, 0); 1918 } 1919 1920 if (level == 0) { 1921 PageDesc *pd = *lp; 1922 1923 for (i = 0; i < V_L2_SIZE; ++i) { 1924 int prot = pd[i].flags; 1925 1926 pa = base | (i << TARGET_PAGE_BITS); 1927 if (prot != data->prot) { 1928 rc = walk_memory_regions_end(data, pa, prot); 1929 if (rc != 0) { 1930 return rc; 1931 } 1932 } 1933 } 1934 } else { 1935 void **pp = *lp; 1936 1937 for (i = 0; i < V_L2_SIZE; ++i) { 1938 pa = base | ((target_ulong)i << 1939 (TARGET_PAGE_BITS + V_L2_BITS * level)); 1940 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 1941 if (rc != 0) { 1942 return rc; 1943 } 1944 } 1945 } 1946 1947 return 0; 1948 } 1949 1950 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 1951 { 1952 struct walk_memory_regions_data data; 1953 uintptr_t i, l1_sz = v_l1_size; 1954 1955 data.fn = fn; 1956 data.priv = priv; 1957 data.start = -1u; 1958 data.prot = 0; 1959 1960 for (i = 0; i < l1_sz; i++) { 1961 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 1962 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 1963 if (rc != 0) { 1964 return rc; 1965 } 1966 } 1967 1968 return walk_memory_regions_end(&data, 0, 0); 1969 } 1970 1971 static int dump_region(void *priv, target_ulong start, 1972 target_ulong end, unsigned long prot) 1973 { 1974 FILE *f = (FILE *)priv; 1975 1976 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 1977 " "TARGET_FMT_lx" %c%c%c\n", 1978 start, end, end - start, 1979 ((prot & PAGE_READ) ? 'r' : '-'), 1980 ((prot & PAGE_WRITE) ? 'w' : '-'), 1981 ((prot & PAGE_EXEC) ? 'x' : '-')); 1982 1983 return 0; 1984 } 1985 1986 /* dump memory mappings */ 1987 void page_dump(FILE *f) 1988 { 1989 const int length = sizeof(target_ulong) * 2; 1990 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 1991 length, "start", length, "end", length, "size", "prot"); 1992 walk_memory_regions(f, dump_region); 1993 } 1994 1995 int page_get_flags(target_ulong address) 1996 { 1997 PageDesc *p; 1998 1999 p = page_find(address >> TARGET_PAGE_BITS); 2000 if (!p) { 2001 return 0; 2002 } 2003 return p->flags; 2004 } 2005 2006 /* Modify the flags of a page and invalidate the code if necessary. 2007 The flag PAGE_WRITE_ORG is positioned automatically depending 2008 on PAGE_WRITE. The mmap_lock should already be held. */ 2009 void page_set_flags(target_ulong start, target_ulong end, int flags) 2010 { 2011 target_ulong addr, len; 2012 2013 /* This function should never be called with addresses outside the 2014 guest address space. If this assert fires, it probably indicates 2015 a missing call to h2g_valid. */ 2016 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2017 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2018 #endif 2019 assert(start < end); 2020 assert_memory_lock(); 2021 2022 start = start & TARGET_PAGE_MASK; 2023 end = TARGET_PAGE_ALIGN(end); 2024 2025 if (flags & PAGE_WRITE) { 2026 flags |= PAGE_WRITE_ORG; 2027 } 2028 2029 for (addr = start, len = end - start; 2030 len != 0; 2031 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2032 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2033 2034 /* If the write protection bit is set, then we invalidate 2035 the code inside. */ 2036 if (!(p->flags & PAGE_WRITE) && 2037 (flags & PAGE_WRITE) && 2038 p->first_tb) { 2039 tb_invalidate_phys_page(addr, 0); 2040 } 2041 p->flags = flags; 2042 } 2043 } 2044 2045 int page_check_range(target_ulong start, target_ulong len, int flags) 2046 { 2047 PageDesc *p; 2048 target_ulong end; 2049 target_ulong addr; 2050 2051 /* This function should never be called with addresses outside the 2052 guest address space. If this assert fires, it probably indicates 2053 a missing call to h2g_valid. */ 2054 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS 2055 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2056 #endif 2057 2058 if (len == 0) { 2059 return 0; 2060 } 2061 if (start + len - 1 < start) { 2062 /* We've wrapped around. */ 2063 return -1; 2064 } 2065 2066 /* must do before we loose bits in the next step */ 2067 end = TARGET_PAGE_ALIGN(start + len); 2068 start = start & TARGET_PAGE_MASK; 2069 2070 for (addr = start, len = end - start; 2071 len != 0; 2072 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2073 p = page_find(addr >> TARGET_PAGE_BITS); 2074 if (!p) { 2075 return -1; 2076 } 2077 if (!(p->flags & PAGE_VALID)) { 2078 return -1; 2079 } 2080 2081 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2082 return -1; 2083 } 2084 if (flags & PAGE_WRITE) { 2085 if (!(p->flags & PAGE_WRITE_ORG)) { 2086 return -1; 2087 } 2088 /* unprotect the page if it was put read-only because it 2089 contains translated code */ 2090 if (!(p->flags & PAGE_WRITE)) { 2091 if (!page_unprotect(addr, 0)) { 2092 return -1; 2093 } 2094 } 2095 } 2096 } 2097 return 0; 2098 } 2099 2100 /* called from signal handler: invalidate the code and unprotect the 2101 * page. Return 0 if the fault was not handled, 1 if it was handled, 2102 * and 2 if it was handled but the caller must cause the TB to be 2103 * immediately exited. (We can only return 2 if the 'pc' argument is 2104 * non-zero.) 2105 */ 2106 int page_unprotect(target_ulong address, uintptr_t pc) 2107 { 2108 unsigned int prot; 2109 bool current_tb_invalidated; 2110 PageDesc *p; 2111 target_ulong host_start, host_end, addr; 2112 2113 /* Technically this isn't safe inside a signal handler. However we 2114 know this only ever happens in a synchronous SEGV handler, so in 2115 practice it seems to be ok. */ 2116 mmap_lock(); 2117 2118 p = page_find(address >> TARGET_PAGE_BITS); 2119 if (!p) { 2120 mmap_unlock(); 2121 return 0; 2122 } 2123 2124 /* if the page was really writable, then we change its 2125 protection back to writable */ 2126 if (p->flags & PAGE_WRITE_ORG) { 2127 current_tb_invalidated = false; 2128 if (p->flags & PAGE_WRITE) { 2129 /* If the page is actually marked WRITE then assume this is because 2130 * this thread raced with another one which got here first and 2131 * set the page to PAGE_WRITE and did the TB invalidate for us. 2132 */ 2133 #ifdef TARGET_HAS_PRECISE_SMC 2134 TranslationBlock *current_tb = tcg_tb_lookup(pc); 2135 if (current_tb) { 2136 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; 2137 } 2138 #endif 2139 } else { 2140 host_start = address & qemu_host_page_mask; 2141 host_end = host_start + qemu_host_page_size; 2142 2143 prot = 0; 2144 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) { 2145 p = page_find(addr >> TARGET_PAGE_BITS); 2146 p->flags |= PAGE_WRITE; 2147 prot |= p->flags; 2148 2149 /* and since the content will be modified, we must invalidate 2150 the corresponding translated code. */ 2151 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2152 #ifdef CONFIG_USER_ONLY 2153 if (DEBUG_TB_CHECK_GATE) { 2154 tb_invalidate_check(addr); 2155 } 2156 #endif 2157 } 2158 mprotect((void *)g2h(host_start), qemu_host_page_size, 2159 prot & PAGE_BITS); 2160 } 2161 mmap_unlock(); 2162 /* If current TB was invalidated return to main loop */ 2163 return current_tb_invalidated ? 2 : 1; 2164 } 2165 mmap_unlock(); 2166 return 0; 2167 } 2168 #endif /* CONFIG_USER_ONLY */ 2169 2170 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2171 void tcg_flush_softmmu_tlb(CPUState *cs) 2172 { 2173 #ifdef CONFIG_SOFTMMU 2174 tlb_flush(cs); 2175 #endif 2176 } 2177