1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/units.h" 22 #include "qemu-common.h" 23 24 #define NO_CPU_IO_DEFS 25 #include "cpu.h" 26 #include "trace.h" 27 #include "disas/disas.h" 28 #include "exec/exec-all.h" 29 #include "tcg/tcg.h" 30 #if defined(CONFIG_USER_ONLY) 31 #include "qemu.h" 32 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 33 #include <sys/param.h> 34 #if __FreeBSD_version >= 700104 35 #define HAVE_KINFO_GETVMMAP 36 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 37 #include <sys/proc.h> 38 #include <machine/profile.h> 39 #define _KERNEL 40 #include <sys/user.h> 41 #undef _KERNEL 42 #undef sigqueue 43 #include <libutil.h> 44 #endif 45 #endif 46 #else 47 #include "exec/ram_addr.h" 48 #endif 49 50 #include "exec/cputlb.h" 51 #include "exec/tb-hash.h" 52 #include "exec/translate-all.h" 53 #include "qemu/bitmap.h" 54 #include "qemu/error-report.h" 55 #include "qemu/qemu-print.h" 56 #include "qemu/timer.h" 57 #include "qemu/main-loop.h" 58 #include "exec/log.h" 59 #include "sysemu/cpus.h" 60 #include "sysemu/cpu-timers.h" 61 #include "sysemu/tcg.h" 62 #include "qapi/error.h" 63 64 /* #define DEBUG_TB_INVALIDATE */ 65 /* #define DEBUG_TB_FLUSH */ 66 /* make various TB consistency checks */ 67 /* #define DEBUG_TB_CHECK */ 68 69 #ifdef DEBUG_TB_INVALIDATE 70 #define DEBUG_TB_INVALIDATE_GATE 1 71 #else 72 #define DEBUG_TB_INVALIDATE_GATE 0 73 #endif 74 75 #ifdef DEBUG_TB_FLUSH 76 #define DEBUG_TB_FLUSH_GATE 1 77 #else 78 #define DEBUG_TB_FLUSH_GATE 0 79 #endif 80 81 #if !defined(CONFIG_USER_ONLY) 82 /* TB consistency checks only implemented for usermode emulation. */ 83 #undef DEBUG_TB_CHECK 84 #endif 85 86 #ifdef DEBUG_TB_CHECK 87 #define DEBUG_TB_CHECK_GATE 1 88 #else 89 #define DEBUG_TB_CHECK_GATE 0 90 #endif 91 92 /* Access to the various translations structures need to be serialised via locks 93 * for consistency. 94 * In user-mode emulation access to the memory related structures are protected 95 * with mmap_lock. 96 * In !user-mode we use per-page locks. 97 */ 98 #ifdef CONFIG_SOFTMMU 99 #define assert_memory_lock() 100 #else 101 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 102 #endif 103 104 #define SMC_BITMAP_USE_THRESHOLD 10 105 106 typedef struct PageDesc { 107 /* list of TBs intersecting this ram page */ 108 uintptr_t first_tb; 109 #ifdef CONFIG_SOFTMMU 110 /* in order to optimize self modifying code, we count the number 111 of lookups we do to a given page to use a bitmap */ 112 unsigned long *code_bitmap; 113 unsigned int code_write_count; 114 #else 115 unsigned long flags; 116 #endif 117 #ifndef CONFIG_USER_ONLY 118 QemuSpin lock; 119 #endif 120 } PageDesc; 121 122 /** 123 * struct page_entry - page descriptor entry 124 * @pd: pointer to the &struct PageDesc of the page this entry represents 125 * @index: page index of the page 126 * @locked: whether the page is locked 127 * 128 * This struct helps us keep track of the locked state of a page, without 129 * bloating &struct PageDesc. 130 * 131 * A page lock protects accesses to all fields of &struct PageDesc. 132 * 133 * See also: &struct page_collection. 134 */ 135 struct page_entry { 136 PageDesc *pd; 137 tb_page_addr_t index; 138 bool locked; 139 }; 140 141 /** 142 * struct page_collection - tracks a set of pages (i.e. &struct page_entry's) 143 * @tree: Binary search tree (BST) of the pages, with key == page index 144 * @max: Pointer to the page in @tree with the highest page index 145 * 146 * To avoid deadlock we lock pages in ascending order of page index. 147 * When operating on a set of pages, we need to keep track of them so that 148 * we can lock them in order and also unlock them later. For this we collect 149 * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the 150 * @tree implementation we use does not provide an O(1) operation to obtain the 151 * highest-ranked element, we use @max to keep track of the inserted page 152 * with the highest index. This is valuable because if a page is not in 153 * the tree and its index is higher than @max's, then we can lock it 154 * without breaking the locking order rule. 155 * 156 * Note on naming: 'struct page_set' would be shorter, but we already have a few 157 * page_set_*() helpers, so page_collection is used instead to avoid confusion. 158 * 159 * See also: page_collection_lock(). 160 */ 161 struct page_collection { 162 GTree *tree; 163 struct page_entry *max; 164 }; 165 166 /* list iterators for lists of tagged pointers in TranslationBlock */ 167 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \ 168 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \ 169 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \ 170 tb = (TranslationBlock *)((uintptr_t)tb & ~1)) 171 172 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \ 173 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next) 174 175 #define TB_FOR_EACH_JMP(head_tb, tb, n) \ 176 TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next) 177 178 /* 179 * In system mode we want L1_MAP to be based on ram offsets, 180 * while in user mode we want it to be based on virtual addresses. 181 * 182 * TODO: For user mode, see the caveat re host vs guest virtual 183 * address spaces near GUEST_ADDR_MAX. 184 */ 185 #if !defined(CONFIG_USER_ONLY) 186 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 187 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 188 #else 189 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 190 #endif 191 #else 192 # define L1_MAP_ADDR_SPACE_BITS MIN(HOST_LONG_BITS, TARGET_ABI_BITS) 193 #endif 194 195 /* Size of the L2 (and L3, etc) page tables. */ 196 #define V_L2_BITS 10 197 #define V_L2_SIZE (1 << V_L2_BITS) 198 199 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 200 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 201 sizeof_field(TranslationBlock, trace_vcpu_dstate) 202 * BITS_PER_BYTE); 203 204 /* 205 * L1 Mapping properties 206 */ 207 static int v_l1_size; 208 static int v_l1_shift; 209 static int v_l2_levels; 210 211 /* The bottom level has pointers to PageDesc, and is indexed by 212 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 213 */ 214 #define V_L1_MIN_BITS 4 215 #define V_L1_MAX_BITS (V_L2_BITS + 3) 216 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 217 218 static void *l1_map[V_L1_MAX_SIZE]; 219 220 /* code generation context */ 221 TCGContext tcg_init_ctx; 222 __thread TCGContext *tcg_ctx; 223 TBContext tb_ctx; 224 bool parallel_cpus; 225 226 static void page_table_config_init(void) 227 { 228 uint32_t v_l1_bits; 229 230 assert(TARGET_PAGE_BITS); 231 /* The bits remaining after N lower levels of page tables. */ 232 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 233 if (v_l1_bits < V_L1_MIN_BITS) { 234 v_l1_bits += V_L2_BITS; 235 } 236 237 v_l1_size = 1 << v_l1_bits; 238 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 239 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 240 241 assert(v_l1_bits <= V_L1_MAX_BITS); 242 assert(v_l1_shift % V_L2_BITS == 0); 243 assert(v_l2_levels >= 0); 244 } 245 246 void cpu_gen_init(void) 247 { 248 tcg_context_init(&tcg_init_ctx); 249 } 250 251 /* Encode VAL as a signed leb128 sequence at P. 252 Return P incremented past the encoded value. */ 253 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 254 { 255 int more, byte; 256 257 do { 258 byte = val & 0x7f; 259 val >>= 7; 260 more = !((val == 0 && (byte & 0x40) == 0) 261 || (val == -1 && (byte & 0x40) != 0)); 262 if (more) { 263 byte |= 0x80; 264 } 265 *p++ = byte; 266 } while (more); 267 268 return p; 269 } 270 271 /* Decode a signed leb128 sequence at *PP; increment *PP past the 272 decoded value. Return the decoded value. */ 273 static target_long decode_sleb128(const uint8_t **pp) 274 { 275 const uint8_t *p = *pp; 276 target_long val = 0; 277 int byte, shift = 0; 278 279 do { 280 byte = *p++; 281 val |= (target_ulong)(byte & 0x7f) << shift; 282 shift += 7; 283 } while (byte & 0x80); 284 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 285 val |= -(target_ulong)1 << shift; 286 } 287 288 *pp = p; 289 return val; 290 } 291 292 /* Encode the data collected about the instructions while compiling TB. 293 Place the data at BLOCK, and return the number of bytes consumed. 294 295 The logical table consists of TARGET_INSN_START_WORDS target_ulong's, 296 which come from the target's insn_start data, followed by a uintptr_t 297 which comes from the host pc of the end of the code implementing the insn. 298 299 Each line of the table is encoded as sleb128 deltas from the previous 300 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 301 That is, the first column is seeded with the guest pc, the last column 302 with the host pc, and the middle columns with zeros. */ 303 304 static int encode_search(TranslationBlock *tb, uint8_t *block) 305 { 306 uint8_t *highwater = tcg_ctx->code_gen_highwater; 307 uint8_t *p = block; 308 int i, j, n; 309 310 for (i = 0, n = tb->icount; i < n; ++i) { 311 target_ulong prev; 312 313 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 314 if (i == 0) { 315 prev = (j == 0 ? tb->pc : 0); 316 } else { 317 prev = tcg_ctx->gen_insn_data[i - 1][j]; 318 } 319 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 320 } 321 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 322 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 323 324 /* Test for (pending) buffer overflow. The assumption is that any 325 one row beginning below the high water mark cannot overrun 326 the buffer completely. Thus we can test for overflow after 327 encoding a row without having to check during encoding. */ 328 if (unlikely(p > highwater)) { 329 return -1; 330 } 331 } 332 333 return p - block; 334 } 335 336 /* The cpu state corresponding to 'searched_pc' is restored. 337 * When reset_icount is true, current TB will be interrupted and 338 * icount should be recalculated. 339 */ 340 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 341 uintptr_t searched_pc, bool reset_icount) 342 { 343 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 344 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 345 CPUArchState *env = cpu->env_ptr; 346 const uint8_t *p = tb->tc.ptr + tb->tc.size; 347 int i, j, num_insns = tb->icount; 348 #ifdef CONFIG_PROFILER 349 TCGProfile *prof = &tcg_ctx->prof; 350 int64_t ti = profile_getclock(); 351 #endif 352 353 searched_pc -= GETPC_ADJ; 354 355 if (searched_pc < host_pc) { 356 return -1; 357 } 358 359 /* Reconstruct the stored insn data while looking for the point at 360 which the end of the insn exceeds the searched_pc. */ 361 for (i = 0; i < num_insns; ++i) { 362 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 363 data[j] += decode_sleb128(&p); 364 } 365 host_pc += decode_sleb128(&p); 366 if (host_pc > searched_pc) { 367 goto found; 368 } 369 } 370 return -1; 371 372 found: 373 if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) { 374 assert(icount_enabled()); 375 /* Reset the cycle counter to the start of the block 376 and shift if to the number of actually executed instructions */ 377 cpu_neg(cpu)->icount_decr.u16.low += num_insns - i; 378 } 379 restore_state_to_opc(env, tb, data); 380 381 #ifdef CONFIG_PROFILER 382 qatomic_set(&prof->restore_time, 383 prof->restore_time + profile_getclock() - ti); 384 qatomic_set(&prof->restore_count, prof->restore_count + 1); 385 #endif 386 return 0; 387 } 388 389 void tb_destroy(TranslationBlock *tb) 390 { 391 qemu_spin_destroy(&tb->jmp_lock); 392 } 393 394 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) 395 { 396 /* 397 * The host_pc has to be in the rx region of the code buffer. 398 * If it is not we will not be able to resolve it here. 399 * The two cases where host_pc will not be correct are: 400 * 401 * - fault during translation (instruction fetch) 402 * - fault from helper (not using GETPC() macro) 403 * 404 * Either way we need return early as we can't resolve it here. 405 */ 406 if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { 407 TranslationBlock *tb = tcg_tb_lookup(host_pc); 408 if (tb) { 409 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); 410 if (tb_cflags(tb) & CF_NOCACHE) { 411 /* one-shot translation, invalidate it immediately */ 412 tb_phys_invalidate(tb, -1); 413 tcg_tb_remove(tb); 414 tb_destroy(tb); 415 } 416 return true; 417 } 418 } 419 return false; 420 } 421 422 static void page_init(void) 423 { 424 page_size_init(); 425 page_table_config_init(); 426 427 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 428 { 429 #ifdef HAVE_KINFO_GETVMMAP 430 struct kinfo_vmentry *freep; 431 int i, cnt; 432 433 freep = kinfo_getvmmap(getpid(), &cnt); 434 if (freep) { 435 mmap_lock(); 436 for (i = 0; i < cnt; i++) { 437 unsigned long startaddr, endaddr; 438 439 startaddr = freep[i].kve_start; 440 endaddr = freep[i].kve_end; 441 if (h2g_valid(startaddr)) { 442 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 443 444 if (h2g_valid(endaddr)) { 445 endaddr = h2g(endaddr); 446 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 447 } else { 448 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 449 endaddr = ~0ul; 450 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 451 #endif 452 } 453 } 454 } 455 free(freep); 456 mmap_unlock(); 457 } 458 #else 459 FILE *f; 460 461 last_brk = (unsigned long)sbrk(0); 462 463 f = fopen("/compat/linux/proc/self/maps", "r"); 464 if (f) { 465 mmap_lock(); 466 467 do { 468 unsigned long startaddr, endaddr; 469 int n; 470 471 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 472 473 if (n == 2 && h2g_valid(startaddr)) { 474 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 475 476 if (h2g_valid(endaddr)) { 477 endaddr = h2g(endaddr); 478 } else { 479 endaddr = ~0ul; 480 } 481 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 482 } 483 } while (!feof(f)); 484 485 fclose(f); 486 mmap_unlock(); 487 } 488 #endif 489 } 490 #endif 491 } 492 493 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 494 { 495 PageDesc *pd; 496 void **lp; 497 int i; 498 499 /* Level 1. Always allocated. */ 500 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 501 502 /* Level 2..N-1. */ 503 for (i = v_l2_levels; i > 0; i--) { 504 void **p = qatomic_rcu_read(lp); 505 506 if (p == NULL) { 507 void *existing; 508 509 if (!alloc) { 510 return NULL; 511 } 512 p = g_new0(void *, V_L2_SIZE); 513 existing = qatomic_cmpxchg(lp, NULL, p); 514 if (unlikely(existing)) { 515 g_free(p); 516 p = existing; 517 } 518 } 519 520 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 521 } 522 523 pd = qatomic_rcu_read(lp); 524 if (pd == NULL) { 525 void *existing; 526 527 if (!alloc) { 528 return NULL; 529 } 530 pd = g_new0(PageDesc, V_L2_SIZE); 531 #ifndef CONFIG_USER_ONLY 532 { 533 int i; 534 535 for (i = 0; i < V_L2_SIZE; i++) { 536 qemu_spin_init(&pd[i].lock); 537 } 538 } 539 #endif 540 existing = qatomic_cmpxchg(lp, NULL, pd); 541 if (unlikely(existing)) { 542 #ifndef CONFIG_USER_ONLY 543 { 544 int i; 545 546 for (i = 0; i < V_L2_SIZE; i++) { 547 qemu_spin_destroy(&pd[i].lock); 548 } 549 } 550 #endif 551 g_free(pd); 552 pd = existing; 553 } 554 } 555 556 return pd + (index & (V_L2_SIZE - 1)); 557 } 558 559 static inline PageDesc *page_find(tb_page_addr_t index) 560 { 561 return page_find_alloc(index, 0); 562 } 563 564 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, 565 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc); 566 567 /* In user-mode page locks aren't used; mmap_lock is enough */ 568 #ifdef CONFIG_USER_ONLY 569 570 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock()) 571 572 static inline void page_lock(PageDesc *pd) 573 { } 574 575 static inline void page_unlock(PageDesc *pd) 576 { } 577 578 static inline void page_lock_tb(const TranslationBlock *tb) 579 { } 580 581 static inline void page_unlock_tb(const TranslationBlock *tb) 582 { } 583 584 struct page_collection * 585 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end) 586 { 587 return NULL; 588 } 589 590 void page_collection_unlock(struct page_collection *set) 591 { } 592 #else /* !CONFIG_USER_ONLY */ 593 594 #ifdef CONFIG_DEBUG_TCG 595 596 static __thread GHashTable *ht_pages_locked_debug; 597 598 static void ht_pages_locked_debug_init(void) 599 { 600 if (ht_pages_locked_debug) { 601 return; 602 } 603 ht_pages_locked_debug = g_hash_table_new(NULL, NULL); 604 } 605 606 static bool page_is_locked(const PageDesc *pd) 607 { 608 PageDesc *found; 609 610 ht_pages_locked_debug_init(); 611 found = g_hash_table_lookup(ht_pages_locked_debug, pd); 612 return !!found; 613 } 614 615 static void page_lock__debug(PageDesc *pd) 616 { 617 ht_pages_locked_debug_init(); 618 g_assert(!page_is_locked(pd)); 619 g_hash_table_insert(ht_pages_locked_debug, pd, pd); 620 } 621 622 static void page_unlock__debug(const PageDesc *pd) 623 { 624 bool removed; 625 626 ht_pages_locked_debug_init(); 627 g_assert(page_is_locked(pd)); 628 removed = g_hash_table_remove(ht_pages_locked_debug, pd); 629 g_assert(removed); 630 } 631 632 static void 633 do_assert_page_locked(const PageDesc *pd, const char *file, int line) 634 { 635 if (unlikely(!page_is_locked(pd))) { 636 error_report("assert_page_lock: PageDesc %p not locked @ %s:%d", 637 pd, file, line); 638 abort(); 639 } 640 } 641 642 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__) 643 644 void assert_no_pages_locked(void) 645 { 646 ht_pages_locked_debug_init(); 647 g_assert(g_hash_table_size(ht_pages_locked_debug) == 0); 648 } 649 650 #else /* !CONFIG_DEBUG_TCG */ 651 652 #define assert_page_locked(pd) 653 654 static inline void page_lock__debug(const PageDesc *pd) 655 { 656 } 657 658 static inline void page_unlock__debug(const PageDesc *pd) 659 { 660 } 661 662 #endif /* CONFIG_DEBUG_TCG */ 663 664 static inline void page_lock(PageDesc *pd) 665 { 666 page_lock__debug(pd); 667 qemu_spin_lock(&pd->lock); 668 } 669 670 static inline void page_unlock(PageDesc *pd) 671 { 672 qemu_spin_unlock(&pd->lock); 673 page_unlock__debug(pd); 674 } 675 676 /* lock the page(s) of a TB in the correct acquisition order */ 677 static inline void page_lock_tb(const TranslationBlock *tb) 678 { 679 page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], 0); 680 } 681 682 static inline void page_unlock_tb(const TranslationBlock *tb) 683 { 684 PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 685 686 page_unlock(p1); 687 if (unlikely(tb->page_addr[1] != -1)) { 688 PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 689 690 if (p2 != p1) { 691 page_unlock(p2); 692 } 693 } 694 } 695 696 static inline struct page_entry * 697 page_entry_new(PageDesc *pd, tb_page_addr_t index) 698 { 699 struct page_entry *pe = g_malloc(sizeof(*pe)); 700 701 pe->index = index; 702 pe->pd = pd; 703 pe->locked = false; 704 return pe; 705 } 706 707 static void page_entry_destroy(gpointer p) 708 { 709 struct page_entry *pe = p; 710 711 g_assert(pe->locked); 712 page_unlock(pe->pd); 713 g_free(pe); 714 } 715 716 /* returns false on success */ 717 static bool page_entry_trylock(struct page_entry *pe) 718 { 719 bool busy; 720 721 busy = qemu_spin_trylock(&pe->pd->lock); 722 if (!busy) { 723 g_assert(!pe->locked); 724 pe->locked = true; 725 page_lock__debug(pe->pd); 726 } 727 return busy; 728 } 729 730 static void do_page_entry_lock(struct page_entry *pe) 731 { 732 page_lock(pe->pd); 733 g_assert(!pe->locked); 734 pe->locked = true; 735 } 736 737 static gboolean page_entry_lock(gpointer key, gpointer value, gpointer data) 738 { 739 struct page_entry *pe = value; 740 741 do_page_entry_lock(pe); 742 return FALSE; 743 } 744 745 static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data) 746 { 747 struct page_entry *pe = value; 748 749 if (pe->locked) { 750 pe->locked = false; 751 page_unlock(pe->pd); 752 } 753 return FALSE; 754 } 755 756 /* 757 * Trylock a page, and if successful, add the page to a collection. 758 * Returns true ("busy") if the page could not be locked; false otherwise. 759 */ 760 static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr) 761 { 762 tb_page_addr_t index = addr >> TARGET_PAGE_BITS; 763 struct page_entry *pe; 764 PageDesc *pd; 765 766 pe = g_tree_lookup(set->tree, &index); 767 if (pe) { 768 return false; 769 } 770 771 pd = page_find(index); 772 if (pd == NULL) { 773 return false; 774 } 775 776 pe = page_entry_new(pd, index); 777 g_tree_insert(set->tree, &pe->index, pe); 778 779 /* 780 * If this is either (1) the first insertion or (2) a page whose index 781 * is higher than any other so far, just lock the page and move on. 782 */ 783 if (set->max == NULL || pe->index > set->max->index) { 784 set->max = pe; 785 do_page_entry_lock(pe); 786 return false; 787 } 788 /* 789 * Try to acquire out-of-order lock; if busy, return busy so that we acquire 790 * locks in order. 791 */ 792 return page_entry_trylock(pe); 793 } 794 795 static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata) 796 { 797 tb_page_addr_t a = *(const tb_page_addr_t *)ap; 798 tb_page_addr_t b = *(const tb_page_addr_t *)bp; 799 800 if (a == b) { 801 return 0; 802 } else if (a < b) { 803 return -1; 804 } 805 return 1; 806 } 807 808 /* 809 * Lock a range of pages ([@start,@end[) as well as the pages of all 810 * intersecting TBs. 811 * Locking order: acquire locks in ascending order of page index. 812 */ 813 struct page_collection * 814 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end) 815 { 816 struct page_collection *set = g_malloc(sizeof(*set)); 817 tb_page_addr_t index; 818 PageDesc *pd; 819 820 start >>= TARGET_PAGE_BITS; 821 end >>= TARGET_PAGE_BITS; 822 g_assert(start <= end); 823 824 set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL, 825 page_entry_destroy); 826 set->max = NULL; 827 assert_no_pages_locked(); 828 829 retry: 830 g_tree_foreach(set->tree, page_entry_lock, NULL); 831 832 for (index = start; index <= end; index++) { 833 TranslationBlock *tb; 834 int n; 835 836 pd = page_find(index); 837 if (pd == NULL) { 838 continue; 839 } 840 if (page_trylock_add(set, index << TARGET_PAGE_BITS)) { 841 g_tree_foreach(set->tree, page_entry_unlock, NULL); 842 goto retry; 843 } 844 assert_page_locked(pd); 845 PAGE_FOR_EACH_TB(pd, tb, n) { 846 if (page_trylock_add(set, tb->page_addr[0]) || 847 (tb->page_addr[1] != -1 && 848 page_trylock_add(set, tb->page_addr[1]))) { 849 /* drop all locks, and reacquire in order */ 850 g_tree_foreach(set->tree, page_entry_unlock, NULL); 851 goto retry; 852 } 853 } 854 } 855 return set; 856 } 857 858 void page_collection_unlock(struct page_collection *set) 859 { 860 /* entries are unlocked and freed via page_entry_destroy */ 861 g_tree_destroy(set->tree); 862 g_free(set); 863 } 864 865 #endif /* !CONFIG_USER_ONLY */ 866 867 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, 868 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc) 869 { 870 PageDesc *p1, *p2; 871 tb_page_addr_t page1; 872 tb_page_addr_t page2; 873 874 assert_memory_lock(); 875 g_assert(phys1 != -1); 876 877 page1 = phys1 >> TARGET_PAGE_BITS; 878 page2 = phys2 >> TARGET_PAGE_BITS; 879 880 p1 = page_find_alloc(page1, alloc); 881 if (ret_p1) { 882 *ret_p1 = p1; 883 } 884 if (likely(phys2 == -1)) { 885 page_lock(p1); 886 return; 887 } else if (page1 == page2) { 888 page_lock(p1); 889 if (ret_p2) { 890 *ret_p2 = p1; 891 } 892 return; 893 } 894 p2 = page_find_alloc(page2, alloc); 895 if (ret_p2) { 896 *ret_p2 = p2; 897 } 898 if (page1 < page2) { 899 page_lock(p1); 900 page_lock(p2); 901 } else { 902 page_lock(p2); 903 page_lock(p1); 904 } 905 } 906 907 /* Minimum size of the code gen buffer. This number is randomly chosen, 908 but not so small that we can't have a fair number of TB's live. */ 909 #define MIN_CODE_GEN_BUFFER_SIZE (1 * MiB) 910 911 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 912 indicated, this is constrained by the range of direct branches on the 913 host cpu, as used by the TCG implementation of goto_tb. */ 914 #if defined(__x86_64__) 915 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 916 #elif defined(__sparc__) 917 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 918 #elif defined(__powerpc64__) 919 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 920 #elif defined(__powerpc__) 921 # define MAX_CODE_GEN_BUFFER_SIZE (32 * MiB) 922 #elif defined(__aarch64__) 923 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 924 #elif defined(__s390x__) 925 /* We have a +- 4GB range on the branches; leave some slop. */ 926 # define MAX_CODE_GEN_BUFFER_SIZE (3 * GiB) 927 #elif defined(__mips__) 928 /* We have a 256MB branch region, but leave room to make sure the 929 main executable is also within that region. */ 930 # define MAX_CODE_GEN_BUFFER_SIZE (128 * MiB) 931 #else 932 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 933 #endif 934 935 #if TCG_TARGET_REG_BITS == 32 936 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32 * MiB) 937 #ifdef CONFIG_USER_ONLY 938 /* 939 * For user mode on smaller 32 bit systems we may run into trouble 940 * allocating big chunks of data in the right place. On these systems 941 * we utilise a static code generation buffer directly in the binary. 942 */ 943 #define USE_STATIC_CODE_GEN_BUFFER 944 #endif 945 #else /* TCG_TARGET_REG_BITS == 64 */ 946 #ifdef CONFIG_USER_ONLY 947 /* 948 * As user-mode emulation typically means running multiple instances 949 * of the translator don't go too nuts with our default code gen 950 * buffer lest we make things too hard for the OS. 951 */ 952 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (128 * MiB) 953 #else 954 /* 955 * We expect most system emulation to run one or two guests per host. 956 * Users running large scale system emulation may want to tweak their 957 * runtime setup via the tb-size control on the command line. 958 */ 959 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (1 * GiB) 960 #endif 961 #endif 962 963 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 964 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 965 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 966 967 static size_t size_code_gen_buffer(size_t tb_size) 968 { 969 /* Size the buffer. */ 970 if (tb_size == 0) { 971 size_t phys_mem = qemu_get_host_physmem(); 972 if (phys_mem == 0) { 973 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 974 } else { 975 tb_size = MIN(DEFAULT_CODE_GEN_BUFFER_SIZE, phys_mem / 8); 976 } 977 } 978 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 979 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 980 } 981 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 982 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 983 } 984 return tb_size; 985 } 986 987 #ifdef __mips__ 988 /* In order to use J and JAL within the code_gen_buffer, we require 989 that the buffer not cross a 256MB boundary. */ 990 static inline bool cross_256mb(void *addr, size_t size) 991 { 992 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 993 } 994 995 /* We weren't able to allocate a buffer without crossing that boundary, 996 so make do with the larger portion of the buffer that doesn't cross. 997 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 998 static inline void *split_cross_256mb(void *buf1, size_t size1) 999 { 1000 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 1001 size_t size2 = buf1 + size1 - buf2; 1002 1003 size1 = buf2 - buf1; 1004 if (size1 < size2) { 1005 size1 = size2; 1006 buf1 = buf2; 1007 } 1008 1009 tcg_ctx->code_gen_buffer_size = size1; 1010 return buf1; 1011 } 1012 #endif 1013 1014 #ifdef USE_STATIC_CODE_GEN_BUFFER 1015 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 1016 __attribute__((aligned(CODE_GEN_ALIGN))); 1017 1018 static bool alloc_code_gen_buffer(size_t tb_size, int splitwx, Error **errp) 1019 { 1020 void *buf, *end; 1021 size_t size; 1022 1023 if (splitwx > 0) { 1024 error_setg(errp, "jit split-wx not supported"); 1025 return false; 1026 } 1027 1028 /* page-align the beginning and end of the buffer */ 1029 buf = static_code_gen_buffer; 1030 end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 1031 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 1032 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 1033 1034 size = end - buf; 1035 1036 /* Honor a command-line option limiting the size of the buffer. */ 1037 if (size > tb_size) { 1038 size = QEMU_ALIGN_DOWN(tb_size, qemu_real_host_page_size); 1039 } 1040 tcg_ctx->code_gen_buffer_size = size; 1041 1042 #ifdef __mips__ 1043 if (cross_256mb(buf, size)) { 1044 buf = split_cross_256mb(buf, size); 1045 size = tcg_ctx->code_gen_buffer_size; 1046 } 1047 #endif 1048 1049 if (qemu_mprotect_rwx(buf, size)) { 1050 error_setg_errno(errp, errno, "mprotect of jit buffer"); 1051 return false; 1052 } 1053 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 1054 1055 tcg_ctx->code_gen_buffer = buf; 1056 return true; 1057 } 1058 #elif defined(_WIN32) 1059 static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) 1060 { 1061 void *buf; 1062 1063 if (splitwx > 0) { 1064 error_setg(errp, "jit split-wx not supported"); 1065 return false; 1066 } 1067 1068 buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 1069 PAGE_EXECUTE_READWRITE); 1070 if (buf == NULL) { 1071 error_setg_win32(errp, GetLastError(), 1072 "allocate %zu bytes for jit buffer", size); 1073 return false; 1074 } 1075 1076 tcg_ctx->code_gen_buffer = buf; 1077 tcg_ctx->code_gen_buffer_size = size; 1078 return true; 1079 } 1080 #else 1081 static bool alloc_code_gen_buffer_anon(size_t size, int prot, 1082 int flags, Error **errp) 1083 { 1084 void *buf; 1085 1086 buf = mmap(NULL, size, prot, flags, -1, 0); 1087 if (buf == MAP_FAILED) { 1088 error_setg_errno(errp, errno, 1089 "allocate %zu bytes for jit buffer", size); 1090 return false; 1091 } 1092 tcg_ctx->code_gen_buffer_size = size; 1093 1094 #ifdef __mips__ 1095 if (cross_256mb(buf, size)) { 1096 /* 1097 * Try again, with the original still mapped, to avoid re-acquiring 1098 * the same 256mb crossing. 1099 */ 1100 size_t size2; 1101 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 1102 switch ((int)(buf2 != MAP_FAILED)) { 1103 case 1: 1104 if (!cross_256mb(buf2, size)) { 1105 /* Success! Use the new buffer. */ 1106 munmap(buf, size); 1107 break; 1108 } 1109 /* Failure. Work with what we had. */ 1110 munmap(buf2, size); 1111 /* fallthru */ 1112 default: 1113 /* Split the original buffer. Free the smaller half. */ 1114 buf2 = split_cross_256mb(buf, size); 1115 size2 = tcg_ctx->code_gen_buffer_size; 1116 if (buf == buf2) { 1117 munmap(buf + size2, size - size2); 1118 } else { 1119 munmap(buf, size - size2); 1120 } 1121 size = size2; 1122 break; 1123 } 1124 buf = buf2; 1125 } 1126 #endif 1127 1128 /* Request large pages for the buffer. */ 1129 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 1130 1131 tcg_ctx->code_gen_buffer = buf; 1132 return true; 1133 } 1134 1135 #ifdef CONFIG_POSIX 1136 #include "qemu/memfd.h" 1137 1138 static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp) 1139 { 1140 void *buf_rw, *buf_rx; 1141 int fd = -1; 1142 1143 buf_rw = qemu_memfd_alloc("tcg-jit", size, 0, &fd, errp); 1144 if (buf_rw == NULL) { 1145 return false; 1146 } 1147 1148 buf_rx = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); 1149 if (buf_rx == MAP_FAILED) { 1150 error_setg_errno(errp, errno, 1151 "failed to map shared memory for execute"); 1152 munmap(buf_rw, size); 1153 close(fd); 1154 return false; 1155 } 1156 close(fd); 1157 1158 tcg_ctx->code_gen_buffer = buf_rw; 1159 tcg_ctx->code_gen_buffer_size = size; 1160 tcg_splitwx_diff = buf_rx - buf_rw; 1161 1162 /* Request large pages for the buffer and the splitwx. */ 1163 qemu_madvise(buf_rw, size, QEMU_MADV_HUGEPAGE); 1164 qemu_madvise(buf_rx, size, QEMU_MADV_HUGEPAGE); 1165 return true; 1166 } 1167 #endif /* CONFIG_POSIX */ 1168 1169 #ifdef CONFIG_DARWIN 1170 #include <mach/mach.h> 1171 1172 extern kern_return_t mach_vm_remap(vm_map_t target_task, 1173 mach_vm_address_t *target_address, 1174 mach_vm_size_t size, 1175 mach_vm_offset_t mask, 1176 int flags, 1177 vm_map_t src_task, 1178 mach_vm_address_t src_address, 1179 boolean_t copy, 1180 vm_prot_t *cur_protection, 1181 vm_prot_t *max_protection, 1182 vm_inherit_t inheritance); 1183 1184 static bool alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp) 1185 { 1186 kern_return_t ret; 1187 mach_vm_address_t buf_rw, buf_rx; 1188 vm_prot_t cur_prot, max_prot; 1189 1190 /* Map the read-write portion via normal anon memory. */ 1191 if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE, 1192 MAP_PRIVATE | MAP_ANONYMOUS, errp)) { 1193 return false; 1194 } 1195 1196 buf_rw = (mach_vm_address_t)tcg_ctx->code_gen_buffer; 1197 buf_rx = 0; 1198 ret = mach_vm_remap(mach_task_self(), 1199 &buf_rx, 1200 size, 1201 0, 1202 VM_FLAGS_ANYWHERE, 1203 mach_task_self(), 1204 buf_rw, 1205 false, 1206 &cur_prot, 1207 &max_prot, 1208 VM_INHERIT_NONE); 1209 if (ret != KERN_SUCCESS) { 1210 /* TODO: Convert "ret" to a human readable error message. */ 1211 error_setg(errp, "vm_remap for jit splitwx failed"); 1212 munmap((void *)buf_rw, size); 1213 return false; 1214 } 1215 1216 if (mprotect((void *)buf_rx, size, PROT_READ | PROT_EXEC) != 0) { 1217 error_setg_errno(errp, errno, "mprotect for jit splitwx"); 1218 munmap((void *)buf_rx, size); 1219 munmap((void *)buf_rw, size); 1220 return false; 1221 } 1222 1223 tcg_splitwx_diff = buf_rx - buf_rw; 1224 return true; 1225 } 1226 #endif /* CONFIG_DARWIN */ 1227 1228 static bool alloc_code_gen_buffer_splitwx(size_t size, Error **errp) 1229 { 1230 if (TCG_TARGET_SUPPORT_MIRROR) { 1231 #ifdef CONFIG_DARWIN 1232 return alloc_code_gen_buffer_splitwx_vmremap(size, errp); 1233 #endif 1234 #ifdef CONFIG_POSIX 1235 return alloc_code_gen_buffer_splitwx_memfd(size, errp); 1236 #endif 1237 } 1238 error_setg(errp, "jit split-wx not supported"); 1239 return false; 1240 } 1241 1242 static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) 1243 { 1244 ERRP_GUARD(); 1245 int prot, flags; 1246 1247 if (splitwx) { 1248 if (alloc_code_gen_buffer_splitwx(size, errp)) { 1249 return true; 1250 } 1251 /* 1252 * If splitwx force-on (1), fail; 1253 * if splitwx default-on (-1), fall through to splitwx off. 1254 */ 1255 if (splitwx > 0) { 1256 return false; 1257 } 1258 error_free_or_abort(errp); 1259 } 1260 1261 prot = PROT_READ | PROT_WRITE | PROT_EXEC; 1262 flags = MAP_PRIVATE | MAP_ANONYMOUS; 1263 #ifdef CONFIG_TCG_INTERPRETER 1264 /* The tcg interpreter does not need execute permission. */ 1265 prot = PROT_READ | PROT_WRITE; 1266 #elif defined(CONFIG_DARWIN) 1267 /* Applicable to both iOS and macOS (Apple Silicon). */ 1268 if (!splitwx) { 1269 flags |= MAP_JIT; 1270 } 1271 #endif 1272 1273 return alloc_code_gen_buffer_anon(size, prot, flags, errp); 1274 } 1275 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 1276 1277 static bool tb_cmp(const void *ap, const void *bp) 1278 { 1279 const TranslationBlock *a = ap; 1280 const TranslationBlock *b = bp; 1281 1282 return a->pc == b->pc && 1283 a->cs_base == b->cs_base && 1284 a->flags == b->flags && 1285 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) && 1286 a->trace_vcpu_dstate == b->trace_vcpu_dstate && 1287 a->page_addr[0] == b->page_addr[0] && 1288 a->page_addr[1] == b->page_addr[1]; 1289 } 1290 1291 static void tb_htable_init(void) 1292 { 1293 unsigned int mode = QHT_MODE_AUTO_RESIZE; 1294 1295 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); 1296 } 1297 1298 /* Must be called before using the QEMU cpus. 'tb_size' is the size 1299 (in bytes) allocated to the translation buffer. Zero means default 1300 size. */ 1301 void tcg_exec_init(unsigned long tb_size, int splitwx) 1302 { 1303 bool ok; 1304 1305 tcg_allowed = true; 1306 cpu_gen_init(); 1307 page_init(); 1308 tb_htable_init(); 1309 1310 ok = alloc_code_gen_buffer(size_code_gen_buffer(tb_size), 1311 splitwx, &error_fatal); 1312 assert(ok); 1313 1314 #if defined(CONFIG_SOFTMMU) 1315 /* There's no guest base to take into account, so go ahead and 1316 initialize the prologue now. */ 1317 tcg_prologue_init(tcg_ctx); 1318 #endif 1319 } 1320 1321 /* call with @p->lock held */ 1322 static inline void invalidate_page_bitmap(PageDesc *p) 1323 { 1324 assert_page_locked(p); 1325 #ifdef CONFIG_SOFTMMU 1326 g_free(p->code_bitmap); 1327 p->code_bitmap = NULL; 1328 p->code_write_count = 0; 1329 #endif 1330 } 1331 1332 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 1333 static void page_flush_tb_1(int level, void **lp) 1334 { 1335 int i; 1336 1337 if (*lp == NULL) { 1338 return; 1339 } 1340 if (level == 0) { 1341 PageDesc *pd = *lp; 1342 1343 for (i = 0; i < V_L2_SIZE; ++i) { 1344 page_lock(&pd[i]); 1345 pd[i].first_tb = (uintptr_t)NULL; 1346 invalidate_page_bitmap(pd + i); 1347 page_unlock(&pd[i]); 1348 } 1349 } else { 1350 void **pp = *lp; 1351 1352 for (i = 0; i < V_L2_SIZE; ++i) { 1353 page_flush_tb_1(level - 1, pp + i); 1354 } 1355 } 1356 } 1357 1358 static void page_flush_tb(void) 1359 { 1360 int i, l1_sz = v_l1_size; 1361 1362 for (i = 0; i < l1_sz; i++) { 1363 page_flush_tb_1(v_l2_levels, l1_map + i); 1364 } 1365 } 1366 1367 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 1368 { 1369 const TranslationBlock *tb = value; 1370 size_t *size = data; 1371 1372 *size += tb->tc.size; 1373 return false; 1374 } 1375 1376 /* flush all the translation blocks */ 1377 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 1378 { 1379 bool did_flush = false; 1380 1381 mmap_lock(); 1382 /* If it is already been done on request of another CPU, 1383 * just retry. 1384 */ 1385 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 1386 goto done; 1387 } 1388 did_flush = true; 1389 1390 if (DEBUG_TB_FLUSH_GATE) { 1391 size_t nb_tbs = tcg_nb_tbs(); 1392 size_t host_size = 0; 1393 1394 tcg_tb_foreach(tb_host_size_iter, &host_size); 1395 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 1396 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 1397 } 1398 1399 CPU_FOREACH(cpu) { 1400 cpu_tb_jmp_cache_clear(cpu); 1401 } 1402 1403 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 1404 page_flush_tb(); 1405 1406 tcg_region_reset_all(); 1407 /* XXX: flush processor icache at this point if cache flush is 1408 expensive */ 1409 qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 1410 1411 done: 1412 mmap_unlock(); 1413 if (did_flush) { 1414 qemu_plugin_flush_cb(); 1415 } 1416 } 1417 1418 void tb_flush(CPUState *cpu) 1419 { 1420 if (tcg_enabled()) { 1421 unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count); 1422 1423 if (cpu_in_exclusive_context(cpu)) { 1424 do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count)); 1425 } else { 1426 async_safe_run_on_cpu(cpu, do_tb_flush, 1427 RUN_ON_CPU_HOST_INT(tb_flush_count)); 1428 } 1429 } 1430 } 1431 1432 /* 1433 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 1434 * so in order to prevent bit rot we compile them unconditionally in user-mode, 1435 * and let the optimizer get rid of them by wrapping their user-only callers 1436 * with if (DEBUG_TB_CHECK_GATE). 1437 */ 1438 #ifdef CONFIG_USER_ONLY 1439 1440 static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp) 1441 { 1442 TranslationBlock *tb = p; 1443 target_ulong addr = *(target_ulong *)userp; 1444 1445 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 1446 printf("ERROR invalidate: address=" TARGET_FMT_lx 1447 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 1448 } 1449 } 1450 1451 /* verify that all the pages have correct rights for code 1452 * 1453 * Called with mmap_lock held. 1454 */ 1455 static void tb_invalidate_check(target_ulong address) 1456 { 1457 address &= TARGET_PAGE_MASK; 1458 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 1459 } 1460 1461 static void do_tb_page_check(void *p, uint32_t hash, void *userp) 1462 { 1463 TranslationBlock *tb = p; 1464 int flags1, flags2; 1465 1466 flags1 = page_get_flags(tb->pc); 1467 flags2 = page_get_flags(tb->pc + tb->size - 1); 1468 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 1469 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 1470 (long)tb->pc, tb->size, flags1, flags2); 1471 } 1472 } 1473 1474 /* verify that all the pages have correct rights for code */ 1475 static void tb_page_check(void) 1476 { 1477 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 1478 } 1479 1480 #endif /* CONFIG_USER_ONLY */ 1481 1482 /* 1483 * user-mode: call with mmap_lock held 1484 * !user-mode: call with @pd->lock held 1485 */ 1486 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb) 1487 { 1488 TranslationBlock *tb1; 1489 uintptr_t *pprev; 1490 unsigned int n1; 1491 1492 assert_page_locked(pd); 1493 pprev = &pd->first_tb; 1494 PAGE_FOR_EACH_TB(pd, tb1, n1) { 1495 if (tb1 == tb) { 1496 *pprev = tb1->page_next[n1]; 1497 return; 1498 } 1499 pprev = &tb1->page_next[n1]; 1500 } 1501 g_assert_not_reached(); 1502 } 1503 1504 /* remove @orig from its @n_orig-th jump list */ 1505 static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig) 1506 { 1507 uintptr_t ptr, ptr_locked; 1508 TranslationBlock *dest; 1509 TranslationBlock *tb; 1510 uintptr_t *pprev; 1511 int n; 1512 1513 /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */ 1514 ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1); 1515 dest = (TranslationBlock *)(ptr & ~1); 1516 if (dest == NULL) { 1517 return; 1518 } 1519 1520 qemu_spin_lock(&dest->jmp_lock); 1521 /* 1522 * While acquiring the lock, the jump might have been removed if the 1523 * destination TB was invalidated; check again. 1524 */ 1525 ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]); 1526 if (ptr_locked != ptr) { 1527 qemu_spin_unlock(&dest->jmp_lock); 1528 /* 1529 * The only possibility is that the jump was unlinked via 1530 * tb_jump_unlink(dest). Seeing here another destination would be a bug, 1531 * because we set the LSB above. 1532 */ 1533 g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID); 1534 return; 1535 } 1536 /* 1537 * We first acquired the lock, and since the destination pointer matches, 1538 * we know for sure that @orig is in the jmp list. 1539 */ 1540 pprev = &dest->jmp_list_head; 1541 TB_FOR_EACH_JMP(dest, tb, n) { 1542 if (tb == orig && n == n_orig) { 1543 *pprev = tb->jmp_list_next[n]; 1544 /* no need to set orig->jmp_dest[n]; setting the LSB was enough */ 1545 qemu_spin_unlock(&dest->jmp_lock); 1546 return; 1547 } 1548 pprev = &tb->jmp_list_next[n]; 1549 } 1550 g_assert_not_reached(); 1551 } 1552 1553 /* reset the jump entry 'n' of a TB so that it is not chained to 1554 another TB */ 1555 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1556 { 1557 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1558 tb_set_jmp_target(tb, n, addr); 1559 } 1560 1561 /* remove any jumps to the TB */ 1562 static inline void tb_jmp_unlink(TranslationBlock *dest) 1563 { 1564 TranslationBlock *tb; 1565 int n; 1566 1567 qemu_spin_lock(&dest->jmp_lock); 1568 1569 TB_FOR_EACH_JMP(dest, tb, n) { 1570 tb_reset_jump(tb, n); 1571 qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1); 1572 /* No need to clear the list entry; setting the dest ptr is enough */ 1573 } 1574 dest->jmp_list_head = (uintptr_t)NULL; 1575 1576 qemu_spin_unlock(&dest->jmp_lock); 1577 } 1578 1579 /* 1580 * In user-mode, call with mmap_lock held. 1581 * In !user-mode, if @rm_from_page_list is set, call with the TB's pages' 1582 * locks held. 1583 */ 1584 static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) 1585 { 1586 CPUState *cpu; 1587 PageDesc *p; 1588 uint32_t h; 1589 tb_page_addr_t phys_pc; 1590 1591 assert_memory_lock(); 1592 1593 /* make sure no further incoming jumps will be chained to this TB */ 1594 qemu_spin_lock(&tb->jmp_lock); 1595 qatomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1596 qemu_spin_unlock(&tb->jmp_lock); 1597 1598 /* remove the TB from the hash list */ 1599 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1600 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb_cflags(tb) & CF_HASH_MASK, 1601 tb->trace_vcpu_dstate); 1602 if (!(tb->cflags & CF_NOCACHE) && 1603 !qht_remove(&tb_ctx.htable, tb, h)) { 1604 return; 1605 } 1606 1607 /* remove the TB from the page list */ 1608 if (rm_from_page_list) { 1609 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1610 tb_page_remove(p, tb); 1611 invalidate_page_bitmap(p); 1612 if (tb->page_addr[1] != -1) { 1613 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1614 tb_page_remove(p, tb); 1615 invalidate_page_bitmap(p); 1616 } 1617 } 1618 1619 /* remove the TB from the hash list */ 1620 h = tb_jmp_cache_hash_func(tb->pc); 1621 CPU_FOREACH(cpu) { 1622 if (qatomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1623 qatomic_set(&cpu->tb_jmp_cache[h], NULL); 1624 } 1625 } 1626 1627 /* suppress this TB from the two jump lists */ 1628 tb_remove_from_jmp_list(tb, 0); 1629 tb_remove_from_jmp_list(tb, 1); 1630 1631 /* suppress any remaining jumps to this TB */ 1632 tb_jmp_unlink(tb); 1633 1634 qatomic_set(&tcg_ctx->tb_phys_invalidate_count, 1635 tcg_ctx->tb_phys_invalidate_count + 1); 1636 } 1637 1638 static void tb_phys_invalidate__locked(TranslationBlock *tb) 1639 { 1640 do_tb_phys_invalidate(tb, true); 1641 } 1642 1643 /* invalidate one TB 1644 * 1645 * Called with mmap_lock held in user-mode. 1646 */ 1647 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1648 { 1649 if (page_addr == -1 && tb->page_addr[0] != -1) { 1650 page_lock_tb(tb); 1651 do_tb_phys_invalidate(tb, true); 1652 page_unlock_tb(tb); 1653 } else { 1654 do_tb_phys_invalidate(tb, false); 1655 } 1656 } 1657 1658 #ifdef CONFIG_SOFTMMU 1659 /* call with @p->lock held */ 1660 static void build_page_bitmap(PageDesc *p) 1661 { 1662 int n, tb_start, tb_end; 1663 TranslationBlock *tb; 1664 1665 assert_page_locked(p); 1666 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1667 1668 PAGE_FOR_EACH_TB(p, tb, n) { 1669 /* NOTE: this is subtle as a TB may span two physical pages */ 1670 if (n == 0) { 1671 /* NOTE: tb_end may be after the end of the page, but 1672 it is not a problem */ 1673 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1674 tb_end = tb_start + tb->size; 1675 if (tb_end > TARGET_PAGE_SIZE) { 1676 tb_end = TARGET_PAGE_SIZE; 1677 } 1678 } else { 1679 tb_start = 0; 1680 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1681 } 1682 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1683 } 1684 } 1685 #endif 1686 1687 /* add the tb in the target page and protect it if necessary 1688 * 1689 * Called with mmap_lock held for user-mode emulation. 1690 * Called with @p->lock held in !user-mode. 1691 */ 1692 static inline void tb_page_add(PageDesc *p, TranslationBlock *tb, 1693 unsigned int n, tb_page_addr_t page_addr) 1694 { 1695 #ifndef CONFIG_USER_ONLY 1696 bool page_already_protected; 1697 #endif 1698 1699 assert_page_locked(p); 1700 1701 tb->page_addr[n] = page_addr; 1702 tb->page_next[n] = p->first_tb; 1703 #ifndef CONFIG_USER_ONLY 1704 page_already_protected = p->first_tb != (uintptr_t)NULL; 1705 #endif 1706 p->first_tb = (uintptr_t)tb | n; 1707 invalidate_page_bitmap(p); 1708 1709 #if defined(CONFIG_USER_ONLY) 1710 if (p->flags & PAGE_WRITE) { 1711 target_ulong addr; 1712 PageDesc *p2; 1713 int prot; 1714 1715 /* force the host page as non writable (writes will have a 1716 page fault + mprotect overhead) */ 1717 page_addr &= qemu_host_page_mask; 1718 prot = 0; 1719 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1720 addr += TARGET_PAGE_SIZE) { 1721 1722 p2 = page_find(addr >> TARGET_PAGE_BITS); 1723 if (!p2) { 1724 continue; 1725 } 1726 prot |= p2->flags; 1727 p2->flags &= ~PAGE_WRITE; 1728 } 1729 mprotect(g2h(page_addr), qemu_host_page_size, 1730 (prot & PAGE_BITS) & ~PAGE_WRITE); 1731 if (DEBUG_TB_INVALIDATE_GATE) { 1732 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1733 } 1734 } 1735 #else 1736 /* if some code is already present, then the pages are already 1737 protected. So we handle the case where only the first TB is 1738 allocated in a physical page */ 1739 if (!page_already_protected) { 1740 tlb_protect_code(page_addr); 1741 } 1742 #endif 1743 } 1744 1745 /* add a new TB and link it to the physical page tables. phys_page2 is 1746 * (-1) to indicate that only one page contains the TB. 1747 * 1748 * Called with mmap_lock held for user-mode emulation. 1749 * 1750 * Returns a pointer @tb, or a pointer to an existing TB that matches @tb. 1751 * Note that in !user-mode, another thread might have already added a TB 1752 * for the same block of guest code that @tb corresponds to. In that case, 1753 * the caller should discard the original @tb, and use instead the returned TB. 1754 */ 1755 static TranslationBlock * 1756 tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1757 tb_page_addr_t phys_page2) 1758 { 1759 PageDesc *p; 1760 PageDesc *p2 = NULL; 1761 1762 assert_memory_lock(); 1763 1764 if (phys_pc == -1) { 1765 /* 1766 * If the TB is not associated with a physical RAM page then 1767 * it must be a temporary one-insn TB, and we have nothing to do 1768 * except fill in the page_addr[] fields. 1769 */ 1770 assert(tb->cflags & CF_NOCACHE); 1771 tb->page_addr[0] = tb->page_addr[1] = -1; 1772 return tb; 1773 } 1774 1775 /* 1776 * Add the TB to the page list, acquiring first the pages's locks. 1777 * We keep the locks held until after inserting the TB in the hash table, 1778 * so that if the insertion fails we know for sure that the TBs are still 1779 * in the page descriptors. 1780 * Note that inserting into the hash table first isn't an option, since 1781 * we can only insert TBs that are fully initialized. 1782 */ 1783 page_lock_pair(&p, phys_pc, &p2, phys_page2, 1); 1784 tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK); 1785 if (p2) { 1786 tb_page_add(p2, tb, 1, phys_page2); 1787 } else { 1788 tb->page_addr[1] = -1; 1789 } 1790 1791 if (!(tb->cflags & CF_NOCACHE)) { 1792 void *existing_tb = NULL; 1793 uint32_t h; 1794 1795 /* add in the hash table */ 1796 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1797 tb->trace_vcpu_dstate); 1798 qht_insert(&tb_ctx.htable, tb, h, &existing_tb); 1799 1800 /* remove TB from the page(s) if we couldn't insert it */ 1801 if (unlikely(existing_tb)) { 1802 tb_page_remove(p, tb); 1803 invalidate_page_bitmap(p); 1804 if (p2) { 1805 tb_page_remove(p2, tb); 1806 invalidate_page_bitmap(p2); 1807 } 1808 tb = existing_tb; 1809 } 1810 } 1811 1812 if (p2 && p2 != p) { 1813 page_unlock(p2); 1814 } 1815 page_unlock(p); 1816 1817 #ifdef CONFIG_USER_ONLY 1818 if (DEBUG_TB_CHECK_GATE) { 1819 tb_page_check(); 1820 } 1821 #endif 1822 return tb; 1823 } 1824 1825 /* Called with mmap_lock held for user mode emulation. */ 1826 TranslationBlock *tb_gen_code(CPUState *cpu, 1827 target_ulong pc, target_ulong cs_base, 1828 uint32_t flags, int cflags) 1829 { 1830 CPUArchState *env = cpu->env_ptr; 1831 TranslationBlock *tb, *existing_tb; 1832 tb_page_addr_t phys_pc, phys_page2; 1833 target_ulong virt_page2; 1834 tcg_insn_unit *gen_code_buf; 1835 int gen_code_size, search_size, max_insns; 1836 #ifdef CONFIG_PROFILER 1837 TCGProfile *prof = &tcg_ctx->prof; 1838 int64_t ti; 1839 #endif 1840 1841 assert_memory_lock(); 1842 1843 phys_pc = get_page_addr_code(env, pc); 1844 1845 if (phys_pc == -1) { 1846 /* Generate a temporary TB with 1 insn in it */ 1847 cflags &= ~CF_COUNT_MASK; 1848 cflags |= CF_NOCACHE | 1; 1849 } 1850 1851 cflags &= ~CF_CLUSTER_MASK; 1852 cflags |= cpu->cluster_index << CF_CLUSTER_SHIFT; 1853 1854 max_insns = cflags & CF_COUNT_MASK; 1855 if (max_insns == 0) { 1856 max_insns = CF_COUNT_MASK; 1857 } 1858 if (max_insns > TCG_MAX_INSNS) { 1859 max_insns = TCG_MAX_INSNS; 1860 } 1861 if (cpu->singlestep_enabled || singlestep) { 1862 max_insns = 1; 1863 } 1864 1865 buffer_overflow: 1866 tb = tcg_tb_alloc(tcg_ctx); 1867 if (unlikely(!tb)) { 1868 /* flush must be done */ 1869 tb_flush(cpu); 1870 mmap_unlock(); 1871 /* Make the execution loop process the flush as soon as possible. */ 1872 cpu->exception_index = EXCP_INTERRUPT; 1873 cpu_loop_exit(cpu); 1874 } 1875 1876 gen_code_buf = tcg_ctx->code_gen_ptr; 1877 tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); 1878 tb->pc = pc; 1879 tb->cs_base = cs_base; 1880 tb->flags = flags; 1881 tb->cflags = cflags; 1882 tb->orig_tb = NULL; 1883 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1884 tcg_ctx->tb_cflags = cflags; 1885 tb_overflow: 1886 1887 #ifdef CONFIG_PROFILER 1888 /* includes aborted translations because of exceptions */ 1889 qatomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1890 ti = profile_getclock(); 1891 #endif 1892 1893 tcg_func_start(tcg_ctx); 1894 1895 tcg_ctx->cpu = env_cpu(env); 1896 gen_intermediate_code(cpu, tb, max_insns); 1897 tcg_ctx->cpu = NULL; 1898 1899 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1900 1901 /* generate machine code */ 1902 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1903 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1904 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1905 if (TCG_TARGET_HAS_direct_jump) { 1906 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1907 tcg_ctx->tb_jmp_target_addr = NULL; 1908 } else { 1909 tcg_ctx->tb_jmp_insn_offset = NULL; 1910 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1911 } 1912 1913 #ifdef CONFIG_PROFILER 1914 qatomic_set(&prof->tb_count, prof->tb_count + 1); 1915 qatomic_set(&prof->interm_time, 1916 prof->interm_time + profile_getclock() - ti); 1917 ti = profile_getclock(); 1918 #endif 1919 1920 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1921 if (unlikely(gen_code_size < 0)) { 1922 switch (gen_code_size) { 1923 case -1: 1924 /* 1925 * Overflow of code_gen_buffer, or the current slice of it. 1926 * 1927 * TODO: We don't need to re-do gen_intermediate_code, nor 1928 * should we re-do the tcg optimization currently hidden 1929 * inside tcg_gen_code. All that should be required is to 1930 * flush the TBs, allocate a new TB, re-initialize it per 1931 * above, and re-do the actual code generation. 1932 */ 1933 goto buffer_overflow; 1934 1935 case -2: 1936 /* 1937 * The code generated for the TranslationBlock is too large. 1938 * The maximum size allowed by the unwind info is 64k. 1939 * There may be stricter constraints from relocations 1940 * in the tcg backend. 1941 * 1942 * Try again with half as many insns as we attempted this time. 1943 * If a single insn overflows, there's a bug somewhere... 1944 */ 1945 max_insns = tb->icount; 1946 assert(max_insns > 1); 1947 max_insns /= 2; 1948 goto tb_overflow; 1949 1950 default: 1951 g_assert_not_reached(); 1952 } 1953 } 1954 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1955 if (unlikely(search_size < 0)) { 1956 goto buffer_overflow; 1957 } 1958 tb->tc.size = gen_code_size; 1959 1960 #ifdef CONFIG_PROFILER 1961 qatomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1962 qatomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1963 qatomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1964 qatomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1965 #endif 1966 1967 #ifdef DEBUG_DISAS 1968 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1969 qemu_log_in_addr_range(tb->pc)) { 1970 FILE *logfile = qemu_log_lock(); 1971 int code_size, data_size; 1972 const tcg_target_ulong *rx_data_gen_ptr; 1973 size_t chunk_start; 1974 int insn = 0; 1975 1976 if (tcg_ctx->data_gen_ptr) { 1977 rx_data_gen_ptr = tcg_splitwx_to_rx(tcg_ctx->data_gen_ptr); 1978 code_size = (const void *)rx_data_gen_ptr - tb->tc.ptr; 1979 data_size = gen_code_size - code_size; 1980 } else { 1981 rx_data_gen_ptr = 0; 1982 code_size = gen_code_size; 1983 data_size = 0; 1984 } 1985 1986 /* Dump header and the first instruction */ 1987 qemu_log("OUT: [size=%d]\n", gen_code_size); 1988 qemu_log(" -- guest addr 0x" TARGET_FMT_lx " + tb prologue\n", 1989 tcg_ctx->gen_insn_data[insn][0]); 1990 chunk_start = tcg_ctx->gen_insn_end_off[insn]; 1991 log_disas(tb->tc.ptr, chunk_start); 1992 1993 /* 1994 * Dump each instruction chunk, wrapping up empty chunks into 1995 * the next instruction. The whole array is offset so the 1996 * first entry is the beginning of the 2nd instruction. 1997 */ 1998 while (insn < tb->icount) { 1999 size_t chunk_end = tcg_ctx->gen_insn_end_off[insn]; 2000 if (chunk_end > chunk_start) { 2001 qemu_log(" -- guest addr 0x" TARGET_FMT_lx "\n", 2002 tcg_ctx->gen_insn_data[insn][0]); 2003 log_disas(tb->tc.ptr + chunk_start, chunk_end - chunk_start); 2004 chunk_start = chunk_end; 2005 } 2006 insn++; 2007 } 2008 2009 if (chunk_start < code_size) { 2010 qemu_log(" -- tb slow paths + alignment\n"); 2011 log_disas(tb->tc.ptr + chunk_start, code_size - chunk_start); 2012 } 2013 2014 /* Finally dump any data we may have after the block */ 2015 if (data_size) { 2016 int i; 2017 qemu_log(" data: [size=%d]\n", data_size); 2018 for (i = 0; i < data_size / sizeof(tcg_target_ulong); i++) { 2019 qemu_log("0x%08" PRIxPTR ": .quad 0x%" TCG_PRIlx "\n", 2020 (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]); 2021 } 2022 } 2023 qemu_log("\n"); 2024 qemu_log_flush(); 2025 qemu_log_unlock(logfile); 2026 } 2027 #endif 2028 2029 qatomic_set(&tcg_ctx->code_gen_ptr, (void *) 2030 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 2031 CODE_GEN_ALIGN)); 2032 2033 /* init jump list */ 2034 qemu_spin_init(&tb->jmp_lock); 2035 tb->jmp_list_head = (uintptr_t)NULL; 2036 tb->jmp_list_next[0] = (uintptr_t)NULL; 2037 tb->jmp_list_next[1] = (uintptr_t)NULL; 2038 tb->jmp_dest[0] = (uintptr_t)NULL; 2039 tb->jmp_dest[1] = (uintptr_t)NULL; 2040 2041 /* init original jump addresses which have been set during tcg_gen_code() */ 2042 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 2043 tb_reset_jump(tb, 0); 2044 } 2045 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 2046 tb_reset_jump(tb, 1); 2047 } 2048 2049 /* check next page if needed */ 2050 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 2051 phys_page2 = -1; 2052 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 2053 phys_page2 = get_page_addr_code(env, virt_page2); 2054 } 2055 /* 2056 * No explicit memory barrier is required -- tb_link_page() makes the 2057 * TB visible in a consistent state. 2058 */ 2059 existing_tb = tb_link_page(tb, phys_pc, phys_page2); 2060 /* if the TB already exists, discard what we just translated */ 2061 if (unlikely(existing_tb != tb)) { 2062 uintptr_t orig_aligned = (uintptr_t)gen_code_buf; 2063 2064 orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize); 2065 qatomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned); 2066 tb_destroy(tb); 2067 return existing_tb; 2068 } 2069 tcg_tb_insert(tb); 2070 return tb; 2071 } 2072 2073 /* 2074 * @p must be non-NULL. 2075 * user-mode: call with mmap_lock held. 2076 * !user-mode: call with all @pages locked. 2077 */ 2078 static void 2079 tb_invalidate_phys_page_range__locked(struct page_collection *pages, 2080 PageDesc *p, tb_page_addr_t start, 2081 tb_page_addr_t end, 2082 uintptr_t retaddr) 2083 { 2084 TranslationBlock *tb; 2085 tb_page_addr_t tb_start, tb_end; 2086 int n; 2087 #ifdef TARGET_HAS_PRECISE_SMC 2088 CPUState *cpu = current_cpu; 2089 CPUArchState *env = NULL; 2090 bool current_tb_not_found = retaddr != 0; 2091 bool current_tb_modified = false; 2092 TranslationBlock *current_tb = NULL; 2093 target_ulong current_pc = 0; 2094 target_ulong current_cs_base = 0; 2095 uint32_t current_flags = 0; 2096 #endif /* TARGET_HAS_PRECISE_SMC */ 2097 2098 assert_page_locked(p); 2099 2100 #if defined(TARGET_HAS_PRECISE_SMC) 2101 if (cpu != NULL) { 2102 env = cpu->env_ptr; 2103 } 2104 #endif 2105 2106 /* we remove all the TBs in the range [start, end[ */ 2107 /* XXX: see if in some cases it could be faster to invalidate all 2108 the code */ 2109 PAGE_FOR_EACH_TB(p, tb, n) { 2110 assert_page_locked(p); 2111 /* NOTE: this is subtle as a TB may span two physical pages */ 2112 if (n == 0) { 2113 /* NOTE: tb_end may be after the end of the page, but 2114 it is not a problem */ 2115 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 2116 tb_end = tb_start + tb->size; 2117 } else { 2118 tb_start = tb->page_addr[1]; 2119 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 2120 } 2121 if (!(tb_end <= start || tb_start >= end)) { 2122 #ifdef TARGET_HAS_PRECISE_SMC 2123 if (current_tb_not_found) { 2124 current_tb_not_found = false; 2125 /* now we have a real cpu fault */ 2126 current_tb = tcg_tb_lookup(retaddr); 2127 } 2128 if (current_tb == tb && 2129 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { 2130 /* 2131 * If we are modifying the current TB, we must stop 2132 * its execution. We could be more precise by checking 2133 * that the modification is after the current PC, but it 2134 * would require a specialized function to partially 2135 * restore the CPU state. 2136 */ 2137 current_tb_modified = true; 2138 cpu_restore_state_from_tb(cpu, current_tb, retaddr, true); 2139 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 2140 ¤t_flags); 2141 } 2142 #endif /* TARGET_HAS_PRECISE_SMC */ 2143 tb_phys_invalidate__locked(tb); 2144 } 2145 } 2146 #if !defined(CONFIG_USER_ONLY) 2147 /* if no code remaining, no need to continue to use slow writes */ 2148 if (!p->first_tb) { 2149 invalidate_page_bitmap(p); 2150 tlb_unprotect_code(start); 2151 } 2152 #endif 2153 #ifdef TARGET_HAS_PRECISE_SMC 2154 if (current_tb_modified) { 2155 page_collection_unlock(pages); 2156 /* Force execution of one insn next time. */ 2157 cpu->cflags_next_tb = 1 | curr_cflags(); 2158 mmap_unlock(); 2159 cpu_loop_exit_noexc(cpu); 2160 } 2161 #endif 2162 } 2163 2164 /* 2165 * Invalidate all TBs which intersect with the target physical address range 2166 * [start;end[. NOTE: start and end must refer to the *same* physical page. 2167 * 'is_cpu_write_access' should be true if called from a real cpu write 2168 * access: the virtual CPU will exit the current TB if code is modified inside 2169 * this TB. 2170 * 2171 * Called with mmap_lock held for user-mode emulation 2172 */ 2173 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end) 2174 { 2175 struct page_collection *pages; 2176 PageDesc *p; 2177 2178 assert_memory_lock(); 2179 2180 p = page_find(start >> TARGET_PAGE_BITS); 2181 if (p == NULL) { 2182 return; 2183 } 2184 pages = page_collection_lock(start, end); 2185 tb_invalidate_phys_page_range__locked(pages, p, start, end, 0); 2186 page_collection_unlock(pages); 2187 } 2188 2189 /* 2190 * Invalidate all TBs which intersect with the target physical address range 2191 * [start;end[. NOTE: start and end may refer to *different* physical pages. 2192 * 'is_cpu_write_access' should be true if called from a real cpu write 2193 * access: the virtual CPU will exit the current TB if code is modified inside 2194 * this TB. 2195 * 2196 * Called with mmap_lock held for user-mode emulation. 2197 */ 2198 #ifdef CONFIG_SOFTMMU 2199 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end) 2200 #else 2201 void tb_invalidate_phys_range(target_ulong start, target_ulong end) 2202 #endif 2203 { 2204 struct page_collection *pages; 2205 tb_page_addr_t next; 2206 2207 assert_memory_lock(); 2208 2209 pages = page_collection_lock(start, end); 2210 for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; 2211 start < end; 2212 start = next, next += TARGET_PAGE_SIZE) { 2213 PageDesc *pd = page_find(start >> TARGET_PAGE_BITS); 2214 tb_page_addr_t bound = MIN(next, end); 2215 2216 if (pd == NULL) { 2217 continue; 2218 } 2219 tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0); 2220 } 2221 page_collection_unlock(pages); 2222 } 2223 2224 #ifdef CONFIG_SOFTMMU 2225 /* len must be <= 8 and start must be a multiple of len. 2226 * Called via softmmu_template.h when code areas are written to with 2227 * iothread mutex not held. 2228 * 2229 * Call with all @pages in the range [@start, @start + len[ locked. 2230 */ 2231 void tb_invalidate_phys_page_fast(struct page_collection *pages, 2232 tb_page_addr_t start, int len, 2233 uintptr_t retaddr) 2234 { 2235 PageDesc *p; 2236 2237 assert_memory_lock(); 2238 2239 p = page_find(start >> TARGET_PAGE_BITS); 2240 if (!p) { 2241 return; 2242 } 2243 2244 assert_page_locked(p); 2245 if (!p->code_bitmap && 2246 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 2247 build_page_bitmap(p); 2248 } 2249 if (p->code_bitmap) { 2250 unsigned int nr; 2251 unsigned long b; 2252 2253 nr = start & ~TARGET_PAGE_MASK; 2254 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 2255 if (b & ((1 << len) - 1)) { 2256 goto do_invalidate; 2257 } 2258 } else { 2259 do_invalidate: 2260 tb_invalidate_phys_page_range__locked(pages, p, start, start + len, 2261 retaddr); 2262 } 2263 } 2264 #else 2265 /* Called with mmap_lock held. If pc is not 0 then it indicates the 2266 * host PC of the faulting store instruction that caused this invalidate. 2267 * Returns true if the caller needs to abort execution of the current 2268 * TB (because it was modified by this store and the guest CPU has 2269 * precise-SMC semantics). 2270 */ 2271 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 2272 { 2273 TranslationBlock *tb; 2274 PageDesc *p; 2275 int n; 2276 #ifdef TARGET_HAS_PRECISE_SMC 2277 TranslationBlock *current_tb = NULL; 2278 CPUState *cpu = current_cpu; 2279 CPUArchState *env = NULL; 2280 int current_tb_modified = 0; 2281 target_ulong current_pc = 0; 2282 target_ulong current_cs_base = 0; 2283 uint32_t current_flags = 0; 2284 #endif 2285 2286 assert_memory_lock(); 2287 2288 addr &= TARGET_PAGE_MASK; 2289 p = page_find(addr >> TARGET_PAGE_BITS); 2290 if (!p) { 2291 return false; 2292 } 2293 2294 #ifdef TARGET_HAS_PRECISE_SMC 2295 if (p->first_tb && pc != 0) { 2296 current_tb = tcg_tb_lookup(pc); 2297 } 2298 if (cpu != NULL) { 2299 env = cpu->env_ptr; 2300 } 2301 #endif 2302 assert_page_locked(p); 2303 PAGE_FOR_EACH_TB(p, tb, n) { 2304 #ifdef TARGET_HAS_PRECISE_SMC 2305 if (current_tb == tb && 2306 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { 2307 /* If we are modifying the current TB, we must stop 2308 its execution. We could be more precise by checking 2309 that the modification is after the current PC, but it 2310 would require a specialized function to partially 2311 restore the CPU state */ 2312 2313 current_tb_modified = 1; 2314 cpu_restore_state_from_tb(cpu, current_tb, pc, true); 2315 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 2316 ¤t_flags); 2317 } 2318 #endif /* TARGET_HAS_PRECISE_SMC */ 2319 tb_phys_invalidate(tb, addr); 2320 } 2321 p->first_tb = (uintptr_t)NULL; 2322 #ifdef TARGET_HAS_PRECISE_SMC 2323 if (current_tb_modified) { 2324 /* Force execution of one insn next time. */ 2325 cpu->cflags_next_tb = 1 | curr_cflags(); 2326 return true; 2327 } 2328 #endif 2329 2330 return false; 2331 } 2332 #endif 2333 2334 /* user-mode: call with mmap_lock held */ 2335 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr) 2336 { 2337 TranslationBlock *tb; 2338 2339 assert_memory_lock(); 2340 2341 tb = tcg_tb_lookup(retaddr); 2342 if (tb) { 2343 /* We can use retranslation to find the PC. */ 2344 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 2345 tb_phys_invalidate(tb, -1); 2346 } else { 2347 /* The exception probably happened in a helper. The CPU state should 2348 have been saved before calling it. Fetch the PC from there. */ 2349 CPUArchState *env = cpu->env_ptr; 2350 target_ulong pc, cs_base; 2351 tb_page_addr_t addr; 2352 uint32_t flags; 2353 2354 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 2355 addr = get_page_addr_code(env, pc); 2356 if (addr != -1) { 2357 tb_invalidate_phys_range(addr, addr + 1); 2358 } 2359 } 2360 } 2361 2362 #ifndef CONFIG_USER_ONLY 2363 /* in deterministic execution mode, instructions doing device I/Os 2364 * must be at the end of the TB. 2365 * 2366 * Called by softmmu_template.h, with iothread mutex not held. 2367 */ 2368 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 2369 { 2370 #if defined(TARGET_MIPS) || defined(TARGET_SH4) 2371 CPUArchState *env = cpu->env_ptr; 2372 #endif 2373 TranslationBlock *tb; 2374 uint32_t n; 2375 2376 tb = tcg_tb_lookup(retaddr); 2377 if (!tb) { 2378 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 2379 (void *)retaddr); 2380 } 2381 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 2382 2383 /* On MIPS and SH, delay slot instructions can only be restarted if 2384 they were already the first instruction in the TB. If this is not 2385 the first instruction in a TB then re-execute the preceding 2386 branch. */ 2387 n = 1; 2388 #if defined(TARGET_MIPS) 2389 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 2390 && env->active_tc.PC != tb->pc) { 2391 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4); 2392 cpu_neg(cpu)->icount_decr.u16.low++; 2393 env->hflags &= ~MIPS_HFLAG_BMASK; 2394 n = 2; 2395 } 2396 #elif defined(TARGET_SH4) 2397 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0 2398 && env->pc != tb->pc) { 2399 env->pc -= 2; 2400 cpu_neg(cpu)->icount_decr.u16.low++; 2401 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL); 2402 n = 2; 2403 } 2404 #endif 2405 2406 /* Generate a new TB executing the I/O insn. */ 2407 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n; 2408 2409 if (tb_cflags(tb) & CF_NOCACHE) { 2410 if (tb->orig_tb) { 2411 /* Invalidate original TB if this TB was generated in 2412 * cpu_exec_nocache() */ 2413 tb_phys_invalidate(tb->orig_tb, -1); 2414 } 2415 tcg_tb_remove(tb); 2416 tb_destroy(tb); 2417 } 2418 2419 qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, 2420 "cpu_io_recompile: rewound execution of TB to " 2421 TARGET_FMT_lx "\n", tb->pc); 2422 2423 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not 2424 * the first in the TB) then we end up generating a whole new TB and 2425 * repeating the fault, which is horribly inefficient. 2426 * Better would be to execute just this insn uncached, or generate a 2427 * second new TB. 2428 */ 2429 cpu_loop_exit_noexc(cpu); 2430 } 2431 2432 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr) 2433 { 2434 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr); 2435 2436 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { 2437 qatomic_set(&cpu->tb_jmp_cache[i0 + i], NULL); 2438 } 2439 } 2440 2441 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr) 2442 { 2443 /* Discard jump cache entries for any tb which might potentially 2444 overlap the flushed page. */ 2445 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); 2446 tb_jmp_cache_clear_page(cpu, addr); 2447 } 2448 2449 static void print_qht_statistics(struct qht_stats hst) 2450 { 2451 uint32_t hgram_opts; 2452 size_t hgram_bins; 2453 char *hgram; 2454 2455 if (!hst.head_buckets) { 2456 return; 2457 } 2458 qemu_printf("TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 2459 hst.used_head_buckets, hst.head_buckets, 2460 (double)hst.used_head_buckets / hst.head_buckets * 100); 2461 2462 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 2463 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 2464 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 2465 hgram_opts |= QDIST_PR_NODECIMAL; 2466 } 2467 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 2468 qemu_printf("TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 2469 qdist_avg(&hst.occupancy) * 100, hgram); 2470 g_free(hgram); 2471 2472 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 2473 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 2474 if (hgram_bins > 10) { 2475 hgram_bins = 10; 2476 } else { 2477 hgram_bins = 0; 2478 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 2479 } 2480 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 2481 qemu_printf("TB hash avg chain %0.3f buckets. Histogram: %s\n", 2482 qdist_avg(&hst.chain), hgram); 2483 g_free(hgram); 2484 } 2485 2486 struct tb_tree_stats { 2487 size_t nb_tbs; 2488 size_t host_size; 2489 size_t target_size; 2490 size_t max_target_size; 2491 size_t direct_jmp_count; 2492 size_t direct_jmp2_count; 2493 size_t cross_page; 2494 }; 2495 2496 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 2497 { 2498 const TranslationBlock *tb = value; 2499 struct tb_tree_stats *tst = data; 2500 2501 tst->nb_tbs++; 2502 tst->host_size += tb->tc.size; 2503 tst->target_size += tb->size; 2504 if (tb->size > tst->max_target_size) { 2505 tst->max_target_size = tb->size; 2506 } 2507 if (tb->page_addr[1] != -1) { 2508 tst->cross_page++; 2509 } 2510 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 2511 tst->direct_jmp_count++; 2512 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 2513 tst->direct_jmp2_count++; 2514 } 2515 } 2516 return false; 2517 } 2518 2519 void dump_exec_info(void) 2520 { 2521 struct tb_tree_stats tst = {}; 2522 struct qht_stats hst; 2523 size_t nb_tbs, flush_full, flush_part, flush_elide; 2524 2525 tcg_tb_foreach(tb_tree_stats_iter, &tst); 2526 nb_tbs = tst.nb_tbs; 2527 /* XXX: avoid using doubles ? */ 2528 qemu_printf("Translation buffer state:\n"); 2529 /* 2530 * Report total code size including the padding and TB structs; 2531 * otherwise users might think "-accel tcg,tb-size" is not honoured. 2532 * For avg host size we use the precise numbers from tb_tree_stats though. 2533 */ 2534 qemu_printf("gen code size %zu/%zu\n", 2535 tcg_code_size(), tcg_code_capacity()); 2536 qemu_printf("TB count %zu\n", nb_tbs); 2537 qemu_printf("TB avg target size %zu max=%zu bytes\n", 2538 nb_tbs ? tst.target_size / nb_tbs : 0, 2539 tst.max_target_size); 2540 qemu_printf("TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 2541 nb_tbs ? tst.host_size / nb_tbs : 0, 2542 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 2543 qemu_printf("cross page TB count %zu (%zu%%)\n", tst.cross_page, 2544 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 2545 qemu_printf("direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 2546 tst.direct_jmp_count, 2547 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 2548 tst.direct_jmp2_count, 2549 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 2550 2551 qht_statistics_init(&tb_ctx.htable, &hst); 2552 print_qht_statistics(hst); 2553 qht_statistics_destroy(&hst); 2554 2555 qemu_printf("\nStatistics:\n"); 2556 qemu_printf("TB flush count %u\n", 2557 qatomic_read(&tb_ctx.tb_flush_count)); 2558 qemu_printf("TB invalidate count %zu\n", 2559 tcg_tb_phys_invalidate_count()); 2560 2561 tlb_flush_counts(&flush_full, &flush_part, &flush_elide); 2562 qemu_printf("TLB full flushes %zu\n", flush_full); 2563 qemu_printf("TLB partial flushes %zu\n", flush_part); 2564 qemu_printf("TLB elided flushes %zu\n", flush_elide); 2565 tcg_dump_info(); 2566 } 2567 2568 void dump_opcount_info(void) 2569 { 2570 tcg_dump_op_count(); 2571 } 2572 2573 #else /* CONFIG_USER_ONLY */ 2574 2575 void cpu_interrupt(CPUState *cpu, int mask) 2576 { 2577 g_assert(qemu_mutex_iothread_locked()); 2578 cpu->interrupt_request |= mask; 2579 qatomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1); 2580 } 2581 2582 /* 2583 * Walks guest process memory "regions" one by one 2584 * and calls callback function 'fn' for each region. 2585 */ 2586 struct walk_memory_regions_data { 2587 walk_memory_regions_fn fn; 2588 void *priv; 2589 target_ulong start; 2590 int prot; 2591 }; 2592 2593 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 2594 target_ulong end, int new_prot) 2595 { 2596 if (data->start != -1u) { 2597 int rc = data->fn(data->priv, data->start, end, data->prot); 2598 if (rc != 0) { 2599 return rc; 2600 } 2601 } 2602 2603 data->start = (new_prot ? end : -1u); 2604 data->prot = new_prot; 2605 2606 return 0; 2607 } 2608 2609 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 2610 target_ulong base, int level, void **lp) 2611 { 2612 target_ulong pa; 2613 int i, rc; 2614 2615 if (*lp == NULL) { 2616 return walk_memory_regions_end(data, base, 0); 2617 } 2618 2619 if (level == 0) { 2620 PageDesc *pd = *lp; 2621 2622 for (i = 0; i < V_L2_SIZE; ++i) { 2623 int prot = pd[i].flags; 2624 2625 pa = base | (i << TARGET_PAGE_BITS); 2626 if (prot != data->prot) { 2627 rc = walk_memory_regions_end(data, pa, prot); 2628 if (rc != 0) { 2629 return rc; 2630 } 2631 } 2632 } 2633 } else { 2634 void **pp = *lp; 2635 2636 for (i = 0; i < V_L2_SIZE; ++i) { 2637 pa = base | ((target_ulong)i << 2638 (TARGET_PAGE_BITS + V_L2_BITS * level)); 2639 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 2640 if (rc != 0) { 2641 return rc; 2642 } 2643 } 2644 } 2645 2646 return 0; 2647 } 2648 2649 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2650 { 2651 struct walk_memory_regions_data data; 2652 uintptr_t i, l1_sz = v_l1_size; 2653 2654 data.fn = fn; 2655 data.priv = priv; 2656 data.start = -1u; 2657 data.prot = 0; 2658 2659 for (i = 0; i < l1_sz; i++) { 2660 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2661 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2662 if (rc != 0) { 2663 return rc; 2664 } 2665 } 2666 2667 return walk_memory_regions_end(&data, 0, 0); 2668 } 2669 2670 static int dump_region(void *priv, target_ulong start, 2671 target_ulong end, unsigned long prot) 2672 { 2673 FILE *f = (FILE *)priv; 2674 2675 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2676 " "TARGET_FMT_lx" %c%c%c\n", 2677 start, end, end - start, 2678 ((prot & PAGE_READ) ? 'r' : '-'), 2679 ((prot & PAGE_WRITE) ? 'w' : '-'), 2680 ((prot & PAGE_EXEC) ? 'x' : '-')); 2681 2682 return 0; 2683 } 2684 2685 /* dump memory mappings */ 2686 void page_dump(FILE *f) 2687 { 2688 const int length = sizeof(target_ulong) * 2; 2689 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2690 length, "start", length, "end", length, "size", "prot"); 2691 walk_memory_regions(f, dump_region); 2692 } 2693 2694 int page_get_flags(target_ulong address) 2695 { 2696 PageDesc *p; 2697 2698 p = page_find(address >> TARGET_PAGE_BITS); 2699 if (!p) { 2700 return 0; 2701 } 2702 return p->flags; 2703 } 2704 2705 /* Modify the flags of a page and invalidate the code if necessary. 2706 The flag PAGE_WRITE_ORG is positioned automatically depending 2707 on PAGE_WRITE. The mmap_lock should already be held. */ 2708 void page_set_flags(target_ulong start, target_ulong end, int flags) 2709 { 2710 target_ulong addr, len; 2711 2712 /* This function should never be called with addresses outside the 2713 guest address space. If this assert fires, it probably indicates 2714 a missing call to h2g_valid. */ 2715 assert(end - 1 <= GUEST_ADDR_MAX); 2716 assert(start < end); 2717 assert_memory_lock(); 2718 2719 start = start & TARGET_PAGE_MASK; 2720 end = TARGET_PAGE_ALIGN(end); 2721 2722 if (flags & PAGE_WRITE) { 2723 flags |= PAGE_WRITE_ORG; 2724 } 2725 2726 for (addr = start, len = end - start; 2727 len != 0; 2728 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2729 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2730 2731 /* If the write protection bit is set, then we invalidate 2732 the code inside. */ 2733 if (!(p->flags & PAGE_WRITE) && 2734 (flags & PAGE_WRITE) && 2735 p->first_tb) { 2736 tb_invalidate_phys_page(addr, 0); 2737 } 2738 p->flags = flags; 2739 } 2740 } 2741 2742 int page_check_range(target_ulong start, target_ulong len, int flags) 2743 { 2744 PageDesc *p; 2745 target_ulong end; 2746 target_ulong addr; 2747 2748 /* This function should never be called with addresses outside the 2749 guest address space. If this assert fires, it probably indicates 2750 a missing call to h2g_valid. */ 2751 if (TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS) { 2752 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2753 } 2754 2755 if (len == 0) { 2756 return 0; 2757 } 2758 if (start + len - 1 < start) { 2759 /* We've wrapped around. */ 2760 return -1; 2761 } 2762 2763 /* must do before we loose bits in the next step */ 2764 end = TARGET_PAGE_ALIGN(start + len); 2765 start = start & TARGET_PAGE_MASK; 2766 2767 for (addr = start, len = end - start; 2768 len != 0; 2769 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2770 p = page_find(addr >> TARGET_PAGE_BITS); 2771 if (!p) { 2772 return -1; 2773 } 2774 if (!(p->flags & PAGE_VALID)) { 2775 return -1; 2776 } 2777 2778 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2779 return -1; 2780 } 2781 if (flags & PAGE_WRITE) { 2782 if (!(p->flags & PAGE_WRITE_ORG)) { 2783 return -1; 2784 } 2785 /* unprotect the page if it was put read-only because it 2786 contains translated code */ 2787 if (!(p->flags & PAGE_WRITE)) { 2788 if (!page_unprotect(addr, 0)) { 2789 return -1; 2790 } 2791 } 2792 } 2793 } 2794 return 0; 2795 } 2796 2797 /* called from signal handler: invalidate the code and unprotect the 2798 * page. Return 0 if the fault was not handled, 1 if it was handled, 2799 * and 2 if it was handled but the caller must cause the TB to be 2800 * immediately exited. (We can only return 2 if the 'pc' argument is 2801 * non-zero.) 2802 */ 2803 int page_unprotect(target_ulong address, uintptr_t pc) 2804 { 2805 unsigned int prot; 2806 bool current_tb_invalidated; 2807 PageDesc *p; 2808 target_ulong host_start, host_end, addr; 2809 2810 /* Technically this isn't safe inside a signal handler. However we 2811 know this only ever happens in a synchronous SEGV handler, so in 2812 practice it seems to be ok. */ 2813 mmap_lock(); 2814 2815 p = page_find(address >> TARGET_PAGE_BITS); 2816 if (!p) { 2817 mmap_unlock(); 2818 return 0; 2819 } 2820 2821 /* if the page was really writable, then we change its 2822 protection back to writable */ 2823 if (p->flags & PAGE_WRITE_ORG) { 2824 current_tb_invalidated = false; 2825 if (p->flags & PAGE_WRITE) { 2826 /* If the page is actually marked WRITE then assume this is because 2827 * this thread raced with another one which got here first and 2828 * set the page to PAGE_WRITE and did the TB invalidate for us. 2829 */ 2830 #ifdef TARGET_HAS_PRECISE_SMC 2831 TranslationBlock *current_tb = tcg_tb_lookup(pc); 2832 if (current_tb) { 2833 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; 2834 } 2835 #endif 2836 } else { 2837 host_start = address & qemu_host_page_mask; 2838 host_end = host_start + qemu_host_page_size; 2839 2840 prot = 0; 2841 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) { 2842 p = page_find(addr >> TARGET_PAGE_BITS); 2843 p->flags |= PAGE_WRITE; 2844 prot |= p->flags; 2845 2846 /* and since the content will be modified, we must invalidate 2847 the corresponding translated code. */ 2848 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2849 #ifdef CONFIG_USER_ONLY 2850 if (DEBUG_TB_CHECK_GATE) { 2851 tb_invalidate_check(addr); 2852 } 2853 #endif 2854 } 2855 mprotect((void *)g2h(host_start), qemu_host_page_size, 2856 prot & PAGE_BITS); 2857 } 2858 mmap_unlock(); 2859 /* If current TB was invalidated return to main loop */ 2860 return current_tb_invalidated ? 2 : 1; 2861 } 2862 mmap_unlock(); 2863 return 0; 2864 } 2865 #endif /* CONFIG_USER_ONLY */ 2866 2867 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2868 void tcg_flush_softmmu_tlb(CPUState *cs) 2869 { 2870 #ifdef CONFIG_SOFTMMU 2871 tlb_flush(cs); 2872 #endif 2873 } 2874