1 /* 2 * internal execution defines for qemu 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #ifndef EXEC_ALL_H 21 #define EXEC_ALL_H 22 23 #include "cpu.h" 24 #include "exec/tb-context.h" 25 #ifdef CONFIG_TCG 26 #include "exec/cpu_ldst.h" 27 #endif 28 #include "sysemu/cpus.h" 29 30 /* allow to see translation results - the slowdown should be negligible, so we leave it */ 31 #define DEBUG_DISAS 32 33 /* Page tracking code uses ram addresses in system mode, and virtual 34 addresses in userspace mode. Define tb_page_addr_t to be an appropriate 35 type. */ 36 #if defined(CONFIG_USER_ONLY) 37 typedef abi_ulong tb_page_addr_t; 38 #define TB_PAGE_ADDR_FMT TARGET_ABI_FMT_lx 39 #else 40 typedef ram_addr_t tb_page_addr_t; 41 #define TB_PAGE_ADDR_FMT RAM_ADDR_FMT 42 #endif 43 44 #include "qemu/log.h" 45 46 void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns); 47 void restore_state_to_opc(CPUArchState *env, TranslationBlock *tb, 48 target_ulong *data); 49 50 void cpu_gen_init(void); 51 52 /** 53 * cpu_restore_state: 54 * @cpu: the vCPU state is to be restore to 55 * @searched_pc: the host PC the fault occurred at 56 * @will_exit: true if the TB executed will be interrupted after some 57 cpu adjustments. Required for maintaining the correct 58 icount valus 59 * @return: true if state was restored, false otherwise 60 * 61 * Attempt to restore the state for a fault occurring in translated 62 * code. If the searched_pc is not in translated code no state is 63 * restored and the function returns false. 64 */ 65 bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc, bool will_exit); 66 67 void QEMU_NORETURN cpu_loop_exit_noexc(CPUState *cpu); 68 void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr); 69 TranslationBlock *tb_gen_code(CPUState *cpu, 70 target_ulong pc, target_ulong cs_base, 71 uint32_t flags, 72 int cflags); 73 74 void QEMU_NORETURN cpu_loop_exit(CPUState *cpu); 75 void QEMU_NORETURN cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc); 76 void QEMU_NORETURN cpu_loop_exit_atomic(CPUState *cpu, uintptr_t pc); 77 78 /** 79 * cpu_loop_exit_requested: 80 * @cpu: The CPU state to be tested 81 * 82 * Indicate if somebody asked for a return of the CPU to the main loop 83 * (e.g., via cpu_exit() or cpu_interrupt()). 84 * 85 * This is helpful for architectures that support interruptible 86 * instructions. After writing back all state to registers/memory, this 87 * call can be used to check if it makes sense to return to the main loop 88 * or to continue executing the interruptible instruction. 89 */ 90 static inline bool cpu_loop_exit_requested(CPUState *cpu) 91 { 92 return (int32_t)atomic_read(&cpu_neg(cpu)->icount_decr.u32) < 0; 93 } 94 95 #if !defined(CONFIG_USER_ONLY) 96 void cpu_reloading_memory_map(void); 97 /** 98 * cpu_address_space_init: 99 * @cpu: CPU to add this address space to 100 * @asidx: integer index of this address space 101 * @prefix: prefix to be used as name of address space 102 * @mr: the root memory region of address space 103 * 104 * Add the specified address space to the CPU's cpu_ases list. 105 * The address space added with @asidx 0 is the one used for the 106 * convenience pointer cpu->as. 107 * The target-specific code which registers ASes is responsible 108 * for defining what semantics address space 0, 1, 2, etc have. 109 * 110 * Before the first call to this function, the caller must set 111 * cpu->num_ases to the total number of address spaces it needs 112 * to support. 113 * 114 * Note that with KVM only one address space is supported. 115 */ 116 void cpu_address_space_init(CPUState *cpu, int asidx, 117 const char *prefix, MemoryRegion *mr); 118 #endif 119 120 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG) 121 /* cputlb.c */ 122 /** 123 * tlb_init - initialize a CPU's TLB 124 * @cpu: CPU whose TLB should be initialized 125 */ 126 void tlb_init(CPUState *cpu); 127 /** 128 * tlb_flush_page: 129 * @cpu: CPU whose TLB should be flushed 130 * @addr: virtual address of page to be flushed 131 * 132 * Flush one page from the TLB of the specified CPU, for all 133 * MMU indexes. 134 */ 135 void tlb_flush_page(CPUState *cpu, target_ulong addr); 136 /** 137 * tlb_flush_page_all_cpus: 138 * @cpu: src CPU of the flush 139 * @addr: virtual address of page to be flushed 140 * 141 * Flush one page from the TLB of the specified CPU, for all 142 * MMU indexes. 143 */ 144 void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr); 145 /** 146 * tlb_flush_page_all_cpus_synced: 147 * @cpu: src CPU of the flush 148 * @addr: virtual address of page to be flushed 149 * 150 * Flush one page from the TLB of the specified CPU, for all MMU 151 * indexes like tlb_flush_page_all_cpus except the source vCPUs work 152 * is scheduled as safe work meaning all flushes will be complete once 153 * the source vCPUs safe work is complete. This will depend on when 154 * the guests translation ends the TB. 155 */ 156 void tlb_flush_page_all_cpus_synced(CPUState *src, target_ulong addr); 157 /** 158 * tlb_flush: 159 * @cpu: CPU whose TLB should be flushed 160 * 161 * Flush the entire TLB for the specified CPU. Most CPU architectures 162 * allow the implementation to drop entries from the TLB at any time 163 * so this is generally safe. If more selective flushing is required 164 * use one of the other functions for efficiency. 165 */ 166 void tlb_flush(CPUState *cpu); 167 /** 168 * tlb_flush_all_cpus: 169 * @cpu: src CPU of the flush 170 */ 171 void tlb_flush_all_cpus(CPUState *src_cpu); 172 /** 173 * tlb_flush_all_cpus_synced: 174 * @cpu: src CPU of the flush 175 * 176 * Like tlb_flush_all_cpus except this except the source vCPUs work is 177 * scheduled as safe work meaning all flushes will be complete once 178 * the source vCPUs safe work is complete. This will depend on when 179 * the guests translation ends the TB. 180 */ 181 void tlb_flush_all_cpus_synced(CPUState *src_cpu); 182 /** 183 * tlb_flush_page_by_mmuidx: 184 * @cpu: CPU whose TLB should be flushed 185 * @addr: virtual address of page to be flushed 186 * @idxmap: bitmap of MMU indexes to flush 187 * 188 * Flush one page from the TLB of the specified CPU, for the specified 189 * MMU indexes. 190 */ 191 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, 192 uint16_t idxmap); 193 /** 194 * tlb_flush_page_by_mmuidx_all_cpus: 195 * @cpu: Originating CPU of the flush 196 * @addr: virtual address of page to be flushed 197 * @idxmap: bitmap of MMU indexes to flush 198 * 199 * Flush one page from the TLB of all CPUs, for the specified 200 * MMU indexes. 201 */ 202 void tlb_flush_page_by_mmuidx_all_cpus(CPUState *cpu, target_ulong addr, 203 uint16_t idxmap); 204 /** 205 * tlb_flush_page_by_mmuidx_all_cpus_synced: 206 * @cpu: Originating CPU of the flush 207 * @addr: virtual address of page to be flushed 208 * @idxmap: bitmap of MMU indexes to flush 209 * 210 * Flush one page from the TLB of all CPUs, for the specified MMU 211 * indexes like tlb_flush_page_by_mmuidx_all_cpus except the source 212 * vCPUs work is scheduled as safe work meaning all flushes will be 213 * complete once the source vCPUs safe work is complete. This will 214 * depend on when the guests translation ends the TB. 215 */ 216 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, target_ulong addr, 217 uint16_t idxmap); 218 /** 219 * tlb_flush_by_mmuidx: 220 * @cpu: CPU whose TLB should be flushed 221 * @wait: If true ensure synchronisation by exiting the cpu_loop 222 * @idxmap: bitmap of MMU indexes to flush 223 * 224 * Flush all entries from the TLB of the specified CPU, for the specified 225 * MMU indexes. 226 */ 227 void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap); 228 /** 229 * tlb_flush_by_mmuidx_all_cpus: 230 * @cpu: Originating CPU of the flush 231 * @idxmap: bitmap of MMU indexes to flush 232 * 233 * Flush all entries from all TLBs of all CPUs, for the specified 234 * MMU indexes. 235 */ 236 void tlb_flush_by_mmuidx_all_cpus(CPUState *cpu, uint16_t idxmap); 237 /** 238 * tlb_flush_by_mmuidx_all_cpus_synced: 239 * @cpu: Originating CPU of the flush 240 * @idxmap: bitmap of MMU indexes to flush 241 * 242 * Flush all entries from all TLBs of all CPUs, for the specified 243 * MMU indexes like tlb_flush_by_mmuidx_all_cpus except except the source 244 * vCPUs work is scheduled as safe work meaning all flushes will be 245 * complete once the source vCPUs safe work is complete. This will 246 * depend on when the guests translation ends the TB. 247 */ 248 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, uint16_t idxmap); 249 /** 250 * tlb_set_page_with_attrs: 251 * @cpu: CPU to add this TLB entry for 252 * @vaddr: virtual address of page to add entry for 253 * @paddr: physical address of the page 254 * @attrs: memory transaction attributes 255 * @prot: access permissions (PAGE_READ/PAGE_WRITE/PAGE_EXEC bits) 256 * @mmu_idx: MMU index to insert TLB entry for 257 * @size: size of the page in bytes 258 * 259 * Add an entry to this CPU's TLB (a mapping from virtual address 260 * @vaddr to physical address @paddr) with the specified memory 261 * transaction attributes. This is generally called by the target CPU 262 * specific code after it has been called through the tlb_fill() 263 * entry point and performed a successful page table walk to find 264 * the physical address and attributes for the virtual address 265 * which provoked the TLB miss. 266 * 267 * At most one entry for a given virtual address is permitted. Only a 268 * single TARGET_PAGE_SIZE region is mapped; the supplied @size is only 269 * used by tlb_flush_page. 270 */ 271 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, 272 hwaddr paddr, MemTxAttrs attrs, 273 int prot, int mmu_idx, target_ulong size); 274 /* tlb_set_page: 275 * 276 * This function is equivalent to calling tlb_set_page_with_attrs() 277 * with an @attrs argument of MEMTXATTRS_UNSPECIFIED. It's provided 278 * as a convenience for CPUs which don't use memory transaction attributes. 279 */ 280 void tlb_set_page(CPUState *cpu, target_ulong vaddr, 281 hwaddr paddr, int prot, 282 int mmu_idx, target_ulong size); 283 #else 284 static inline void tlb_init(CPUState *cpu) 285 { 286 } 287 static inline void tlb_flush_page(CPUState *cpu, target_ulong addr) 288 { 289 } 290 static inline void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr) 291 { 292 } 293 static inline void tlb_flush_page_all_cpus_synced(CPUState *src, 294 target_ulong addr) 295 { 296 } 297 static inline void tlb_flush(CPUState *cpu) 298 { 299 } 300 static inline void tlb_flush_all_cpus(CPUState *src_cpu) 301 { 302 } 303 static inline void tlb_flush_all_cpus_synced(CPUState *src_cpu) 304 { 305 } 306 static inline void tlb_flush_page_by_mmuidx(CPUState *cpu, 307 target_ulong addr, uint16_t idxmap) 308 { 309 } 310 311 static inline void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap) 312 { 313 } 314 static inline void tlb_flush_page_by_mmuidx_all_cpus(CPUState *cpu, 315 target_ulong addr, 316 uint16_t idxmap) 317 { 318 } 319 static inline void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, 320 target_ulong addr, 321 uint16_t idxmap) 322 { 323 } 324 static inline void tlb_flush_by_mmuidx_all_cpus(CPUState *cpu, uint16_t idxmap) 325 { 326 } 327 328 static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, 329 uint16_t idxmap) 330 { 331 } 332 #endif 333 void *probe_access(CPUArchState *env, target_ulong addr, int size, 334 MMUAccessType access_type, int mmu_idx, uintptr_t retaddr); 335 336 static inline void *probe_write(CPUArchState *env, target_ulong addr, int size, 337 int mmu_idx, uintptr_t retaddr) 338 { 339 return probe_access(env, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); 340 } 341 342 static inline void *probe_read(CPUArchState *env, target_ulong addr, int size, 343 int mmu_idx, uintptr_t retaddr) 344 { 345 return probe_access(env, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr); 346 } 347 348 #define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */ 349 350 /* Estimated block size for TB allocation. */ 351 /* ??? The following is based on a 2015 survey of x86_64 host output. 352 Better would seem to be some sort of dynamically sized TB array, 353 adapting to the block sizes actually being produced. */ 354 #if defined(CONFIG_SOFTMMU) 355 #define CODE_GEN_AVG_BLOCK_SIZE 400 356 #else 357 #define CODE_GEN_AVG_BLOCK_SIZE 150 358 #endif 359 360 /* 361 * Translation Cache-related fields of a TB. 362 * This struct exists just for convenience; we keep track of TB's in a binary 363 * search tree, and the only fields needed to compare TB's in the tree are 364 * @ptr and @size. 365 * Note: the address of search data can be obtained by adding @size to @ptr. 366 */ 367 struct tb_tc { 368 void *ptr; /* pointer to the translated code */ 369 size_t size; 370 }; 371 372 struct TranslationBlock { 373 target_ulong pc; /* simulated PC corresponding to this block (EIP + CS base) */ 374 target_ulong cs_base; /* CS base for this block */ 375 uint32_t flags; /* flags defining in which context the code was generated */ 376 uint16_t size; /* size of target code for this block (1 <= 377 size <= TARGET_PAGE_SIZE) */ 378 uint16_t icount; 379 uint32_t cflags; /* compile flags */ 380 #define CF_COUNT_MASK 0x00007fff 381 #define CF_LAST_IO 0x00008000 /* Last insn may be an IO access. */ 382 #define CF_NOCACHE 0x00010000 /* To be freed after execution */ 383 #define CF_USE_ICOUNT 0x00020000 384 #define CF_INVALID 0x00040000 /* TB is stale. Set with @jmp_lock held */ 385 #define CF_PARALLEL 0x00080000 /* Generate code for a parallel context */ 386 #define CF_CLUSTER_MASK 0xff000000 /* Top 8 bits are cluster ID */ 387 #define CF_CLUSTER_SHIFT 24 388 /* cflags' mask for hashing/comparison */ 389 #define CF_HASH_MASK \ 390 (CF_COUNT_MASK | CF_LAST_IO | CF_USE_ICOUNT | CF_PARALLEL | CF_CLUSTER_MASK) 391 392 /* Per-vCPU dynamic tracing state used to generate this TB */ 393 uint32_t trace_vcpu_dstate; 394 395 struct tb_tc tc; 396 397 /* original tb when cflags has CF_NOCACHE */ 398 struct TranslationBlock *orig_tb; 399 /* first and second physical page containing code. The lower bit 400 of the pointer tells the index in page_next[]. 401 The list is protected by the TB's page('s) lock(s) */ 402 uintptr_t page_next[2]; 403 tb_page_addr_t page_addr[2]; 404 405 /* jmp_lock placed here to fill a 4-byte hole. Its documentation is below */ 406 QemuSpin jmp_lock; 407 408 /* The following data are used to directly call another TB from 409 * the code of this one. This can be done either by emitting direct or 410 * indirect native jump instructions. These jumps are reset so that the TB 411 * just continues its execution. The TB can be linked to another one by 412 * setting one of the jump targets (or patching the jump instruction). Only 413 * two of such jumps are supported. 414 */ 415 uint16_t jmp_reset_offset[2]; /* offset of original jump target */ 416 #define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */ 417 uintptr_t jmp_target_arg[2]; /* target address or offset */ 418 419 /* 420 * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps. 421 * Each TB can have two outgoing jumps, and therefore can participate 422 * in two lists. The list entries are kept in jmp_list_next[2]. The least 423 * significant bit (LSB) of the pointers in these lists is used to encode 424 * which of the two list entries is to be used in the pointed TB. 425 * 426 * List traversals are protected by jmp_lock. The destination TB of each 427 * outgoing jump is kept in jmp_dest[] so that the appropriate jmp_lock 428 * can be acquired from any origin TB. 429 * 430 * jmp_dest[] are tagged pointers as well. The LSB is set when the TB is 431 * being invalidated, so that no further outgoing jumps from it can be set. 432 * 433 * jmp_lock also protects the CF_INVALID cflag; a jump must not be chained 434 * to a destination TB that has CF_INVALID set. 435 */ 436 uintptr_t jmp_list_head; 437 uintptr_t jmp_list_next[2]; 438 uintptr_t jmp_dest[2]; 439 }; 440 441 extern bool parallel_cpus; 442 443 /* Hide the atomic_read to make code a little easier on the eyes */ 444 static inline uint32_t tb_cflags(const TranslationBlock *tb) 445 { 446 return atomic_read(&tb->cflags); 447 } 448 449 /* current cflags for hashing/comparison */ 450 static inline uint32_t curr_cflags(void) 451 { 452 return (parallel_cpus ? CF_PARALLEL : 0) 453 | (use_icount ? CF_USE_ICOUNT : 0); 454 } 455 456 /* TranslationBlock invalidate API */ 457 #if defined(CONFIG_USER_ONLY) 458 void tb_invalidate_phys_addr(target_ulong addr); 459 void tb_invalidate_phys_range(target_ulong start, target_ulong end); 460 #else 461 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs); 462 #endif 463 void tb_flush(CPUState *cpu); 464 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); 465 TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, 466 target_ulong cs_base, uint32_t flags, 467 uint32_t cf_mask); 468 void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr); 469 470 /* GETPC is the true target of the return instruction that we'll execute. */ 471 #if defined(CONFIG_TCG_INTERPRETER) 472 extern uintptr_t tci_tb_ptr; 473 # define GETPC() tci_tb_ptr 474 #else 475 # define GETPC() \ 476 ((uintptr_t)__builtin_extract_return_addr(__builtin_return_address(0))) 477 #endif 478 479 /* The true return address will often point to a host insn that is part of 480 the next translated guest insn. Adjust the address backward to point to 481 the middle of the call insn. Subtracting one would do the job except for 482 several compressed mode architectures (arm, mips) which set the low bit 483 to indicate the compressed mode; subtracting two works around that. It 484 is also the case that there are no host isas that contain a call insn 485 smaller than 4 bytes, so we don't worry about special-casing this. */ 486 #define GETPC_ADJ 2 487 488 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_DEBUG_TCG) 489 void assert_no_pages_locked(void); 490 #else 491 static inline void assert_no_pages_locked(void) 492 { 493 } 494 #endif 495 496 #if !defined(CONFIG_USER_ONLY) 497 498 /** 499 * iotlb_to_section: 500 * @cpu: CPU performing the access 501 * @index: TCG CPU IOTLB entry 502 * 503 * Given a TCG CPU IOTLB entry, return the MemoryRegionSection that 504 * it refers to. @index will have been initially created and returned 505 * by memory_region_section_get_iotlb(). 506 */ 507 struct MemoryRegionSection *iotlb_to_section(CPUState *cpu, 508 hwaddr index, MemTxAttrs attrs); 509 #endif 510 511 #if defined(CONFIG_USER_ONLY) 512 void mmap_lock(void); 513 void mmap_unlock(void); 514 bool have_mmap_lock(void); 515 516 /** 517 * get_page_addr_code() - user-mode version 518 * @env: CPUArchState 519 * @addr: guest virtual address of guest code 520 * 521 * Returns @addr. 522 */ 523 static inline tb_page_addr_t get_page_addr_code(CPUArchState *env, 524 target_ulong addr) 525 { 526 return addr; 527 } 528 529 /** 530 * get_page_addr_code_hostp() - user-mode version 531 * @env: CPUArchState 532 * @addr: guest virtual address of guest code 533 * 534 * Returns @addr. 535 * 536 * If @hostp is non-NULL, sets *@hostp to the host address where @addr's content 537 * is kept. 538 */ 539 static inline tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, 540 target_ulong addr, 541 void **hostp) 542 { 543 if (hostp) { 544 *hostp = g2h(addr); 545 } 546 return addr; 547 } 548 #else 549 static inline void mmap_lock(void) {} 550 static inline void mmap_unlock(void) {} 551 552 /** 553 * get_page_addr_code() - full-system version 554 * @env: CPUArchState 555 * @addr: guest virtual address of guest code 556 * 557 * If we cannot translate and execute from the entire RAM page, or if 558 * the region is not backed by RAM, returns -1. Otherwise, returns the 559 * ram_addr_t corresponding to the guest code at @addr. 560 * 561 * Note: this function can trigger an exception. 562 */ 563 tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr); 564 565 /** 566 * get_page_addr_code_hostp() - full-system version 567 * @env: CPUArchState 568 * @addr: guest virtual address of guest code 569 * 570 * See get_page_addr_code() (full-system version) for documentation on the 571 * return value. 572 * 573 * Sets *@hostp (when @hostp is non-NULL) as follows. 574 * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp 575 * to the host address where @addr's content is kept. 576 * 577 * Note: this function can trigger an exception. 578 */ 579 tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, 580 void **hostp); 581 582 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length); 583 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr); 584 585 /* exec.c */ 586 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr); 587 588 MemoryRegionSection * 589 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr, 590 hwaddr *xlat, hwaddr *plen, 591 MemTxAttrs attrs, int *prot); 592 hwaddr memory_region_section_get_iotlb(CPUState *cpu, 593 MemoryRegionSection *section); 594 #endif 595 596 /* vl.c */ 597 extern int singlestep; 598 599 #endif 600