xref: /openbmc/qemu/accel/tcg/cpu-exec.c (revision 3161f9f4)
1 /*
2  *  emulator main execution loop
3  *
4  *  Copyright (c) 2003-2005 Fabrice Bellard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/qemu-print.h"
22 #include "qapi/error.h"
23 #include "qapi/type-helpers.h"
24 #include "hw/core/tcg-cpu-ops.h"
25 #include "trace.h"
26 #include "disas/disas.h"
27 #include "exec/exec-all.h"
28 #include "tcg/tcg.h"
29 #include "qemu/atomic.h"
30 #include "qemu/rcu.h"
31 #include "exec/log.h"
32 #include "qemu/main-loop.h"
33 #if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
34 #include "hw/i386/apic.h"
35 #endif
36 #include "sysemu/cpus.h"
37 #include "exec/cpu-all.h"
38 #include "sysemu/cpu-timers.h"
39 #include "exec/replay-core.h"
40 #include "sysemu/tcg.h"
41 #include "exec/helper-proto-common.h"
42 #include "tb-jmp-cache.h"
43 #include "tb-hash.h"
44 #include "tb-context.h"
45 #include "internal-common.h"
46 #include "internal-target.h"
47 
48 /* -icount align implementation. */
49 
50 typedef struct SyncClocks {
51     int64_t diff_clk;
52     int64_t last_cpu_icount;
53     int64_t realtime_clock;
54 } SyncClocks;
55 
56 #if !defined(CONFIG_USER_ONLY)
57 /* Allow the guest to have a max 3ms advance.
58  * The difference between the 2 clocks could therefore
59  * oscillate around 0.
60  */
61 #define VM_CLOCK_ADVANCE 3000000
62 #define THRESHOLD_REDUCE 1.5
63 #define MAX_DELAY_PRINT_RATE 2000000000LL
64 #define MAX_NB_PRINTS 100
65 
66 int64_t max_delay;
67 int64_t max_advance;
68 
69 static void align_clocks(SyncClocks *sc, CPUState *cpu)
70 {
71     int64_t cpu_icount;
72 
73     if (!icount_align_option) {
74         return;
75     }
76 
77     cpu_icount = cpu->icount_extra + cpu->neg.icount_decr.u16.low;
78     sc->diff_clk += icount_to_ns(sc->last_cpu_icount - cpu_icount);
79     sc->last_cpu_icount = cpu_icount;
80 
81     if (sc->diff_clk > VM_CLOCK_ADVANCE) {
82 #ifndef _WIN32
83         struct timespec sleep_delay, rem_delay;
84         sleep_delay.tv_sec = sc->diff_clk / 1000000000LL;
85         sleep_delay.tv_nsec = sc->diff_clk % 1000000000LL;
86         if (nanosleep(&sleep_delay, &rem_delay) < 0) {
87             sc->diff_clk = rem_delay.tv_sec * 1000000000LL + rem_delay.tv_nsec;
88         } else {
89             sc->diff_clk = 0;
90         }
91 #else
92         Sleep(sc->diff_clk / SCALE_MS);
93         sc->diff_clk = 0;
94 #endif
95     }
96 }
97 
98 static void print_delay(const SyncClocks *sc)
99 {
100     static float threshold_delay;
101     static int64_t last_realtime_clock;
102     static int nb_prints;
103 
104     if (icount_align_option &&
105         sc->realtime_clock - last_realtime_clock >= MAX_DELAY_PRINT_RATE &&
106         nb_prints < MAX_NB_PRINTS) {
107         if ((-sc->diff_clk / (float)1000000000LL > threshold_delay) ||
108             (-sc->diff_clk / (float)1000000000LL <
109              (threshold_delay - THRESHOLD_REDUCE))) {
110             threshold_delay = (-sc->diff_clk / 1000000000LL) + 1;
111             qemu_printf("Warning: The guest is now late by %.1f to %.1f seconds\n",
112                         threshold_delay - 1,
113                         threshold_delay);
114             nb_prints++;
115             last_realtime_clock = sc->realtime_clock;
116         }
117     }
118 }
119 
120 static void init_delay_params(SyncClocks *sc, CPUState *cpu)
121 {
122     if (!icount_align_option) {
123         return;
124     }
125     sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT);
126     sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - sc->realtime_clock;
127     sc->last_cpu_icount
128         = cpu->icount_extra + cpu->neg.icount_decr.u16.low;
129     if (sc->diff_clk < max_delay) {
130         max_delay = sc->diff_clk;
131     }
132     if (sc->diff_clk > max_advance) {
133         max_advance = sc->diff_clk;
134     }
135 
136     /* Print every 2s max if the guest is late. We limit the number
137        of printed messages to NB_PRINT_MAX(currently 100) */
138     print_delay(sc);
139 }
140 #else
141 static void align_clocks(SyncClocks *sc, const CPUState *cpu)
142 {
143 }
144 
145 static void init_delay_params(SyncClocks *sc, const CPUState *cpu)
146 {
147 }
148 #endif /* CONFIG USER ONLY */
149 
150 uint32_t curr_cflags(CPUState *cpu)
151 {
152     uint32_t cflags = cpu->tcg_cflags;
153 
154     /*
155      * Record gdb single-step.  We should be exiting the TB by raising
156      * EXCP_DEBUG, but to simplify other tests, disable chaining too.
157      *
158      * For singlestep and -d nochain, suppress goto_tb so that
159      * we can log -d cpu,exec after every TB.
160      */
161     if (unlikely(cpu->singlestep_enabled)) {
162         cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | CF_SINGLE_STEP | 1;
163     } else if (qatomic_read(&one_insn_per_tb)) {
164         cflags |= CF_NO_GOTO_TB | 1;
165     } else if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
166         cflags |= CF_NO_GOTO_TB;
167     }
168 
169     return cflags;
170 }
171 
172 struct tb_desc {
173     vaddr pc;
174     uint64_t cs_base;
175     CPUArchState *env;
176     tb_page_addr_t page_addr0;
177     uint32_t flags;
178     uint32_t cflags;
179 };
180 
181 static bool tb_lookup_cmp(const void *p, const void *d)
182 {
183     const TranslationBlock *tb = p;
184     const struct tb_desc *desc = d;
185 
186     if ((tb_cflags(tb) & CF_PCREL || tb->pc == desc->pc) &&
187         tb_page_addr0(tb) == desc->page_addr0 &&
188         tb->cs_base == desc->cs_base &&
189         tb->flags == desc->flags &&
190         tb_cflags(tb) == desc->cflags) {
191         /* check next page if needed */
192         tb_page_addr_t tb_phys_page1 = tb_page_addr1(tb);
193         if (tb_phys_page1 == -1) {
194             return true;
195         } else {
196             tb_page_addr_t phys_page1;
197             vaddr virt_page1;
198 
199             /*
200              * We know that the first page matched, and an otherwise valid TB
201              * encountered an incomplete instruction at the end of that page,
202              * therefore we know that generating a new TB from the current PC
203              * must also require reading from the next page -- even if the
204              * second pages do not match, and therefore the resulting insn
205              * is different for the new TB.  Therefore any exception raised
206              * here by the faulting lookup is not premature.
207              */
208             virt_page1 = TARGET_PAGE_ALIGN(desc->pc);
209             phys_page1 = get_page_addr_code(desc->env, virt_page1);
210             if (tb_phys_page1 == phys_page1) {
211                 return true;
212             }
213         }
214     }
215     return false;
216 }
217 
218 static TranslationBlock *tb_htable_lookup(CPUState *cpu, vaddr pc,
219                                           uint64_t cs_base, uint32_t flags,
220                                           uint32_t cflags)
221 {
222     tb_page_addr_t phys_pc;
223     struct tb_desc desc;
224     uint32_t h;
225 
226     desc.env = cpu_env(cpu);
227     desc.cs_base = cs_base;
228     desc.flags = flags;
229     desc.cflags = cflags;
230     desc.pc = pc;
231     phys_pc = get_page_addr_code(desc.env, pc);
232     if (phys_pc == -1) {
233         return NULL;
234     }
235     desc.page_addr0 = phys_pc;
236     h = tb_hash_func(phys_pc, (cflags & CF_PCREL ? 0 : pc),
237                      flags, cs_base, cflags);
238     return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp);
239 }
240 
241 /* Might cause an exception, so have a longjmp destination ready */
242 static inline TranslationBlock *tb_lookup(CPUState *cpu, vaddr pc,
243                                           uint64_t cs_base, uint32_t flags,
244                                           uint32_t cflags)
245 {
246     TranslationBlock *tb;
247     CPUJumpCache *jc;
248     uint32_t hash;
249 
250     /* we should never be trying to look up an INVALID tb */
251     tcg_debug_assert(!(cflags & CF_INVALID));
252 
253     hash = tb_jmp_cache_hash_func(pc);
254     jc = cpu->tb_jmp_cache;
255 
256     if (cflags & CF_PCREL) {
257         /* Use acquire to ensure current load of pc from jc. */
258         tb = qatomic_load_acquire(&jc->array[hash].tb);
259 
260         if (likely(tb &&
261                    jc->array[hash].pc == pc &&
262                    tb->cs_base == cs_base &&
263                    tb->flags == flags &&
264                    tb_cflags(tb) == cflags)) {
265             return tb;
266         }
267         tb = tb_htable_lookup(cpu, pc, cs_base, flags, cflags);
268         if (tb == NULL) {
269             return NULL;
270         }
271         jc->array[hash].pc = pc;
272         /* Ensure pc is written first. */
273         qatomic_store_release(&jc->array[hash].tb, tb);
274     } else {
275         /* Use rcu_read to ensure current load of pc from *tb. */
276         tb = qatomic_rcu_read(&jc->array[hash].tb);
277 
278         if (likely(tb &&
279                    tb->pc == pc &&
280                    tb->cs_base == cs_base &&
281                    tb->flags == flags &&
282                    tb_cflags(tb) == cflags)) {
283             return tb;
284         }
285         tb = tb_htable_lookup(cpu, pc, cs_base, flags, cflags);
286         if (tb == NULL) {
287             return NULL;
288         }
289         /* Use the pc value already stored in tb->pc. */
290         qatomic_set(&jc->array[hash].tb, tb);
291     }
292 
293     return tb;
294 }
295 
296 static void log_cpu_exec(vaddr pc, CPUState *cpu,
297                          const TranslationBlock *tb)
298 {
299     if (qemu_log_in_addr_range(pc)) {
300         qemu_log_mask(CPU_LOG_EXEC,
301                       "Trace %d: %p [%08" PRIx64
302                       "/%016" VADDR_PRIx "/%08x/%08x] %s\n",
303                       cpu->cpu_index, tb->tc.ptr, tb->cs_base, pc,
304                       tb->flags, tb->cflags, lookup_symbol(pc));
305 
306         if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
307             FILE *logfile = qemu_log_trylock();
308             if (logfile) {
309                 int flags = 0;
310 
311                 if (qemu_loglevel_mask(CPU_LOG_TB_FPU)) {
312                     flags |= CPU_DUMP_FPU;
313                 }
314 #if defined(TARGET_I386)
315                 flags |= CPU_DUMP_CCOP;
316 #endif
317                 if (qemu_loglevel_mask(CPU_LOG_TB_VPU)) {
318                     flags |= CPU_DUMP_VPU;
319                 }
320                 cpu_dump_state(cpu, logfile, flags);
321                 qemu_log_unlock(logfile);
322             }
323         }
324     }
325 }
326 
327 static bool check_for_breakpoints_slow(CPUState *cpu, vaddr pc,
328                                        uint32_t *cflags)
329 {
330     CPUBreakpoint *bp;
331     bool match_page = false;
332 
333     /*
334      * Singlestep overrides breakpoints.
335      * This requirement is visible in the record-replay tests, where
336      * we would fail to make forward progress in reverse-continue.
337      *
338      * TODO: gdb singlestep should only override gdb breakpoints,
339      * so that one could (gdb) singlestep into the guest kernel's
340      * architectural breakpoint handler.
341      */
342     if (cpu->singlestep_enabled) {
343         return false;
344     }
345 
346     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
347         /*
348          * If we have an exact pc match, trigger the breakpoint.
349          * Otherwise, note matches within the page.
350          */
351         if (pc == bp->pc) {
352             bool match_bp = false;
353 
354             if (bp->flags & BP_GDB) {
355                 match_bp = true;
356             } else if (bp->flags & BP_CPU) {
357 #ifdef CONFIG_USER_ONLY
358                 g_assert_not_reached();
359 #else
360                 CPUClass *cc = CPU_GET_CLASS(cpu);
361                 assert(cc->tcg_ops->debug_check_breakpoint);
362                 match_bp = cc->tcg_ops->debug_check_breakpoint(cpu);
363 #endif
364             }
365 
366             if (match_bp) {
367                 cpu->exception_index = EXCP_DEBUG;
368                 return true;
369             }
370         } else if (((pc ^ bp->pc) & TARGET_PAGE_MASK) == 0) {
371             match_page = true;
372         }
373     }
374 
375     /*
376      * Within the same page as a breakpoint, single-step,
377      * returning to helper_lookup_tb_ptr after each insn looking
378      * for the actual breakpoint.
379      *
380      * TODO: Perhaps better to record all of the TBs associated
381      * with a given virtual page that contains a breakpoint, and
382      * then invalidate them when a new overlapping breakpoint is
383      * set on the page.  Non-overlapping TBs would not be
384      * invalidated, nor would any TB need to be invalidated as
385      * breakpoints are removed.
386      */
387     if (match_page) {
388         *cflags = (*cflags & ~CF_COUNT_MASK) | CF_NO_GOTO_TB | 1;
389     }
390     return false;
391 }
392 
393 static inline bool check_for_breakpoints(CPUState *cpu, vaddr pc,
394                                          uint32_t *cflags)
395 {
396     return unlikely(!QTAILQ_EMPTY(&cpu->breakpoints)) &&
397         check_for_breakpoints_slow(cpu, pc, cflags);
398 }
399 
400 /**
401  * helper_lookup_tb_ptr: quick check for next tb
402  * @env: current cpu state
403  *
404  * Look for an existing TB matching the current cpu state.
405  * If found, return the code pointer.  If not found, return
406  * the tcg epilogue so that we return into cpu_tb_exec.
407  */
408 const void *HELPER(lookup_tb_ptr)(CPUArchState *env)
409 {
410     CPUState *cpu = env_cpu(env);
411     TranslationBlock *tb;
412     vaddr pc;
413     uint64_t cs_base;
414     uint32_t flags, cflags;
415 
416     cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
417 
418     cflags = curr_cflags(cpu);
419     if (check_for_breakpoints(cpu, pc, &cflags)) {
420         cpu_loop_exit(cpu);
421     }
422 
423     tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
424     if (tb == NULL) {
425         return tcg_code_gen_epilogue;
426     }
427 
428     if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) {
429         log_cpu_exec(pc, cpu, tb);
430     }
431 
432     return tb->tc.ptr;
433 }
434 
435 /* Execute a TB, and fix up the CPU state afterwards if necessary */
436 /*
437  * Disable CFI checks.
438  * TCG creates binary blobs at runtime, with the transformed code.
439  * A TB is a blob of binary code, created at runtime and called with an
440  * indirect function call. Since such function did not exist at compile time,
441  * the CFI runtime has no way to verify its signature and would fail.
442  * TCG is not considered a security-sensitive part of QEMU so this does not
443  * affect the impact of CFI in environment with high security requirements
444  */
445 static inline TranslationBlock * QEMU_DISABLE_CFI
446 cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit)
447 {
448     CPUArchState *env = cpu_env(cpu);
449     uintptr_t ret;
450     TranslationBlock *last_tb;
451     const void *tb_ptr = itb->tc.ptr;
452 
453     if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) {
454         log_cpu_exec(log_pc(cpu, itb), cpu, itb);
455     }
456 
457     qemu_thread_jit_execute();
458     ret = tcg_qemu_tb_exec(env, tb_ptr);
459     cpu->neg.can_do_io = true;
460     qemu_plugin_disable_mem_helpers(cpu);
461     /*
462      * TODO: Delay swapping back to the read-write region of the TB
463      * until we actually need to modify the TB.  The read-only copy,
464      * coming from the rx region, shares the same host TLB entry as
465      * the code that executed the exit_tb opcode that arrived here.
466      * If we insist on touching both the RX and the RW pages, we
467      * double the host TLB pressure.
468      */
469     last_tb = tcg_splitwx_to_rw((void *)(ret & ~TB_EXIT_MASK));
470     *tb_exit = ret & TB_EXIT_MASK;
471 
472     trace_exec_tb_exit(last_tb, *tb_exit);
473 
474     if (*tb_exit > TB_EXIT_IDX1) {
475         /* We didn't start executing this TB (eg because the instruction
476          * counter hit zero); we must restore the guest PC to the address
477          * of the start of the TB.
478          */
479         CPUClass *cc = CPU_GET_CLASS(cpu);
480 
481         if (cc->tcg_ops->synchronize_from_tb) {
482             cc->tcg_ops->synchronize_from_tb(cpu, last_tb);
483         } else {
484             tcg_debug_assert(!(tb_cflags(last_tb) & CF_PCREL));
485             assert(cc->set_pc);
486             cc->set_pc(cpu, last_tb->pc);
487         }
488         if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
489             vaddr pc = log_pc(cpu, last_tb);
490             if (qemu_log_in_addr_range(pc)) {
491                 qemu_log("Stopped execution of TB chain before %p [%016"
492                          VADDR_PRIx "] %s\n",
493                          last_tb->tc.ptr, pc, lookup_symbol(pc));
494             }
495         }
496     }
497 
498     /*
499      * If gdb single-step, and we haven't raised another exception,
500      * raise a debug exception.  Single-step with another exception
501      * is handled in cpu_handle_exception.
502      */
503     if (unlikely(cpu->singlestep_enabled) && cpu->exception_index == -1) {
504         cpu->exception_index = EXCP_DEBUG;
505         cpu_loop_exit(cpu);
506     }
507 
508     return last_tb;
509 }
510 
511 
512 static void cpu_exec_enter(CPUState *cpu)
513 {
514     CPUClass *cc = CPU_GET_CLASS(cpu);
515 
516     if (cc->tcg_ops->cpu_exec_enter) {
517         cc->tcg_ops->cpu_exec_enter(cpu);
518     }
519 }
520 
521 static void cpu_exec_exit(CPUState *cpu)
522 {
523     CPUClass *cc = CPU_GET_CLASS(cpu);
524 
525     if (cc->tcg_ops->cpu_exec_exit) {
526         cc->tcg_ops->cpu_exec_exit(cpu);
527     }
528 }
529 
530 static void cpu_exec_longjmp_cleanup(CPUState *cpu)
531 {
532     /* Non-buggy compilers preserve this; assert the correct value. */
533     g_assert(cpu == current_cpu);
534 
535 #ifdef CONFIG_USER_ONLY
536     clear_helper_retaddr();
537     if (have_mmap_lock()) {
538         mmap_unlock();
539     }
540 #else
541     /*
542      * For softmmu, a tlb_fill fault during translation will land here,
543      * and we need to release any page locks held.  In system mode we
544      * have one tcg_ctx per thread, so we know it was this cpu doing
545      * the translation.
546      *
547      * Alternative 1: Install a cleanup to be called via an exception
548      * handling safe longjmp.  It seems plausible that all our hosts
549      * support such a thing.  We'd have to properly register unwind info
550      * for the JIT for EH, rather that just for GDB.
551      *
552      * Alternative 2: Set and restore cpu->jmp_env in tb_gen_code to
553      * capture the cpu_loop_exit longjmp, perform the cleanup, and
554      * jump again to arrive here.
555      */
556     if (tcg_ctx->gen_tb) {
557         tb_unlock_pages(tcg_ctx->gen_tb);
558         tcg_ctx->gen_tb = NULL;
559     }
560 #endif
561     if (qemu_mutex_iothread_locked()) {
562         qemu_mutex_unlock_iothread();
563     }
564     assert_no_pages_locked();
565 }
566 
567 void cpu_exec_step_atomic(CPUState *cpu)
568 {
569     CPUArchState *env = cpu_env(cpu);
570     TranslationBlock *tb;
571     vaddr pc;
572     uint64_t cs_base;
573     uint32_t flags, cflags;
574     int tb_exit;
575 
576     if (sigsetjmp(cpu->jmp_env, 0) == 0) {
577         start_exclusive();
578         g_assert(cpu == current_cpu);
579         g_assert(!cpu->running);
580         cpu->running = true;
581 
582         cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
583 
584         cflags = curr_cflags(cpu);
585         /* Execute in a serial context. */
586         cflags &= ~CF_PARALLEL;
587         /* After 1 insn, return and release the exclusive lock. */
588         cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | 1;
589         /*
590          * No need to check_for_breakpoints here.
591          * We only arrive in cpu_exec_step_atomic after beginning execution
592          * of an insn that includes an atomic operation we can't handle.
593          * Any breakpoint for this insn will have been recognized earlier.
594          */
595 
596         tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
597         if (tb == NULL) {
598             mmap_lock();
599             tb = tb_gen_code(cpu, pc, cs_base, flags, cflags);
600             mmap_unlock();
601         }
602 
603         cpu_exec_enter(cpu);
604         /* execute the generated code */
605         trace_exec_tb(tb, pc);
606         cpu_tb_exec(cpu, tb, &tb_exit);
607         cpu_exec_exit(cpu);
608     } else {
609         cpu_exec_longjmp_cleanup(cpu);
610     }
611 
612     /*
613      * As we start the exclusive region before codegen we must still
614      * be in the region if we longjump out of either the codegen or
615      * the execution.
616      */
617     g_assert(cpu_in_exclusive_context(cpu));
618     cpu->running = false;
619     end_exclusive();
620 }
621 
622 void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
623 {
624     /*
625      * Get the rx view of the structure, from which we find the
626      * executable code address, and tb_target_set_jmp_target can
627      * produce a pc-relative displacement to jmp_target_addr[n].
628      */
629     const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
630     uintptr_t offset = tb->jmp_insn_offset[n];
631     uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
632     uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
633 
634     tb->jmp_target_addr[n] = addr;
635     tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
636 }
637 
638 static inline void tb_add_jump(TranslationBlock *tb, int n,
639                                TranslationBlock *tb_next)
640 {
641     uintptr_t old;
642 
643     qemu_thread_jit_write();
644     assert(n < ARRAY_SIZE(tb->jmp_list_next));
645     qemu_spin_lock(&tb_next->jmp_lock);
646 
647     /* make sure the destination TB is valid */
648     if (tb_next->cflags & CF_INVALID) {
649         goto out_unlock_next;
650     }
651     /* Atomically claim the jump destination slot only if it was NULL */
652     old = qatomic_cmpxchg(&tb->jmp_dest[n], (uintptr_t)NULL,
653                           (uintptr_t)tb_next);
654     if (old) {
655         goto out_unlock_next;
656     }
657 
658     /* patch the native jump address */
659     tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc.ptr);
660 
661     /* add in TB jmp list */
662     tb->jmp_list_next[n] = tb_next->jmp_list_head;
663     tb_next->jmp_list_head = (uintptr_t)tb | n;
664 
665     qemu_spin_unlock(&tb_next->jmp_lock);
666 
667     qemu_log_mask(CPU_LOG_EXEC, "Linking TBs %p index %d -> %p\n",
668                   tb->tc.ptr, n, tb_next->tc.ptr);
669     return;
670 
671  out_unlock_next:
672     qemu_spin_unlock(&tb_next->jmp_lock);
673     return;
674 }
675 
676 static inline bool cpu_handle_halt(CPUState *cpu)
677 {
678 #ifndef CONFIG_USER_ONLY
679     if (cpu->halted) {
680 #if defined(TARGET_I386)
681         if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
682             X86CPU *x86_cpu = X86_CPU(cpu);
683             qemu_mutex_lock_iothread();
684             apic_poll_irq(x86_cpu->apic_state);
685             cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL);
686             qemu_mutex_unlock_iothread();
687         }
688 #endif /* TARGET_I386 */
689         if (!cpu_has_work(cpu)) {
690             return true;
691         }
692 
693         cpu->halted = 0;
694     }
695 #endif /* !CONFIG_USER_ONLY */
696 
697     return false;
698 }
699 
700 static inline void cpu_handle_debug_exception(CPUState *cpu)
701 {
702     CPUClass *cc = CPU_GET_CLASS(cpu);
703     CPUWatchpoint *wp;
704 
705     if (!cpu->watchpoint_hit) {
706         QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
707             wp->flags &= ~BP_WATCHPOINT_HIT;
708         }
709     }
710 
711     if (cc->tcg_ops->debug_excp_handler) {
712         cc->tcg_ops->debug_excp_handler(cpu);
713     }
714 }
715 
716 static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
717 {
718     if (cpu->exception_index < 0) {
719 #ifndef CONFIG_USER_ONLY
720         if (replay_has_exception()
721             && cpu->neg.icount_decr.u16.low + cpu->icount_extra == 0) {
722             /* Execute just one insn to trigger exception pending in the log */
723             cpu->cflags_next_tb = (curr_cflags(cpu) & ~CF_USE_ICOUNT)
724                 | CF_NOIRQ | 1;
725         }
726 #endif
727         return false;
728     }
729     if (cpu->exception_index >= EXCP_INTERRUPT) {
730         /* exit request from the cpu execution loop */
731         *ret = cpu->exception_index;
732         if (*ret == EXCP_DEBUG) {
733             cpu_handle_debug_exception(cpu);
734         }
735         cpu->exception_index = -1;
736         return true;
737     } else {
738 #if defined(CONFIG_USER_ONLY)
739         /* if user mode only, we simulate a fake exception
740            which will be handled outside the cpu execution
741            loop */
742 #if defined(TARGET_I386)
743         CPUClass *cc = CPU_GET_CLASS(cpu);
744         cc->tcg_ops->fake_user_interrupt(cpu);
745 #endif /* TARGET_I386 */
746         *ret = cpu->exception_index;
747         cpu->exception_index = -1;
748         return true;
749 #else
750         if (replay_exception()) {
751             CPUClass *cc = CPU_GET_CLASS(cpu);
752             qemu_mutex_lock_iothread();
753             cc->tcg_ops->do_interrupt(cpu);
754             qemu_mutex_unlock_iothread();
755             cpu->exception_index = -1;
756 
757             if (unlikely(cpu->singlestep_enabled)) {
758                 /*
759                  * After processing the exception, ensure an EXCP_DEBUG is
760                  * raised when single-stepping so that GDB doesn't miss the
761                  * next instruction.
762                  */
763                 *ret = EXCP_DEBUG;
764                 cpu_handle_debug_exception(cpu);
765                 return true;
766             }
767         } else if (!replay_has_interrupt()) {
768             /* give a chance to iothread in replay mode */
769             *ret = EXCP_INTERRUPT;
770             return true;
771         }
772 #endif
773     }
774 
775     return false;
776 }
777 
778 #ifndef CONFIG_USER_ONLY
779 /*
780  * CPU_INTERRUPT_POLL is a virtual event which gets converted into a
781  * "real" interrupt event later. It does not need to be recorded for
782  * replay purposes.
783  */
784 static inline bool need_replay_interrupt(int interrupt_request)
785 {
786 #if defined(TARGET_I386)
787     return !(interrupt_request & CPU_INTERRUPT_POLL);
788 #else
789     return true;
790 #endif
791 }
792 #endif /* !CONFIG_USER_ONLY */
793 
794 static inline bool cpu_handle_interrupt(CPUState *cpu,
795                                         TranslationBlock **last_tb)
796 {
797     /*
798      * If we have requested custom cflags with CF_NOIRQ we should
799      * skip checking here. Any pending interrupts will get picked up
800      * by the next TB we execute under normal cflags.
801      */
802     if (cpu->cflags_next_tb != -1 && cpu->cflags_next_tb & CF_NOIRQ) {
803         return false;
804     }
805 
806     /* Clear the interrupt flag now since we're processing
807      * cpu->interrupt_request and cpu->exit_request.
808      * Ensure zeroing happens before reading cpu->exit_request or
809      * cpu->interrupt_request (see also smp_wmb in cpu_exit())
810      */
811     qatomic_set_mb(&cpu->neg.icount_decr.u16.high, 0);
812 
813     if (unlikely(qatomic_read(&cpu->interrupt_request))) {
814         int interrupt_request;
815         qemu_mutex_lock_iothread();
816         interrupt_request = cpu->interrupt_request;
817         if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
818             /* Mask out external interrupts for this step. */
819             interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
820         }
821         if (interrupt_request & CPU_INTERRUPT_DEBUG) {
822             cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
823             cpu->exception_index = EXCP_DEBUG;
824             qemu_mutex_unlock_iothread();
825             return true;
826         }
827 #if !defined(CONFIG_USER_ONLY)
828         if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) {
829             /* Do nothing */
830         } else if (interrupt_request & CPU_INTERRUPT_HALT) {
831             replay_interrupt();
832             cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
833             cpu->halted = 1;
834             cpu->exception_index = EXCP_HLT;
835             qemu_mutex_unlock_iothread();
836             return true;
837         }
838 #if defined(TARGET_I386)
839         else if (interrupt_request & CPU_INTERRUPT_INIT) {
840             X86CPU *x86_cpu = X86_CPU(cpu);
841             CPUArchState *env = &x86_cpu->env;
842             replay_interrupt();
843             cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0, 0);
844             do_cpu_init(x86_cpu);
845             cpu->exception_index = EXCP_HALTED;
846             qemu_mutex_unlock_iothread();
847             return true;
848         }
849 #else
850         else if (interrupt_request & CPU_INTERRUPT_RESET) {
851             replay_interrupt();
852             cpu_reset(cpu);
853             qemu_mutex_unlock_iothread();
854             return true;
855         }
856 #endif /* !TARGET_I386 */
857         /* The target hook has 3 exit conditions:
858            False when the interrupt isn't processed,
859            True when it is, and we should restart on a new TB,
860            and via longjmp via cpu_loop_exit.  */
861         else {
862             CPUClass *cc = CPU_GET_CLASS(cpu);
863 
864             if (cc->tcg_ops->cpu_exec_interrupt &&
865                 cc->tcg_ops->cpu_exec_interrupt(cpu, interrupt_request)) {
866                 if (need_replay_interrupt(interrupt_request)) {
867                     replay_interrupt();
868                 }
869                 /*
870                  * After processing the interrupt, ensure an EXCP_DEBUG is
871                  * raised when single-stepping so that GDB doesn't miss the
872                  * next instruction.
873                  */
874                 if (unlikely(cpu->singlestep_enabled)) {
875                     cpu->exception_index = EXCP_DEBUG;
876                     qemu_mutex_unlock_iothread();
877                     return true;
878                 }
879                 cpu->exception_index = -1;
880                 *last_tb = NULL;
881             }
882             /* The target hook may have updated the 'cpu->interrupt_request';
883              * reload the 'interrupt_request' value */
884             interrupt_request = cpu->interrupt_request;
885         }
886 #endif /* !CONFIG_USER_ONLY */
887         if (interrupt_request & CPU_INTERRUPT_EXITTB) {
888             cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
889             /* ensure that no TB jump will be modified as
890                the program flow was changed */
891             *last_tb = NULL;
892         }
893 
894         /* If we exit via cpu_loop_exit/longjmp it is reset in cpu_exec */
895         qemu_mutex_unlock_iothread();
896     }
897 
898     /* Finally, check if we need to exit to the main loop.  */
899     if (unlikely(qatomic_read(&cpu->exit_request))
900         || (icount_enabled()
901             && (cpu->cflags_next_tb == -1 || cpu->cflags_next_tb & CF_USE_ICOUNT)
902             && cpu->neg.icount_decr.u16.low + cpu->icount_extra == 0)) {
903         qatomic_set(&cpu->exit_request, 0);
904         if (cpu->exception_index == -1) {
905             cpu->exception_index = EXCP_INTERRUPT;
906         }
907         return true;
908     }
909 
910     return false;
911 }
912 
913 static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
914                                     vaddr pc, TranslationBlock **last_tb,
915                                     int *tb_exit)
916 {
917     int32_t insns_left;
918 
919     trace_exec_tb(tb, pc);
920     tb = cpu_tb_exec(cpu, tb, tb_exit);
921     if (*tb_exit != TB_EXIT_REQUESTED) {
922         *last_tb = tb;
923         return;
924     }
925 
926     *last_tb = NULL;
927     insns_left = qatomic_read(&cpu->neg.icount_decr.u32);
928     if (insns_left < 0) {
929         /* Something asked us to stop executing chained TBs; just
930          * continue round the main loop. Whatever requested the exit
931          * will also have set something else (eg exit_request or
932          * interrupt_request) which will be handled by
933          * cpu_handle_interrupt.  cpu_handle_interrupt will also
934          * clear cpu->icount_decr.u16.high.
935          */
936         return;
937     }
938 
939     /* Instruction counter expired.  */
940     assert(icount_enabled());
941 #ifndef CONFIG_USER_ONLY
942     /* Ensure global icount has gone forward */
943     icount_update(cpu);
944     /* Refill decrementer and continue execution.  */
945     insns_left = MIN(0xffff, cpu->icount_budget);
946     cpu->neg.icount_decr.u16.low = insns_left;
947     cpu->icount_extra = cpu->icount_budget - insns_left;
948 
949     /*
950      * If the next tb has more instructions than we have left to
951      * execute we need to ensure we find/generate a TB with exactly
952      * insns_left instructions in it.
953      */
954     if (insns_left > 0 && insns_left < tb->icount)  {
955         assert(insns_left <= CF_COUNT_MASK);
956         assert(cpu->icount_extra == 0);
957         cpu->cflags_next_tb = (tb->cflags & ~CF_COUNT_MASK) | insns_left;
958     }
959 #endif
960 }
961 
962 /* main execution loop */
963 
964 static int __attribute__((noinline))
965 cpu_exec_loop(CPUState *cpu, SyncClocks *sc)
966 {
967     int ret;
968 
969     /* if an exception is pending, we execute it here */
970     while (!cpu_handle_exception(cpu, &ret)) {
971         TranslationBlock *last_tb = NULL;
972         int tb_exit = 0;
973 
974         while (!cpu_handle_interrupt(cpu, &last_tb)) {
975             TranslationBlock *tb;
976             vaddr pc;
977             uint64_t cs_base;
978             uint32_t flags, cflags;
979 
980             cpu_get_tb_cpu_state(cpu_env(cpu), &pc, &cs_base, &flags);
981 
982             /*
983              * When requested, use an exact setting for cflags for the next
984              * execution.  This is used for icount, precise smc, and stop-
985              * after-access watchpoints.  Since this request should never
986              * have CF_INVALID set, -1 is a convenient invalid value that
987              * does not require tcg headers for cpu_common_reset.
988              */
989             cflags = cpu->cflags_next_tb;
990             if (cflags == -1) {
991                 cflags = curr_cflags(cpu);
992             } else {
993                 cpu->cflags_next_tb = -1;
994             }
995 
996             if (check_for_breakpoints(cpu, pc, &cflags)) {
997                 break;
998             }
999 
1000             tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
1001             if (tb == NULL) {
1002                 CPUJumpCache *jc;
1003                 uint32_t h;
1004 
1005                 mmap_lock();
1006                 tb = tb_gen_code(cpu, pc, cs_base, flags, cflags);
1007                 mmap_unlock();
1008 
1009                 /*
1010                  * We add the TB in the virtual pc hash table
1011                  * for the fast lookup
1012                  */
1013                 h = tb_jmp_cache_hash_func(pc);
1014                 jc = cpu->tb_jmp_cache;
1015                 if (cflags & CF_PCREL) {
1016                     jc->array[h].pc = pc;
1017                     /* Ensure pc is written first. */
1018                     qatomic_store_release(&jc->array[h].tb, tb);
1019                 } else {
1020                     /* Use the pc value already stored in tb->pc. */
1021                     qatomic_set(&jc->array[h].tb, tb);
1022                 }
1023             }
1024 
1025 #ifndef CONFIG_USER_ONLY
1026             /*
1027              * We don't take care of direct jumps when address mapping
1028              * changes in system emulation.  So it's not safe to make a
1029              * direct jump to a TB spanning two pages because the mapping
1030              * for the second page can change.
1031              */
1032             if (tb_page_addr1(tb) != -1) {
1033                 last_tb = NULL;
1034             }
1035 #endif
1036             /* See if we can patch the calling TB. */
1037             if (last_tb) {
1038                 tb_add_jump(last_tb, tb_exit, tb);
1039             }
1040 
1041             cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit);
1042 
1043             /* Try to align the host and virtual clocks
1044                if the guest is in advance */
1045             align_clocks(sc, cpu);
1046         }
1047     }
1048     return ret;
1049 }
1050 
1051 static int cpu_exec_setjmp(CPUState *cpu, SyncClocks *sc)
1052 {
1053     /* Prepare setjmp context for exception handling. */
1054     if (unlikely(sigsetjmp(cpu->jmp_env, 0) != 0)) {
1055         cpu_exec_longjmp_cleanup(cpu);
1056     }
1057 
1058     return cpu_exec_loop(cpu, sc);
1059 }
1060 
1061 int cpu_exec(CPUState *cpu)
1062 {
1063     int ret;
1064     SyncClocks sc = { 0 };
1065 
1066     /* replay_interrupt may need current_cpu */
1067     current_cpu = cpu;
1068 
1069     if (cpu_handle_halt(cpu)) {
1070         return EXCP_HALTED;
1071     }
1072 
1073     rcu_read_lock();
1074     cpu_exec_enter(cpu);
1075 
1076     /*
1077      * Calculate difference between guest clock and host clock.
1078      * This delay includes the delay of the last cycle, so
1079      * what we have to do is sleep until it is 0. As for the
1080      * advance/delay we gain here, we try to fix it next time.
1081      */
1082     init_delay_params(&sc, cpu);
1083 
1084     ret = cpu_exec_setjmp(cpu, &sc);
1085 
1086     cpu_exec_exit(cpu);
1087     rcu_read_unlock();
1088 
1089     return ret;
1090 }
1091 
1092 bool tcg_exec_realizefn(CPUState *cpu, Error **errp)
1093 {
1094     static bool tcg_target_initialized;
1095     CPUClass *cc = CPU_GET_CLASS(cpu);
1096 
1097     if (!tcg_target_initialized) {
1098         cc->tcg_ops->initialize();
1099         tcg_target_initialized = true;
1100     }
1101 
1102     cpu->tb_jmp_cache = g_new0(CPUJumpCache, 1);
1103     tlb_init(cpu);
1104 #ifndef CONFIG_USER_ONLY
1105     tcg_iommu_init_notifier_list(cpu);
1106 #endif /* !CONFIG_USER_ONLY */
1107     /* qemu_plugin_vcpu_init_hook delayed until cpu_index assigned. */
1108 
1109     return true;
1110 }
1111 
1112 /* undo the initializations in reverse order */
1113 void tcg_exec_unrealizefn(CPUState *cpu)
1114 {
1115 #ifndef CONFIG_USER_ONLY
1116     tcg_iommu_free_notifier_list(cpu);
1117 #endif /* !CONFIG_USER_ONLY */
1118 
1119     tlb_destroy(cpu);
1120     g_free_rcu(cpu->tb_jmp_cache, rcu);
1121 }
1122