xref: /openbmc/qemu/accel/tcg/translate-all.c (revision f06176be76ffa96098737665ac770cac0f7bfdb8)
1 /*
2  *  Host code generation
3  *
4  *  Copyright (c) 2003 Fabrice Bellard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/units.h"
22 #include "qemu-common.h"
23 
24 #define NO_CPU_IO_DEFS
25 #include "cpu.h"
26 #include "trace.h"
27 #include "disas/disas.h"
28 #include "exec/exec-all.h"
29 #include "tcg/tcg.h"
30 #if defined(CONFIG_USER_ONLY)
31 #include "qemu.h"
32 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
33 #include <sys/param.h>
34 #if __FreeBSD_version >= 700104
35 #define HAVE_KINFO_GETVMMAP
36 #define sigqueue sigqueue_freebsd  /* avoid redefinition */
37 #include <sys/proc.h>
38 #include <machine/profile.h>
39 #define _KERNEL
40 #include <sys/user.h>
41 #undef _KERNEL
42 #undef sigqueue
43 #include <libutil.h>
44 #endif
45 #endif
46 #else
47 #include "exec/ram_addr.h"
48 #endif
49 
50 #include "exec/cputlb.h"
51 #include "exec/tb-hash.h"
52 #include "exec/translate-all.h"
53 #include "qemu/bitmap.h"
54 #include "qemu/error-report.h"
55 #include "qemu/qemu-print.h"
56 #include "qemu/timer.h"
57 #include "qemu/main-loop.h"
58 #include "exec/log.h"
59 #include "sysemu/cpus.h"
60 #include "sysemu/cpu-timers.h"
61 #include "sysemu/tcg.h"
62 #include "qapi/error.h"
63 
64 /* #define DEBUG_TB_INVALIDATE */
65 /* #define DEBUG_TB_FLUSH */
66 /* make various TB consistency checks */
67 /* #define DEBUG_TB_CHECK */
68 
69 #ifdef DEBUG_TB_INVALIDATE
70 #define DEBUG_TB_INVALIDATE_GATE 1
71 #else
72 #define DEBUG_TB_INVALIDATE_GATE 0
73 #endif
74 
75 #ifdef DEBUG_TB_FLUSH
76 #define DEBUG_TB_FLUSH_GATE 1
77 #else
78 #define DEBUG_TB_FLUSH_GATE 0
79 #endif
80 
81 #if !defined(CONFIG_USER_ONLY)
82 /* TB consistency checks only implemented for usermode emulation.  */
83 #undef DEBUG_TB_CHECK
84 #endif
85 
86 #ifdef DEBUG_TB_CHECK
87 #define DEBUG_TB_CHECK_GATE 1
88 #else
89 #define DEBUG_TB_CHECK_GATE 0
90 #endif
91 
92 /* Access to the various translations structures need to be serialised via locks
93  * for consistency.
94  * In user-mode emulation access to the memory related structures are protected
95  * with mmap_lock.
96  * In !user-mode we use per-page locks.
97  */
98 #ifdef CONFIG_SOFTMMU
99 #define assert_memory_lock()
100 #else
101 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
102 #endif
103 
104 #define SMC_BITMAP_USE_THRESHOLD 10
105 
106 typedef struct PageDesc {
107     /* list of TBs intersecting this ram page */
108     uintptr_t first_tb;
109 #ifdef CONFIG_SOFTMMU
110     /* in order to optimize self modifying code, we count the number
111        of lookups we do to a given page to use a bitmap */
112     unsigned long *code_bitmap;
113     unsigned int code_write_count;
114 #else
115     unsigned long flags;
116 #endif
117 #ifndef CONFIG_USER_ONLY
118     QemuSpin lock;
119 #endif
120 } PageDesc;
121 
122 /**
123  * struct page_entry - page descriptor entry
124  * @pd:     pointer to the &struct PageDesc of the page this entry represents
125  * @index:  page index of the page
126  * @locked: whether the page is locked
127  *
128  * This struct helps us keep track of the locked state of a page, without
129  * bloating &struct PageDesc.
130  *
131  * A page lock protects accesses to all fields of &struct PageDesc.
132  *
133  * See also: &struct page_collection.
134  */
135 struct page_entry {
136     PageDesc *pd;
137     tb_page_addr_t index;
138     bool locked;
139 };
140 
141 /**
142  * struct page_collection - tracks a set of pages (i.e. &struct page_entry's)
143  * @tree:   Binary search tree (BST) of the pages, with key == page index
144  * @max:    Pointer to the page in @tree with the highest page index
145  *
146  * To avoid deadlock we lock pages in ascending order of page index.
147  * When operating on a set of pages, we need to keep track of them so that
148  * we can lock them in order and also unlock them later. For this we collect
149  * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the
150  * @tree implementation we use does not provide an O(1) operation to obtain the
151  * highest-ranked element, we use @max to keep track of the inserted page
152  * with the highest index. This is valuable because if a page is not in
153  * the tree and its index is higher than @max's, then we can lock it
154  * without breaking the locking order rule.
155  *
156  * Note on naming: 'struct page_set' would be shorter, but we already have a few
157  * page_set_*() helpers, so page_collection is used instead to avoid confusion.
158  *
159  * See also: page_collection_lock().
160  */
161 struct page_collection {
162     GTree *tree;
163     struct page_entry *max;
164 };
165 
166 /* list iterators for lists of tagged pointers in TranslationBlock */
167 #define TB_FOR_EACH_TAGGED(head, tb, n, field)                          \
168     for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1);        \
169          tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
170              tb = (TranslationBlock *)((uintptr_t)tb & ~1))
171 
172 #define PAGE_FOR_EACH_TB(pagedesc, tb, n)                       \
173     TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
174 
175 #define TB_FOR_EACH_JMP(head_tb, tb, n)                                 \
176     TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
177 
178 /*
179  * In system mode we want L1_MAP to be based on ram offsets,
180  * while in user mode we want it to be based on virtual addresses.
181  *
182  * TODO: For user mode, see the caveat re host vs guest virtual
183  * address spaces near GUEST_ADDR_MAX.
184  */
185 #if !defined(CONFIG_USER_ONLY)
186 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
187 # define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
188 #else
189 # define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
190 #endif
191 #else
192 # define L1_MAP_ADDR_SPACE_BITS  MIN(HOST_LONG_BITS, TARGET_ABI_BITS)
193 #endif
194 
195 /* Size of the L2 (and L3, etc) page tables.  */
196 #define V_L2_BITS 10
197 #define V_L2_SIZE (1 << V_L2_BITS)
198 
199 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
200 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
201                   sizeof_field(TranslationBlock, trace_vcpu_dstate)
202                   * BITS_PER_BYTE);
203 
204 /*
205  * L1 Mapping properties
206  */
207 static int v_l1_size;
208 static int v_l1_shift;
209 static int v_l2_levels;
210 
211 /* The bottom level has pointers to PageDesc, and is indexed by
212  * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
213  */
214 #define V_L1_MIN_BITS 4
215 #define V_L1_MAX_BITS (V_L2_BITS + 3)
216 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
217 
218 static void *l1_map[V_L1_MAX_SIZE];
219 
220 /* code generation context */
221 TCGContext tcg_init_ctx;
222 __thread TCGContext *tcg_ctx;
223 TBContext tb_ctx;
224 bool parallel_cpus;
225 
226 static void page_table_config_init(void)
227 {
228     uint32_t v_l1_bits;
229 
230     assert(TARGET_PAGE_BITS);
231     /* The bits remaining after N lower levels of page tables.  */
232     v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS;
233     if (v_l1_bits < V_L1_MIN_BITS) {
234         v_l1_bits += V_L2_BITS;
235     }
236 
237     v_l1_size = 1 << v_l1_bits;
238     v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits;
239     v_l2_levels = v_l1_shift / V_L2_BITS - 1;
240 
241     assert(v_l1_bits <= V_L1_MAX_BITS);
242     assert(v_l1_shift % V_L2_BITS == 0);
243     assert(v_l2_levels >= 0);
244 }
245 
246 void cpu_gen_init(void)
247 {
248     tcg_context_init(&tcg_init_ctx);
249 }
250 
251 /* Encode VAL as a signed leb128 sequence at P.
252    Return P incremented past the encoded value.  */
253 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
254 {
255     int more, byte;
256 
257     do {
258         byte = val & 0x7f;
259         val >>= 7;
260         more = !((val == 0 && (byte & 0x40) == 0)
261                  || (val == -1 && (byte & 0x40) != 0));
262         if (more) {
263             byte |= 0x80;
264         }
265         *p++ = byte;
266     } while (more);
267 
268     return p;
269 }
270 
271 /* Decode a signed leb128 sequence at *PP; increment *PP past the
272    decoded value.  Return the decoded value.  */
273 static target_long decode_sleb128(const uint8_t **pp)
274 {
275     const uint8_t *p = *pp;
276     target_long val = 0;
277     int byte, shift = 0;
278 
279     do {
280         byte = *p++;
281         val |= (target_ulong)(byte & 0x7f) << shift;
282         shift += 7;
283     } while (byte & 0x80);
284     if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
285         val |= -(target_ulong)1 << shift;
286     }
287 
288     *pp = p;
289     return val;
290 }
291 
292 /* Encode the data collected about the instructions while compiling TB.
293    Place the data at BLOCK, and return the number of bytes consumed.
294 
295    The logical table consists of TARGET_INSN_START_WORDS target_ulong's,
296    which come from the target's insn_start data, followed by a uintptr_t
297    which comes from the host pc of the end of the code implementing the insn.
298 
299    Each line of the table is encoded as sleb128 deltas from the previous
300    line.  The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
301    That is, the first column is seeded with the guest pc, the last column
302    with the host pc, and the middle columns with zeros.  */
303 
304 static int encode_search(TranslationBlock *tb, uint8_t *block)
305 {
306     uint8_t *highwater = tcg_ctx->code_gen_highwater;
307     uint8_t *p = block;
308     int i, j, n;
309 
310     for (i = 0, n = tb->icount; i < n; ++i) {
311         target_ulong prev;
312 
313         for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
314             if (i == 0) {
315                 prev = (j == 0 ? tb->pc : 0);
316             } else {
317                 prev = tcg_ctx->gen_insn_data[i - 1][j];
318             }
319             p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev);
320         }
321         prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]);
322         p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev);
323 
324         /* Test for (pending) buffer overflow.  The assumption is that any
325            one row beginning below the high water mark cannot overrun
326            the buffer completely.  Thus we can test for overflow after
327            encoding a row without having to check during encoding.  */
328         if (unlikely(p > highwater)) {
329             return -1;
330         }
331     }
332 
333     return p - block;
334 }
335 
336 /* The cpu state corresponding to 'searched_pc' is restored.
337  * When reset_icount is true, current TB will be interrupted and
338  * icount should be recalculated.
339  */
340 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
341                                      uintptr_t searched_pc, bool reset_icount)
342 {
343     target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
344     uintptr_t host_pc = (uintptr_t)tb->tc.ptr;
345     CPUArchState *env = cpu->env_ptr;
346     const uint8_t *p = tb->tc.ptr + tb->tc.size;
347     int i, j, num_insns = tb->icount;
348 #ifdef CONFIG_PROFILER
349     TCGProfile *prof = &tcg_ctx->prof;
350     int64_t ti = profile_getclock();
351 #endif
352 
353     searched_pc -= GETPC_ADJ;
354 
355     if (searched_pc < host_pc) {
356         return -1;
357     }
358 
359     /* Reconstruct the stored insn data while looking for the point at
360        which the end of the insn exceeds the searched_pc.  */
361     for (i = 0; i < num_insns; ++i) {
362         for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
363             data[j] += decode_sleb128(&p);
364         }
365         host_pc += decode_sleb128(&p);
366         if (host_pc > searched_pc) {
367             goto found;
368         }
369     }
370     return -1;
371 
372  found:
373     if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) {
374         assert(icount_enabled());
375         /* Reset the cycle counter to the start of the block
376            and shift if to the number of actually executed instructions */
377         cpu_neg(cpu)->icount_decr.u16.low += num_insns - i;
378     }
379     restore_state_to_opc(env, tb, data);
380 
381 #ifdef CONFIG_PROFILER
382     qatomic_set(&prof->restore_time,
383                 prof->restore_time + profile_getclock() - ti);
384     qatomic_set(&prof->restore_count, prof->restore_count + 1);
385 #endif
386     return 0;
387 }
388 
389 void tb_destroy(TranslationBlock *tb)
390 {
391     qemu_spin_destroy(&tb->jmp_lock);
392 }
393 
394 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit)
395 {
396     /*
397      * The host_pc has to be in the rx region of the code buffer.
398      * If it is not we will not be able to resolve it here.
399      * The two cases where host_pc will not be correct are:
400      *
401      *  - fault during translation (instruction fetch)
402      *  - fault from helper (not using GETPC() macro)
403      *
404      * Either way we need return early as we can't resolve it here.
405      */
406     if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) {
407         TranslationBlock *tb = tcg_tb_lookup(host_pc);
408         if (tb) {
409             cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit);
410             if (tb_cflags(tb) & CF_NOCACHE) {
411                 /* one-shot translation, invalidate it immediately */
412                 tb_phys_invalidate(tb, -1);
413                 tcg_tb_remove(tb);
414                 tb_destroy(tb);
415             }
416             return true;
417         }
418     }
419     return false;
420 }
421 
422 static void page_init(void)
423 {
424     page_size_init();
425     page_table_config_init();
426 
427 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
428     {
429 #ifdef HAVE_KINFO_GETVMMAP
430         struct kinfo_vmentry *freep;
431         int i, cnt;
432 
433         freep = kinfo_getvmmap(getpid(), &cnt);
434         if (freep) {
435             mmap_lock();
436             for (i = 0; i < cnt; i++) {
437                 unsigned long startaddr, endaddr;
438 
439                 startaddr = freep[i].kve_start;
440                 endaddr = freep[i].kve_end;
441                 if (h2g_valid(startaddr)) {
442                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
443 
444                     if (h2g_valid(endaddr)) {
445                         endaddr = h2g(endaddr);
446                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
447                     } else {
448 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
449                         endaddr = ~0ul;
450                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
451 #endif
452                     }
453                 }
454             }
455             free(freep);
456             mmap_unlock();
457         }
458 #else
459         FILE *f;
460 
461         last_brk = (unsigned long)sbrk(0);
462 
463         f = fopen("/compat/linux/proc/self/maps", "r");
464         if (f) {
465             mmap_lock();
466 
467             do {
468                 unsigned long startaddr, endaddr;
469                 int n;
470 
471                 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
472 
473                 if (n == 2 && h2g_valid(startaddr)) {
474                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
475 
476                     if (h2g_valid(endaddr)) {
477                         endaddr = h2g(endaddr);
478                     } else {
479                         endaddr = ~0ul;
480                     }
481                     page_set_flags(startaddr, endaddr, PAGE_RESERVED);
482                 }
483             } while (!feof(f));
484 
485             fclose(f);
486             mmap_unlock();
487         }
488 #endif
489     }
490 #endif
491 }
492 
493 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
494 {
495     PageDesc *pd;
496     void **lp;
497     int i;
498 
499     /* Level 1.  Always allocated.  */
500     lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1));
501 
502     /* Level 2..N-1.  */
503     for (i = v_l2_levels; i > 0; i--) {
504         void **p = qatomic_rcu_read(lp);
505 
506         if (p == NULL) {
507             void *existing;
508 
509             if (!alloc) {
510                 return NULL;
511             }
512             p = g_new0(void *, V_L2_SIZE);
513             existing = qatomic_cmpxchg(lp, NULL, p);
514             if (unlikely(existing)) {
515                 g_free(p);
516                 p = existing;
517             }
518         }
519 
520         lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
521     }
522 
523     pd = qatomic_rcu_read(lp);
524     if (pd == NULL) {
525         void *existing;
526 
527         if (!alloc) {
528             return NULL;
529         }
530         pd = g_new0(PageDesc, V_L2_SIZE);
531 #ifndef CONFIG_USER_ONLY
532         {
533             int i;
534 
535             for (i = 0; i < V_L2_SIZE; i++) {
536                 qemu_spin_init(&pd[i].lock);
537             }
538         }
539 #endif
540         existing = qatomic_cmpxchg(lp, NULL, pd);
541         if (unlikely(existing)) {
542 #ifndef CONFIG_USER_ONLY
543             {
544                 int i;
545 
546                 for (i = 0; i < V_L2_SIZE; i++) {
547                     qemu_spin_destroy(&pd[i].lock);
548                 }
549             }
550 #endif
551             g_free(pd);
552             pd = existing;
553         }
554     }
555 
556     return pd + (index & (V_L2_SIZE - 1));
557 }
558 
559 static inline PageDesc *page_find(tb_page_addr_t index)
560 {
561     return page_find_alloc(index, 0);
562 }
563 
564 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
565                            PageDesc **ret_p2, tb_page_addr_t phys2, int alloc);
566 
567 /* In user-mode page locks aren't used; mmap_lock is enough */
568 #ifdef CONFIG_USER_ONLY
569 
570 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
571 
572 static inline void page_lock(PageDesc *pd)
573 { }
574 
575 static inline void page_unlock(PageDesc *pd)
576 { }
577 
578 static inline void page_lock_tb(const TranslationBlock *tb)
579 { }
580 
581 static inline void page_unlock_tb(const TranslationBlock *tb)
582 { }
583 
584 struct page_collection *
585 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
586 {
587     return NULL;
588 }
589 
590 void page_collection_unlock(struct page_collection *set)
591 { }
592 #else /* !CONFIG_USER_ONLY */
593 
594 #ifdef CONFIG_DEBUG_TCG
595 
596 static __thread GHashTable *ht_pages_locked_debug;
597 
598 static void ht_pages_locked_debug_init(void)
599 {
600     if (ht_pages_locked_debug) {
601         return;
602     }
603     ht_pages_locked_debug = g_hash_table_new(NULL, NULL);
604 }
605 
606 static bool page_is_locked(const PageDesc *pd)
607 {
608     PageDesc *found;
609 
610     ht_pages_locked_debug_init();
611     found = g_hash_table_lookup(ht_pages_locked_debug, pd);
612     return !!found;
613 }
614 
615 static void page_lock__debug(PageDesc *pd)
616 {
617     ht_pages_locked_debug_init();
618     g_assert(!page_is_locked(pd));
619     g_hash_table_insert(ht_pages_locked_debug, pd, pd);
620 }
621 
622 static void page_unlock__debug(const PageDesc *pd)
623 {
624     bool removed;
625 
626     ht_pages_locked_debug_init();
627     g_assert(page_is_locked(pd));
628     removed = g_hash_table_remove(ht_pages_locked_debug, pd);
629     g_assert(removed);
630 }
631 
632 static void
633 do_assert_page_locked(const PageDesc *pd, const char *file, int line)
634 {
635     if (unlikely(!page_is_locked(pd))) {
636         error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
637                      pd, file, line);
638         abort();
639     }
640 }
641 
642 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
643 
644 void assert_no_pages_locked(void)
645 {
646     ht_pages_locked_debug_init();
647     g_assert(g_hash_table_size(ht_pages_locked_debug) == 0);
648 }
649 
650 #else /* !CONFIG_DEBUG_TCG */
651 
652 #define assert_page_locked(pd)
653 
654 static inline void page_lock__debug(const PageDesc *pd)
655 {
656 }
657 
658 static inline void page_unlock__debug(const PageDesc *pd)
659 {
660 }
661 
662 #endif /* CONFIG_DEBUG_TCG */
663 
664 static inline void page_lock(PageDesc *pd)
665 {
666     page_lock__debug(pd);
667     qemu_spin_lock(&pd->lock);
668 }
669 
670 static inline void page_unlock(PageDesc *pd)
671 {
672     qemu_spin_unlock(&pd->lock);
673     page_unlock__debug(pd);
674 }
675 
676 /* lock the page(s) of a TB in the correct acquisition order */
677 static inline void page_lock_tb(const TranslationBlock *tb)
678 {
679     page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], 0);
680 }
681 
682 static inline void page_unlock_tb(const TranslationBlock *tb)
683 {
684     PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
685 
686     page_unlock(p1);
687     if (unlikely(tb->page_addr[1] != -1)) {
688         PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
689 
690         if (p2 != p1) {
691             page_unlock(p2);
692         }
693     }
694 }
695 
696 static inline struct page_entry *
697 page_entry_new(PageDesc *pd, tb_page_addr_t index)
698 {
699     struct page_entry *pe = g_malloc(sizeof(*pe));
700 
701     pe->index = index;
702     pe->pd = pd;
703     pe->locked = false;
704     return pe;
705 }
706 
707 static void page_entry_destroy(gpointer p)
708 {
709     struct page_entry *pe = p;
710 
711     g_assert(pe->locked);
712     page_unlock(pe->pd);
713     g_free(pe);
714 }
715 
716 /* returns false on success */
717 static bool page_entry_trylock(struct page_entry *pe)
718 {
719     bool busy;
720 
721     busy = qemu_spin_trylock(&pe->pd->lock);
722     if (!busy) {
723         g_assert(!pe->locked);
724         pe->locked = true;
725         page_lock__debug(pe->pd);
726     }
727     return busy;
728 }
729 
730 static void do_page_entry_lock(struct page_entry *pe)
731 {
732     page_lock(pe->pd);
733     g_assert(!pe->locked);
734     pe->locked = true;
735 }
736 
737 static gboolean page_entry_lock(gpointer key, gpointer value, gpointer data)
738 {
739     struct page_entry *pe = value;
740 
741     do_page_entry_lock(pe);
742     return FALSE;
743 }
744 
745 static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data)
746 {
747     struct page_entry *pe = value;
748 
749     if (pe->locked) {
750         pe->locked = false;
751         page_unlock(pe->pd);
752     }
753     return FALSE;
754 }
755 
756 /*
757  * Trylock a page, and if successful, add the page to a collection.
758  * Returns true ("busy") if the page could not be locked; false otherwise.
759  */
760 static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
761 {
762     tb_page_addr_t index = addr >> TARGET_PAGE_BITS;
763     struct page_entry *pe;
764     PageDesc *pd;
765 
766     pe = g_tree_lookup(set->tree, &index);
767     if (pe) {
768         return false;
769     }
770 
771     pd = page_find(index);
772     if (pd == NULL) {
773         return false;
774     }
775 
776     pe = page_entry_new(pd, index);
777     g_tree_insert(set->tree, &pe->index, pe);
778 
779     /*
780      * If this is either (1) the first insertion or (2) a page whose index
781      * is higher than any other so far, just lock the page and move on.
782      */
783     if (set->max == NULL || pe->index > set->max->index) {
784         set->max = pe;
785         do_page_entry_lock(pe);
786         return false;
787     }
788     /*
789      * Try to acquire out-of-order lock; if busy, return busy so that we acquire
790      * locks in order.
791      */
792     return page_entry_trylock(pe);
793 }
794 
795 static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata)
796 {
797     tb_page_addr_t a = *(const tb_page_addr_t *)ap;
798     tb_page_addr_t b = *(const tb_page_addr_t *)bp;
799 
800     if (a == b) {
801         return 0;
802     } else if (a < b) {
803         return -1;
804     }
805     return 1;
806 }
807 
808 /*
809  * Lock a range of pages ([@start,@end[) as well as the pages of all
810  * intersecting TBs.
811  * Locking order: acquire locks in ascending order of page index.
812  */
813 struct page_collection *
814 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
815 {
816     struct page_collection *set = g_malloc(sizeof(*set));
817     tb_page_addr_t index;
818     PageDesc *pd;
819 
820     start >>= TARGET_PAGE_BITS;
821     end   >>= TARGET_PAGE_BITS;
822     g_assert(start <= end);
823 
824     set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL,
825                                 page_entry_destroy);
826     set->max = NULL;
827     assert_no_pages_locked();
828 
829  retry:
830     g_tree_foreach(set->tree, page_entry_lock, NULL);
831 
832     for (index = start; index <= end; index++) {
833         TranslationBlock *tb;
834         int n;
835 
836         pd = page_find(index);
837         if (pd == NULL) {
838             continue;
839         }
840         if (page_trylock_add(set, index << TARGET_PAGE_BITS)) {
841             g_tree_foreach(set->tree, page_entry_unlock, NULL);
842             goto retry;
843         }
844         assert_page_locked(pd);
845         PAGE_FOR_EACH_TB(pd, tb, n) {
846             if (page_trylock_add(set, tb->page_addr[0]) ||
847                 (tb->page_addr[1] != -1 &&
848                  page_trylock_add(set, tb->page_addr[1]))) {
849                 /* drop all locks, and reacquire in order */
850                 g_tree_foreach(set->tree, page_entry_unlock, NULL);
851                 goto retry;
852             }
853         }
854     }
855     return set;
856 }
857 
858 void page_collection_unlock(struct page_collection *set)
859 {
860     /* entries are unlocked and freed via page_entry_destroy */
861     g_tree_destroy(set->tree);
862     g_free(set);
863 }
864 
865 #endif /* !CONFIG_USER_ONLY */
866 
867 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
868                            PageDesc **ret_p2, tb_page_addr_t phys2, int alloc)
869 {
870     PageDesc *p1, *p2;
871     tb_page_addr_t page1;
872     tb_page_addr_t page2;
873 
874     assert_memory_lock();
875     g_assert(phys1 != -1);
876 
877     page1 = phys1 >> TARGET_PAGE_BITS;
878     page2 = phys2 >> TARGET_PAGE_BITS;
879 
880     p1 = page_find_alloc(page1, alloc);
881     if (ret_p1) {
882         *ret_p1 = p1;
883     }
884     if (likely(phys2 == -1)) {
885         page_lock(p1);
886         return;
887     } else if (page1 == page2) {
888         page_lock(p1);
889         if (ret_p2) {
890             *ret_p2 = p1;
891         }
892         return;
893     }
894     p2 = page_find_alloc(page2, alloc);
895     if (ret_p2) {
896         *ret_p2 = p2;
897     }
898     if (page1 < page2) {
899         page_lock(p1);
900         page_lock(p2);
901     } else {
902         page_lock(p2);
903         page_lock(p1);
904     }
905 }
906 
907 /* Minimum size of the code gen buffer.  This number is randomly chosen,
908    but not so small that we can't have a fair number of TB's live.  */
909 #define MIN_CODE_GEN_BUFFER_SIZE     (1 * MiB)
910 
911 /* Maximum size of the code gen buffer we'd like to use.  Unless otherwise
912    indicated, this is constrained by the range of direct branches on the
913    host cpu, as used by the TCG implementation of goto_tb.  */
914 #if defined(__x86_64__)
915 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
916 #elif defined(__sparc__)
917 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
918 #elif defined(__powerpc64__)
919 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
920 #elif defined(__powerpc__)
921 # define MAX_CODE_GEN_BUFFER_SIZE  (32 * MiB)
922 #elif defined(__aarch64__)
923 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
924 #elif defined(__s390x__)
925   /* We have a +- 4GB range on the branches; leave some slop.  */
926 # define MAX_CODE_GEN_BUFFER_SIZE  (3 * GiB)
927 #elif defined(__mips__)
928   /* We have a 256MB branch region, but leave room to make sure the
929      main executable is also within that region.  */
930 # define MAX_CODE_GEN_BUFFER_SIZE  (128 * MiB)
931 #else
932 # define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
933 #endif
934 
935 #if TCG_TARGET_REG_BITS == 32
936 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32 * MiB)
937 #ifdef CONFIG_USER_ONLY
938 /*
939  * For user mode on smaller 32 bit systems we may run into trouble
940  * allocating big chunks of data in the right place. On these systems
941  * we utilise a static code generation buffer directly in the binary.
942  */
943 #define USE_STATIC_CODE_GEN_BUFFER
944 #endif
945 #else /* TCG_TARGET_REG_BITS == 64 */
946 #ifdef CONFIG_USER_ONLY
947 /*
948  * As user-mode emulation typically means running multiple instances
949  * of the translator don't go too nuts with our default code gen
950  * buffer lest we make things too hard for the OS.
951  */
952 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (128 * MiB)
953 #else
954 /*
955  * We expect most system emulation to run one or two guests per host.
956  * Users running large scale system emulation may want to tweak their
957  * runtime setup via the tb-size control on the command line.
958  */
959 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (1 * GiB)
960 #endif
961 #endif
962 
963 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
964   (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
965    ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
966 
967 static size_t size_code_gen_buffer(size_t tb_size)
968 {
969     /* Size the buffer.  */
970     if (tb_size == 0) {
971         size_t phys_mem = qemu_get_host_physmem();
972         if (phys_mem == 0) {
973             tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
974         } else {
975             tb_size = MIN(DEFAULT_CODE_GEN_BUFFER_SIZE, phys_mem / 8);
976         }
977     }
978     if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
979         tb_size = MIN_CODE_GEN_BUFFER_SIZE;
980     }
981     if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
982         tb_size = MAX_CODE_GEN_BUFFER_SIZE;
983     }
984     return tb_size;
985 }
986 
987 #ifdef __mips__
988 /* In order to use J and JAL within the code_gen_buffer, we require
989    that the buffer not cross a 256MB boundary.  */
990 static inline bool cross_256mb(void *addr, size_t size)
991 {
992     return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
993 }
994 
995 /* We weren't able to allocate a buffer without crossing that boundary,
996    so make do with the larger portion of the buffer that doesn't cross.
997    Returns the new base of the buffer, and adjusts code_gen_buffer_size.  */
998 static inline void *split_cross_256mb(void *buf1, size_t size1)
999 {
1000     void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
1001     size_t size2 = buf1 + size1 - buf2;
1002 
1003     size1 = buf2 - buf1;
1004     if (size1 < size2) {
1005         size1 = size2;
1006         buf1 = buf2;
1007     }
1008 
1009     tcg_ctx->code_gen_buffer_size = size1;
1010     return buf1;
1011 }
1012 #endif
1013 
1014 #ifdef USE_STATIC_CODE_GEN_BUFFER
1015 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
1016     __attribute__((aligned(CODE_GEN_ALIGN)));
1017 
1018 static bool alloc_code_gen_buffer(size_t tb_size, int splitwx, Error **errp)
1019 {
1020     void *buf, *end;
1021     size_t size;
1022 
1023     if (splitwx > 0) {
1024         error_setg(errp, "jit split-wx not supported");
1025         return false;
1026     }
1027 
1028     /* page-align the beginning and end of the buffer */
1029     buf = static_code_gen_buffer;
1030     end = static_code_gen_buffer + sizeof(static_code_gen_buffer);
1031     buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size);
1032     end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size);
1033 
1034     size = end - buf;
1035 
1036     /* Honor a command-line option limiting the size of the buffer.  */
1037     if (size > tb_size) {
1038         size = QEMU_ALIGN_DOWN(tb_size, qemu_real_host_page_size);
1039     }
1040     tcg_ctx->code_gen_buffer_size = size;
1041 
1042 #ifdef __mips__
1043     if (cross_256mb(buf, size)) {
1044         buf = split_cross_256mb(buf, size);
1045         size = tcg_ctx->code_gen_buffer_size;
1046     }
1047 #endif
1048 
1049     if (qemu_mprotect_rwx(buf, size)) {
1050         error_setg_errno(errp, errno, "mprotect of jit buffer");
1051         return false;
1052     }
1053     qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1054 
1055     tcg_ctx->code_gen_buffer = buf;
1056     return true;
1057 }
1058 #elif defined(_WIN32)
1059 static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp)
1060 {
1061     void *buf;
1062 
1063     if (splitwx > 0) {
1064         error_setg(errp, "jit split-wx not supported");
1065         return false;
1066     }
1067 
1068     buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT,
1069                              PAGE_EXECUTE_READWRITE);
1070     if (buf == NULL) {
1071         error_setg_win32(errp, GetLastError(),
1072                          "allocate %zu bytes for jit buffer", size);
1073         return false;
1074     }
1075 
1076     tcg_ctx->code_gen_buffer = buf;
1077     tcg_ctx->code_gen_buffer_size = size;
1078     return true;
1079 }
1080 #else
1081 static bool alloc_code_gen_buffer_anon(size_t size, int prot,
1082                                        int flags, Error **errp)
1083 {
1084     void *buf;
1085 
1086     buf = mmap(NULL, size, prot, flags, -1, 0);
1087     if (buf == MAP_FAILED) {
1088         error_setg_errno(errp, errno,
1089                          "allocate %zu bytes for jit buffer", size);
1090         return false;
1091     }
1092     tcg_ctx->code_gen_buffer_size = size;
1093 
1094 #ifdef __mips__
1095     if (cross_256mb(buf, size)) {
1096         /*
1097          * Try again, with the original still mapped, to avoid re-acquiring
1098          * the same 256mb crossing.
1099          */
1100         size_t size2;
1101         void *buf2 = mmap(NULL, size, prot, flags, -1, 0);
1102         switch ((int)(buf2 != MAP_FAILED)) {
1103         case 1:
1104             if (!cross_256mb(buf2, size)) {
1105                 /* Success!  Use the new buffer.  */
1106                 munmap(buf, size);
1107                 break;
1108             }
1109             /* Failure.  Work with what we had.  */
1110             munmap(buf2, size);
1111             /* fallthru */
1112         default:
1113             /* Split the original buffer.  Free the smaller half.  */
1114             buf2 = split_cross_256mb(buf, size);
1115             size2 = tcg_ctx->code_gen_buffer_size;
1116             if (buf == buf2) {
1117                 munmap(buf + size2, size - size2);
1118             } else {
1119                 munmap(buf, size - size2);
1120             }
1121             size = size2;
1122             break;
1123         }
1124         buf = buf2;
1125     }
1126 #endif
1127 
1128     /* Request large pages for the buffer.  */
1129     qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1130 
1131     tcg_ctx->code_gen_buffer = buf;
1132     return true;
1133 }
1134 
1135 #ifdef CONFIG_POSIX
1136 #include "qemu/memfd.h"
1137 
1138 static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp)
1139 {
1140     void *buf_rw, *buf_rx;
1141     int fd = -1;
1142 
1143     buf_rw = qemu_memfd_alloc("tcg-jit", size, 0, &fd, errp);
1144     if (buf_rw == NULL) {
1145         return false;
1146     }
1147 
1148     buf_rx = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0);
1149     if (buf_rx == MAP_FAILED) {
1150         error_setg_errno(errp, errno,
1151                          "failed to map shared memory for execute");
1152         munmap(buf_rw, size);
1153         close(fd);
1154         return false;
1155     }
1156     close(fd);
1157 
1158     tcg_ctx->code_gen_buffer = buf_rw;
1159     tcg_ctx->code_gen_buffer_size = size;
1160     tcg_splitwx_diff = buf_rx - buf_rw;
1161 
1162     /* Request large pages for the buffer and the splitwx.  */
1163     qemu_madvise(buf_rw, size, QEMU_MADV_HUGEPAGE);
1164     qemu_madvise(buf_rx, size, QEMU_MADV_HUGEPAGE);
1165     return true;
1166 }
1167 #endif /* CONFIG_POSIX */
1168 
1169 #ifdef CONFIG_DARWIN
1170 #include <mach/mach.h>
1171 
1172 extern kern_return_t mach_vm_remap(vm_map_t target_task,
1173                                    mach_vm_address_t *target_address,
1174                                    mach_vm_size_t size,
1175                                    mach_vm_offset_t mask,
1176                                    int flags,
1177                                    vm_map_t src_task,
1178                                    mach_vm_address_t src_address,
1179                                    boolean_t copy,
1180                                    vm_prot_t *cur_protection,
1181                                    vm_prot_t *max_protection,
1182                                    vm_inherit_t inheritance);
1183 
1184 static bool alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp)
1185 {
1186     kern_return_t ret;
1187     mach_vm_address_t buf_rw, buf_rx;
1188     vm_prot_t cur_prot, max_prot;
1189 
1190     /* Map the read-write portion via normal anon memory. */
1191     if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE,
1192                                     MAP_PRIVATE | MAP_ANONYMOUS, errp)) {
1193         return false;
1194     }
1195 
1196     buf_rw = (mach_vm_address_t)tcg_ctx->code_gen_buffer;
1197     buf_rx = 0;
1198     ret = mach_vm_remap(mach_task_self(),
1199                         &buf_rx,
1200                         size,
1201                         0,
1202                         VM_FLAGS_ANYWHERE,
1203                         mach_task_self(),
1204                         buf_rw,
1205                         false,
1206                         &cur_prot,
1207                         &max_prot,
1208                         VM_INHERIT_NONE);
1209     if (ret != KERN_SUCCESS) {
1210         /* TODO: Convert "ret" to a human readable error message. */
1211         error_setg(errp, "vm_remap for jit splitwx failed");
1212         munmap((void *)buf_rw, size);
1213         return false;
1214     }
1215 
1216     if (mprotect((void *)buf_rx, size, PROT_READ | PROT_EXEC) != 0) {
1217         error_setg_errno(errp, errno, "mprotect for jit splitwx");
1218         munmap((void *)buf_rx, size);
1219         munmap((void *)buf_rw, size);
1220         return false;
1221     }
1222 
1223     tcg_splitwx_diff = buf_rx - buf_rw;
1224     return true;
1225 }
1226 #endif /* CONFIG_DARWIN */
1227 
1228 static bool alloc_code_gen_buffer_splitwx(size_t size, Error **errp)
1229 {
1230     if (TCG_TARGET_SUPPORT_MIRROR) {
1231 #ifdef CONFIG_DARWIN
1232         return alloc_code_gen_buffer_splitwx_vmremap(size, errp);
1233 #endif
1234 #ifdef CONFIG_POSIX
1235         return alloc_code_gen_buffer_splitwx_memfd(size, errp);
1236 #endif
1237     }
1238     error_setg(errp, "jit split-wx not supported");
1239     return false;
1240 }
1241 
1242 static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp)
1243 {
1244     ERRP_GUARD();
1245     int prot, flags;
1246 
1247     if (splitwx) {
1248         if (alloc_code_gen_buffer_splitwx(size, errp)) {
1249             return true;
1250         }
1251         /*
1252          * If splitwx force-on (1), fail;
1253          * if splitwx default-on (-1), fall through to splitwx off.
1254          */
1255         if (splitwx > 0) {
1256             return false;
1257         }
1258         error_free_or_abort(errp);
1259     }
1260 
1261     prot = PROT_READ | PROT_WRITE | PROT_EXEC;
1262     flags = MAP_PRIVATE | MAP_ANONYMOUS;
1263 #ifdef CONFIG_TCG_INTERPRETER
1264     /* The tcg interpreter does not need execute permission. */
1265     prot = PROT_READ | PROT_WRITE;
1266 #elif defined(CONFIG_DARWIN)
1267     /* Applicable to both iOS and macOS (Apple Silicon). */
1268     if (!splitwx) {
1269         flags |= MAP_JIT;
1270     }
1271 #endif
1272 
1273     return alloc_code_gen_buffer_anon(size, prot, flags, errp);
1274 }
1275 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
1276 
1277 static bool tb_cmp(const void *ap, const void *bp)
1278 {
1279     const TranslationBlock *a = ap;
1280     const TranslationBlock *b = bp;
1281 
1282     return a->pc == b->pc &&
1283         a->cs_base == b->cs_base &&
1284         a->flags == b->flags &&
1285         (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) &&
1286         a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
1287         a->page_addr[0] == b->page_addr[0] &&
1288         a->page_addr[1] == b->page_addr[1];
1289 }
1290 
1291 static void tb_htable_init(void)
1292 {
1293     unsigned int mode = QHT_MODE_AUTO_RESIZE;
1294 
1295     qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
1296 }
1297 
1298 /* Must be called before using the QEMU cpus. 'tb_size' is the size
1299    (in bytes) allocated to the translation buffer. Zero means default
1300    size. */
1301 void tcg_exec_init(unsigned long tb_size, int splitwx)
1302 {
1303     bool ok;
1304 
1305     tcg_allowed = true;
1306     cpu_gen_init();
1307     page_init();
1308     tb_htable_init();
1309 
1310     ok = alloc_code_gen_buffer(size_code_gen_buffer(tb_size),
1311                                splitwx, &error_fatal);
1312     assert(ok);
1313 
1314 #if defined(CONFIG_SOFTMMU)
1315     /* There's no guest base to take into account, so go ahead and
1316        initialize the prologue now.  */
1317     tcg_prologue_init(tcg_ctx);
1318 #endif
1319 }
1320 
1321 /* call with @p->lock held */
1322 static inline void invalidate_page_bitmap(PageDesc *p)
1323 {
1324     assert_page_locked(p);
1325 #ifdef CONFIG_SOFTMMU
1326     g_free(p->code_bitmap);
1327     p->code_bitmap = NULL;
1328     p->code_write_count = 0;
1329 #endif
1330 }
1331 
1332 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
1333 static void page_flush_tb_1(int level, void **lp)
1334 {
1335     int i;
1336 
1337     if (*lp == NULL) {
1338         return;
1339     }
1340     if (level == 0) {
1341         PageDesc *pd = *lp;
1342 
1343         for (i = 0; i < V_L2_SIZE; ++i) {
1344             page_lock(&pd[i]);
1345             pd[i].first_tb = (uintptr_t)NULL;
1346             invalidate_page_bitmap(pd + i);
1347             page_unlock(&pd[i]);
1348         }
1349     } else {
1350         void **pp = *lp;
1351 
1352         for (i = 0; i < V_L2_SIZE; ++i) {
1353             page_flush_tb_1(level - 1, pp + i);
1354         }
1355     }
1356 }
1357 
1358 static void page_flush_tb(void)
1359 {
1360     int i, l1_sz = v_l1_size;
1361 
1362     for (i = 0; i < l1_sz; i++) {
1363         page_flush_tb_1(v_l2_levels, l1_map + i);
1364     }
1365 }
1366 
1367 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data)
1368 {
1369     const TranslationBlock *tb = value;
1370     size_t *size = data;
1371 
1372     *size += tb->tc.size;
1373     return false;
1374 }
1375 
1376 /* flush all the translation blocks */
1377 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
1378 {
1379     bool did_flush = false;
1380 
1381     mmap_lock();
1382     /* If it is already been done on request of another CPU,
1383      * just retry.
1384      */
1385     if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
1386         goto done;
1387     }
1388     did_flush = true;
1389 
1390     if (DEBUG_TB_FLUSH_GATE) {
1391         size_t nb_tbs = tcg_nb_tbs();
1392         size_t host_size = 0;
1393 
1394         tcg_tb_foreach(tb_host_size_iter, &host_size);
1395         printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
1396                tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0);
1397     }
1398 
1399     CPU_FOREACH(cpu) {
1400         cpu_tb_jmp_cache_clear(cpu);
1401     }
1402 
1403     qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
1404     page_flush_tb();
1405 
1406     tcg_region_reset_all();
1407     /* XXX: flush processor icache at this point if cache flush is
1408        expensive */
1409     qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
1410 
1411 done:
1412     mmap_unlock();
1413     if (did_flush) {
1414         qemu_plugin_flush_cb();
1415     }
1416 }
1417 
1418 void tb_flush(CPUState *cpu)
1419 {
1420     if (tcg_enabled()) {
1421         unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count);
1422 
1423         if (cpu_in_exclusive_context(cpu)) {
1424             do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
1425         } else {
1426             async_safe_run_on_cpu(cpu, do_tb_flush,
1427                                   RUN_ON_CPU_HOST_INT(tb_flush_count));
1428         }
1429     }
1430 }
1431 
1432 /*
1433  * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
1434  * so in order to prevent bit rot we compile them unconditionally in user-mode,
1435  * and let the optimizer get rid of them by wrapping their user-only callers
1436  * with if (DEBUG_TB_CHECK_GATE).
1437  */
1438 #ifdef CONFIG_USER_ONLY
1439 
1440 static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp)
1441 {
1442     TranslationBlock *tb = p;
1443     target_ulong addr = *(target_ulong *)userp;
1444 
1445     if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
1446         printf("ERROR invalidate: address=" TARGET_FMT_lx
1447                " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
1448     }
1449 }
1450 
1451 /* verify that all the pages have correct rights for code
1452  *
1453  * Called with mmap_lock held.
1454  */
1455 static void tb_invalidate_check(target_ulong address)
1456 {
1457     address &= TARGET_PAGE_MASK;
1458     qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address);
1459 }
1460 
1461 static void do_tb_page_check(void *p, uint32_t hash, void *userp)
1462 {
1463     TranslationBlock *tb = p;
1464     int flags1, flags2;
1465 
1466     flags1 = page_get_flags(tb->pc);
1467     flags2 = page_get_flags(tb->pc + tb->size - 1);
1468     if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
1469         printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1470                (long)tb->pc, tb->size, flags1, flags2);
1471     }
1472 }
1473 
1474 /* verify that all the pages have correct rights for code */
1475 static void tb_page_check(void)
1476 {
1477     qht_iter(&tb_ctx.htable, do_tb_page_check, NULL);
1478 }
1479 
1480 #endif /* CONFIG_USER_ONLY */
1481 
1482 /*
1483  * user-mode: call with mmap_lock held
1484  * !user-mode: call with @pd->lock held
1485  */
1486 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
1487 {
1488     TranslationBlock *tb1;
1489     uintptr_t *pprev;
1490     unsigned int n1;
1491 
1492     assert_page_locked(pd);
1493     pprev = &pd->first_tb;
1494     PAGE_FOR_EACH_TB(pd, tb1, n1) {
1495         if (tb1 == tb) {
1496             *pprev = tb1->page_next[n1];
1497             return;
1498         }
1499         pprev = &tb1->page_next[n1];
1500     }
1501     g_assert_not_reached();
1502 }
1503 
1504 /* remove @orig from its @n_orig-th jump list */
1505 static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
1506 {
1507     uintptr_t ptr, ptr_locked;
1508     TranslationBlock *dest;
1509     TranslationBlock *tb;
1510     uintptr_t *pprev;
1511     int n;
1512 
1513     /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
1514     ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1);
1515     dest = (TranslationBlock *)(ptr & ~1);
1516     if (dest == NULL) {
1517         return;
1518     }
1519 
1520     qemu_spin_lock(&dest->jmp_lock);
1521     /*
1522      * While acquiring the lock, the jump might have been removed if the
1523      * destination TB was invalidated; check again.
1524      */
1525     ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]);
1526     if (ptr_locked != ptr) {
1527         qemu_spin_unlock(&dest->jmp_lock);
1528         /*
1529          * The only possibility is that the jump was unlinked via
1530          * tb_jump_unlink(dest). Seeing here another destination would be a bug,
1531          * because we set the LSB above.
1532          */
1533         g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
1534         return;
1535     }
1536     /*
1537      * We first acquired the lock, and since the destination pointer matches,
1538      * we know for sure that @orig is in the jmp list.
1539      */
1540     pprev = &dest->jmp_list_head;
1541     TB_FOR_EACH_JMP(dest, tb, n) {
1542         if (tb == orig && n == n_orig) {
1543             *pprev = tb->jmp_list_next[n];
1544             /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
1545             qemu_spin_unlock(&dest->jmp_lock);
1546             return;
1547         }
1548         pprev = &tb->jmp_list_next[n];
1549     }
1550     g_assert_not_reached();
1551 }
1552 
1553 /* reset the jump entry 'n' of a TB so that it is not chained to
1554    another TB */
1555 static inline void tb_reset_jump(TranslationBlock *tb, int n)
1556 {
1557     uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
1558     tb_set_jmp_target(tb, n, addr);
1559 }
1560 
1561 /* remove any jumps to the TB */
1562 static inline void tb_jmp_unlink(TranslationBlock *dest)
1563 {
1564     TranslationBlock *tb;
1565     int n;
1566 
1567     qemu_spin_lock(&dest->jmp_lock);
1568 
1569     TB_FOR_EACH_JMP(dest, tb, n) {
1570         tb_reset_jump(tb, n);
1571         qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
1572         /* No need to clear the list entry; setting the dest ptr is enough */
1573     }
1574     dest->jmp_list_head = (uintptr_t)NULL;
1575 
1576     qemu_spin_unlock(&dest->jmp_lock);
1577 }
1578 
1579 /*
1580  * In user-mode, call with mmap_lock held.
1581  * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
1582  * locks held.
1583  */
1584 static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
1585 {
1586     CPUState *cpu;
1587     PageDesc *p;
1588     uint32_t h;
1589     tb_page_addr_t phys_pc;
1590 
1591     assert_memory_lock();
1592 
1593     /* make sure no further incoming jumps will be chained to this TB */
1594     qemu_spin_lock(&tb->jmp_lock);
1595     qatomic_set(&tb->cflags, tb->cflags | CF_INVALID);
1596     qemu_spin_unlock(&tb->jmp_lock);
1597 
1598     /* remove the TB from the hash list */
1599     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1600     h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb_cflags(tb) & CF_HASH_MASK,
1601                      tb->trace_vcpu_dstate);
1602     if (!(tb->cflags & CF_NOCACHE) &&
1603         !qht_remove(&tb_ctx.htable, tb, h)) {
1604         return;
1605     }
1606 
1607     /* remove the TB from the page list */
1608     if (rm_from_page_list) {
1609         p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1610         tb_page_remove(p, tb);
1611         invalidate_page_bitmap(p);
1612         if (tb->page_addr[1] != -1) {
1613             p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1614             tb_page_remove(p, tb);
1615             invalidate_page_bitmap(p);
1616         }
1617     }
1618 
1619     /* remove the TB from the hash list */
1620     h = tb_jmp_cache_hash_func(tb->pc);
1621     CPU_FOREACH(cpu) {
1622         if (qatomic_read(&cpu->tb_jmp_cache[h]) == tb) {
1623             qatomic_set(&cpu->tb_jmp_cache[h], NULL);
1624         }
1625     }
1626 
1627     /* suppress this TB from the two jump lists */
1628     tb_remove_from_jmp_list(tb, 0);
1629     tb_remove_from_jmp_list(tb, 1);
1630 
1631     /* suppress any remaining jumps to this TB */
1632     tb_jmp_unlink(tb);
1633 
1634     qatomic_set(&tcg_ctx->tb_phys_invalidate_count,
1635                tcg_ctx->tb_phys_invalidate_count + 1);
1636 }
1637 
1638 static void tb_phys_invalidate__locked(TranslationBlock *tb)
1639 {
1640     do_tb_phys_invalidate(tb, true);
1641 }
1642 
1643 /* invalidate one TB
1644  *
1645  * Called with mmap_lock held in user-mode.
1646  */
1647 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1648 {
1649     if (page_addr == -1 && tb->page_addr[0] != -1) {
1650         page_lock_tb(tb);
1651         do_tb_phys_invalidate(tb, true);
1652         page_unlock_tb(tb);
1653     } else {
1654         do_tb_phys_invalidate(tb, false);
1655     }
1656 }
1657 
1658 #ifdef CONFIG_SOFTMMU
1659 /* call with @p->lock held */
1660 static void build_page_bitmap(PageDesc *p)
1661 {
1662     int n, tb_start, tb_end;
1663     TranslationBlock *tb;
1664 
1665     assert_page_locked(p);
1666     p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1667 
1668     PAGE_FOR_EACH_TB(p, tb, n) {
1669         /* NOTE: this is subtle as a TB may span two physical pages */
1670         if (n == 0) {
1671             /* NOTE: tb_end may be after the end of the page, but
1672                it is not a problem */
1673             tb_start = tb->pc & ~TARGET_PAGE_MASK;
1674             tb_end = tb_start + tb->size;
1675             if (tb_end > TARGET_PAGE_SIZE) {
1676                 tb_end = TARGET_PAGE_SIZE;
1677              }
1678         } else {
1679             tb_start = 0;
1680             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1681         }
1682         bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1683     }
1684 }
1685 #endif
1686 
1687 /* add the tb in the target page and protect it if necessary
1688  *
1689  * Called with mmap_lock held for user-mode emulation.
1690  * Called with @p->lock held in !user-mode.
1691  */
1692 static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
1693                                unsigned int n, tb_page_addr_t page_addr)
1694 {
1695 #ifndef CONFIG_USER_ONLY
1696     bool page_already_protected;
1697 #endif
1698 
1699     assert_page_locked(p);
1700 
1701     tb->page_addr[n] = page_addr;
1702     tb->page_next[n] = p->first_tb;
1703 #ifndef CONFIG_USER_ONLY
1704     page_already_protected = p->first_tb != (uintptr_t)NULL;
1705 #endif
1706     p->first_tb = (uintptr_t)tb | n;
1707     invalidate_page_bitmap(p);
1708 
1709 #if defined(CONFIG_USER_ONLY)
1710     if (p->flags & PAGE_WRITE) {
1711         target_ulong addr;
1712         PageDesc *p2;
1713         int prot;
1714 
1715         /* force the host page as non writable (writes will have a
1716            page fault + mprotect overhead) */
1717         page_addr &= qemu_host_page_mask;
1718         prot = 0;
1719         for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1720             addr += TARGET_PAGE_SIZE) {
1721 
1722             p2 = page_find(addr >> TARGET_PAGE_BITS);
1723             if (!p2) {
1724                 continue;
1725             }
1726             prot |= p2->flags;
1727             p2->flags &= ~PAGE_WRITE;
1728           }
1729         mprotect(g2h(page_addr), qemu_host_page_size,
1730                  (prot & PAGE_BITS) & ~PAGE_WRITE);
1731         if (DEBUG_TB_INVALIDATE_GATE) {
1732             printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr);
1733         }
1734     }
1735 #else
1736     /* if some code is already present, then the pages are already
1737        protected. So we handle the case where only the first TB is
1738        allocated in a physical page */
1739     if (!page_already_protected) {
1740         tlb_protect_code(page_addr);
1741     }
1742 #endif
1743 }
1744 
1745 /* add a new TB and link it to the physical page tables. phys_page2 is
1746  * (-1) to indicate that only one page contains the TB.
1747  *
1748  * Called with mmap_lock held for user-mode emulation.
1749  *
1750  * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
1751  * Note that in !user-mode, another thread might have already added a TB
1752  * for the same block of guest code that @tb corresponds to. In that case,
1753  * the caller should discard the original @tb, and use instead the returned TB.
1754  */
1755 static TranslationBlock *
1756 tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1757              tb_page_addr_t phys_page2)
1758 {
1759     PageDesc *p;
1760     PageDesc *p2 = NULL;
1761 
1762     assert_memory_lock();
1763 
1764     if (phys_pc == -1) {
1765         /*
1766          * If the TB is not associated with a physical RAM page then
1767          * it must be a temporary one-insn TB, and we have nothing to do
1768          * except fill in the page_addr[] fields.
1769          */
1770         assert(tb->cflags & CF_NOCACHE);
1771         tb->page_addr[0] = tb->page_addr[1] = -1;
1772         return tb;
1773     }
1774 
1775     /*
1776      * Add the TB to the page list, acquiring first the pages's locks.
1777      * We keep the locks held until after inserting the TB in the hash table,
1778      * so that if the insertion fails we know for sure that the TBs are still
1779      * in the page descriptors.
1780      * Note that inserting into the hash table first isn't an option, since
1781      * we can only insert TBs that are fully initialized.
1782      */
1783     page_lock_pair(&p, phys_pc, &p2, phys_page2, 1);
1784     tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK);
1785     if (p2) {
1786         tb_page_add(p2, tb, 1, phys_page2);
1787     } else {
1788         tb->page_addr[1] = -1;
1789     }
1790 
1791     if (!(tb->cflags & CF_NOCACHE)) {
1792         void *existing_tb = NULL;
1793         uint32_t h;
1794 
1795         /* add in the hash table */
1796         h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK,
1797                          tb->trace_vcpu_dstate);
1798         qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
1799 
1800         /* remove TB from the page(s) if we couldn't insert it */
1801         if (unlikely(existing_tb)) {
1802             tb_page_remove(p, tb);
1803             invalidate_page_bitmap(p);
1804             if (p2) {
1805                 tb_page_remove(p2, tb);
1806                 invalidate_page_bitmap(p2);
1807             }
1808             tb = existing_tb;
1809         }
1810     }
1811 
1812     if (p2 && p2 != p) {
1813         page_unlock(p2);
1814     }
1815     page_unlock(p);
1816 
1817 #ifdef CONFIG_USER_ONLY
1818     if (DEBUG_TB_CHECK_GATE) {
1819         tb_page_check();
1820     }
1821 #endif
1822     return tb;
1823 }
1824 
1825 /* Called with mmap_lock held for user mode emulation.  */
1826 TranslationBlock *tb_gen_code(CPUState *cpu,
1827                               target_ulong pc, target_ulong cs_base,
1828                               uint32_t flags, int cflags)
1829 {
1830     CPUArchState *env = cpu->env_ptr;
1831     TranslationBlock *tb, *existing_tb;
1832     tb_page_addr_t phys_pc, phys_page2;
1833     target_ulong virt_page2;
1834     tcg_insn_unit *gen_code_buf;
1835     int gen_code_size, search_size, max_insns;
1836 #ifdef CONFIG_PROFILER
1837     TCGProfile *prof = &tcg_ctx->prof;
1838     int64_t ti;
1839 #endif
1840 
1841     assert_memory_lock();
1842 
1843     phys_pc = get_page_addr_code(env, pc);
1844 
1845     if (phys_pc == -1) {
1846         /* Generate a temporary TB with 1 insn in it */
1847         cflags &= ~CF_COUNT_MASK;
1848         cflags |= CF_NOCACHE | 1;
1849     }
1850 
1851     cflags &= ~CF_CLUSTER_MASK;
1852     cflags |= cpu->cluster_index << CF_CLUSTER_SHIFT;
1853 
1854     max_insns = cflags & CF_COUNT_MASK;
1855     if (max_insns == 0) {
1856         max_insns = CF_COUNT_MASK;
1857     }
1858     if (max_insns > TCG_MAX_INSNS) {
1859         max_insns = TCG_MAX_INSNS;
1860     }
1861     if (cpu->singlestep_enabled || singlestep) {
1862         max_insns = 1;
1863     }
1864 
1865  buffer_overflow:
1866     tb = tcg_tb_alloc(tcg_ctx);
1867     if (unlikely(!tb)) {
1868         /* flush must be done */
1869         tb_flush(cpu);
1870         mmap_unlock();
1871         /* Make the execution loop process the flush as soon as possible.  */
1872         cpu->exception_index = EXCP_INTERRUPT;
1873         cpu_loop_exit(cpu);
1874     }
1875 
1876     gen_code_buf = tcg_ctx->code_gen_ptr;
1877     tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf);
1878     tb->pc = pc;
1879     tb->cs_base = cs_base;
1880     tb->flags = flags;
1881     tb->cflags = cflags;
1882     tb->orig_tb = NULL;
1883     tb->trace_vcpu_dstate = *cpu->trace_dstate;
1884     tcg_ctx->tb_cflags = cflags;
1885  tb_overflow:
1886 
1887 #ifdef CONFIG_PROFILER
1888     /* includes aborted translations because of exceptions */
1889     qatomic_set(&prof->tb_count1, prof->tb_count1 + 1);
1890     ti = profile_getclock();
1891 #endif
1892 
1893     tcg_func_start(tcg_ctx);
1894 
1895     tcg_ctx->cpu = env_cpu(env);
1896     gen_intermediate_code(cpu, tb, max_insns);
1897     tcg_ctx->cpu = NULL;
1898 
1899     trace_translate_block(tb, tb->pc, tb->tc.ptr);
1900 
1901     /* generate machine code */
1902     tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1903     tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1904     tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
1905     if (TCG_TARGET_HAS_direct_jump) {
1906         tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
1907         tcg_ctx->tb_jmp_target_addr = NULL;
1908     } else {
1909         tcg_ctx->tb_jmp_insn_offset = NULL;
1910         tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
1911     }
1912 
1913 #ifdef CONFIG_PROFILER
1914     qatomic_set(&prof->tb_count, prof->tb_count + 1);
1915     qatomic_set(&prof->interm_time,
1916                 prof->interm_time + profile_getclock() - ti);
1917     ti = profile_getclock();
1918 #endif
1919 
1920     gen_code_size = tcg_gen_code(tcg_ctx, tb);
1921     if (unlikely(gen_code_size < 0)) {
1922         switch (gen_code_size) {
1923         case -1:
1924             /*
1925              * Overflow of code_gen_buffer, or the current slice of it.
1926              *
1927              * TODO: We don't need to re-do gen_intermediate_code, nor
1928              * should we re-do the tcg optimization currently hidden
1929              * inside tcg_gen_code.  All that should be required is to
1930              * flush the TBs, allocate a new TB, re-initialize it per
1931              * above, and re-do the actual code generation.
1932              */
1933             goto buffer_overflow;
1934 
1935         case -2:
1936             /*
1937              * The code generated for the TranslationBlock is too large.
1938              * The maximum size allowed by the unwind info is 64k.
1939              * There may be stricter constraints from relocations
1940              * in the tcg backend.
1941              *
1942              * Try again with half as many insns as we attempted this time.
1943              * If a single insn overflows, there's a bug somewhere...
1944              */
1945             max_insns = tb->icount;
1946             assert(max_insns > 1);
1947             max_insns /= 2;
1948             goto tb_overflow;
1949 
1950         default:
1951             g_assert_not_reached();
1952         }
1953     }
1954     search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1955     if (unlikely(search_size < 0)) {
1956         goto buffer_overflow;
1957     }
1958     tb->tc.size = gen_code_size;
1959 
1960 #ifdef CONFIG_PROFILER
1961     qatomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti);
1962     qatomic_set(&prof->code_in_len, prof->code_in_len + tb->size);
1963     qatomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size);
1964     qatomic_set(&prof->search_out_len, prof->search_out_len + search_size);
1965 #endif
1966 
1967 #ifdef DEBUG_DISAS
1968     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1969         qemu_log_in_addr_range(tb->pc)) {
1970         FILE *logfile = qemu_log_lock();
1971         int code_size, data_size;
1972         const tcg_target_ulong *rx_data_gen_ptr;
1973         size_t chunk_start;
1974         int insn = 0;
1975 
1976         if (tcg_ctx->data_gen_ptr) {
1977             rx_data_gen_ptr = tcg_splitwx_to_rx(tcg_ctx->data_gen_ptr);
1978             code_size = (const void *)rx_data_gen_ptr - tb->tc.ptr;
1979             data_size = gen_code_size - code_size;
1980         } else {
1981             rx_data_gen_ptr = 0;
1982             code_size = gen_code_size;
1983             data_size = 0;
1984         }
1985 
1986         /* Dump header and the first instruction */
1987         qemu_log("OUT: [size=%d]\n", gen_code_size);
1988         qemu_log("  -- guest addr 0x" TARGET_FMT_lx " + tb prologue\n",
1989                  tcg_ctx->gen_insn_data[insn][0]);
1990         chunk_start = tcg_ctx->gen_insn_end_off[insn];
1991         log_disas(tb->tc.ptr, chunk_start);
1992 
1993         /*
1994          * Dump each instruction chunk, wrapping up empty chunks into
1995          * the next instruction. The whole array is offset so the
1996          * first entry is the beginning of the 2nd instruction.
1997          */
1998         while (insn < tb->icount) {
1999             size_t chunk_end = tcg_ctx->gen_insn_end_off[insn];
2000             if (chunk_end > chunk_start) {
2001                 qemu_log("  -- guest addr 0x" TARGET_FMT_lx "\n",
2002                          tcg_ctx->gen_insn_data[insn][0]);
2003                 log_disas(tb->tc.ptr + chunk_start, chunk_end - chunk_start);
2004                 chunk_start = chunk_end;
2005             }
2006             insn++;
2007         }
2008 
2009         if (chunk_start < code_size) {
2010             qemu_log("  -- tb slow paths + alignment\n");
2011             log_disas(tb->tc.ptr + chunk_start, code_size - chunk_start);
2012         }
2013 
2014         /* Finally dump any data we may have after the block */
2015         if (data_size) {
2016             int i;
2017             qemu_log("  data: [size=%d]\n", data_size);
2018             for (i = 0; i < data_size / sizeof(tcg_target_ulong); i++) {
2019                 qemu_log("0x%08" PRIxPTR ":  .quad  0x%" TCG_PRIlx "\n",
2020                          (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]);
2021             }
2022         }
2023         qemu_log("\n");
2024         qemu_log_flush();
2025         qemu_log_unlock(logfile);
2026     }
2027 #endif
2028 
2029     qatomic_set(&tcg_ctx->code_gen_ptr, (void *)
2030         ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
2031                  CODE_GEN_ALIGN));
2032 
2033     /* init jump list */
2034     qemu_spin_init(&tb->jmp_lock);
2035     tb->jmp_list_head = (uintptr_t)NULL;
2036     tb->jmp_list_next[0] = (uintptr_t)NULL;
2037     tb->jmp_list_next[1] = (uintptr_t)NULL;
2038     tb->jmp_dest[0] = (uintptr_t)NULL;
2039     tb->jmp_dest[1] = (uintptr_t)NULL;
2040 
2041     /* init original jump addresses which have been set during tcg_gen_code() */
2042     if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
2043         tb_reset_jump(tb, 0);
2044     }
2045     if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
2046         tb_reset_jump(tb, 1);
2047     }
2048 
2049     /* check next page if needed */
2050     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
2051     phys_page2 = -1;
2052     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
2053         phys_page2 = get_page_addr_code(env, virt_page2);
2054     }
2055     /*
2056      * No explicit memory barrier is required -- tb_link_page() makes the
2057      * TB visible in a consistent state.
2058      */
2059     existing_tb = tb_link_page(tb, phys_pc, phys_page2);
2060     /* if the TB already exists, discard what we just translated */
2061     if (unlikely(existing_tb != tb)) {
2062         uintptr_t orig_aligned = (uintptr_t)gen_code_buf;
2063 
2064         orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize);
2065         qatomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned);
2066         tb_destroy(tb);
2067         return existing_tb;
2068     }
2069     tcg_tb_insert(tb);
2070     return tb;
2071 }
2072 
2073 /*
2074  * @p must be non-NULL.
2075  * user-mode: call with mmap_lock held.
2076  * !user-mode: call with all @pages locked.
2077  */
2078 static void
2079 tb_invalidate_phys_page_range__locked(struct page_collection *pages,
2080                                       PageDesc *p, tb_page_addr_t start,
2081                                       tb_page_addr_t end,
2082                                       uintptr_t retaddr)
2083 {
2084     TranslationBlock *tb;
2085     tb_page_addr_t tb_start, tb_end;
2086     int n;
2087 #ifdef TARGET_HAS_PRECISE_SMC
2088     CPUState *cpu = current_cpu;
2089     CPUArchState *env = NULL;
2090     bool current_tb_not_found = retaddr != 0;
2091     bool current_tb_modified = false;
2092     TranslationBlock *current_tb = NULL;
2093     target_ulong current_pc = 0;
2094     target_ulong current_cs_base = 0;
2095     uint32_t current_flags = 0;
2096 #endif /* TARGET_HAS_PRECISE_SMC */
2097 
2098     assert_page_locked(p);
2099 
2100 #if defined(TARGET_HAS_PRECISE_SMC)
2101     if (cpu != NULL) {
2102         env = cpu->env_ptr;
2103     }
2104 #endif
2105 
2106     /* we remove all the TBs in the range [start, end[ */
2107     /* XXX: see if in some cases it could be faster to invalidate all
2108        the code */
2109     PAGE_FOR_EACH_TB(p, tb, n) {
2110         assert_page_locked(p);
2111         /* NOTE: this is subtle as a TB may span two physical pages */
2112         if (n == 0) {
2113             /* NOTE: tb_end may be after the end of the page, but
2114                it is not a problem */
2115             tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
2116             tb_end = tb_start + tb->size;
2117         } else {
2118             tb_start = tb->page_addr[1];
2119             tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
2120         }
2121         if (!(tb_end <= start || tb_start >= end)) {
2122 #ifdef TARGET_HAS_PRECISE_SMC
2123             if (current_tb_not_found) {
2124                 current_tb_not_found = false;
2125                 /* now we have a real cpu fault */
2126                 current_tb = tcg_tb_lookup(retaddr);
2127             }
2128             if (current_tb == tb &&
2129                 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
2130                 /*
2131                  * If we are modifying the current TB, we must stop
2132                  * its execution. We could be more precise by checking
2133                  * that the modification is after the current PC, but it
2134                  * would require a specialized function to partially
2135                  * restore the CPU state.
2136                  */
2137                 current_tb_modified = true;
2138                 cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
2139                 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
2140                                      &current_flags);
2141             }
2142 #endif /* TARGET_HAS_PRECISE_SMC */
2143             tb_phys_invalidate__locked(tb);
2144         }
2145     }
2146 #if !defined(CONFIG_USER_ONLY)
2147     /* if no code remaining, no need to continue to use slow writes */
2148     if (!p->first_tb) {
2149         invalidate_page_bitmap(p);
2150         tlb_unprotect_code(start);
2151     }
2152 #endif
2153 #ifdef TARGET_HAS_PRECISE_SMC
2154     if (current_tb_modified) {
2155         page_collection_unlock(pages);
2156         /* Force execution of one insn next time.  */
2157         cpu->cflags_next_tb = 1 | curr_cflags();
2158         mmap_unlock();
2159         cpu_loop_exit_noexc(cpu);
2160     }
2161 #endif
2162 }
2163 
2164 /*
2165  * Invalidate all TBs which intersect with the target physical address range
2166  * [start;end[. NOTE: start and end must refer to the *same* physical page.
2167  * 'is_cpu_write_access' should be true if called from a real cpu write
2168  * access: the virtual CPU will exit the current TB if code is modified inside
2169  * this TB.
2170  *
2171  * Called with mmap_lock held for user-mode emulation
2172  */
2173 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
2174 {
2175     struct page_collection *pages;
2176     PageDesc *p;
2177 
2178     assert_memory_lock();
2179 
2180     p = page_find(start >> TARGET_PAGE_BITS);
2181     if (p == NULL) {
2182         return;
2183     }
2184     pages = page_collection_lock(start, end);
2185     tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
2186     page_collection_unlock(pages);
2187 }
2188 
2189 /*
2190  * Invalidate all TBs which intersect with the target physical address range
2191  * [start;end[. NOTE: start and end may refer to *different* physical pages.
2192  * 'is_cpu_write_access' should be true if called from a real cpu write
2193  * access: the virtual CPU will exit the current TB if code is modified inside
2194  * this TB.
2195  *
2196  * Called with mmap_lock held for user-mode emulation.
2197  */
2198 #ifdef CONFIG_SOFTMMU
2199 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
2200 #else
2201 void tb_invalidate_phys_range(target_ulong start, target_ulong end)
2202 #endif
2203 {
2204     struct page_collection *pages;
2205     tb_page_addr_t next;
2206 
2207     assert_memory_lock();
2208 
2209     pages = page_collection_lock(start, end);
2210     for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
2211          start < end;
2212          start = next, next += TARGET_PAGE_SIZE) {
2213         PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
2214         tb_page_addr_t bound = MIN(next, end);
2215 
2216         if (pd == NULL) {
2217             continue;
2218         }
2219         tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
2220     }
2221     page_collection_unlock(pages);
2222 }
2223 
2224 #ifdef CONFIG_SOFTMMU
2225 /* len must be <= 8 and start must be a multiple of len.
2226  * Called via softmmu_template.h when code areas are written to with
2227  * iothread mutex not held.
2228  *
2229  * Call with all @pages in the range [@start, @start + len[ locked.
2230  */
2231 void tb_invalidate_phys_page_fast(struct page_collection *pages,
2232                                   tb_page_addr_t start, int len,
2233                                   uintptr_t retaddr)
2234 {
2235     PageDesc *p;
2236 
2237     assert_memory_lock();
2238 
2239     p = page_find(start >> TARGET_PAGE_BITS);
2240     if (!p) {
2241         return;
2242     }
2243 
2244     assert_page_locked(p);
2245     if (!p->code_bitmap &&
2246         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
2247         build_page_bitmap(p);
2248     }
2249     if (p->code_bitmap) {
2250         unsigned int nr;
2251         unsigned long b;
2252 
2253         nr = start & ~TARGET_PAGE_MASK;
2254         b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
2255         if (b & ((1 << len) - 1)) {
2256             goto do_invalidate;
2257         }
2258     } else {
2259     do_invalidate:
2260         tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
2261                                               retaddr);
2262     }
2263 }
2264 #else
2265 /* Called with mmap_lock held. If pc is not 0 then it indicates the
2266  * host PC of the faulting store instruction that caused this invalidate.
2267  * Returns true if the caller needs to abort execution of the current
2268  * TB (because it was modified by this store and the guest CPU has
2269  * precise-SMC semantics).
2270  */
2271 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
2272 {
2273     TranslationBlock *tb;
2274     PageDesc *p;
2275     int n;
2276 #ifdef TARGET_HAS_PRECISE_SMC
2277     TranslationBlock *current_tb = NULL;
2278     CPUState *cpu = current_cpu;
2279     CPUArchState *env = NULL;
2280     int current_tb_modified = 0;
2281     target_ulong current_pc = 0;
2282     target_ulong current_cs_base = 0;
2283     uint32_t current_flags = 0;
2284 #endif
2285 
2286     assert_memory_lock();
2287 
2288     addr &= TARGET_PAGE_MASK;
2289     p = page_find(addr >> TARGET_PAGE_BITS);
2290     if (!p) {
2291         return false;
2292     }
2293 
2294 #ifdef TARGET_HAS_PRECISE_SMC
2295     if (p->first_tb && pc != 0) {
2296         current_tb = tcg_tb_lookup(pc);
2297     }
2298     if (cpu != NULL) {
2299         env = cpu->env_ptr;
2300     }
2301 #endif
2302     assert_page_locked(p);
2303     PAGE_FOR_EACH_TB(p, tb, n) {
2304 #ifdef TARGET_HAS_PRECISE_SMC
2305         if (current_tb == tb &&
2306             (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
2307                 /* If we are modifying the current TB, we must stop
2308                    its execution. We could be more precise by checking
2309                    that the modification is after the current PC, but it
2310                    would require a specialized function to partially
2311                    restore the CPU state */
2312 
2313             current_tb_modified = 1;
2314             cpu_restore_state_from_tb(cpu, current_tb, pc, true);
2315             cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
2316                                  &current_flags);
2317         }
2318 #endif /* TARGET_HAS_PRECISE_SMC */
2319         tb_phys_invalidate(tb, addr);
2320     }
2321     p->first_tb = (uintptr_t)NULL;
2322 #ifdef TARGET_HAS_PRECISE_SMC
2323     if (current_tb_modified) {
2324         /* Force execution of one insn next time.  */
2325         cpu->cflags_next_tb = 1 | curr_cflags();
2326         return true;
2327     }
2328 #endif
2329 
2330     return false;
2331 }
2332 #endif
2333 
2334 /* user-mode: call with mmap_lock held */
2335 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
2336 {
2337     TranslationBlock *tb;
2338 
2339     assert_memory_lock();
2340 
2341     tb = tcg_tb_lookup(retaddr);
2342     if (tb) {
2343         /* We can use retranslation to find the PC.  */
2344         cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2345         tb_phys_invalidate(tb, -1);
2346     } else {
2347         /* The exception probably happened in a helper.  The CPU state should
2348            have been saved before calling it. Fetch the PC from there.  */
2349         CPUArchState *env = cpu->env_ptr;
2350         target_ulong pc, cs_base;
2351         tb_page_addr_t addr;
2352         uint32_t flags;
2353 
2354         cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
2355         addr = get_page_addr_code(env, pc);
2356         if (addr != -1) {
2357             tb_invalidate_phys_range(addr, addr + 1);
2358         }
2359     }
2360 }
2361 
2362 #ifndef CONFIG_USER_ONLY
2363 /* in deterministic execution mode, instructions doing device I/Os
2364  * must be at the end of the TB.
2365  *
2366  * Called by softmmu_template.h, with iothread mutex not held.
2367  */
2368 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
2369 {
2370 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
2371     CPUArchState *env = cpu->env_ptr;
2372 #endif
2373     TranslationBlock *tb;
2374     uint32_t n;
2375 
2376     tb = tcg_tb_lookup(retaddr);
2377     if (!tb) {
2378         cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
2379                   (void *)retaddr);
2380     }
2381     cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2382 
2383     /* On MIPS and SH, delay slot instructions can only be restarted if
2384        they were already the first instruction in the TB.  If this is not
2385        the first instruction in a TB then re-execute the preceding
2386        branch.  */
2387     n = 1;
2388 #if defined(TARGET_MIPS)
2389     if ((env->hflags & MIPS_HFLAG_BMASK) != 0
2390         && env->active_tc.PC != tb->pc) {
2391         env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
2392         cpu_neg(cpu)->icount_decr.u16.low++;
2393         env->hflags &= ~MIPS_HFLAG_BMASK;
2394         n = 2;
2395     }
2396 #elif defined(TARGET_SH4)
2397     if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
2398         && env->pc != tb->pc) {
2399         env->pc -= 2;
2400         cpu_neg(cpu)->icount_decr.u16.low++;
2401         env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
2402         n = 2;
2403     }
2404 #endif
2405 
2406     /* Generate a new TB executing the I/O insn.  */
2407     cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n;
2408 
2409     if (tb_cflags(tb) & CF_NOCACHE) {
2410         if (tb->orig_tb) {
2411             /* Invalidate original TB if this TB was generated in
2412              * cpu_exec_nocache() */
2413             tb_phys_invalidate(tb->orig_tb, -1);
2414         }
2415         tcg_tb_remove(tb);
2416         tb_destroy(tb);
2417     }
2418 
2419     qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc,
2420                            "cpu_io_recompile: rewound execution of TB to "
2421                            TARGET_FMT_lx "\n", tb->pc);
2422 
2423     /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
2424      * the first in the TB) then we end up generating a whole new TB and
2425      *  repeating the fault, which is horribly inefficient.
2426      *  Better would be to execute just this insn uncached, or generate a
2427      *  second new TB.
2428      */
2429     cpu_loop_exit_noexc(cpu);
2430 }
2431 
2432 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr)
2433 {
2434     unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr);
2435 
2436     for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
2437         qatomic_set(&cpu->tb_jmp_cache[i0 + i], NULL);
2438     }
2439 }
2440 
2441 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
2442 {
2443     /* Discard jump cache entries for any tb which might potentially
2444        overlap the flushed page.  */
2445     tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
2446     tb_jmp_cache_clear_page(cpu, addr);
2447 }
2448 
2449 static void print_qht_statistics(struct qht_stats hst)
2450 {
2451     uint32_t hgram_opts;
2452     size_t hgram_bins;
2453     char *hgram;
2454 
2455     if (!hst.head_buckets) {
2456         return;
2457     }
2458     qemu_printf("TB hash buckets     %zu/%zu (%0.2f%% head buckets used)\n",
2459                 hst.used_head_buckets, hst.head_buckets,
2460                 (double)hst.used_head_buckets / hst.head_buckets * 100);
2461 
2462     hgram_opts =  QDIST_PR_BORDER | QDIST_PR_LABELS;
2463     hgram_opts |= QDIST_PR_100X   | QDIST_PR_PERCENT;
2464     if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
2465         hgram_opts |= QDIST_PR_NODECIMAL;
2466     }
2467     hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
2468     qemu_printf("TB hash occupancy   %0.2f%% avg chain occ. Histogram: %s\n",
2469                 qdist_avg(&hst.occupancy) * 100, hgram);
2470     g_free(hgram);
2471 
2472     hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
2473     hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
2474     if (hgram_bins > 10) {
2475         hgram_bins = 10;
2476     } else {
2477         hgram_bins = 0;
2478         hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
2479     }
2480     hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
2481     qemu_printf("TB hash avg chain   %0.3f buckets. Histogram: %s\n",
2482                 qdist_avg(&hst.chain), hgram);
2483     g_free(hgram);
2484 }
2485 
2486 struct tb_tree_stats {
2487     size_t nb_tbs;
2488     size_t host_size;
2489     size_t target_size;
2490     size_t max_target_size;
2491     size_t direct_jmp_count;
2492     size_t direct_jmp2_count;
2493     size_t cross_page;
2494 };
2495 
2496 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
2497 {
2498     const TranslationBlock *tb = value;
2499     struct tb_tree_stats *tst = data;
2500 
2501     tst->nb_tbs++;
2502     tst->host_size += tb->tc.size;
2503     tst->target_size += tb->size;
2504     if (tb->size > tst->max_target_size) {
2505         tst->max_target_size = tb->size;
2506     }
2507     if (tb->page_addr[1] != -1) {
2508         tst->cross_page++;
2509     }
2510     if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
2511         tst->direct_jmp_count++;
2512         if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
2513             tst->direct_jmp2_count++;
2514         }
2515     }
2516     return false;
2517 }
2518 
2519 void dump_exec_info(void)
2520 {
2521     struct tb_tree_stats tst = {};
2522     struct qht_stats hst;
2523     size_t nb_tbs, flush_full, flush_part, flush_elide;
2524 
2525     tcg_tb_foreach(tb_tree_stats_iter, &tst);
2526     nb_tbs = tst.nb_tbs;
2527     /* XXX: avoid using doubles ? */
2528     qemu_printf("Translation buffer state:\n");
2529     /*
2530      * Report total code size including the padding and TB structs;
2531      * otherwise users might think "-accel tcg,tb-size" is not honoured.
2532      * For avg host size we use the precise numbers from tb_tree_stats though.
2533      */
2534     qemu_printf("gen code size       %zu/%zu\n",
2535                 tcg_code_size(), tcg_code_capacity());
2536     qemu_printf("TB count            %zu\n", nb_tbs);
2537     qemu_printf("TB avg target size  %zu max=%zu bytes\n",
2538                 nb_tbs ? tst.target_size / nb_tbs : 0,
2539                 tst.max_target_size);
2540     qemu_printf("TB avg host size    %zu bytes (expansion ratio: %0.1f)\n",
2541                 nb_tbs ? tst.host_size / nb_tbs : 0,
2542                 tst.target_size ? (double)tst.host_size / tst.target_size : 0);
2543     qemu_printf("cross page TB count %zu (%zu%%)\n", tst.cross_page,
2544                 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0);
2545     qemu_printf("direct jump count   %zu (%zu%%) (2 jumps=%zu %zu%%)\n",
2546                 tst.direct_jmp_count,
2547                 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0,
2548                 tst.direct_jmp2_count,
2549                 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0);
2550 
2551     qht_statistics_init(&tb_ctx.htable, &hst);
2552     print_qht_statistics(hst);
2553     qht_statistics_destroy(&hst);
2554 
2555     qemu_printf("\nStatistics:\n");
2556     qemu_printf("TB flush count      %u\n",
2557                 qatomic_read(&tb_ctx.tb_flush_count));
2558     qemu_printf("TB invalidate count %zu\n",
2559                 tcg_tb_phys_invalidate_count());
2560 
2561     tlb_flush_counts(&flush_full, &flush_part, &flush_elide);
2562     qemu_printf("TLB full flushes    %zu\n", flush_full);
2563     qemu_printf("TLB partial flushes %zu\n", flush_part);
2564     qemu_printf("TLB elided flushes  %zu\n", flush_elide);
2565     tcg_dump_info();
2566 }
2567 
2568 void dump_opcount_info(void)
2569 {
2570     tcg_dump_op_count();
2571 }
2572 
2573 #else /* CONFIG_USER_ONLY */
2574 
2575 void cpu_interrupt(CPUState *cpu, int mask)
2576 {
2577     g_assert(qemu_mutex_iothread_locked());
2578     cpu->interrupt_request |= mask;
2579     qatomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
2580 }
2581 
2582 /*
2583  * Walks guest process memory "regions" one by one
2584  * and calls callback function 'fn' for each region.
2585  */
2586 struct walk_memory_regions_data {
2587     walk_memory_regions_fn fn;
2588     void *priv;
2589     target_ulong start;
2590     int prot;
2591 };
2592 
2593 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2594                                    target_ulong end, int new_prot)
2595 {
2596     if (data->start != -1u) {
2597         int rc = data->fn(data->priv, data->start, end, data->prot);
2598         if (rc != 0) {
2599             return rc;
2600         }
2601     }
2602 
2603     data->start = (new_prot ? end : -1u);
2604     data->prot = new_prot;
2605 
2606     return 0;
2607 }
2608 
2609 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2610                                  target_ulong base, int level, void **lp)
2611 {
2612     target_ulong pa;
2613     int i, rc;
2614 
2615     if (*lp == NULL) {
2616         return walk_memory_regions_end(data, base, 0);
2617     }
2618 
2619     if (level == 0) {
2620         PageDesc *pd = *lp;
2621 
2622         for (i = 0; i < V_L2_SIZE; ++i) {
2623             int prot = pd[i].flags;
2624 
2625             pa = base | (i << TARGET_PAGE_BITS);
2626             if (prot != data->prot) {
2627                 rc = walk_memory_regions_end(data, pa, prot);
2628                 if (rc != 0) {
2629                     return rc;
2630                 }
2631             }
2632         }
2633     } else {
2634         void **pp = *lp;
2635 
2636         for (i = 0; i < V_L2_SIZE; ++i) {
2637             pa = base | ((target_ulong)i <<
2638                 (TARGET_PAGE_BITS + V_L2_BITS * level));
2639             rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2640             if (rc != 0) {
2641                 return rc;
2642             }
2643         }
2644     }
2645 
2646     return 0;
2647 }
2648 
2649 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2650 {
2651     struct walk_memory_regions_data data;
2652     uintptr_t i, l1_sz = v_l1_size;
2653 
2654     data.fn = fn;
2655     data.priv = priv;
2656     data.start = -1u;
2657     data.prot = 0;
2658 
2659     for (i = 0; i < l1_sz; i++) {
2660         target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS);
2661         int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i);
2662         if (rc != 0) {
2663             return rc;
2664         }
2665     }
2666 
2667     return walk_memory_regions_end(&data, 0, 0);
2668 }
2669 
2670 static int dump_region(void *priv, target_ulong start,
2671     target_ulong end, unsigned long prot)
2672 {
2673     FILE *f = (FILE *)priv;
2674 
2675     (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
2676         " "TARGET_FMT_lx" %c%c%c\n",
2677         start, end, end - start,
2678         ((prot & PAGE_READ) ? 'r' : '-'),
2679         ((prot & PAGE_WRITE) ? 'w' : '-'),
2680         ((prot & PAGE_EXEC) ? 'x' : '-'));
2681 
2682     return 0;
2683 }
2684 
2685 /* dump memory mappings */
2686 void page_dump(FILE *f)
2687 {
2688     const int length = sizeof(target_ulong) * 2;
2689     (void) fprintf(f, "%-*s %-*s %-*s %s\n",
2690             length, "start", length, "end", length, "size", "prot");
2691     walk_memory_regions(f, dump_region);
2692 }
2693 
2694 int page_get_flags(target_ulong address)
2695 {
2696     PageDesc *p;
2697 
2698     p = page_find(address >> TARGET_PAGE_BITS);
2699     if (!p) {
2700         return 0;
2701     }
2702     return p->flags;
2703 }
2704 
2705 /* Modify the flags of a page and invalidate the code if necessary.
2706    The flag PAGE_WRITE_ORG is positioned automatically depending
2707    on PAGE_WRITE.  The mmap_lock should already be held.  */
2708 void page_set_flags(target_ulong start, target_ulong end, int flags)
2709 {
2710     target_ulong addr, len;
2711 
2712     /* This function should never be called with addresses outside the
2713        guest address space.  If this assert fires, it probably indicates
2714        a missing call to h2g_valid.  */
2715     assert(end - 1 <= GUEST_ADDR_MAX);
2716     assert(start < end);
2717     assert_memory_lock();
2718 
2719     start = start & TARGET_PAGE_MASK;
2720     end = TARGET_PAGE_ALIGN(end);
2721 
2722     if (flags & PAGE_WRITE) {
2723         flags |= PAGE_WRITE_ORG;
2724     }
2725 
2726     for (addr = start, len = end - start;
2727          len != 0;
2728          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2729         PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2730 
2731         /* If the write protection bit is set, then we invalidate
2732            the code inside.  */
2733         if (!(p->flags & PAGE_WRITE) &&
2734             (flags & PAGE_WRITE) &&
2735             p->first_tb) {
2736             tb_invalidate_phys_page(addr, 0);
2737         }
2738         p->flags = flags;
2739     }
2740 }
2741 
2742 int page_check_range(target_ulong start, target_ulong len, int flags)
2743 {
2744     PageDesc *p;
2745     target_ulong end;
2746     target_ulong addr;
2747 
2748     /* This function should never be called with addresses outside the
2749        guest address space.  If this assert fires, it probably indicates
2750        a missing call to h2g_valid.  */
2751     if (TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS) {
2752         assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2753     }
2754 
2755     if (len == 0) {
2756         return 0;
2757     }
2758     if (start + len - 1 < start) {
2759         /* We've wrapped around.  */
2760         return -1;
2761     }
2762 
2763     /* must do before we loose bits in the next step */
2764     end = TARGET_PAGE_ALIGN(start + len);
2765     start = start & TARGET_PAGE_MASK;
2766 
2767     for (addr = start, len = end - start;
2768          len != 0;
2769          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2770         p = page_find(addr >> TARGET_PAGE_BITS);
2771         if (!p) {
2772             return -1;
2773         }
2774         if (!(p->flags & PAGE_VALID)) {
2775             return -1;
2776         }
2777 
2778         if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
2779             return -1;
2780         }
2781         if (flags & PAGE_WRITE) {
2782             if (!(p->flags & PAGE_WRITE_ORG)) {
2783                 return -1;
2784             }
2785             /* unprotect the page if it was put read-only because it
2786                contains translated code */
2787             if (!(p->flags & PAGE_WRITE)) {
2788                 if (!page_unprotect(addr, 0)) {
2789                     return -1;
2790                 }
2791             }
2792         }
2793     }
2794     return 0;
2795 }
2796 
2797 /* called from signal handler: invalidate the code and unprotect the
2798  * page. Return 0 if the fault was not handled, 1 if it was handled,
2799  * and 2 if it was handled but the caller must cause the TB to be
2800  * immediately exited. (We can only return 2 if the 'pc' argument is
2801  * non-zero.)
2802  */
2803 int page_unprotect(target_ulong address, uintptr_t pc)
2804 {
2805     unsigned int prot;
2806     bool current_tb_invalidated;
2807     PageDesc *p;
2808     target_ulong host_start, host_end, addr;
2809 
2810     /* Technically this isn't safe inside a signal handler.  However we
2811        know this only ever happens in a synchronous SEGV handler, so in
2812        practice it seems to be ok.  */
2813     mmap_lock();
2814 
2815     p = page_find(address >> TARGET_PAGE_BITS);
2816     if (!p) {
2817         mmap_unlock();
2818         return 0;
2819     }
2820 
2821     /* if the page was really writable, then we change its
2822        protection back to writable */
2823     if (p->flags & PAGE_WRITE_ORG) {
2824         current_tb_invalidated = false;
2825         if (p->flags & PAGE_WRITE) {
2826             /* If the page is actually marked WRITE then assume this is because
2827              * this thread raced with another one which got here first and
2828              * set the page to PAGE_WRITE and did the TB invalidate for us.
2829              */
2830 #ifdef TARGET_HAS_PRECISE_SMC
2831             TranslationBlock *current_tb = tcg_tb_lookup(pc);
2832             if (current_tb) {
2833                 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID;
2834             }
2835 #endif
2836         } else {
2837             host_start = address & qemu_host_page_mask;
2838             host_end = host_start + qemu_host_page_size;
2839 
2840             prot = 0;
2841             for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) {
2842                 p = page_find(addr >> TARGET_PAGE_BITS);
2843                 p->flags |= PAGE_WRITE;
2844                 prot |= p->flags;
2845 
2846                 /* and since the content will be modified, we must invalidate
2847                    the corresponding translated code. */
2848                 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
2849 #ifdef CONFIG_USER_ONLY
2850                 if (DEBUG_TB_CHECK_GATE) {
2851                     tb_invalidate_check(addr);
2852                 }
2853 #endif
2854             }
2855             mprotect((void *)g2h(host_start), qemu_host_page_size,
2856                      prot & PAGE_BITS);
2857         }
2858         mmap_unlock();
2859         /* If current TB was invalidated return to main loop */
2860         return current_tb_invalidated ? 2 : 1;
2861     }
2862     mmap_unlock();
2863     return 0;
2864 }
2865 #endif /* CONFIG_USER_ONLY */
2866 
2867 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2868 void tcg_flush_softmmu_tlb(CPUState *cs)
2869 {
2870 #ifdef CONFIG_SOFTMMU
2871     tlb_flush(cs);
2872 #endif
2873 }
2874