xref: /openbmc/qemu/accel/tcg/translate-all.c (revision 7fb3949d)
1 /*
2  *  Host code generation
3  *
4  *  Copyright (c) 2003 Fabrice Bellard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/units.h"
22 #include "qemu-common.h"
23 
24 #define NO_CPU_IO_DEFS
25 #include "cpu.h"
26 #include "trace.h"
27 #include "disas/disas.h"
28 #include "exec/exec-all.h"
29 #include "tcg/tcg.h"
30 #if defined(CONFIG_USER_ONLY)
31 #include "qemu.h"
32 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
33 #include <sys/param.h>
34 #if __FreeBSD_version >= 700104
35 #define HAVE_KINFO_GETVMMAP
36 #define sigqueue sigqueue_freebsd  /* avoid redefinition */
37 #include <sys/proc.h>
38 #include <machine/profile.h>
39 #define _KERNEL
40 #include <sys/user.h>
41 #undef _KERNEL
42 #undef sigqueue
43 #include <libutil.h>
44 #endif
45 #endif
46 #else
47 #include "exec/ram_addr.h"
48 #endif
49 
50 #include "exec/cputlb.h"
51 #include "exec/tb-hash.h"
52 #include "translate-all.h"
53 #include "qemu/bitmap.h"
54 #include "qemu/error-report.h"
55 #include "qemu/qemu-print.h"
56 #include "qemu/timer.h"
57 #include "qemu/main-loop.h"
58 #include "exec/log.h"
59 #include "sysemu/cpus.h"
60 #include "sysemu/tcg.h"
61 
62 /* #define DEBUG_TB_INVALIDATE */
63 /* #define DEBUG_TB_FLUSH */
64 /* make various TB consistency checks */
65 /* #define DEBUG_TB_CHECK */
66 
67 #ifdef DEBUG_TB_INVALIDATE
68 #define DEBUG_TB_INVALIDATE_GATE 1
69 #else
70 #define DEBUG_TB_INVALIDATE_GATE 0
71 #endif
72 
73 #ifdef DEBUG_TB_FLUSH
74 #define DEBUG_TB_FLUSH_GATE 1
75 #else
76 #define DEBUG_TB_FLUSH_GATE 0
77 #endif
78 
79 #if !defined(CONFIG_USER_ONLY)
80 /* TB consistency checks only implemented for usermode emulation.  */
81 #undef DEBUG_TB_CHECK
82 #endif
83 
84 #ifdef DEBUG_TB_CHECK
85 #define DEBUG_TB_CHECK_GATE 1
86 #else
87 #define DEBUG_TB_CHECK_GATE 0
88 #endif
89 
90 /* Access to the various translations structures need to be serialised via locks
91  * for consistency.
92  * In user-mode emulation access to the memory related structures are protected
93  * with mmap_lock.
94  * In !user-mode we use per-page locks.
95  */
96 #ifdef CONFIG_SOFTMMU
97 #define assert_memory_lock()
98 #else
99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
100 #endif
101 
102 #define SMC_BITMAP_USE_THRESHOLD 10
103 
104 typedef struct PageDesc {
105     /* list of TBs intersecting this ram page */
106     uintptr_t first_tb;
107 #ifdef CONFIG_SOFTMMU
108     /* in order to optimize self modifying code, we count the number
109        of lookups we do to a given page to use a bitmap */
110     unsigned long *code_bitmap;
111     unsigned int code_write_count;
112 #else
113     unsigned long flags;
114 #endif
115 #ifndef CONFIG_USER_ONLY
116     QemuSpin lock;
117 #endif
118 } PageDesc;
119 
120 /**
121  * struct page_entry - page descriptor entry
122  * @pd:     pointer to the &struct PageDesc of the page this entry represents
123  * @index:  page index of the page
124  * @locked: whether the page is locked
125  *
126  * This struct helps us keep track of the locked state of a page, without
127  * bloating &struct PageDesc.
128  *
129  * A page lock protects accesses to all fields of &struct PageDesc.
130  *
131  * See also: &struct page_collection.
132  */
133 struct page_entry {
134     PageDesc *pd;
135     tb_page_addr_t index;
136     bool locked;
137 };
138 
139 /**
140  * struct page_collection - tracks a set of pages (i.e. &struct page_entry's)
141  * @tree:   Binary search tree (BST) of the pages, with key == page index
142  * @max:    Pointer to the page in @tree with the highest page index
143  *
144  * To avoid deadlock we lock pages in ascending order of page index.
145  * When operating on a set of pages, we need to keep track of them so that
146  * we can lock them in order and also unlock them later. For this we collect
147  * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the
148  * @tree implementation we use does not provide an O(1) operation to obtain the
149  * highest-ranked element, we use @max to keep track of the inserted page
150  * with the highest index. This is valuable because if a page is not in
151  * the tree and its index is higher than @max's, then we can lock it
152  * without breaking the locking order rule.
153  *
154  * Note on naming: 'struct page_set' would be shorter, but we already have a few
155  * page_set_*() helpers, so page_collection is used instead to avoid confusion.
156  *
157  * See also: page_collection_lock().
158  */
159 struct page_collection {
160     GTree *tree;
161     struct page_entry *max;
162 };
163 
164 /* list iterators for lists of tagged pointers in TranslationBlock */
165 #define TB_FOR_EACH_TAGGED(head, tb, n, field)                          \
166     for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1);        \
167          tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
168              tb = (TranslationBlock *)((uintptr_t)tb & ~1))
169 
170 #define PAGE_FOR_EACH_TB(pagedesc, tb, n)                       \
171     TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
172 
173 #define TB_FOR_EACH_JMP(head_tb, tb, n)                                 \
174     TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
175 
176 /*
177  * In system mode we want L1_MAP to be based on ram offsets,
178  * while in user mode we want it to be based on virtual addresses.
179  *
180  * TODO: For user mode, see the caveat re host vs guest virtual
181  * address spaces near GUEST_ADDR_MAX.
182  */
183 #if !defined(CONFIG_USER_ONLY)
184 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
185 # define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
186 #else
187 # define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
188 #endif
189 #else
190 # define L1_MAP_ADDR_SPACE_BITS  MIN(HOST_LONG_BITS, TARGET_ABI_BITS)
191 #endif
192 
193 /* Size of the L2 (and L3, etc) page tables.  */
194 #define V_L2_BITS 10
195 #define V_L2_SIZE (1 << V_L2_BITS)
196 
197 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
198 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
199                   sizeof_field(TranslationBlock, trace_vcpu_dstate)
200                   * BITS_PER_BYTE);
201 
202 /*
203  * L1 Mapping properties
204  */
205 static int v_l1_size;
206 static int v_l1_shift;
207 static int v_l2_levels;
208 
209 /* The bottom level has pointers to PageDesc, and is indexed by
210  * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
211  */
212 #define V_L1_MIN_BITS 4
213 #define V_L1_MAX_BITS (V_L2_BITS + 3)
214 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
215 
216 static void *l1_map[V_L1_MAX_SIZE];
217 
218 /* code generation context */
219 TCGContext tcg_init_ctx;
220 __thread TCGContext *tcg_ctx;
221 TBContext tb_ctx;
222 bool parallel_cpus;
223 
224 static void page_table_config_init(void)
225 {
226     uint32_t v_l1_bits;
227 
228     assert(TARGET_PAGE_BITS);
229     /* The bits remaining after N lower levels of page tables.  */
230     v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS;
231     if (v_l1_bits < V_L1_MIN_BITS) {
232         v_l1_bits += V_L2_BITS;
233     }
234 
235     v_l1_size = 1 << v_l1_bits;
236     v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits;
237     v_l2_levels = v_l1_shift / V_L2_BITS - 1;
238 
239     assert(v_l1_bits <= V_L1_MAX_BITS);
240     assert(v_l1_shift % V_L2_BITS == 0);
241     assert(v_l2_levels >= 0);
242 }
243 
244 void cpu_gen_init(void)
245 {
246     tcg_context_init(&tcg_init_ctx);
247 }
248 
249 /* Encode VAL as a signed leb128 sequence at P.
250    Return P incremented past the encoded value.  */
251 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
252 {
253     int more, byte;
254 
255     do {
256         byte = val & 0x7f;
257         val >>= 7;
258         more = !((val == 0 && (byte & 0x40) == 0)
259                  || (val == -1 && (byte & 0x40) != 0));
260         if (more) {
261             byte |= 0x80;
262         }
263         *p++ = byte;
264     } while (more);
265 
266     return p;
267 }
268 
269 /* Decode a signed leb128 sequence at *PP; increment *PP past the
270    decoded value.  Return the decoded value.  */
271 static target_long decode_sleb128(uint8_t **pp)
272 {
273     uint8_t *p = *pp;
274     target_long val = 0;
275     int byte, shift = 0;
276 
277     do {
278         byte = *p++;
279         val |= (target_ulong)(byte & 0x7f) << shift;
280         shift += 7;
281     } while (byte & 0x80);
282     if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
283         val |= -(target_ulong)1 << shift;
284     }
285 
286     *pp = p;
287     return val;
288 }
289 
290 /* Encode the data collected about the instructions while compiling TB.
291    Place the data at BLOCK, and return the number of bytes consumed.
292 
293    The logical table consists of TARGET_INSN_START_WORDS target_ulong's,
294    which come from the target's insn_start data, followed by a uintptr_t
295    which comes from the host pc of the end of the code implementing the insn.
296 
297    Each line of the table is encoded as sleb128 deltas from the previous
298    line.  The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
299    That is, the first column is seeded with the guest pc, the last column
300    with the host pc, and the middle columns with zeros.  */
301 
302 static int encode_search(TranslationBlock *tb, uint8_t *block)
303 {
304     uint8_t *highwater = tcg_ctx->code_gen_highwater;
305     uint8_t *p = block;
306     int i, j, n;
307 
308     for (i = 0, n = tb->icount; i < n; ++i) {
309         target_ulong prev;
310 
311         for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
312             if (i == 0) {
313                 prev = (j == 0 ? tb->pc : 0);
314             } else {
315                 prev = tcg_ctx->gen_insn_data[i - 1][j];
316             }
317             p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev);
318         }
319         prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]);
320         p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev);
321 
322         /* Test for (pending) buffer overflow.  The assumption is that any
323            one row beginning below the high water mark cannot overrun
324            the buffer completely.  Thus we can test for overflow after
325            encoding a row without having to check during encoding.  */
326         if (unlikely(p > highwater)) {
327             return -1;
328         }
329     }
330 
331     return p - block;
332 }
333 
334 /* The cpu state corresponding to 'searched_pc' is restored.
335  * When reset_icount is true, current TB will be interrupted and
336  * icount should be recalculated.
337  */
338 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
339                                      uintptr_t searched_pc, bool reset_icount)
340 {
341     target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
342     uintptr_t host_pc = (uintptr_t)tb->tc.ptr;
343     CPUArchState *env = cpu->env_ptr;
344     uint8_t *p = tb->tc.ptr + tb->tc.size;
345     int i, j, num_insns = tb->icount;
346 #ifdef CONFIG_PROFILER
347     TCGProfile *prof = &tcg_ctx->prof;
348     int64_t ti = profile_getclock();
349 #endif
350 
351     searched_pc -= GETPC_ADJ;
352 
353     if (searched_pc < host_pc) {
354         return -1;
355     }
356 
357     /* Reconstruct the stored insn data while looking for the point at
358        which the end of the insn exceeds the searched_pc.  */
359     for (i = 0; i < num_insns; ++i) {
360         for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
361             data[j] += decode_sleb128(&p);
362         }
363         host_pc += decode_sleb128(&p);
364         if (host_pc > searched_pc) {
365             goto found;
366         }
367     }
368     return -1;
369 
370  found:
371     if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) {
372         assert(use_icount);
373         /* Reset the cycle counter to the start of the block
374            and shift if to the number of actually executed instructions */
375         cpu_neg(cpu)->icount_decr.u16.low += num_insns - i;
376     }
377     restore_state_to_opc(env, tb, data);
378 
379 #ifdef CONFIG_PROFILER
380     atomic_set(&prof->restore_time,
381                 prof->restore_time + profile_getclock() - ti);
382     atomic_set(&prof->restore_count, prof->restore_count + 1);
383 #endif
384     return 0;
385 }
386 
387 void tb_destroy(TranslationBlock *tb)
388 {
389     qemu_spin_destroy(&tb->jmp_lock);
390 }
391 
392 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit)
393 {
394     TranslationBlock *tb;
395     bool r = false;
396     uintptr_t check_offset;
397 
398     /* The host_pc has to be in the region of current code buffer. If
399      * it is not we will not be able to resolve it here. The two cases
400      * where host_pc will not be correct are:
401      *
402      *  - fault during translation (instruction fetch)
403      *  - fault from helper (not using GETPC() macro)
404      *
405      * Either way we need return early as we can't resolve it here.
406      *
407      * We are using unsigned arithmetic so if host_pc <
408      * tcg_init_ctx.code_gen_buffer check_offset will wrap to way
409      * above the code_gen_buffer_size
410      */
411     check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer;
412 
413     if (check_offset < tcg_init_ctx.code_gen_buffer_size) {
414         tb = tcg_tb_lookup(host_pc);
415         if (tb) {
416             cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit);
417             if (tb_cflags(tb) & CF_NOCACHE) {
418                 /* one-shot translation, invalidate it immediately */
419                 tb_phys_invalidate(tb, -1);
420                 tcg_tb_remove(tb);
421                 tb_destroy(tb);
422             }
423             r = true;
424         }
425     }
426 
427     return r;
428 }
429 
430 static void page_init(void)
431 {
432     page_size_init();
433     page_table_config_init();
434 
435 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
436     {
437 #ifdef HAVE_KINFO_GETVMMAP
438         struct kinfo_vmentry *freep;
439         int i, cnt;
440 
441         freep = kinfo_getvmmap(getpid(), &cnt);
442         if (freep) {
443             mmap_lock();
444             for (i = 0; i < cnt; i++) {
445                 unsigned long startaddr, endaddr;
446 
447                 startaddr = freep[i].kve_start;
448                 endaddr = freep[i].kve_end;
449                 if (h2g_valid(startaddr)) {
450                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
451 
452                     if (h2g_valid(endaddr)) {
453                         endaddr = h2g(endaddr);
454                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
455                     } else {
456 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
457                         endaddr = ~0ul;
458                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
459 #endif
460                     }
461                 }
462             }
463             free(freep);
464             mmap_unlock();
465         }
466 #else
467         FILE *f;
468 
469         last_brk = (unsigned long)sbrk(0);
470 
471         f = fopen("/compat/linux/proc/self/maps", "r");
472         if (f) {
473             mmap_lock();
474 
475             do {
476                 unsigned long startaddr, endaddr;
477                 int n;
478 
479                 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
480 
481                 if (n == 2 && h2g_valid(startaddr)) {
482                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
483 
484                     if (h2g_valid(endaddr)) {
485                         endaddr = h2g(endaddr);
486                     } else {
487                         endaddr = ~0ul;
488                     }
489                     page_set_flags(startaddr, endaddr, PAGE_RESERVED);
490                 }
491             } while (!feof(f));
492 
493             fclose(f);
494             mmap_unlock();
495         }
496 #endif
497     }
498 #endif
499 }
500 
501 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
502 {
503     PageDesc *pd;
504     void **lp;
505     int i;
506 
507     /* Level 1.  Always allocated.  */
508     lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1));
509 
510     /* Level 2..N-1.  */
511     for (i = v_l2_levels; i > 0; i--) {
512         void **p = atomic_rcu_read(lp);
513 
514         if (p == NULL) {
515             void *existing;
516 
517             if (!alloc) {
518                 return NULL;
519             }
520             p = g_new0(void *, V_L2_SIZE);
521             existing = atomic_cmpxchg(lp, NULL, p);
522             if (unlikely(existing)) {
523                 g_free(p);
524                 p = existing;
525             }
526         }
527 
528         lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
529     }
530 
531     pd = atomic_rcu_read(lp);
532     if (pd == NULL) {
533         void *existing;
534 
535         if (!alloc) {
536             return NULL;
537         }
538         pd = g_new0(PageDesc, V_L2_SIZE);
539 #ifndef CONFIG_USER_ONLY
540         {
541             int i;
542 
543             for (i = 0; i < V_L2_SIZE; i++) {
544                 qemu_spin_init(&pd[i].lock);
545             }
546         }
547 #endif
548         existing = atomic_cmpxchg(lp, NULL, pd);
549         if (unlikely(existing)) {
550 #ifndef CONFIG_USER_ONLY
551             {
552                 int i;
553 
554                 for (i = 0; i < V_L2_SIZE; i++) {
555                     qemu_spin_destroy(&pd[i].lock);
556                 }
557             }
558 #endif
559             g_free(pd);
560             pd = existing;
561         }
562     }
563 
564     return pd + (index & (V_L2_SIZE - 1));
565 }
566 
567 static inline PageDesc *page_find(tb_page_addr_t index)
568 {
569     return page_find_alloc(index, 0);
570 }
571 
572 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
573                            PageDesc **ret_p2, tb_page_addr_t phys2, int alloc);
574 
575 /* In user-mode page locks aren't used; mmap_lock is enough */
576 #ifdef CONFIG_USER_ONLY
577 
578 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
579 
580 static inline void page_lock(PageDesc *pd)
581 { }
582 
583 static inline void page_unlock(PageDesc *pd)
584 { }
585 
586 static inline void page_lock_tb(const TranslationBlock *tb)
587 { }
588 
589 static inline void page_unlock_tb(const TranslationBlock *tb)
590 { }
591 
592 struct page_collection *
593 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
594 {
595     return NULL;
596 }
597 
598 void page_collection_unlock(struct page_collection *set)
599 { }
600 #else /* !CONFIG_USER_ONLY */
601 
602 #ifdef CONFIG_DEBUG_TCG
603 
604 static __thread GHashTable *ht_pages_locked_debug;
605 
606 static void ht_pages_locked_debug_init(void)
607 {
608     if (ht_pages_locked_debug) {
609         return;
610     }
611     ht_pages_locked_debug = g_hash_table_new(NULL, NULL);
612 }
613 
614 static bool page_is_locked(const PageDesc *pd)
615 {
616     PageDesc *found;
617 
618     ht_pages_locked_debug_init();
619     found = g_hash_table_lookup(ht_pages_locked_debug, pd);
620     return !!found;
621 }
622 
623 static void page_lock__debug(PageDesc *pd)
624 {
625     ht_pages_locked_debug_init();
626     g_assert(!page_is_locked(pd));
627     g_hash_table_insert(ht_pages_locked_debug, pd, pd);
628 }
629 
630 static void page_unlock__debug(const PageDesc *pd)
631 {
632     bool removed;
633 
634     ht_pages_locked_debug_init();
635     g_assert(page_is_locked(pd));
636     removed = g_hash_table_remove(ht_pages_locked_debug, pd);
637     g_assert(removed);
638 }
639 
640 static void
641 do_assert_page_locked(const PageDesc *pd, const char *file, int line)
642 {
643     if (unlikely(!page_is_locked(pd))) {
644         error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
645                      pd, file, line);
646         abort();
647     }
648 }
649 
650 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
651 
652 void assert_no_pages_locked(void)
653 {
654     ht_pages_locked_debug_init();
655     g_assert(g_hash_table_size(ht_pages_locked_debug) == 0);
656 }
657 
658 #else /* !CONFIG_DEBUG_TCG */
659 
660 #define assert_page_locked(pd)
661 
662 static inline void page_lock__debug(const PageDesc *pd)
663 {
664 }
665 
666 static inline void page_unlock__debug(const PageDesc *pd)
667 {
668 }
669 
670 #endif /* CONFIG_DEBUG_TCG */
671 
672 static inline void page_lock(PageDesc *pd)
673 {
674     page_lock__debug(pd);
675     qemu_spin_lock(&pd->lock);
676 }
677 
678 static inline void page_unlock(PageDesc *pd)
679 {
680     qemu_spin_unlock(&pd->lock);
681     page_unlock__debug(pd);
682 }
683 
684 /* lock the page(s) of a TB in the correct acquisition order */
685 static inline void page_lock_tb(const TranslationBlock *tb)
686 {
687     page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], 0);
688 }
689 
690 static inline void page_unlock_tb(const TranslationBlock *tb)
691 {
692     PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
693 
694     page_unlock(p1);
695     if (unlikely(tb->page_addr[1] != -1)) {
696         PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
697 
698         if (p2 != p1) {
699             page_unlock(p2);
700         }
701     }
702 }
703 
704 static inline struct page_entry *
705 page_entry_new(PageDesc *pd, tb_page_addr_t index)
706 {
707     struct page_entry *pe = g_malloc(sizeof(*pe));
708 
709     pe->index = index;
710     pe->pd = pd;
711     pe->locked = false;
712     return pe;
713 }
714 
715 static void page_entry_destroy(gpointer p)
716 {
717     struct page_entry *pe = p;
718 
719     g_assert(pe->locked);
720     page_unlock(pe->pd);
721     g_free(pe);
722 }
723 
724 /* returns false on success */
725 static bool page_entry_trylock(struct page_entry *pe)
726 {
727     bool busy;
728 
729     busy = qemu_spin_trylock(&pe->pd->lock);
730     if (!busy) {
731         g_assert(!pe->locked);
732         pe->locked = true;
733         page_lock__debug(pe->pd);
734     }
735     return busy;
736 }
737 
738 static void do_page_entry_lock(struct page_entry *pe)
739 {
740     page_lock(pe->pd);
741     g_assert(!pe->locked);
742     pe->locked = true;
743 }
744 
745 static gboolean page_entry_lock(gpointer key, gpointer value, gpointer data)
746 {
747     struct page_entry *pe = value;
748 
749     do_page_entry_lock(pe);
750     return FALSE;
751 }
752 
753 static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data)
754 {
755     struct page_entry *pe = value;
756 
757     if (pe->locked) {
758         pe->locked = false;
759         page_unlock(pe->pd);
760     }
761     return FALSE;
762 }
763 
764 /*
765  * Trylock a page, and if successful, add the page to a collection.
766  * Returns true ("busy") if the page could not be locked; false otherwise.
767  */
768 static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
769 {
770     tb_page_addr_t index = addr >> TARGET_PAGE_BITS;
771     struct page_entry *pe;
772     PageDesc *pd;
773 
774     pe = g_tree_lookup(set->tree, &index);
775     if (pe) {
776         return false;
777     }
778 
779     pd = page_find(index);
780     if (pd == NULL) {
781         return false;
782     }
783 
784     pe = page_entry_new(pd, index);
785     g_tree_insert(set->tree, &pe->index, pe);
786 
787     /*
788      * If this is either (1) the first insertion or (2) a page whose index
789      * is higher than any other so far, just lock the page and move on.
790      */
791     if (set->max == NULL || pe->index > set->max->index) {
792         set->max = pe;
793         do_page_entry_lock(pe);
794         return false;
795     }
796     /*
797      * Try to acquire out-of-order lock; if busy, return busy so that we acquire
798      * locks in order.
799      */
800     return page_entry_trylock(pe);
801 }
802 
803 static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata)
804 {
805     tb_page_addr_t a = *(const tb_page_addr_t *)ap;
806     tb_page_addr_t b = *(const tb_page_addr_t *)bp;
807 
808     if (a == b) {
809         return 0;
810     } else if (a < b) {
811         return -1;
812     }
813     return 1;
814 }
815 
816 /*
817  * Lock a range of pages ([@start,@end[) as well as the pages of all
818  * intersecting TBs.
819  * Locking order: acquire locks in ascending order of page index.
820  */
821 struct page_collection *
822 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
823 {
824     struct page_collection *set = g_malloc(sizeof(*set));
825     tb_page_addr_t index;
826     PageDesc *pd;
827 
828     start >>= TARGET_PAGE_BITS;
829     end   >>= TARGET_PAGE_BITS;
830     g_assert(start <= end);
831 
832     set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL,
833                                 page_entry_destroy);
834     set->max = NULL;
835     assert_no_pages_locked();
836 
837  retry:
838     g_tree_foreach(set->tree, page_entry_lock, NULL);
839 
840     for (index = start; index <= end; index++) {
841         TranslationBlock *tb;
842         int n;
843 
844         pd = page_find(index);
845         if (pd == NULL) {
846             continue;
847         }
848         if (page_trylock_add(set, index << TARGET_PAGE_BITS)) {
849             g_tree_foreach(set->tree, page_entry_unlock, NULL);
850             goto retry;
851         }
852         assert_page_locked(pd);
853         PAGE_FOR_EACH_TB(pd, tb, n) {
854             if (page_trylock_add(set, tb->page_addr[0]) ||
855                 (tb->page_addr[1] != -1 &&
856                  page_trylock_add(set, tb->page_addr[1]))) {
857                 /* drop all locks, and reacquire in order */
858                 g_tree_foreach(set->tree, page_entry_unlock, NULL);
859                 goto retry;
860             }
861         }
862     }
863     return set;
864 }
865 
866 void page_collection_unlock(struct page_collection *set)
867 {
868     /* entries are unlocked and freed via page_entry_destroy */
869     g_tree_destroy(set->tree);
870     g_free(set);
871 }
872 
873 #endif /* !CONFIG_USER_ONLY */
874 
875 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
876                            PageDesc **ret_p2, tb_page_addr_t phys2, int alloc)
877 {
878     PageDesc *p1, *p2;
879     tb_page_addr_t page1;
880     tb_page_addr_t page2;
881 
882     assert_memory_lock();
883     g_assert(phys1 != -1);
884 
885     page1 = phys1 >> TARGET_PAGE_BITS;
886     page2 = phys2 >> TARGET_PAGE_BITS;
887 
888     p1 = page_find_alloc(page1, alloc);
889     if (ret_p1) {
890         *ret_p1 = p1;
891     }
892     if (likely(phys2 == -1)) {
893         page_lock(p1);
894         return;
895     } else if (page1 == page2) {
896         page_lock(p1);
897         if (ret_p2) {
898             *ret_p2 = p1;
899         }
900         return;
901     }
902     p2 = page_find_alloc(page2, alloc);
903     if (ret_p2) {
904         *ret_p2 = p2;
905     }
906     if (page1 < page2) {
907         page_lock(p1);
908         page_lock(p2);
909     } else {
910         page_lock(p2);
911         page_lock(p1);
912     }
913 }
914 
915 /* Minimum size of the code gen buffer.  This number is randomly chosen,
916    but not so small that we can't have a fair number of TB's live.  */
917 #define MIN_CODE_GEN_BUFFER_SIZE     (1 * MiB)
918 
919 /* Maximum size of the code gen buffer we'd like to use.  Unless otherwise
920    indicated, this is constrained by the range of direct branches on the
921    host cpu, as used by the TCG implementation of goto_tb.  */
922 #if defined(__x86_64__)
923 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
924 #elif defined(__sparc__)
925 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
926 #elif defined(__powerpc64__)
927 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
928 #elif defined(__powerpc__)
929 # define MAX_CODE_GEN_BUFFER_SIZE  (32 * MiB)
930 #elif defined(__aarch64__)
931 # define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
932 #elif defined(__s390x__)
933   /* We have a +- 4GB range on the branches; leave some slop.  */
934 # define MAX_CODE_GEN_BUFFER_SIZE  (3 * GiB)
935 #elif defined(__mips__)
936   /* We have a 256MB branch region, but leave room to make sure the
937      main executable is also within that region.  */
938 # define MAX_CODE_GEN_BUFFER_SIZE  (128 * MiB)
939 #else
940 # define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
941 #endif
942 
943 #if TCG_TARGET_REG_BITS == 32
944 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32 * MiB)
945 #ifdef CONFIG_USER_ONLY
946 /*
947  * For user mode on smaller 32 bit systems we may run into trouble
948  * allocating big chunks of data in the right place. On these systems
949  * we utilise a static code generation buffer directly in the binary.
950  */
951 #define USE_STATIC_CODE_GEN_BUFFER
952 #endif
953 #else /* TCG_TARGET_REG_BITS == 64 */
954 #ifdef CONFIG_USER_ONLY
955 /*
956  * As user-mode emulation typically means running multiple instances
957  * of the translator don't go too nuts with our default code gen
958  * buffer lest we make things too hard for the OS.
959  */
960 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (128 * MiB)
961 #else
962 /*
963  * We expect most system emulation to run one or two guests per host.
964  * Users running large scale system emulation may want to tweak their
965  * runtime setup via the tb-size control on the command line.
966  */
967 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (1 * GiB)
968 #endif
969 #endif
970 
971 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
972   (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
973    ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
974 
975 static inline size_t size_code_gen_buffer(size_t tb_size)
976 {
977     /* Size the buffer.  */
978     if (tb_size == 0) {
979         tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
980     }
981     if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
982         tb_size = MIN_CODE_GEN_BUFFER_SIZE;
983     }
984     if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
985         tb_size = MAX_CODE_GEN_BUFFER_SIZE;
986     }
987     return tb_size;
988 }
989 
990 #ifdef __mips__
991 /* In order to use J and JAL within the code_gen_buffer, we require
992    that the buffer not cross a 256MB boundary.  */
993 static inline bool cross_256mb(void *addr, size_t size)
994 {
995     return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
996 }
997 
998 /* We weren't able to allocate a buffer without crossing that boundary,
999    so make do with the larger portion of the buffer that doesn't cross.
1000    Returns the new base of the buffer, and adjusts code_gen_buffer_size.  */
1001 static inline void *split_cross_256mb(void *buf1, size_t size1)
1002 {
1003     void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
1004     size_t size2 = buf1 + size1 - buf2;
1005 
1006     size1 = buf2 - buf1;
1007     if (size1 < size2) {
1008         size1 = size2;
1009         buf1 = buf2;
1010     }
1011 
1012     tcg_ctx->code_gen_buffer_size = size1;
1013     return buf1;
1014 }
1015 #endif
1016 
1017 #ifdef USE_STATIC_CODE_GEN_BUFFER
1018 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
1019     __attribute__((aligned(CODE_GEN_ALIGN)));
1020 
1021 static inline void *alloc_code_gen_buffer(void)
1022 {
1023     void *buf = static_code_gen_buffer;
1024     void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer);
1025     size_t size;
1026 
1027     /* page-align the beginning and end of the buffer */
1028     buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size);
1029     end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size);
1030 
1031     size = end - buf;
1032 
1033     /* Honor a command-line option limiting the size of the buffer.  */
1034     if (size > tcg_ctx->code_gen_buffer_size) {
1035         size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size,
1036                                qemu_real_host_page_size);
1037     }
1038     tcg_ctx->code_gen_buffer_size = size;
1039 
1040 #ifdef __mips__
1041     if (cross_256mb(buf, size)) {
1042         buf = split_cross_256mb(buf, size);
1043         size = tcg_ctx->code_gen_buffer_size;
1044     }
1045 #endif
1046 
1047     if (qemu_mprotect_rwx(buf, size)) {
1048         abort();
1049     }
1050     qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1051 
1052     return buf;
1053 }
1054 #elif defined(_WIN32)
1055 static inline void *alloc_code_gen_buffer(void)
1056 {
1057     size_t size = tcg_ctx->code_gen_buffer_size;
1058     return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT,
1059                         PAGE_EXECUTE_READWRITE);
1060 }
1061 #else
1062 static inline void *alloc_code_gen_buffer(void)
1063 {
1064     int prot = PROT_WRITE | PROT_READ | PROT_EXEC;
1065     int flags = MAP_PRIVATE | MAP_ANONYMOUS;
1066     size_t size = tcg_ctx->code_gen_buffer_size;
1067     void *buf;
1068 
1069     buf = mmap(NULL, size, prot, flags, -1, 0);
1070     if (buf == MAP_FAILED) {
1071         return NULL;
1072     }
1073 
1074 #ifdef __mips__
1075     if (cross_256mb(buf, size)) {
1076         /*
1077          * Try again, with the original still mapped, to avoid re-acquiring
1078          * the same 256mb crossing.
1079          */
1080         size_t size2;
1081         void *buf2 = mmap(NULL, size, prot, flags, -1, 0);
1082         switch ((int)(buf2 != MAP_FAILED)) {
1083         case 1:
1084             if (!cross_256mb(buf2, size)) {
1085                 /* Success!  Use the new buffer.  */
1086                 munmap(buf, size);
1087                 break;
1088             }
1089             /* Failure.  Work with what we had.  */
1090             munmap(buf2, size);
1091             /* fallthru */
1092         default:
1093             /* Split the original buffer.  Free the smaller half.  */
1094             buf2 = split_cross_256mb(buf, size);
1095             size2 = tcg_ctx->code_gen_buffer_size;
1096             if (buf == buf2) {
1097                 munmap(buf + size2, size - size2);
1098             } else {
1099                 munmap(buf, size - size2);
1100             }
1101             size = size2;
1102             break;
1103         }
1104         buf = buf2;
1105     }
1106 #endif
1107 
1108     /* Request large pages for the buffer.  */
1109     qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1110 
1111     return buf;
1112 }
1113 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
1114 
1115 static inline void code_gen_alloc(size_t tb_size)
1116 {
1117     tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size);
1118     tcg_ctx->code_gen_buffer = alloc_code_gen_buffer();
1119     if (tcg_ctx->code_gen_buffer == NULL) {
1120         fprintf(stderr, "Could not allocate dynamic translator buffer\n");
1121         exit(1);
1122     }
1123 }
1124 
1125 static bool tb_cmp(const void *ap, const void *bp)
1126 {
1127     const TranslationBlock *a = ap;
1128     const TranslationBlock *b = bp;
1129 
1130     return a->pc == b->pc &&
1131         a->cs_base == b->cs_base &&
1132         a->flags == b->flags &&
1133         (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) &&
1134         a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
1135         a->page_addr[0] == b->page_addr[0] &&
1136         a->page_addr[1] == b->page_addr[1];
1137 }
1138 
1139 static void tb_htable_init(void)
1140 {
1141     unsigned int mode = QHT_MODE_AUTO_RESIZE;
1142 
1143     qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
1144 }
1145 
1146 /* Must be called before using the QEMU cpus. 'tb_size' is the size
1147    (in bytes) allocated to the translation buffer. Zero means default
1148    size. */
1149 void tcg_exec_init(unsigned long tb_size)
1150 {
1151     tcg_allowed = true;
1152     cpu_gen_init();
1153     page_init();
1154     tb_htable_init();
1155     code_gen_alloc(tb_size);
1156 #if defined(CONFIG_SOFTMMU)
1157     /* There's no guest base to take into account, so go ahead and
1158        initialize the prologue now.  */
1159     tcg_prologue_init(tcg_ctx);
1160 #endif
1161 }
1162 
1163 /* call with @p->lock held */
1164 static inline void invalidate_page_bitmap(PageDesc *p)
1165 {
1166     assert_page_locked(p);
1167 #ifdef CONFIG_SOFTMMU
1168     g_free(p->code_bitmap);
1169     p->code_bitmap = NULL;
1170     p->code_write_count = 0;
1171 #endif
1172 }
1173 
1174 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
1175 static void page_flush_tb_1(int level, void **lp)
1176 {
1177     int i;
1178 
1179     if (*lp == NULL) {
1180         return;
1181     }
1182     if (level == 0) {
1183         PageDesc *pd = *lp;
1184 
1185         for (i = 0; i < V_L2_SIZE; ++i) {
1186             page_lock(&pd[i]);
1187             pd[i].first_tb = (uintptr_t)NULL;
1188             invalidate_page_bitmap(pd + i);
1189             page_unlock(&pd[i]);
1190         }
1191     } else {
1192         void **pp = *lp;
1193 
1194         for (i = 0; i < V_L2_SIZE; ++i) {
1195             page_flush_tb_1(level - 1, pp + i);
1196         }
1197     }
1198 }
1199 
1200 static void page_flush_tb(void)
1201 {
1202     int i, l1_sz = v_l1_size;
1203 
1204     for (i = 0; i < l1_sz; i++) {
1205         page_flush_tb_1(v_l2_levels, l1_map + i);
1206     }
1207 }
1208 
1209 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data)
1210 {
1211     const TranslationBlock *tb = value;
1212     size_t *size = data;
1213 
1214     *size += tb->tc.size;
1215     return false;
1216 }
1217 
1218 /* flush all the translation blocks */
1219 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
1220 {
1221     bool did_flush = false;
1222 
1223     mmap_lock();
1224     /* If it is already been done on request of another CPU,
1225      * just retry.
1226      */
1227     if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
1228         goto done;
1229     }
1230     did_flush = true;
1231 
1232     if (DEBUG_TB_FLUSH_GATE) {
1233         size_t nb_tbs = tcg_nb_tbs();
1234         size_t host_size = 0;
1235 
1236         tcg_tb_foreach(tb_host_size_iter, &host_size);
1237         printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
1238                tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0);
1239     }
1240 
1241     CPU_FOREACH(cpu) {
1242         cpu_tb_jmp_cache_clear(cpu);
1243     }
1244 
1245     qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
1246     page_flush_tb();
1247 
1248     tcg_region_reset_all();
1249     /* XXX: flush processor icache at this point if cache flush is
1250        expensive */
1251     atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
1252 
1253 done:
1254     mmap_unlock();
1255     if (did_flush) {
1256         qemu_plugin_flush_cb();
1257     }
1258 }
1259 
1260 void tb_flush(CPUState *cpu)
1261 {
1262     if (tcg_enabled()) {
1263         unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count);
1264 
1265         if (cpu_in_exclusive_context(cpu)) {
1266             do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
1267         } else {
1268             async_safe_run_on_cpu(cpu, do_tb_flush,
1269                                   RUN_ON_CPU_HOST_INT(tb_flush_count));
1270         }
1271     }
1272 }
1273 
1274 /*
1275  * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
1276  * so in order to prevent bit rot we compile them unconditionally in user-mode,
1277  * and let the optimizer get rid of them by wrapping their user-only callers
1278  * with if (DEBUG_TB_CHECK_GATE).
1279  */
1280 #ifdef CONFIG_USER_ONLY
1281 
1282 static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp)
1283 {
1284     TranslationBlock *tb = p;
1285     target_ulong addr = *(target_ulong *)userp;
1286 
1287     if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
1288         printf("ERROR invalidate: address=" TARGET_FMT_lx
1289                " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
1290     }
1291 }
1292 
1293 /* verify that all the pages have correct rights for code
1294  *
1295  * Called with mmap_lock held.
1296  */
1297 static void tb_invalidate_check(target_ulong address)
1298 {
1299     address &= TARGET_PAGE_MASK;
1300     qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address);
1301 }
1302 
1303 static void do_tb_page_check(void *p, uint32_t hash, void *userp)
1304 {
1305     TranslationBlock *tb = p;
1306     int flags1, flags2;
1307 
1308     flags1 = page_get_flags(tb->pc);
1309     flags2 = page_get_flags(tb->pc + tb->size - 1);
1310     if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
1311         printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1312                (long)tb->pc, tb->size, flags1, flags2);
1313     }
1314 }
1315 
1316 /* verify that all the pages have correct rights for code */
1317 static void tb_page_check(void)
1318 {
1319     qht_iter(&tb_ctx.htable, do_tb_page_check, NULL);
1320 }
1321 
1322 #endif /* CONFIG_USER_ONLY */
1323 
1324 /*
1325  * user-mode: call with mmap_lock held
1326  * !user-mode: call with @pd->lock held
1327  */
1328 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
1329 {
1330     TranslationBlock *tb1;
1331     uintptr_t *pprev;
1332     unsigned int n1;
1333 
1334     assert_page_locked(pd);
1335     pprev = &pd->first_tb;
1336     PAGE_FOR_EACH_TB(pd, tb1, n1) {
1337         if (tb1 == tb) {
1338             *pprev = tb1->page_next[n1];
1339             return;
1340         }
1341         pprev = &tb1->page_next[n1];
1342     }
1343     g_assert_not_reached();
1344 }
1345 
1346 /* remove @orig from its @n_orig-th jump list */
1347 static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
1348 {
1349     uintptr_t ptr, ptr_locked;
1350     TranslationBlock *dest;
1351     TranslationBlock *tb;
1352     uintptr_t *pprev;
1353     int n;
1354 
1355     /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
1356     ptr = atomic_or_fetch(&orig->jmp_dest[n_orig], 1);
1357     dest = (TranslationBlock *)(ptr & ~1);
1358     if (dest == NULL) {
1359         return;
1360     }
1361 
1362     qemu_spin_lock(&dest->jmp_lock);
1363     /*
1364      * While acquiring the lock, the jump might have been removed if the
1365      * destination TB was invalidated; check again.
1366      */
1367     ptr_locked = atomic_read(&orig->jmp_dest[n_orig]);
1368     if (ptr_locked != ptr) {
1369         qemu_spin_unlock(&dest->jmp_lock);
1370         /*
1371          * The only possibility is that the jump was unlinked via
1372          * tb_jump_unlink(dest). Seeing here another destination would be a bug,
1373          * because we set the LSB above.
1374          */
1375         g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
1376         return;
1377     }
1378     /*
1379      * We first acquired the lock, and since the destination pointer matches,
1380      * we know for sure that @orig is in the jmp list.
1381      */
1382     pprev = &dest->jmp_list_head;
1383     TB_FOR_EACH_JMP(dest, tb, n) {
1384         if (tb == orig && n == n_orig) {
1385             *pprev = tb->jmp_list_next[n];
1386             /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
1387             qemu_spin_unlock(&dest->jmp_lock);
1388             return;
1389         }
1390         pprev = &tb->jmp_list_next[n];
1391     }
1392     g_assert_not_reached();
1393 }
1394 
1395 /* reset the jump entry 'n' of a TB so that it is not chained to
1396    another TB */
1397 static inline void tb_reset_jump(TranslationBlock *tb, int n)
1398 {
1399     uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
1400     tb_set_jmp_target(tb, n, addr);
1401 }
1402 
1403 /* remove any jumps to the TB */
1404 static inline void tb_jmp_unlink(TranslationBlock *dest)
1405 {
1406     TranslationBlock *tb;
1407     int n;
1408 
1409     qemu_spin_lock(&dest->jmp_lock);
1410 
1411     TB_FOR_EACH_JMP(dest, tb, n) {
1412         tb_reset_jump(tb, n);
1413         atomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
1414         /* No need to clear the list entry; setting the dest ptr is enough */
1415     }
1416     dest->jmp_list_head = (uintptr_t)NULL;
1417 
1418     qemu_spin_unlock(&dest->jmp_lock);
1419 }
1420 
1421 /*
1422  * In user-mode, call with mmap_lock held.
1423  * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
1424  * locks held.
1425  */
1426 static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
1427 {
1428     CPUState *cpu;
1429     PageDesc *p;
1430     uint32_t h;
1431     tb_page_addr_t phys_pc;
1432 
1433     assert_memory_lock();
1434 
1435     /* make sure no further incoming jumps will be chained to this TB */
1436     qemu_spin_lock(&tb->jmp_lock);
1437     atomic_set(&tb->cflags, tb->cflags | CF_INVALID);
1438     qemu_spin_unlock(&tb->jmp_lock);
1439 
1440     /* remove the TB from the hash list */
1441     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1442     h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb_cflags(tb) & CF_HASH_MASK,
1443                      tb->trace_vcpu_dstate);
1444     if (!(tb->cflags & CF_NOCACHE) &&
1445         !qht_remove(&tb_ctx.htable, tb, h)) {
1446         return;
1447     }
1448 
1449     /* remove the TB from the page list */
1450     if (rm_from_page_list) {
1451         p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1452         tb_page_remove(p, tb);
1453         invalidate_page_bitmap(p);
1454         if (tb->page_addr[1] != -1) {
1455             p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1456             tb_page_remove(p, tb);
1457             invalidate_page_bitmap(p);
1458         }
1459     }
1460 
1461     /* remove the TB from the hash list */
1462     h = tb_jmp_cache_hash_func(tb->pc);
1463     CPU_FOREACH(cpu) {
1464         if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) {
1465             atomic_set(&cpu->tb_jmp_cache[h], NULL);
1466         }
1467     }
1468 
1469     /* suppress this TB from the two jump lists */
1470     tb_remove_from_jmp_list(tb, 0);
1471     tb_remove_from_jmp_list(tb, 1);
1472 
1473     /* suppress any remaining jumps to this TB */
1474     tb_jmp_unlink(tb);
1475 
1476     atomic_set(&tcg_ctx->tb_phys_invalidate_count,
1477                tcg_ctx->tb_phys_invalidate_count + 1);
1478 }
1479 
1480 static void tb_phys_invalidate__locked(TranslationBlock *tb)
1481 {
1482     do_tb_phys_invalidate(tb, true);
1483 }
1484 
1485 /* invalidate one TB
1486  *
1487  * Called with mmap_lock held in user-mode.
1488  */
1489 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1490 {
1491     if (page_addr == -1 && tb->page_addr[0] != -1) {
1492         page_lock_tb(tb);
1493         do_tb_phys_invalidate(tb, true);
1494         page_unlock_tb(tb);
1495     } else {
1496         do_tb_phys_invalidate(tb, false);
1497     }
1498 }
1499 
1500 #ifdef CONFIG_SOFTMMU
1501 /* call with @p->lock held */
1502 static void build_page_bitmap(PageDesc *p)
1503 {
1504     int n, tb_start, tb_end;
1505     TranslationBlock *tb;
1506 
1507     assert_page_locked(p);
1508     p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1509 
1510     PAGE_FOR_EACH_TB(p, tb, n) {
1511         /* NOTE: this is subtle as a TB may span two physical pages */
1512         if (n == 0) {
1513             /* NOTE: tb_end may be after the end of the page, but
1514                it is not a problem */
1515             tb_start = tb->pc & ~TARGET_PAGE_MASK;
1516             tb_end = tb_start + tb->size;
1517             if (tb_end > TARGET_PAGE_SIZE) {
1518                 tb_end = TARGET_PAGE_SIZE;
1519              }
1520         } else {
1521             tb_start = 0;
1522             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1523         }
1524         bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1525     }
1526 }
1527 #endif
1528 
1529 /* add the tb in the target page and protect it if necessary
1530  *
1531  * Called with mmap_lock held for user-mode emulation.
1532  * Called with @p->lock held in !user-mode.
1533  */
1534 static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
1535                                unsigned int n, tb_page_addr_t page_addr)
1536 {
1537 #ifndef CONFIG_USER_ONLY
1538     bool page_already_protected;
1539 #endif
1540 
1541     assert_page_locked(p);
1542 
1543     tb->page_addr[n] = page_addr;
1544     tb->page_next[n] = p->first_tb;
1545 #ifndef CONFIG_USER_ONLY
1546     page_already_protected = p->first_tb != (uintptr_t)NULL;
1547 #endif
1548     p->first_tb = (uintptr_t)tb | n;
1549     invalidate_page_bitmap(p);
1550 
1551 #if defined(CONFIG_USER_ONLY)
1552     if (p->flags & PAGE_WRITE) {
1553         target_ulong addr;
1554         PageDesc *p2;
1555         int prot;
1556 
1557         /* force the host page as non writable (writes will have a
1558            page fault + mprotect overhead) */
1559         page_addr &= qemu_host_page_mask;
1560         prot = 0;
1561         for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1562             addr += TARGET_PAGE_SIZE) {
1563 
1564             p2 = page_find(addr >> TARGET_PAGE_BITS);
1565             if (!p2) {
1566                 continue;
1567             }
1568             prot |= p2->flags;
1569             p2->flags &= ~PAGE_WRITE;
1570           }
1571         mprotect(g2h(page_addr), qemu_host_page_size,
1572                  (prot & PAGE_BITS) & ~PAGE_WRITE);
1573         if (DEBUG_TB_INVALIDATE_GATE) {
1574             printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr);
1575         }
1576     }
1577 #else
1578     /* if some code is already present, then the pages are already
1579        protected. So we handle the case where only the first TB is
1580        allocated in a physical page */
1581     if (!page_already_protected) {
1582         tlb_protect_code(page_addr);
1583     }
1584 #endif
1585 }
1586 
1587 /* add a new TB and link it to the physical page tables. phys_page2 is
1588  * (-1) to indicate that only one page contains the TB.
1589  *
1590  * Called with mmap_lock held for user-mode emulation.
1591  *
1592  * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
1593  * Note that in !user-mode, another thread might have already added a TB
1594  * for the same block of guest code that @tb corresponds to. In that case,
1595  * the caller should discard the original @tb, and use instead the returned TB.
1596  */
1597 static TranslationBlock *
1598 tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1599              tb_page_addr_t phys_page2)
1600 {
1601     PageDesc *p;
1602     PageDesc *p2 = NULL;
1603 
1604     assert_memory_lock();
1605 
1606     if (phys_pc == -1) {
1607         /*
1608          * If the TB is not associated with a physical RAM page then
1609          * it must be a temporary one-insn TB, and we have nothing to do
1610          * except fill in the page_addr[] fields.
1611          */
1612         assert(tb->cflags & CF_NOCACHE);
1613         tb->page_addr[0] = tb->page_addr[1] = -1;
1614         return tb;
1615     }
1616 
1617     /*
1618      * Add the TB to the page list, acquiring first the pages's locks.
1619      * We keep the locks held until after inserting the TB in the hash table,
1620      * so that if the insertion fails we know for sure that the TBs are still
1621      * in the page descriptors.
1622      * Note that inserting into the hash table first isn't an option, since
1623      * we can only insert TBs that are fully initialized.
1624      */
1625     page_lock_pair(&p, phys_pc, &p2, phys_page2, 1);
1626     tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK);
1627     if (p2) {
1628         tb_page_add(p2, tb, 1, phys_page2);
1629     } else {
1630         tb->page_addr[1] = -1;
1631     }
1632 
1633     if (!(tb->cflags & CF_NOCACHE)) {
1634         void *existing_tb = NULL;
1635         uint32_t h;
1636 
1637         /* add in the hash table */
1638         h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK,
1639                          tb->trace_vcpu_dstate);
1640         qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
1641 
1642         /* remove TB from the page(s) if we couldn't insert it */
1643         if (unlikely(existing_tb)) {
1644             tb_page_remove(p, tb);
1645             invalidate_page_bitmap(p);
1646             if (p2) {
1647                 tb_page_remove(p2, tb);
1648                 invalidate_page_bitmap(p2);
1649             }
1650             tb = existing_tb;
1651         }
1652     }
1653 
1654     if (p2 && p2 != p) {
1655         page_unlock(p2);
1656     }
1657     page_unlock(p);
1658 
1659 #ifdef CONFIG_USER_ONLY
1660     if (DEBUG_TB_CHECK_GATE) {
1661         tb_page_check();
1662     }
1663 #endif
1664     return tb;
1665 }
1666 
1667 /* Called with mmap_lock held for user mode emulation.  */
1668 TranslationBlock *tb_gen_code(CPUState *cpu,
1669                               target_ulong pc, target_ulong cs_base,
1670                               uint32_t flags, int cflags)
1671 {
1672     CPUArchState *env = cpu->env_ptr;
1673     TranslationBlock *tb, *existing_tb;
1674     tb_page_addr_t phys_pc, phys_page2;
1675     target_ulong virt_page2;
1676     tcg_insn_unit *gen_code_buf;
1677     int gen_code_size, search_size, max_insns;
1678 #ifdef CONFIG_PROFILER
1679     TCGProfile *prof = &tcg_ctx->prof;
1680     int64_t ti;
1681 #endif
1682 
1683     assert_memory_lock();
1684 
1685     phys_pc = get_page_addr_code(env, pc);
1686 
1687     if (phys_pc == -1) {
1688         /* Generate a temporary TB with 1 insn in it */
1689         cflags &= ~CF_COUNT_MASK;
1690         cflags |= CF_NOCACHE | 1;
1691     }
1692 
1693     cflags &= ~CF_CLUSTER_MASK;
1694     cflags |= cpu->cluster_index << CF_CLUSTER_SHIFT;
1695 
1696     max_insns = cflags & CF_COUNT_MASK;
1697     if (max_insns == 0) {
1698         max_insns = CF_COUNT_MASK;
1699     }
1700     if (max_insns > TCG_MAX_INSNS) {
1701         max_insns = TCG_MAX_INSNS;
1702     }
1703     if (cpu->singlestep_enabled || singlestep) {
1704         max_insns = 1;
1705     }
1706 
1707  buffer_overflow:
1708     tb = tcg_tb_alloc(tcg_ctx);
1709     if (unlikely(!tb)) {
1710         /* flush must be done */
1711         tb_flush(cpu);
1712         mmap_unlock();
1713         /* Make the execution loop process the flush as soon as possible.  */
1714         cpu->exception_index = EXCP_INTERRUPT;
1715         cpu_loop_exit(cpu);
1716     }
1717 
1718     gen_code_buf = tcg_ctx->code_gen_ptr;
1719     tb->tc.ptr = gen_code_buf;
1720     tb->pc = pc;
1721     tb->cs_base = cs_base;
1722     tb->flags = flags;
1723     tb->cflags = cflags;
1724     tb->orig_tb = NULL;
1725     tb->trace_vcpu_dstate = *cpu->trace_dstate;
1726     tcg_ctx->tb_cflags = cflags;
1727  tb_overflow:
1728 
1729 #ifdef CONFIG_PROFILER
1730     /* includes aborted translations because of exceptions */
1731     atomic_set(&prof->tb_count1, prof->tb_count1 + 1);
1732     ti = profile_getclock();
1733 #endif
1734 
1735     tcg_func_start(tcg_ctx);
1736 
1737     tcg_ctx->cpu = env_cpu(env);
1738     gen_intermediate_code(cpu, tb, max_insns);
1739     tcg_ctx->cpu = NULL;
1740 
1741     trace_translate_block(tb, tb->pc, tb->tc.ptr);
1742 
1743     /* generate machine code */
1744     tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1745     tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1746     tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
1747     if (TCG_TARGET_HAS_direct_jump) {
1748         tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
1749         tcg_ctx->tb_jmp_target_addr = NULL;
1750     } else {
1751         tcg_ctx->tb_jmp_insn_offset = NULL;
1752         tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
1753     }
1754 
1755 #ifdef CONFIG_PROFILER
1756     atomic_set(&prof->tb_count, prof->tb_count + 1);
1757     atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti);
1758     ti = profile_getclock();
1759 #endif
1760 
1761     gen_code_size = tcg_gen_code(tcg_ctx, tb);
1762     if (unlikely(gen_code_size < 0)) {
1763         switch (gen_code_size) {
1764         case -1:
1765             /*
1766              * Overflow of code_gen_buffer, or the current slice of it.
1767              *
1768              * TODO: We don't need to re-do gen_intermediate_code, nor
1769              * should we re-do the tcg optimization currently hidden
1770              * inside tcg_gen_code.  All that should be required is to
1771              * flush the TBs, allocate a new TB, re-initialize it per
1772              * above, and re-do the actual code generation.
1773              */
1774             goto buffer_overflow;
1775 
1776         case -2:
1777             /*
1778              * The code generated for the TranslationBlock is too large.
1779              * The maximum size allowed by the unwind info is 64k.
1780              * There may be stricter constraints from relocations
1781              * in the tcg backend.
1782              *
1783              * Try again with half as many insns as we attempted this time.
1784              * If a single insn overflows, there's a bug somewhere...
1785              */
1786             max_insns = tb->icount;
1787             assert(max_insns > 1);
1788             max_insns /= 2;
1789             goto tb_overflow;
1790 
1791         default:
1792             g_assert_not_reached();
1793         }
1794     }
1795     search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1796     if (unlikely(search_size < 0)) {
1797         goto buffer_overflow;
1798     }
1799     tb->tc.size = gen_code_size;
1800 
1801 #ifdef CONFIG_PROFILER
1802     atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti);
1803     atomic_set(&prof->code_in_len, prof->code_in_len + tb->size);
1804     atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size);
1805     atomic_set(&prof->search_out_len, prof->search_out_len + search_size);
1806 #endif
1807 
1808 #ifdef DEBUG_DISAS
1809     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1810         qemu_log_in_addr_range(tb->pc)) {
1811         FILE *logfile = qemu_log_lock();
1812         int code_size, data_size = 0;
1813         g_autoptr(GString) note = g_string_new("[tb header & initial instruction]");
1814         size_t chunk_start = 0;
1815         int insn = 0;
1816         qemu_log("OUT: [size=%d]\n", gen_code_size);
1817         if (tcg_ctx->data_gen_ptr) {
1818             code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr;
1819             data_size = gen_code_size - code_size;
1820         } else {
1821             code_size = gen_code_size;
1822         }
1823 
1824         /* Dump header and the first instruction */
1825         chunk_start = tcg_ctx->gen_insn_end_off[insn];
1826         log_disas(tb->tc.ptr, chunk_start, note->str);
1827 
1828         /*
1829          * Dump each instruction chunk, wrapping up empty chunks into
1830          * the next instruction. The whole array is offset so the
1831          * first entry is the beginning of the 2nd instruction.
1832          */
1833         while (insn <= tb->icount && chunk_start < code_size) {
1834             size_t chunk_end = tcg_ctx->gen_insn_end_off[insn];
1835             if (chunk_end > chunk_start) {
1836                 g_string_printf(note, "[guest addr: " TARGET_FMT_lx "]",
1837                                 tcg_ctx->gen_insn_data[insn][0]);
1838                 log_disas(tb->tc.ptr + chunk_start, chunk_end - chunk_start,
1839                           note->str);
1840                 chunk_start = chunk_end;
1841             }
1842             insn++;
1843         }
1844 
1845         /* Finally dump any data we may have after the block */
1846         if (data_size) {
1847             int i;
1848             qemu_log("  data: [size=%d]\n", data_size);
1849             for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1850                 if (sizeof(tcg_target_ulong) == 8) {
1851                     qemu_log("0x%08" PRIxPTR ":  .quad  0x%016" PRIx64 "\n",
1852                              (uintptr_t)tcg_ctx->data_gen_ptr + i,
1853                              *(uint64_t *)(tcg_ctx->data_gen_ptr + i));
1854                 } else {
1855                     qemu_log("0x%08" PRIxPTR ":  .long  0x%08x\n",
1856                              (uintptr_t)tcg_ctx->data_gen_ptr + i,
1857                              *(uint32_t *)(tcg_ctx->data_gen_ptr + i));
1858                 }
1859             }
1860         }
1861         qemu_log("\n");
1862         qemu_log_flush();
1863         qemu_log_unlock(logfile);
1864     }
1865 #endif
1866 
1867     atomic_set(&tcg_ctx->code_gen_ptr, (void *)
1868         ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1869                  CODE_GEN_ALIGN));
1870 
1871     /* init jump list */
1872     qemu_spin_init(&tb->jmp_lock);
1873     tb->jmp_list_head = (uintptr_t)NULL;
1874     tb->jmp_list_next[0] = (uintptr_t)NULL;
1875     tb->jmp_list_next[1] = (uintptr_t)NULL;
1876     tb->jmp_dest[0] = (uintptr_t)NULL;
1877     tb->jmp_dest[1] = (uintptr_t)NULL;
1878 
1879     /* init original jump addresses which have been set during tcg_gen_code() */
1880     if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1881         tb_reset_jump(tb, 0);
1882     }
1883     if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1884         tb_reset_jump(tb, 1);
1885     }
1886 
1887     /* check next page if needed */
1888     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1889     phys_page2 = -1;
1890     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1891         phys_page2 = get_page_addr_code(env, virt_page2);
1892     }
1893     /*
1894      * No explicit memory barrier is required -- tb_link_page() makes the
1895      * TB visible in a consistent state.
1896      */
1897     existing_tb = tb_link_page(tb, phys_pc, phys_page2);
1898     /* if the TB already exists, discard what we just translated */
1899     if (unlikely(existing_tb != tb)) {
1900         uintptr_t orig_aligned = (uintptr_t)gen_code_buf;
1901 
1902         orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize);
1903         atomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned);
1904         tb_destroy(tb);
1905         return existing_tb;
1906     }
1907     tcg_tb_insert(tb);
1908     return tb;
1909 }
1910 
1911 /*
1912  * @p must be non-NULL.
1913  * user-mode: call with mmap_lock held.
1914  * !user-mode: call with all @pages locked.
1915  */
1916 static void
1917 tb_invalidate_phys_page_range__locked(struct page_collection *pages,
1918                                       PageDesc *p, tb_page_addr_t start,
1919                                       tb_page_addr_t end,
1920                                       uintptr_t retaddr)
1921 {
1922     TranslationBlock *tb;
1923     tb_page_addr_t tb_start, tb_end;
1924     int n;
1925 #ifdef TARGET_HAS_PRECISE_SMC
1926     CPUState *cpu = current_cpu;
1927     CPUArchState *env = NULL;
1928     bool current_tb_not_found = retaddr != 0;
1929     bool current_tb_modified = false;
1930     TranslationBlock *current_tb = NULL;
1931     target_ulong current_pc = 0;
1932     target_ulong current_cs_base = 0;
1933     uint32_t current_flags = 0;
1934 #endif /* TARGET_HAS_PRECISE_SMC */
1935 
1936     assert_page_locked(p);
1937 
1938 #if defined(TARGET_HAS_PRECISE_SMC)
1939     if (cpu != NULL) {
1940         env = cpu->env_ptr;
1941     }
1942 #endif
1943 
1944     /* we remove all the TBs in the range [start, end[ */
1945     /* XXX: see if in some cases it could be faster to invalidate all
1946        the code */
1947     PAGE_FOR_EACH_TB(p, tb, n) {
1948         assert_page_locked(p);
1949         /* NOTE: this is subtle as a TB may span two physical pages */
1950         if (n == 0) {
1951             /* NOTE: tb_end may be after the end of the page, but
1952                it is not a problem */
1953             tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1954             tb_end = tb_start + tb->size;
1955         } else {
1956             tb_start = tb->page_addr[1];
1957             tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1958         }
1959         if (!(tb_end <= start || tb_start >= end)) {
1960 #ifdef TARGET_HAS_PRECISE_SMC
1961             if (current_tb_not_found) {
1962                 current_tb_not_found = false;
1963                 /* now we have a real cpu fault */
1964                 current_tb = tcg_tb_lookup(retaddr);
1965             }
1966             if (current_tb == tb &&
1967                 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
1968                 /*
1969                  * If we are modifying the current TB, we must stop
1970                  * its execution. We could be more precise by checking
1971                  * that the modification is after the current PC, but it
1972                  * would require a specialized function to partially
1973                  * restore the CPU state.
1974                  */
1975                 current_tb_modified = true;
1976                 cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
1977                 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1978                                      &current_flags);
1979             }
1980 #endif /* TARGET_HAS_PRECISE_SMC */
1981             tb_phys_invalidate__locked(tb);
1982         }
1983     }
1984 #if !defined(CONFIG_USER_ONLY)
1985     /* if no code remaining, no need to continue to use slow writes */
1986     if (!p->first_tb) {
1987         invalidate_page_bitmap(p);
1988         tlb_unprotect_code(start);
1989     }
1990 #endif
1991 #ifdef TARGET_HAS_PRECISE_SMC
1992     if (current_tb_modified) {
1993         page_collection_unlock(pages);
1994         /* Force execution of one insn next time.  */
1995         cpu->cflags_next_tb = 1 | curr_cflags();
1996         mmap_unlock();
1997         cpu_loop_exit_noexc(cpu);
1998     }
1999 #endif
2000 }
2001 
2002 /*
2003  * Invalidate all TBs which intersect with the target physical address range
2004  * [start;end[. NOTE: start and end must refer to the *same* physical page.
2005  * 'is_cpu_write_access' should be true if called from a real cpu write
2006  * access: the virtual CPU will exit the current TB if code is modified inside
2007  * this TB.
2008  *
2009  * Called with mmap_lock held for user-mode emulation
2010  */
2011 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
2012 {
2013     struct page_collection *pages;
2014     PageDesc *p;
2015 
2016     assert_memory_lock();
2017 
2018     p = page_find(start >> TARGET_PAGE_BITS);
2019     if (p == NULL) {
2020         return;
2021     }
2022     pages = page_collection_lock(start, end);
2023     tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
2024     page_collection_unlock(pages);
2025 }
2026 
2027 /*
2028  * Invalidate all TBs which intersect with the target physical address range
2029  * [start;end[. NOTE: start and end may refer to *different* physical pages.
2030  * 'is_cpu_write_access' should be true if called from a real cpu write
2031  * access: the virtual CPU will exit the current TB if code is modified inside
2032  * this TB.
2033  *
2034  * Called with mmap_lock held for user-mode emulation.
2035  */
2036 #ifdef CONFIG_SOFTMMU
2037 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
2038 #else
2039 void tb_invalidate_phys_range(target_ulong start, target_ulong end)
2040 #endif
2041 {
2042     struct page_collection *pages;
2043     tb_page_addr_t next;
2044 
2045     assert_memory_lock();
2046 
2047     pages = page_collection_lock(start, end);
2048     for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
2049          start < end;
2050          start = next, next += TARGET_PAGE_SIZE) {
2051         PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
2052         tb_page_addr_t bound = MIN(next, end);
2053 
2054         if (pd == NULL) {
2055             continue;
2056         }
2057         tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
2058     }
2059     page_collection_unlock(pages);
2060 }
2061 
2062 #ifdef CONFIG_SOFTMMU
2063 /* len must be <= 8 and start must be a multiple of len.
2064  * Called via softmmu_template.h when code areas are written to with
2065  * iothread mutex not held.
2066  *
2067  * Call with all @pages in the range [@start, @start + len[ locked.
2068  */
2069 void tb_invalidate_phys_page_fast(struct page_collection *pages,
2070                                   tb_page_addr_t start, int len,
2071                                   uintptr_t retaddr)
2072 {
2073     PageDesc *p;
2074 
2075     assert_memory_lock();
2076 
2077     p = page_find(start >> TARGET_PAGE_BITS);
2078     if (!p) {
2079         return;
2080     }
2081 
2082     assert_page_locked(p);
2083     if (!p->code_bitmap &&
2084         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
2085         build_page_bitmap(p);
2086     }
2087     if (p->code_bitmap) {
2088         unsigned int nr;
2089         unsigned long b;
2090 
2091         nr = start & ~TARGET_PAGE_MASK;
2092         b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
2093         if (b & ((1 << len) - 1)) {
2094             goto do_invalidate;
2095         }
2096     } else {
2097     do_invalidate:
2098         tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
2099                                               retaddr);
2100     }
2101 }
2102 #else
2103 /* Called with mmap_lock held. If pc is not 0 then it indicates the
2104  * host PC of the faulting store instruction that caused this invalidate.
2105  * Returns true if the caller needs to abort execution of the current
2106  * TB (because it was modified by this store and the guest CPU has
2107  * precise-SMC semantics).
2108  */
2109 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
2110 {
2111     TranslationBlock *tb;
2112     PageDesc *p;
2113     int n;
2114 #ifdef TARGET_HAS_PRECISE_SMC
2115     TranslationBlock *current_tb = NULL;
2116     CPUState *cpu = current_cpu;
2117     CPUArchState *env = NULL;
2118     int current_tb_modified = 0;
2119     target_ulong current_pc = 0;
2120     target_ulong current_cs_base = 0;
2121     uint32_t current_flags = 0;
2122 #endif
2123 
2124     assert_memory_lock();
2125 
2126     addr &= TARGET_PAGE_MASK;
2127     p = page_find(addr >> TARGET_PAGE_BITS);
2128     if (!p) {
2129         return false;
2130     }
2131 
2132 #ifdef TARGET_HAS_PRECISE_SMC
2133     if (p->first_tb && pc != 0) {
2134         current_tb = tcg_tb_lookup(pc);
2135     }
2136     if (cpu != NULL) {
2137         env = cpu->env_ptr;
2138     }
2139 #endif
2140     assert_page_locked(p);
2141     PAGE_FOR_EACH_TB(p, tb, n) {
2142 #ifdef TARGET_HAS_PRECISE_SMC
2143         if (current_tb == tb &&
2144             (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
2145                 /* If we are modifying the current TB, we must stop
2146                    its execution. We could be more precise by checking
2147                    that the modification is after the current PC, but it
2148                    would require a specialized function to partially
2149                    restore the CPU state */
2150 
2151             current_tb_modified = 1;
2152             cpu_restore_state_from_tb(cpu, current_tb, pc, true);
2153             cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
2154                                  &current_flags);
2155         }
2156 #endif /* TARGET_HAS_PRECISE_SMC */
2157         tb_phys_invalidate(tb, addr);
2158     }
2159     p->first_tb = (uintptr_t)NULL;
2160 #ifdef TARGET_HAS_PRECISE_SMC
2161     if (current_tb_modified) {
2162         /* Force execution of one insn next time.  */
2163         cpu->cflags_next_tb = 1 | curr_cflags();
2164         return true;
2165     }
2166 #endif
2167 
2168     return false;
2169 }
2170 #endif
2171 
2172 /* user-mode: call with mmap_lock held */
2173 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
2174 {
2175     TranslationBlock *tb;
2176 
2177     assert_memory_lock();
2178 
2179     tb = tcg_tb_lookup(retaddr);
2180     if (tb) {
2181         /* We can use retranslation to find the PC.  */
2182         cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2183         tb_phys_invalidate(tb, -1);
2184     } else {
2185         /* The exception probably happened in a helper.  The CPU state should
2186            have been saved before calling it. Fetch the PC from there.  */
2187         CPUArchState *env = cpu->env_ptr;
2188         target_ulong pc, cs_base;
2189         tb_page_addr_t addr;
2190         uint32_t flags;
2191 
2192         cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
2193         addr = get_page_addr_code(env, pc);
2194         if (addr != -1) {
2195             tb_invalidate_phys_range(addr, addr + 1);
2196         }
2197     }
2198 }
2199 
2200 #ifndef CONFIG_USER_ONLY
2201 /* in deterministic execution mode, instructions doing device I/Os
2202  * must be at the end of the TB.
2203  *
2204  * Called by softmmu_template.h, with iothread mutex not held.
2205  */
2206 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
2207 {
2208 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
2209     CPUArchState *env = cpu->env_ptr;
2210 #endif
2211     TranslationBlock *tb;
2212     uint32_t n;
2213 
2214     tb = tcg_tb_lookup(retaddr);
2215     if (!tb) {
2216         cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
2217                   (void *)retaddr);
2218     }
2219     cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2220 
2221     /* On MIPS and SH, delay slot instructions can only be restarted if
2222        they were already the first instruction in the TB.  If this is not
2223        the first instruction in a TB then re-execute the preceding
2224        branch.  */
2225     n = 1;
2226 #if defined(TARGET_MIPS)
2227     if ((env->hflags & MIPS_HFLAG_BMASK) != 0
2228         && env->active_tc.PC != tb->pc) {
2229         env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
2230         cpu_neg(cpu)->icount_decr.u16.low++;
2231         env->hflags &= ~MIPS_HFLAG_BMASK;
2232         n = 2;
2233     }
2234 #elif defined(TARGET_SH4)
2235     if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
2236         && env->pc != tb->pc) {
2237         env->pc -= 2;
2238         cpu_neg(cpu)->icount_decr.u16.low++;
2239         env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
2240         n = 2;
2241     }
2242 #endif
2243 
2244     /* Generate a new TB executing the I/O insn.  */
2245     cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n;
2246 
2247     if (tb_cflags(tb) & CF_NOCACHE) {
2248         if (tb->orig_tb) {
2249             /* Invalidate original TB if this TB was generated in
2250              * cpu_exec_nocache() */
2251             tb_phys_invalidate(tb->orig_tb, -1);
2252         }
2253         tcg_tb_remove(tb);
2254         tb_destroy(tb);
2255     }
2256 
2257     /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
2258      * the first in the TB) then we end up generating a whole new TB and
2259      *  repeating the fault, which is horribly inefficient.
2260      *  Better would be to execute just this insn uncached, or generate a
2261      *  second new TB.
2262      */
2263     cpu_loop_exit_noexc(cpu);
2264 }
2265 
2266 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr)
2267 {
2268     unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr);
2269 
2270     for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
2271         atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL);
2272     }
2273 }
2274 
2275 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
2276 {
2277     /* Discard jump cache entries for any tb which might potentially
2278        overlap the flushed page.  */
2279     tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
2280     tb_jmp_cache_clear_page(cpu, addr);
2281 }
2282 
2283 static void print_qht_statistics(struct qht_stats hst)
2284 {
2285     uint32_t hgram_opts;
2286     size_t hgram_bins;
2287     char *hgram;
2288 
2289     if (!hst.head_buckets) {
2290         return;
2291     }
2292     qemu_printf("TB hash buckets     %zu/%zu (%0.2f%% head buckets used)\n",
2293                 hst.used_head_buckets, hst.head_buckets,
2294                 (double)hst.used_head_buckets / hst.head_buckets * 100);
2295 
2296     hgram_opts =  QDIST_PR_BORDER | QDIST_PR_LABELS;
2297     hgram_opts |= QDIST_PR_100X   | QDIST_PR_PERCENT;
2298     if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
2299         hgram_opts |= QDIST_PR_NODECIMAL;
2300     }
2301     hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
2302     qemu_printf("TB hash occupancy   %0.2f%% avg chain occ. Histogram: %s\n",
2303                 qdist_avg(&hst.occupancy) * 100, hgram);
2304     g_free(hgram);
2305 
2306     hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
2307     hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
2308     if (hgram_bins > 10) {
2309         hgram_bins = 10;
2310     } else {
2311         hgram_bins = 0;
2312         hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
2313     }
2314     hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
2315     qemu_printf("TB hash avg chain   %0.3f buckets. Histogram: %s\n",
2316                 qdist_avg(&hst.chain), hgram);
2317     g_free(hgram);
2318 }
2319 
2320 struct tb_tree_stats {
2321     size_t nb_tbs;
2322     size_t host_size;
2323     size_t target_size;
2324     size_t max_target_size;
2325     size_t direct_jmp_count;
2326     size_t direct_jmp2_count;
2327     size_t cross_page;
2328 };
2329 
2330 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
2331 {
2332     const TranslationBlock *tb = value;
2333     struct tb_tree_stats *tst = data;
2334 
2335     tst->nb_tbs++;
2336     tst->host_size += tb->tc.size;
2337     tst->target_size += tb->size;
2338     if (tb->size > tst->max_target_size) {
2339         tst->max_target_size = tb->size;
2340     }
2341     if (tb->page_addr[1] != -1) {
2342         tst->cross_page++;
2343     }
2344     if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
2345         tst->direct_jmp_count++;
2346         if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
2347             tst->direct_jmp2_count++;
2348         }
2349     }
2350     return false;
2351 }
2352 
2353 void dump_exec_info(void)
2354 {
2355     struct tb_tree_stats tst = {};
2356     struct qht_stats hst;
2357     size_t nb_tbs, flush_full, flush_part, flush_elide;
2358 
2359     tcg_tb_foreach(tb_tree_stats_iter, &tst);
2360     nb_tbs = tst.nb_tbs;
2361     /* XXX: avoid using doubles ? */
2362     qemu_printf("Translation buffer state:\n");
2363     /*
2364      * Report total code size including the padding and TB structs;
2365      * otherwise users might think "-tb-size" is not honoured.
2366      * For avg host size we use the precise numbers from tb_tree_stats though.
2367      */
2368     qemu_printf("gen code size       %zu/%zu\n",
2369                 tcg_code_size(), tcg_code_capacity());
2370     qemu_printf("TB count            %zu\n", nb_tbs);
2371     qemu_printf("TB avg target size  %zu max=%zu bytes\n",
2372                 nb_tbs ? tst.target_size / nb_tbs : 0,
2373                 tst.max_target_size);
2374     qemu_printf("TB avg host size    %zu bytes (expansion ratio: %0.1f)\n",
2375                 nb_tbs ? tst.host_size / nb_tbs : 0,
2376                 tst.target_size ? (double)tst.host_size / tst.target_size : 0);
2377     qemu_printf("cross page TB count %zu (%zu%%)\n", tst.cross_page,
2378                 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0);
2379     qemu_printf("direct jump count   %zu (%zu%%) (2 jumps=%zu %zu%%)\n",
2380                 tst.direct_jmp_count,
2381                 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0,
2382                 tst.direct_jmp2_count,
2383                 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0);
2384 
2385     qht_statistics_init(&tb_ctx.htable, &hst);
2386     print_qht_statistics(hst);
2387     qht_statistics_destroy(&hst);
2388 
2389     qemu_printf("\nStatistics:\n");
2390     qemu_printf("TB flush count      %u\n",
2391                 atomic_read(&tb_ctx.tb_flush_count));
2392     qemu_printf("TB invalidate count %zu\n",
2393                 tcg_tb_phys_invalidate_count());
2394 
2395     tlb_flush_counts(&flush_full, &flush_part, &flush_elide);
2396     qemu_printf("TLB full flushes    %zu\n", flush_full);
2397     qemu_printf("TLB partial flushes %zu\n", flush_part);
2398     qemu_printf("TLB elided flushes  %zu\n", flush_elide);
2399     tcg_dump_info();
2400 }
2401 
2402 void dump_opcount_info(void)
2403 {
2404     tcg_dump_op_count();
2405 }
2406 
2407 #else /* CONFIG_USER_ONLY */
2408 
2409 void cpu_interrupt(CPUState *cpu, int mask)
2410 {
2411     g_assert(qemu_mutex_iothread_locked());
2412     cpu->interrupt_request |= mask;
2413     atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
2414 }
2415 
2416 /*
2417  * Walks guest process memory "regions" one by one
2418  * and calls callback function 'fn' for each region.
2419  */
2420 struct walk_memory_regions_data {
2421     walk_memory_regions_fn fn;
2422     void *priv;
2423     target_ulong start;
2424     int prot;
2425 };
2426 
2427 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2428                                    target_ulong end, int new_prot)
2429 {
2430     if (data->start != -1u) {
2431         int rc = data->fn(data->priv, data->start, end, data->prot);
2432         if (rc != 0) {
2433             return rc;
2434         }
2435     }
2436 
2437     data->start = (new_prot ? end : -1u);
2438     data->prot = new_prot;
2439 
2440     return 0;
2441 }
2442 
2443 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2444                                  target_ulong base, int level, void **lp)
2445 {
2446     target_ulong pa;
2447     int i, rc;
2448 
2449     if (*lp == NULL) {
2450         return walk_memory_regions_end(data, base, 0);
2451     }
2452 
2453     if (level == 0) {
2454         PageDesc *pd = *lp;
2455 
2456         for (i = 0; i < V_L2_SIZE; ++i) {
2457             int prot = pd[i].flags;
2458 
2459             pa = base | (i << TARGET_PAGE_BITS);
2460             if (prot != data->prot) {
2461                 rc = walk_memory_regions_end(data, pa, prot);
2462                 if (rc != 0) {
2463                     return rc;
2464                 }
2465             }
2466         }
2467     } else {
2468         void **pp = *lp;
2469 
2470         for (i = 0; i < V_L2_SIZE; ++i) {
2471             pa = base | ((target_ulong)i <<
2472                 (TARGET_PAGE_BITS + V_L2_BITS * level));
2473             rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2474             if (rc != 0) {
2475                 return rc;
2476             }
2477         }
2478     }
2479 
2480     return 0;
2481 }
2482 
2483 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2484 {
2485     struct walk_memory_regions_data data;
2486     uintptr_t i, l1_sz = v_l1_size;
2487 
2488     data.fn = fn;
2489     data.priv = priv;
2490     data.start = -1u;
2491     data.prot = 0;
2492 
2493     for (i = 0; i < l1_sz; i++) {
2494         target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS);
2495         int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i);
2496         if (rc != 0) {
2497             return rc;
2498         }
2499     }
2500 
2501     return walk_memory_regions_end(&data, 0, 0);
2502 }
2503 
2504 static int dump_region(void *priv, target_ulong start,
2505     target_ulong end, unsigned long prot)
2506 {
2507     FILE *f = (FILE *)priv;
2508 
2509     (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
2510         " "TARGET_FMT_lx" %c%c%c\n",
2511         start, end, end - start,
2512         ((prot & PAGE_READ) ? 'r' : '-'),
2513         ((prot & PAGE_WRITE) ? 'w' : '-'),
2514         ((prot & PAGE_EXEC) ? 'x' : '-'));
2515 
2516     return 0;
2517 }
2518 
2519 /* dump memory mappings */
2520 void page_dump(FILE *f)
2521 {
2522     const int length = sizeof(target_ulong) * 2;
2523     (void) fprintf(f, "%-*s %-*s %-*s %s\n",
2524             length, "start", length, "end", length, "size", "prot");
2525     walk_memory_regions(f, dump_region);
2526 }
2527 
2528 int page_get_flags(target_ulong address)
2529 {
2530     PageDesc *p;
2531 
2532     p = page_find(address >> TARGET_PAGE_BITS);
2533     if (!p) {
2534         return 0;
2535     }
2536     return p->flags;
2537 }
2538 
2539 /* Modify the flags of a page and invalidate the code if necessary.
2540    The flag PAGE_WRITE_ORG is positioned automatically depending
2541    on PAGE_WRITE.  The mmap_lock should already be held.  */
2542 void page_set_flags(target_ulong start, target_ulong end, int flags)
2543 {
2544     target_ulong addr, len;
2545 
2546     /* This function should never be called with addresses outside the
2547        guest address space.  If this assert fires, it probably indicates
2548        a missing call to h2g_valid.  */
2549     assert(end - 1 <= GUEST_ADDR_MAX);
2550     assert(start < end);
2551     assert_memory_lock();
2552 
2553     start = start & TARGET_PAGE_MASK;
2554     end = TARGET_PAGE_ALIGN(end);
2555 
2556     if (flags & PAGE_WRITE) {
2557         flags |= PAGE_WRITE_ORG;
2558     }
2559 
2560     for (addr = start, len = end - start;
2561          len != 0;
2562          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2563         PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2564 
2565         /* If the write protection bit is set, then we invalidate
2566            the code inside.  */
2567         if (!(p->flags & PAGE_WRITE) &&
2568             (flags & PAGE_WRITE) &&
2569             p->first_tb) {
2570             tb_invalidate_phys_page(addr, 0);
2571         }
2572         p->flags = flags;
2573     }
2574 }
2575 
2576 int page_check_range(target_ulong start, target_ulong len, int flags)
2577 {
2578     PageDesc *p;
2579     target_ulong end;
2580     target_ulong addr;
2581 
2582     /* This function should never be called with addresses outside the
2583        guest address space.  If this assert fires, it probably indicates
2584        a missing call to h2g_valid.  */
2585     if (TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS) {
2586         assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2587     }
2588 
2589     if (len == 0) {
2590         return 0;
2591     }
2592     if (start + len - 1 < start) {
2593         /* We've wrapped around.  */
2594         return -1;
2595     }
2596 
2597     /* must do before we loose bits in the next step */
2598     end = TARGET_PAGE_ALIGN(start + len);
2599     start = start & TARGET_PAGE_MASK;
2600 
2601     for (addr = start, len = end - start;
2602          len != 0;
2603          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2604         p = page_find(addr >> TARGET_PAGE_BITS);
2605         if (!p) {
2606             return -1;
2607         }
2608         if (!(p->flags & PAGE_VALID)) {
2609             return -1;
2610         }
2611 
2612         if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
2613             return -1;
2614         }
2615         if (flags & PAGE_WRITE) {
2616             if (!(p->flags & PAGE_WRITE_ORG)) {
2617                 return -1;
2618             }
2619             /* unprotect the page if it was put read-only because it
2620                contains translated code */
2621             if (!(p->flags & PAGE_WRITE)) {
2622                 if (!page_unprotect(addr, 0)) {
2623                     return -1;
2624                 }
2625             }
2626         }
2627     }
2628     return 0;
2629 }
2630 
2631 /* called from signal handler: invalidate the code and unprotect the
2632  * page. Return 0 if the fault was not handled, 1 if it was handled,
2633  * and 2 if it was handled but the caller must cause the TB to be
2634  * immediately exited. (We can only return 2 if the 'pc' argument is
2635  * non-zero.)
2636  */
2637 int page_unprotect(target_ulong address, uintptr_t pc)
2638 {
2639     unsigned int prot;
2640     bool current_tb_invalidated;
2641     PageDesc *p;
2642     target_ulong host_start, host_end, addr;
2643 
2644     /* Technically this isn't safe inside a signal handler.  However we
2645        know this only ever happens in a synchronous SEGV handler, so in
2646        practice it seems to be ok.  */
2647     mmap_lock();
2648 
2649     p = page_find(address >> TARGET_PAGE_BITS);
2650     if (!p) {
2651         mmap_unlock();
2652         return 0;
2653     }
2654 
2655     /* if the page was really writable, then we change its
2656        protection back to writable */
2657     if (p->flags & PAGE_WRITE_ORG) {
2658         current_tb_invalidated = false;
2659         if (p->flags & PAGE_WRITE) {
2660             /* If the page is actually marked WRITE then assume this is because
2661              * this thread raced with another one which got here first and
2662              * set the page to PAGE_WRITE and did the TB invalidate for us.
2663              */
2664 #ifdef TARGET_HAS_PRECISE_SMC
2665             TranslationBlock *current_tb = tcg_tb_lookup(pc);
2666             if (current_tb) {
2667                 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID;
2668             }
2669 #endif
2670         } else {
2671             host_start = address & qemu_host_page_mask;
2672             host_end = host_start + qemu_host_page_size;
2673 
2674             prot = 0;
2675             for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) {
2676                 p = page_find(addr >> TARGET_PAGE_BITS);
2677                 p->flags |= PAGE_WRITE;
2678                 prot |= p->flags;
2679 
2680                 /* and since the content will be modified, we must invalidate
2681                    the corresponding translated code. */
2682                 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
2683 #ifdef CONFIG_USER_ONLY
2684                 if (DEBUG_TB_CHECK_GATE) {
2685                     tb_invalidate_check(addr);
2686                 }
2687 #endif
2688             }
2689             mprotect((void *)g2h(host_start), qemu_host_page_size,
2690                      prot & PAGE_BITS);
2691         }
2692         mmap_unlock();
2693         /* If current TB was invalidated return to main loop */
2694         return current_tb_invalidated ? 2 : 1;
2695     }
2696     mmap_unlock();
2697     return 0;
2698 }
2699 #endif /* CONFIG_USER_ONLY */
2700 
2701 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2702 void tcg_flush_softmmu_tlb(CPUState *cs)
2703 {
2704 #ifdef CONFIG_SOFTMMU
2705     tlb_flush(cs);
2706 #endif
2707 }
2708