xref: /openbmc/qemu/accel/tcg/cputlb.c (revision 03582094)
1 /*
2  *  Common CPU TLB handling
3  *
4  *  Copyright (c) 2003 Fabrice Bellard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "hw/core/tcg-cpu-ops.h"
23 #include "exec/exec-all.h"
24 #include "exec/page-protection.h"
25 #include "exec/memory.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
28 #include "exec/tb-flush.h"
29 #include "exec/memory-internal.h"
30 #include "exec/ram_addr.h"
31 #include "exec/mmu-access-type.h"
32 #include "exec/tlb-common.h"
33 #include "exec/vaddr.h"
34 #include "tcg/tcg.h"
35 #include "qemu/error-report.h"
36 #include "exec/log.h"
37 #include "exec/helper-proto-common.h"
38 #include "qemu/atomic.h"
39 #include "qemu/atomic128.h"
40 #include "exec/translate-all.h"
41 #include "trace.h"
42 #include "tb-hash.h"
43 #include "internal-common.h"
44 #include "internal-target.h"
45 #ifdef CONFIG_PLUGIN
46 #include "qemu/plugin-memory.h"
47 #endif
48 #include "tcg/tcg-ldst.h"
49 #include "tcg/oversized-guest.h"
50 
51 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
52 /* #define DEBUG_TLB */
53 /* #define DEBUG_TLB_LOG */
54 
55 #ifdef DEBUG_TLB
56 # define DEBUG_TLB_GATE 1
57 # ifdef DEBUG_TLB_LOG
58 #  define DEBUG_TLB_LOG_GATE 1
59 # else
60 #  define DEBUG_TLB_LOG_GATE 0
61 # endif
62 #else
63 # define DEBUG_TLB_GATE 0
64 # define DEBUG_TLB_LOG_GATE 0
65 #endif
66 
67 #define tlb_debug(fmt, ...) do { \
68     if (DEBUG_TLB_LOG_GATE) { \
69         qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
70                       ## __VA_ARGS__); \
71     } else if (DEBUG_TLB_GATE) { \
72         fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
73     } \
74 } while (0)
75 
76 #define assert_cpu_is_self(cpu) do {                              \
77         if (DEBUG_TLB_GATE) {                                     \
78             g_assert(!(cpu)->created || qemu_cpu_is_self(cpu));   \
79         }                                                         \
80     } while (0)
81 
82 /* run_on_cpu_data.target_ptr should always be big enough for a
83  * vaddr even on 32 bit builds
84  */
85 QEMU_BUILD_BUG_ON(sizeof(vaddr) > sizeof(run_on_cpu_data));
86 
87 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
88  */
89 QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
90 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
91 
92 static inline size_t tlb_n_entries(CPUTLBDescFast *fast)
93 {
94     return (fast->mask >> CPU_TLB_ENTRY_BITS) + 1;
95 }
96 
97 static inline size_t sizeof_tlb(CPUTLBDescFast *fast)
98 {
99     return fast->mask + (1 << CPU_TLB_ENTRY_BITS);
100 }
101 
102 static inline uint64_t tlb_read_idx(const CPUTLBEntry *entry,
103                                     MMUAccessType access_type)
104 {
105     /* Do not rearrange the CPUTLBEntry structure members. */
106     QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_read) !=
107                       MMU_DATA_LOAD * sizeof(uint64_t));
108     QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_write) !=
109                       MMU_DATA_STORE * sizeof(uint64_t));
110     QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_code) !=
111                       MMU_INST_FETCH * sizeof(uint64_t));
112 
113 #if TARGET_LONG_BITS == 32
114     /* Use qatomic_read, in case of addr_write; only care about low bits. */
115     const uint32_t *ptr = (uint32_t *)&entry->addr_idx[access_type];
116     ptr += HOST_BIG_ENDIAN;
117     return qatomic_read(ptr);
118 #else
119     const uint64_t *ptr = &entry->addr_idx[access_type];
120 # if TCG_OVERSIZED_GUEST
121     return *ptr;
122 # else
123     /* ofs might correspond to .addr_write, so use qatomic_read */
124     return qatomic_read(ptr);
125 # endif
126 #endif
127 }
128 
129 static inline uint64_t tlb_addr_write(const CPUTLBEntry *entry)
130 {
131     return tlb_read_idx(entry, MMU_DATA_STORE);
132 }
133 
134 /* Find the TLB index corresponding to the mmu_idx + address pair.  */
135 static inline uintptr_t tlb_index(CPUState *cpu, uintptr_t mmu_idx,
136                                   vaddr addr)
137 {
138     uintptr_t size_mask = cpu->neg.tlb.f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS;
139 
140     return (addr >> TARGET_PAGE_BITS) & size_mask;
141 }
142 
143 /* Find the TLB entry corresponding to the mmu_idx + address pair.  */
144 static inline CPUTLBEntry *tlb_entry(CPUState *cpu, uintptr_t mmu_idx,
145                                      vaddr addr)
146 {
147     return &cpu->neg.tlb.f[mmu_idx].table[tlb_index(cpu, mmu_idx, addr)];
148 }
149 
150 static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns,
151                              size_t max_entries)
152 {
153     desc->window_begin_ns = ns;
154     desc->window_max_entries = max_entries;
155 }
156 
157 static void tb_jmp_cache_clear_page(CPUState *cpu, vaddr page_addr)
158 {
159     CPUJumpCache *jc = cpu->tb_jmp_cache;
160     int i, i0;
161 
162     if (unlikely(!jc)) {
163         return;
164     }
165 
166     i0 = tb_jmp_cache_hash_page(page_addr);
167     for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
168         qatomic_set(&jc->array[i0 + i].tb, NULL);
169     }
170 }
171 
172 /**
173  * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
174  * @desc: The CPUTLBDesc portion of the TLB
175  * @fast: The CPUTLBDescFast portion of the same TLB
176  *
177  * Called with tlb_lock_held.
178  *
179  * We have two main constraints when resizing a TLB: (1) we only resize it
180  * on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
181  * the array or unnecessarily flushing it), which means we do not control how
182  * frequently the resizing can occur; (2) we don't have access to the guest's
183  * future scheduling decisions, and therefore have to decide the magnitude of
184  * the resize based on past observations.
185  *
186  * In general, a memory-hungry process can benefit greatly from an appropriately
187  * sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
188  * we just have to make the TLB as large as possible; while an oversized TLB
189  * results in minimal TLB miss rates, it also takes longer to be flushed
190  * (flushes can be _very_ frequent), and the reduced locality can also hurt
191  * performance.
192  *
193  * To achieve near-optimal performance for all kinds of workloads, we:
194  *
195  * 1. Aggressively increase the size of the TLB when the use rate of the
196  * TLB being flushed is high, since it is likely that in the near future this
197  * memory-hungry process will execute again, and its memory hungriness will
198  * probably be similar.
199  *
200  * 2. Slowly reduce the size of the TLB as the use rate declines over a
201  * reasonably large time window. The rationale is that if in such a time window
202  * we have not observed a high TLB use rate, it is likely that we won't observe
203  * it in the near future. In that case, once a time window expires we downsize
204  * the TLB to match the maximum use rate observed in the window.
205  *
206  * 3. Try to keep the maximum use rate in a time window in the 30-70% range,
207  * since in that range performance is likely near-optimal. Recall that the TLB
208  * is direct mapped, so we want the use rate to be low (or at least not too
209  * high), since otherwise we are likely to have a significant amount of
210  * conflict misses.
211  */
212 static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast,
213                                   int64_t now)
214 {
215     size_t old_size = tlb_n_entries(fast);
216     size_t rate;
217     size_t new_size = old_size;
218     int64_t window_len_ms = 100;
219     int64_t window_len_ns = window_len_ms * 1000 * 1000;
220     bool window_expired = now > desc->window_begin_ns + window_len_ns;
221 
222     if (desc->n_used_entries > desc->window_max_entries) {
223         desc->window_max_entries = desc->n_used_entries;
224     }
225     rate = desc->window_max_entries * 100 / old_size;
226 
227     if (rate > 70) {
228         new_size = MIN(old_size << 1, 1 << CPU_TLB_DYN_MAX_BITS);
229     } else if (rate < 30 && window_expired) {
230         size_t ceil = pow2ceil(desc->window_max_entries);
231         size_t expected_rate = desc->window_max_entries * 100 / ceil;
232 
233         /*
234          * Avoid undersizing when the max number of entries seen is just below
235          * a pow2. For instance, if max_entries == 1025, the expected use rate
236          * would be 1025/2048==50%. However, if max_entries == 1023, we'd get
237          * 1023/1024==99.9% use rate, so we'd likely end up doubling the size
238          * later. Thus, make sure that the expected use rate remains below 70%.
239          * (and since we double the size, that means the lowest rate we'd
240          * expect to get is 35%, which is still in the 30-70% range where
241          * we consider that the size is appropriate.)
242          */
243         if (expected_rate > 70) {
244             ceil *= 2;
245         }
246         new_size = MAX(ceil, 1 << CPU_TLB_DYN_MIN_BITS);
247     }
248 
249     if (new_size == old_size) {
250         if (window_expired) {
251             tlb_window_reset(desc, now, desc->n_used_entries);
252         }
253         return;
254     }
255 
256     g_free(fast->table);
257     g_free(desc->fulltlb);
258 
259     tlb_window_reset(desc, now, 0);
260     /* desc->n_used_entries is cleared by the caller */
261     fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
262     fast->table = g_try_new(CPUTLBEntry, new_size);
263     desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size);
264 
265     /*
266      * If the allocations fail, try smaller sizes. We just freed some
267      * memory, so going back to half of new_size has a good chance of working.
268      * Increased memory pressure elsewhere in the system might cause the
269      * allocations to fail though, so we progressively reduce the allocation
270      * size, aborting if we cannot even allocate the smallest TLB we support.
271      */
272     while (fast->table == NULL || desc->fulltlb == NULL) {
273         if (new_size == (1 << CPU_TLB_DYN_MIN_BITS)) {
274             error_report("%s: %s", __func__, strerror(errno));
275             abort();
276         }
277         new_size = MAX(new_size >> 1, 1 << CPU_TLB_DYN_MIN_BITS);
278         fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
279 
280         g_free(fast->table);
281         g_free(desc->fulltlb);
282         fast->table = g_try_new(CPUTLBEntry, new_size);
283         desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size);
284     }
285 }
286 
287 static void tlb_mmu_flush_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast)
288 {
289     desc->n_used_entries = 0;
290     desc->large_page_addr = -1;
291     desc->large_page_mask = -1;
292     desc->vindex = 0;
293     memset(fast->table, -1, sizeof_tlb(fast));
294     memset(desc->vtable, -1, sizeof(desc->vtable));
295 }
296 
297 static void tlb_flush_one_mmuidx_locked(CPUState *cpu, int mmu_idx,
298                                         int64_t now)
299 {
300     CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx];
301     CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx];
302 
303     tlb_mmu_resize_locked(desc, fast, now);
304     tlb_mmu_flush_locked(desc, fast);
305 }
306 
307 static void tlb_mmu_init(CPUTLBDesc *desc, CPUTLBDescFast *fast, int64_t now)
308 {
309     size_t n_entries = 1 << CPU_TLB_DYN_DEFAULT_BITS;
310 
311     tlb_window_reset(desc, now, 0);
312     desc->n_used_entries = 0;
313     fast->mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS;
314     fast->table = g_new(CPUTLBEntry, n_entries);
315     desc->fulltlb = g_new(CPUTLBEntryFull, n_entries);
316     tlb_mmu_flush_locked(desc, fast);
317 }
318 
319 static inline void tlb_n_used_entries_inc(CPUState *cpu, uintptr_t mmu_idx)
320 {
321     cpu->neg.tlb.d[mmu_idx].n_used_entries++;
322 }
323 
324 static inline void tlb_n_used_entries_dec(CPUState *cpu, uintptr_t mmu_idx)
325 {
326     cpu->neg.tlb.d[mmu_idx].n_used_entries--;
327 }
328 
329 void tlb_init(CPUState *cpu)
330 {
331     int64_t now = get_clock_realtime();
332     int i;
333 
334     qemu_spin_init(&cpu->neg.tlb.c.lock);
335 
336     /* All tlbs are initialized flushed. */
337     cpu->neg.tlb.c.dirty = 0;
338 
339     for (i = 0; i < NB_MMU_MODES; i++) {
340         tlb_mmu_init(&cpu->neg.tlb.d[i], &cpu->neg.tlb.f[i], now);
341     }
342 }
343 
344 void tlb_destroy(CPUState *cpu)
345 {
346     int i;
347 
348     qemu_spin_destroy(&cpu->neg.tlb.c.lock);
349     for (i = 0; i < NB_MMU_MODES; i++) {
350         CPUTLBDesc *desc = &cpu->neg.tlb.d[i];
351         CPUTLBDescFast *fast = &cpu->neg.tlb.f[i];
352 
353         g_free(fast->table);
354         g_free(desc->fulltlb);
355     }
356 }
357 
358 /* flush_all_helper: run fn across all cpus
359  *
360  * If the wait flag is set then the src cpu's helper will be queued as
361  * "safe" work and the loop exited creating a synchronisation point
362  * where all queued work will be finished before execution starts
363  * again.
364  */
365 static void flush_all_helper(CPUState *src, run_on_cpu_func fn,
366                              run_on_cpu_data d)
367 {
368     CPUState *cpu;
369 
370     CPU_FOREACH(cpu) {
371         if (cpu != src) {
372             async_run_on_cpu(cpu, fn, d);
373         }
374     }
375 }
376 
377 static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
378 {
379     uint16_t asked = data.host_int;
380     uint16_t all_dirty, work, to_clean;
381     int64_t now = get_clock_realtime();
382 
383     assert_cpu_is_self(cpu);
384 
385     tlb_debug("mmu_idx:0x%04" PRIx16 "\n", asked);
386 
387     qemu_spin_lock(&cpu->neg.tlb.c.lock);
388 
389     all_dirty = cpu->neg.tlb.c.dirty;
390     to_clean = asked & all_dirty;
391     all_dirty &= ~to_clean;
392     cpu->neg.tlb.c.dirty = all_dirty;
393 
394     for (work = to_clean; work != 0; work &= work - 1) {
395         int mmu_idx = ctz32(work);
396         tlb_flush_one_mmuidx_locked(cpu, mmu_idx, now);
397     }
398 
399     qemu_spin_unlock(&cpu->neg.tlb.c.lock);
400 
401     tcg_flush_jmp_cache(cpu);
402 
403     if (to_clean == ALL_MMUIDX_BITS) {
404         qatomic_set(&cpu->neg.tlb.c.full_flush_count,
405                     cpu->neg.tlb.c.full_flush_count + 1);
406     } else {
407         qatomic_set(&cpu->neg.tlb.c.part_flush_count,
408                     cpu->neg.tlb.c.part_flush_count + ctpop16(to_clean));
409         if (to_clean != asked) {
410             qatomic_set(&cpu->neg.tlb.c.elide_flush_count,
411                         cpu->neg.tlb.c.elide_flush_count +
412                         ctpop16(asked & ~to_clean));
413         }
414     }
415 }
416 
417 void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
418 {
419     tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);
420 
421     assert_cpu_is_self(cpu);
422 
423     tlb_flush_by_mmuidx_async_work(cpu, RUN_ON_CPU_HOST_INT(idxmap));
424 }
425 
426 void tlb_flush(CPUState *cpu)
427 {
428     tlb_flush_by_mmuidx(cpu, ALL_MMUIDX_BITS);
429 }
430 
431 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu, uint16_t idxmap)
432 {
433     const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
434 
435     tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
436 
437     flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
438     async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
439 }
440 
441 void tlb_flush_all_cpus_synced(CPUState *src_cpu)
442 {
443     tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, ALL_MMUIDX_BITS);
444 }
445 
446 static bool tlb_hit_page_mask_anyprot(CPUTLBEntry *tlb_entry,
447                                       vaddr page, vaddr mask)
448 {
449     page &= mask;
450     mask &= TARGET_PAGE_MASK | TLB_INVALID_MASK;
451 
452     return (page == (tlb_entry->addr_read & mask) ||
453             page == (tlb_addr_write(tlb_entry) & mask) ||
454             page == (tlb_entry->addr_code & mask));
455 }
456 
457 static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry, vaddr page)
458 {
459     return tlb_hit_page_mask_anyprot(tlb_entry, page, -1);
460 }
461 
462 /**
463  * tlb_entry_is_empty - return true if the entry is not in use
464  * @te: pointer to CPUTLBEntry
465  */
466 static inline bool tlb_entry_is_empty(const CPUTLBEntry *te)
467 {
468     return te->addr_read == -1 && te->addr_write == -1 && te->addr_code == -1;
469 }
470 
471 /* Called with tlb_c.lock held */
472 static bool tlb_flush_entry_mask_locked(CPUTLBEntry *tlb_entry,
473                                         vaddr page,
474                                         vaddr mask)
475 {
476     if (tlb_hit_page_mask_anyprot(tlb_entry, page, mask)) {
477         memset(tlb_entry, -1, sizeof(*tlb_entry));
478         return true;
479     }
480     return false;
481 }
482 
483 static inline bool tlb_flush_entry_locked(CPUTLBEntry *tlb_entry, vaddr page)
484 {
485     return tlb_flush_entry_mask_locked(tlb_entry, page, -1);
486 }
487 
488 /* Called with tlb_c.lock held */
489 static void tlb_flush_vtlb_page_mask_locked(CPUState *cpu, int mmu_idx,
490                                             vaddr page,
491                                             vaddr mask)
492 {
493     CPUTLBDesc *d = &cpu->neg.tlb.d[mmu_idx];
494     int k;
495 
496     assert_cpu_is_self(cpu);
497     for (k = 0; k < CPU_VTLB_SIZE; k++) {
498         if (tlb_flush_entry_mask_locked(&d->vtable[k], page, mask)) {
499             tlb_n_used_entries_dec(cpu, mmu_idx);
500         }
501     }
502 }
503 
504 static inline void tlb_flush_vtlb_page_locked(CPUState *cpu, int mmu_idx,
505                                               vaddr page)
506 {
507     tlb_flush_vtlb_page_mask_locked(cpu, mmu_idx, page, -1);
508 }
509 
510 static void tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page)
511 {
512     vaddr lp_addr = cpu->neg.tlb.d[midx].large_page_addr;
513     vaddr lp_mask = cpu->neg.tlb.d[midx].large_page_mask;
514 
515     /* Check if we need to flush due to large pages.  */
516     if ((page & lp_mask) == lp_addr) {
517         tlb_debug("forcing full flush midx %d (%016"
518                   VADDR_PRIx "/%016" VADDR_PRIx ")\n",
519                   midx, lp_addr, lp_mask);
520         tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
521     } else {
522         if (tlb_flush_entry_locked(tlb_entry(cpu, midx, page), page)) {
523             tlb_n_used_entries_dec(cpu, midx);
524         }
525         tlb_flush_vtlb_page_locked(cpu, midx, page);
526     }
527 }
528 
529 /**
530  * tlb_flush_page_by_mmuidx_async_0:
531  * @cpu: cpu on which to flush
532  * @addr: page of virtual address to flush
533  * @idxmap: set of mmu_idx to flush
534  *
535  * Helper for tlb_flush_page_by_mmuidx and friends, flush one page
536  * at @addr from the tlbs indicated by @idxmap from @cpu.
537  */
538 static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu,
539                                              vaddr addr,
540                                              uint16_t idxmap)
541 {
542     int mmu_idx;
543 
544     assert_cpu_is_self(cpu);
545 
546     tlb_debug("page addr: %016" VADDR_PRIx " mmu_map:0x%x\n", addr, idxmap);
547 
548     qemu_spin_lock(&cpu->neg.tlb.c.lock);
549     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
550         if ((idxmap >> mmu_idx) & 1) {
551             tlb_flush_page_locked(cpu, mmu_idx, addr);
552         }
553     }
554     qemu_spin_unlock(&cpu->neg.tlb.c.lock);
555 
556     /*
557      * Discard jump cache entries for any tb which might potentially
558      * overlap the flushed page, which includes the previous.
559      */
560     tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
561     tb_jmp_cache_clear_page(cpu, addr);
562 }
563 
564 /**
565  * tlb_flush_page_by_mmuidx_async_1:
566  * @cpu: cpu on which to flush
567  * @data: encoded addr + idxmap
568  *
569  * Helper for tlb_flush_page_by_mmuidx and friends, called through
570  * async_run_on_cpu.  The idxmap parameter is encoded in the page
571  * offset of the target_ptr field.  This limits the set of mmu_idx
572  * that can be passed via this method.
573  */
574 static void tlb_flush_page_by_mmuidx_async_1(CPUState *cpu,
575                                              run_on_cpu_data data)
576 {
577     vaddr addr_and_idxmap = data.target_ptr;
578     vaddr addr = addr_and_idxmap & TARGET_PAGE_MASK;
579     uint16_t idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK;
580 
581     tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap);
582 }
583 
584 typedef struct {
585     vaddr addr;
586     uint16_t idxmap;
587 } TLBFlushPageByMMUIdxData;
588 
589 /**
590  * tlb_flush_page_by_mmuidx_async_2:
591  * @cpu: cpu on which to flush
592  * @data: allocated addr + idxmap
593  *
594  * Helper for tlb_flush_page_by_mmuidx and friends, called through
595  * async_run_on_cpu.  The addr+idxmap parameters are stored in a
596  * TLBFlushPageByMMUIdxData structure that has been allocated
597  * specifically for this helper.  Free the structure when done.
598  */
599 static void tlb_flush_page_by_mmuidx_async_2(CPUState *cpu,
600                                              run_on_cpu_data data)
601 {
602     TLBFlushPageByMMUIdxData *d = data.host_ptr;
603 
604     tlb_flush_page_by_mmuidx_async_0(cpu, d->addr, d->idxmap);
605     g_free(d);
606 }
607 
608 void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, uint16_t idxmap)
609 {
610     tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%" PRIx16 "\n", addr, idxmap);
611 
612     assert_cpu_is_self(cpu);
613 
614     /* This should already be page aligned */
615     addr &= TARGET_PAGE_MASK;
616 
617     tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap);
618 }
619 
620 void tlb_flush_page(CPUState *cpu, vaddr addr)
621 {
622     tlb_flush_page_by_mmuidx(cpu, addr, ALL_MMUIDX_BITS);
623 }
624 
625 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
626                                               vaddr addr,
627                                               uint16_t idxmap)
628 {
629     tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap);
630 
631     /* This should already be page aligned */
632     addr &= TARGET_PAGE_MASK;
633 
634     /*
635      * Allocate memory to hold addr+idxmap only when needed.
636      * See tlb_flush_page_by_mmuidx for details.
637      */
638     if (idxmap < TARGET_PAGE_SIZE) {
639         flush_all_helper(src_cpu, tlb_flush_page_by_mmuidx_async_1,
640                          RUN_ON_CPU_TARGET_PTR(addr | idxmap));
641         async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_1,
642                               RUN_ON_CPU_TARGET_PTR(addr | idxmap));
643     } else {
644         CPUState *dst_cpu;
645         TLBFlushPageByMMUIdxData *d;
646 
647         /* Allocate a separate data block for each destination cpu.  */
648         CPU_FOREACH(dst_cpu) {
649             if (dst_cpu != src_cpu) {
650                 d = g_new(TLBFlushPageByMMUIdxData, 1);
651                 d->addr = addr;
652                 d->idxmap = idxmap;
653                 async_run_on_cpu(dst_cpu, tlb_flush_page_by_mmuidx_async_2,
654                                  RUN_ON_CPU_HOST_PTR(d));
655             }
656         }
657 
658         d = g_new(TLBFlushPageByMMUIdxData, 1);
659         d->addr = addr;
660         d->idxmap = idxmap;
661         async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_2,
662                               RUN_ON_CPU_HOST_PTR(d));
663     }
664 }
665 
666 void tlb_flush_page_all_cpus_synced(CPUState *src, vaddr addr)
667 {
668     tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS);
669 }
670 
671 static void tlb_flush_range_locked(CPUState *cpu, int midx,
672                                    vaddr addr, vaddr len,
673                                    unsigned bits)
674 {
675     CPUTLBDesc *d = &cpu->neg.tlb.d[midx];
676     CPUTLBDescFast *f = &cpu->neg.tlb.f[midx];
677     vaddr mask = MAKE_64BIT_MASK(0, bits);
678 
679     /*
680      * If @bits is smaller than the tlb size, there may be multiple entries
681      * within the TLB; otherwise all addresses that match under @mask hit
682      * the same TLB entry.
683      * TODO: Perhaps allow bits to be a few bits less than the size.
684      * For now, just flush the entire TLB.
685      *
686      * If @len is larger than the tlb size, then it will take longer to
687      * test all of the entries in the TLB than it will to flush it all.
688      */
689     if (mask < f->mask || len > f->mask) {
690         tlb_debug("forcing full flush midx %d ("
691                   "%016" VADDR_PRIx "/%016" VADDR_PRIx "+%016" VADDR_PRIx ")\n",
692                   midx, addr, mask, len);
693         tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
694         return;
695     }
696 
697     /*
698      * Check if we need to flush due to large pages.
699      * Because large_page_mask contains all 1's from the msb,
700      * we only need to test the end of the range.
701      */
702     if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) {
703         tlb_debug("forcing full flush midx %d ("
704                   "%016" VADDR_PRIx "/%016" VADDR_PRIx ")\n",
705                   midx, d->large_page_addr, d->large_page_mask);
706         tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
707         return;
708     }
709 
710     for (vaddr i = 0; i < len; i += TARGET_PAGE_SIZE) {
711         vaddr page = addr + i;
712         CPUTLBEntry *entry = tlb_entry(cpu, midx, page);
713 
714         if (tlb_flush_entry_mask_locked(entry, page, mask)) {
715             tlb_n_used_entries_dec(cpu, midx);
716         }
717         tlb_flush_vtlb_page_mask_locked(cpu, midx, page, mask);
718     }
719 }
720 
721 typedef struct {
722     vaddr addr;
723     vaddr len;
724     uint16_t idxmap;
725     uint16_t bits;
726 } TLBFlushRangeData;
727 
728 static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
729                                               TLBFlushRangeData d)
730 {
731     int mmu_idx;
732 
733     assert_cpu_is_self(cpu);
734 
735     tlb_debug("range: %016" VADDR_PRIx "/%u+%016" VADDR_PRIx " mmu_map:0x%x\n",
736               d.addr, d.bits, d.len, d.idxmap);
737 
738     qemu_spin_lock(&cpu->neg.tlb.c.lock);
739     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
740         if ((d.idxmap >> mmu_idx) & 1) {
741             tlb_flush_range_locked(cpu, mmu_idx, d.addr, d.len, d.bits);
742         }
743     }
744     qemu_spin_unlock(&cpu->neg.tlb.c.lock);
745 
746     /*
747      * If the length is larger than the jump cache size, then it will take
748      * longer to clear each entry individually than it will to clear it all.
749      */
750     if (d.len >= (TARGET_PAGE_SIZE * TB_JMP_CACHE_SIZE)) {
751         tcg_flush_jmp_cache(cpu);
752         return;
753     }
754 
755     /*
756      * Discard jump cache entries for any tb which might potentially
757      * overlap the flushed pages, which includes the previous.
758      */
759     d.addr -= TARGET_PAGE_SIZE;
760     for (vaddr i = 0, n = d.len / TARGET_PAGE_SIZE + 1; i < n; i++) {
761         tb_jmp_cache_clear_page(cpu, d.addr);
762         d.addr += TARGET_PAGE_SIZE;
763     }
764 }
765 
766 static void tlb_flush_range_by_mmuidx_async_1(CPUState *cpu,
767                                               run_on_cpu_data data)
768 {
769     TLBFlushRangeData *d = data.host_ptr;
770     tlb_flush_range_by_mmuidx_async_0(cpu, *d);
771     g_free(d);
772 }
773 
774 void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr,
775                                vaddr len, uint16_t idxmap,
776                                unsigned bits)
777 {
778     TLBFlushRangeData d;
779 
780     assert_cpu_is_self(cpu);
781 
782     /*
783      * If all bits are significant, and len is small,
784      * this devolves to tlb_flush_page.
785      */
786     if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
787         tlb_flush_page_by_mmuidx(cpu, addr, idxmap);
788         return;
789     }
790     /* If no page bits are significant, this devolves to tlb_flush. */
791     if (bits < TARGET_PAGE_BITS) {
792         tlb_flush_by_mmuidx(cpu, idxmap);
793         return;
794     }
795 
796     /* This should already be page aligned */
797     d.addr = addr & TARGET_PAGE_MASK;
798     d.len = len;
799     d.idxmap = idxmap;
800     d.bits = bits;
801 
802     tlb_flush_range_by_mmuidx_async_0(cpu, d);
803 }
804 
805 void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr,
806                                    uint16_t idxmap, unsigned bits)
807 {
808     tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits);
809 }
810 
811 void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
812                                                vaddr addr,
813                                                vaddr len,
814                                                uint16_t idxmap,
815                                                unsigned bits)
816 {
817     TLBFlushRangeData d, *p;
818     CPUState *dst_cpu;
819 
820     /*
821      * If all bits are significant, and len is small,
822      * this devolves to tlb_flush_page.
823      */
824     if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
825         tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu, addr, idxmap);
826         return;
827     }
828     /* If no page bits are significant, this devolves to tlb_flush. */
829     if (bits < TARGET_PAGE_BITS) {
830         tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, idxmap);
831         return;
832     }
833 
834     /* This should already be page aligned */
835     d.addr = addr & TARGET_PAGE_MASK;
836     d.len = len;
837     d.idxmap = idxmap;
838     d.bits = bits;
839 
840     /* Allocate a separate data block for each destination cpu.  */
841     CPU_FOREACH(dst_cpu) {
842         if (dst_cpu != src_cpu) {
843             p = g_memdup(&d, sizeof(d));
844             async_run_on_cpu(dst_cpu, tlb_flush_range_by_mmuidx_async_1,
845                              RUN_ON_CPU_HOST_PTR(p));
846         }
847     }
848 
849     p = g_memdup(&d, sizeof(d));
850     async_safe_run_on_cpu(src_cpu, tlb_flush_range_by_mmuidx_async_1,
851                           RUN_ON_CPU_HOST_PTR(p));
852 }
853 
854 void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
855                                                    vaddr addr,
856                                                    uint16_t idxmap,
857                                                    unsigned bits)
858 {
859     tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu, addr, TARGET_PAGE_SIZE,
860                                               idxmap, bits);
861 }
862 
863 /* update the TLBs so that writes to code in the virtual page 'addr'
864    can be detected */
865 void tlb_protect_code(ram_addr_t ram_addr)
866 {
867     cpu_physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK,
868                                              TARGET_PAGE_SIZE,
869                                              DIRTY_MEMORY_CODE);
870 }
871 
872 /* update the TLB so that writes in physical page 'phys_addr' are no longer
873    tested for self modifying code */
874 void tlb_unprotect_code(ram_addr_t ram_addr)
875 {
876     cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
877 }
878 
879 
880 /*
881  * Dirty write flag handling
882  *
883  * When the TCG code writes to a location it looks up the address in
884  * the TLB and uses that data to compute the final address. If any of
885  * the lower bits of the address are set then the slow path is forced.
886  * There are a number of reasons to do this but for normal RAM the
887  * most usual is detecting writes to code regions which may invalidate
888  * generated code.
889  *
890  * Other vCPUs might be reading their TLBs during guest execution, so we update
891  * te->addr_write with qatomic_set. We don't need to worry about this for
892  * oversized guests as MTTCG is disabled for them.
893  *
894  * Called with tlb_c.lock held.
895  */
896 static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,
897                                          uintptr_t start, uintptr_t length)
898 {
899     uintptr_t addr = tlb_entry->addr_write;
900 
901     if ((addr & (TLB_INVALID_MASK | TLB_MMIO |
902                  TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) {
903         addr &= TARGET_PAGE_MASK;
904         addr += tlb_entry->addend;
905         if ((addr - start) < length) {
906 #if TARGET_LONG_BITS == 32
907             uint32_t *ptr_write = (uint32_t *)&tlb_entry->addr_write;
908             ptr_write += HOST_BIG_ENDIAN;
909             qatomic_set(ptr_write, *ptr_write | TLB_NOTDIRTY);
910 #elif TCG_OVERSIZED_GUEST
911             tlb_entry->addr_write |= TLB_NOTDIRTY;
912 #else
913             qatomic_set(&tlb_entry->addr_write,
914                         tlb_entry->addr_write | TLB_NOTDIRTY);
915 #endif
916         }
917     }
918 }
919 
920 /*
921  * Called with tlb_c.lock held.
922  * Called only from the vCPU context, i.e. the TLB's owner thread.
923  */
924 static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s)
925 {
926     *d = *s;
927 }
928 
929 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
930  * the target vCPU).
931  * We must take tlb_c.lock to avoid racing with another vCPU update. The only
932  * thing actually updated is the target TLB entry ->addr_write flags.
933  */
934 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
935 {
936     int mmu_idx;
937 
938     qemu_spin_lock(&cpu->neg.tlb.c.lock);
939     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
940         unsigned int i;
941         unsigned int n = tlb_n_entries(&cpu->neg.tlb.f[mmu_idx]);
942 
943         for (i = 0; i < n; i++) {
944             tlb_reset_dirty_range_locked(&cpu->neg.tlb.f[mmu_idx].table[i],
945                                          start1, length);
946         }
947 
948         for (i = 0; i < CPU_VTLB_SIZE; i++) {
949             tlb_reset_dirty_range_locked(&cpu->neg.tlb.d[mmu_idx].vtable[i],
950                                          start1, length);
951         }
952     }
953     qemu_spin_unlock(&cpu->neg.tlb.c.lock);
954 }
955 
956 /* Called with tlb_c.lock held */
957 static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,
958                                          vaddr addr)
959 {
960     if (tlb_entry->addr_write == (addr | TLB_NOTDIRTY)) {
961         tlb_entry->addr_write = addr;
962     }
963 }
964 
965 /* update the TLB corresponding to virtual page vaddr
966    so that it is no longer dirty */
967 static void tlb_set_dirty(CPUState *cpu, vaddr addr)
968 {
969     int mmu_idx;
970 
971     assert_cpu_is_self(cpu);
972 
973     addr &= TARGET_PAGE_MASK;
974     qemu_spin_lock(&cpu->neg.tlb.c.lock);
975     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
976         tlb_set_dirty1_locked(tlb_entry(cpu, mmu_idx, addr), addr);
977     }
978 
979     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
980         int k;
981         for (k = 0; k < CPU_VTLB_SIZE; k++) {
982             tlb_set_dirty1_locked(&cpu->neg.tlb.d[mmu_idx].vtable[k], addr);
983         }
984     }
985     qemu_spin_unlock(&cpu->neg.tlb.c.lock);
986 }
987 
988 /* Our TLB does not support large pages, so remember the area covered by
989    large pages and trigger a full TLB flush if these are invalidated.  */
990 static void tlb_add_large_page(CPUState *cpu, int mmu_idx,
991                                vaddr addr, uint64_t size)
992 {
993     vaddr lp_addr = cpu->neg.tlb.d[mmu_idx].large_page_addr;
994     vaddr lp_mask = ~(size - 1);
995 
996     if (lp_addr == (vaddr)-1) {
997         /* No previous large page.  */
998         lp_addr = addr;
999     } else {
1000         /* Extend the existing region to include the new page.
1001            This is a compromise between unnecessary flushes and
1002            the cost of maintaining a full variable size TLB.  */
1003         lp_mask &= cpu->neg.tlb.d[mmu_idx].large_page_mask;
1004         while (((lp_addr ^ addr) & lp_mask) != 0) {
1005             lp_mask <<= 1;
1006         }
1007     }
1008     cpu->neg.tlb.d[mmu_idx].large_page_addr = lp_addr & lp_mask;
1009     cpu->neg.tlb.d[mmu_idx].large_page_mask = lp_mask;
1010 }
1011 
1012 static inline void tlb_set_compare(CPUTLBEntryFull *full, CPUTLBEntry *ent,
1013                                    vaddr address, int flags,
1014                                    MMUAccessType access_type, bool enable)
1015 {
1016     if (enable) {
1017         address |= flags & TLB_FLAGS_MASK;
1018         flags &= TLB_SLOW_FLAGS_MASK;
1019         if (flags) {
1020             address |= TLB_FORCE_SLOW;
1021         }
1022     } else {
1023         address = -1;
1024         flags = 0;
1025     }
1026     ent->addr_idx[access_type] = address;
1027     full->slow_flags[access_type] = flags;
1028 }
1029 
1030 /*
1031  * Add a new TLB entry. At most one entry for a given virtual address
1032  * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
1033  * supplied size is only used by tlb_flush_page.
1034  *
1035  * Called from TCG-generated code, which is under an RCU read-side
1036  * critical section.
1037  */
1038 void tlb_set_page_full(CPUState *cpu, int mmu_idx,
1039                        vaddr addr, CPUTLBEntryFull *full)
1040 {
1041     CPUTLB *tlb = &cpu->neg.tlb;
1042     CPUTLBDesc *desc = &tlb->d[mmu_idx];
1043     MemoryRegionSection *section;
1044     unsigned int index, read_flags, write_flags;
1045     uintptr_t addend;
1046     CPUTLBEntry *te, tn;
1047     hwaddr iotlb, xlat, sz, paddr_page;
1048     vaddr addr_page;
1049     int asidx, wp_flags, prot;
1050     bool is_ram, is_romd;
1051 
1052     assert_cpu_is_self(cpu);
1053 
1054     if (full->lg_page_size <= TARGET_PAGE_BITS) {
1055         sz = TARGET_PAGE_SIZE;
1056     } else {
1057         sz = (hwaddr)1 << full->lg_page_size;
1058         tlb_add_large_page(cpu, mmu_idx, addr, sz);
1059     }
1060     addr_page = addr & TARGET_PAGE_MASK;
1061     paddr_page = full->phys_addr & TARGET_PAGE_MASK;
1062 
1063     prot = full->prot;
1064     asidx = cpu_asidx_from_attrs(cpu, full->attrs);
1065     section = address_space_translate_for_iotlb(cpu, asidx, paddr_page,
1066                                                 &xlat, &sz, full->attrs, &prot);
1067     assert(sz >= TARGET_PAGE_SIZE);
1068 
1069     tlb_debug("vaddr=%016" VADDR_PRIx " paddr=0x" HWADDR_FMT_plx
1070               " prot=%x idx=%d\n",
1071               addr, full->phys_addr, prot, mmu_idx);
1072 
1073     read_flags = full->tlb_fill_flags;
1074     if (full->lg_page_size < TARGET_PAGE_BITS) {
1075         /* Repeat the MMU check and TLB fill on every access.  */
1076         read_flags |= TLB_INVALID_MASK;
1077     }
1078 
1079     is_ram = memory_region_is_ram(section->mr);
1080     is_romd = memory_region_is_romd(section->mr);
1081 
1082     if (is_ram || is_romd) {
1083         /* RAM and ROMD both have associated host memory. */
1084         addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
1085     } else {
1086         /* I/O does not; force the host address to NULL. */
1087         addend = 0;
1088     }
1089 
1090     write_flags = read_flags;
1091     if (is_ram) {
1092         iotlb = memory_region_get_ram_addr(section->mr) + xlat;
1093         assert(!(iotlb & ~TARGET_PAGE_MASK));
1094         /*
1095          * Computing is_clean is expensive; avoid all that unless
1096          * the page is actually writable.
1097          */
1098         if (prot & PAGE_WRITE) {
1099             if (section->readonly) {
1100                 write_flags |= TLB_DISCARD_WRITE;
1101             } else if (cpu_physical_memory_is_clean(iotlb)) {
1102                 write_flags |= TLB_NOTDIRTY;
1103             }
1104         }
1105     } else {
1106         /* I/O or ROMD */
1107         iotlb = memory_region_section_get_iotlb(cpu, section) + xlat;
1108         /*
1109          * Writes to romd devices must go through MMIO to enable write.
1110          * Reads to romd devices go through the ram_ptr found above,
1111          * but of course reads to I/O must go through MMIO.
1112          */
1113         write_flags |= TLB_MMIO;
1114         if (!is_romd) {
1115             read_flags = write_flags;
1116         }
1117     }
1118 
1119     wp_flags = cpu_watchpoint_address_matches(cpu, addr_page,
1120                                               TARGET_PAGE_SIZE);
1121 
1122     index = tlb_index(cpu, mmu_idx, addr_page);
1123     te = tlb_entry(cpu, mmu_idx, addr_page);
1124 
1125     /*
1126      * Hold the TLB lock for the rest of the function. We could acquire/release
1127      * the lock several times in the function, but it is faster to amortize the
1128      * acquisition cost by acquiring it just once. Note that this leads to
1129      * a longer critical section, but this is not a concern since the TLB lock
1130      * is unlikely to be contended.
1131      */
1132     qemu_spin_lock(&tlb->c.lock);
1133 
1134     /* Note that the tlb is no longer clean.  */
1135     tlb->c.dirty |= 1 << mmu_idx;
1136 
1137     /* Make sure there's no cached translation for the new page.  */
1138     tlb_flush_vtlb_page_locked(cpu, mmu_idx, addr_page);
1139 
1140     /*
1141      * Only evict the old entry to the victim tlb if it's for a
1142      * different page; otherwise just overwrite the stale data.
1143      */
1144     if (!tlb_hit_page_anyprot(te, addr_page) && !tlb_entry_is_empty(te)) {
1145         unsigned vidx = desc->vindex++ % CPU_VTLB_SIZE;
1146         CPUTLBEntry *tv = &desc->vtable[vidx];
1147 
1148         /* Evict the old entry into the victim tlb.  */
1149         copy_tlb_helper_locked(tv, te);
1150         desc->vfulltlb[vidx] = desc->fulltlb[index];
1151         tlb_n_used_entries_dec(cpu, mmu_idx);
1152     }
1153 
1154     /* refill the tlb */
1155     /*
1156      * When memory region is ram, iotlb contains a TARGET_PAGE_BITS
1157      * aligned ram_addr_t of the page base of the target RAM.
1158      * Otherwise, iotlb contains
1159      *  - a physical section number in the lower TARGET_PAGE_BITS
1160      *  - the offset within section->mr of the page base (I/O, ROMD) with the
1161      *    TARGET_PAGE_BITS masked off.
1162      * We subtract addr_page (which is page aligned and thus won't
1163      * disturb the low bits) to give an offset which can be added to the
1164      * (non-page-aligned) vaddr of the eventual memory access to get
1165      * the MemoryRegion offset for the access. Note that the vaddr we
1166      * subtract here is that of the page base, and not the same as the
1167      * vaddr we add back in io_prepare()/get_page_addr_code().
1168      */
1169     desc->fulltlb[index] = *full;
1170     full = &desc->fulltlb[index];
1171     full->xlat_section = iotlb - addr_page;
1172     full->phys_addr = paddr_page;
1173 
1174     /* Now calculate the new entry */
1175     tn.addend = addend - addr_page;
1176 
1177     tlb_set_compare(full, &tn, addr_page, read_flags,
1178                     MMU_INST_FETCH, prot & PAGE_EXEC);
1179 
1180     if (wp_flags & BP_MEM_READ) {
1181         read_flags |= TLB_WATCHPOINT;
1182     }
1183     tlb_set_compare(full, &tn, addr_page, read_flags,
1184                     MMU_DATA_LOAD, prot & PAGE_READ);
1185 
1186     if (prot & PAGE_WRITE_INV) {
1187         write_flags |= TLB_INVALID_MASK;
1188     }
1189     if (wp_flags & BP_MEM_WRITE) {
1190         write_flags |= TLB_WATCHPOINT;
1191     }
1192     tlb_set_compare(full, &tn, addr_page, write_flags,
1193                     MMU_DATA_STORE, prot & PAGE_WRITE);
1194 
1195     copy_tlb_helper_locked(te, &tn);
1196     tlb_n_used_entries_inc(cpu, mmu_idx);
1197     qemu_spin_unlock(&tlb->c.lock);
1198 }
1199 
1200 void tlb_set_page_with_attrs(CPUState *cpu, vaddr addr,
1201                              hwaddr paddr, MemTxAttrs attrs, int prot,
1202                              int mmu_idx, uint64_t size)
1203 {
1204     CPUTLBEntryFull full = {
1205         .phys_addr = paddr,
1206         .attrs = attrs,
1207         .prot = prot,
1208         .lg_page_size = ctz64(size)
1209     };
1210 
1211     assert(is_power_of_2(size));
1212     tlb_set_page_full(cpu, mmu_idx, addr, &full);
1213 }
1214 
1215 void tlb_set_page(CPUState *cpu, vaddr addr,
1216                   hwaddr paddr, int prot,
1217                   int mmu_idx, uint64_t size)
1218 {
1219     tlb_set_page_with_attrs(cpu, addr, paddr, MEMTXATTRS_UNSPECIFIED,
1220                             prot, mmu_idx, size);
1221 }
1222 
1223 /*
1224  * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the
1225  * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must
1226  * be discarded and looked up again (e.g. via tlb_entry()).
1227  */
1228 static void tlb_fill(CPUState *cpu, vaddr addr, int size,
1229                      MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
1230 {
1231     bool ok;
1232 
1233     /*
1234      * This is not a probe, so only valid return is success; failure
1235      * should result in exception + longjmp to the cpu loop.
1236      */
1237     ok = cpu->cc->tcg_ops->tlb_fill(cpu, addr, size,
1238                                     access_type, mmu_idx, false, retaddr);
1239     assert(ok);
1240 }
1241 
1242 static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr,
1243                                         MMUAccessType access_type,
1244                                         int mmu_idx, uintptr_t retaddr)
1245 {
1246     cpu->cc->tcg_ops->do_unaligned_access(cpu, addr, access_type,
1247                                           mmu_idx, retaddr);
1248 }
1249 
1250 static MemoryRegionSection *
1251 io_prepare(hwaddr *out_offset, CPUState *cpu, hwaddr xlat,
1252            MemTxAttrs attrs, vaddr addr, uintptr_t retaddr)
1253 {
1254     MemoryRegionSection *section;
1255     hwaddr mr_offset;
1256 
1257     section = iotlb_to_section(cpu, xlat, attrs);
1258     mr_offset = (xlat & TARGET_PAGE_MASK) + addr;
1259     cpu->mem_io_pc = retaddr;
1260     if (!cpu->neg.can_do_io) {
1261         cpu_io_recompile(cpu, retaddr);
1262     }
1263 
1264     *out_offset = mr_offset;
1265     return section;
1266 }
1267 
1268 static void io_failed(CPUState *cpu, CPUTLBEntryFull *full, vaddr addr,
1269                       unsigned size, MMUAccessType access_type, int mmu_idx,
1270                       MemTxResult response, uintptr_t retaddr)
1271 {
1272     if (!cpu->ignore_memory_transaction_failures
1273         && cpu->cc->tcg_ops->do_transaction_failed) {
1274         hwaddr physaddr = full->phys_addr | (addr & ~TARGET_PAGE_MASK);
1275 
1276         cpu->cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size,
1277                                                 access_type, mmu_idx,
1278                                                 full->attrs, response, retaddr);
1279     }
1280 }
1281 
1282 /* Return true if ADDR is present in the victim tlb, and has been copied
1283    back to the main tlb.  */
1284 static bool victim_tlb_hit(CPUState *cpu, size_t mmu_idx, size_t index,
1285                            MMUAccessType access_type, vaddr page)
1286 {
1287     size_t vidx;
1288 
1289     assert_cpu_is_self(cpu);
1290     for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
1291         CPUTLBEntry *vtlb = &cpu->neg.tlb.d[mmu_idx].vtable[vidx];
1292         uint64_t cmp = tlb_read_idx(vtlb, access_type);
1293 
1294         if (cmp == page) {
1295             /* Found entry in victim tlb, swap tlb and iotlb.  */
1296             CPUTLBEntry tmptlb, *tlb = &cpu->neg.tlb.f[mmu_idx].table[index];
1297 
1298             qemu_spin_lock(&cpu->neg.tlb.c.lock);
1299             copy_tlb_helper_locked(&tmptlb, tlb);
1300             copy_tlb_helper_locked(tlb, vtlb);
1301             copy_tlb_helper_locked(vtlb, &tmptlb);
1302             qemu_spin_unlock(&cpu->neg.tlb.c.lock);
1303 
1304             CPUTLBEntryFull *f1 = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1305             CPUTLBEntryFull *f2 = &cpu->neg.tlb.d[mmu_idx].vfulltlb[vidx];
1306             CPUTLBEntryFull tmpf;
1307             tmpf = *f1; *f1 = *f2; *f2 = tmpf;
1308             return true;
1309         }
1310     }
1311     return false;
1312 }
1313 
1314 static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size,
1315                            CPUTLBEntryFull *full, uintptr_t retaddr)
1316 {
1317     ram_addr_t ram_addr = mem_vaddr + full->xlat_section;
1318 
1319     trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);
1320 
1321     if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1322         tb_invalidate_phys_range_fast(ram_addr, size, retaddr);
1323     }
1324 
1325     /*
1326      * Set both VGA and migration bits for simplicity and to remove
1327      * the notdirty callback faster.
1328      */
1329     cpu_physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE);
1330 
1331     /* We remove the notdirty callback only if the code has been flushed. */
1332     if (!cpu_physical_memory_is_clean(ram_addr)) {
1333         trace_memory_notdirty_set_dirty(mem_vaddr);
1334         tlb_set_dirty(cpu, mem_vaddr);
1335     }
1336 }
1337 
1338 static int probe_access_internal(CPUState *cpu, vaddr addr,
1339                                  int fault_size, MMUAccessType access_type,
1340                                  int mmu_idx, bool nonfault,
1341                                  void **phost, CPUTLBEntryFull **pfull,
1342                                  uintptr_t retaddr, bool check_mem_cbs)
1343 {
1344     uintptr_t index = tlb_index(cpu, mmu_idx, addr);
1345     CPUTLBEntry *entry = tlb_entry(cpu, mmu_idx, addr);
1346     uint64_t tlb_addr = tlb_read_idx(entry, access_type);
1347     vaddr page_addr = addr & TARGET_PAGE_MASK;
1348     int flags = TLB_FLAGS_MASK & ~TLB_FORCE_SLOW;
1349     bool force_mmio = check_mem_cbs && cpu_plugin_mem_cbs_enabled(cpu);
1350     CPUTLBEntryFull *full;
1351 
1352     if (!tlb_hit_page(tlb_addr, page_addr)) {
1353         if (!victim_tlb_hit(cpu, mmu_idx, index, access_type, page_addr)) {
1354             if (!cpu->cc->tcg_ops->tlb_fill(cpu, addr, fault_size, access_type,
1355                                             mmu_idx, nonfault, retaddr)) {
1356                 /* Non-faulting page table read failed.  */
1357                 *phost = NULL;
1358                 *pfull = NULL;
1359                 return TLB_INVALID_MASK;
1360             }
1361 
1362             /* TLB resize via tlb_fill may have moved the entry.  */
1363             index = tlb_index(cpu, mmu_idx, addr);
1364             entry = tlb_entry(cpu, mmu_idx, addr);
1365 
1366             /*
1367              * With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately,
1368              * to force the next access through tlb_fill.  We've just
1369              * called tlb_fill, so we know that this entry *is* valid.
1370              */
1371             flags &= ~TLB_INVALID_MASK;
1372         }
1373         tlb_addr = tlb_read_idx(entry, access_type);
1374     }
1375     flags &= tlb_addr;
1376 
1377     *pfull = full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1378     flags |= full->slow_flags[access_type];
1379 
1380     /* Fold all "mmio-like" bits into TLB_MMIO.  This is not RAM.  */
1381     if (unlikely(flags & ~(TLB_WATCHPOINT | TLB_NOTDIRTY | TLB_CHECK_ALIGNED))
1382         || (access_type != MMU_INST_FETCH && force_mmio)) {
1383         *phost = NULL;
1384         return TLB_MMIO;
1385     }
1386 
1387     /* Everything else is RAM. */
1388     *phost = (void *)((uintptr_t)addr + entry->addend);
1389     return flags;
1390 }
1391 
1392 int probe_access_full(CPUArchState *env, vaddr addr, int size,
1393                       MMUAccessType access_type, int mmu_idx,
1394                       bool nonfault, void **phost, CPUTLBEntryFull **pfull,
1395                       uintptr_t retaddr)
1396 {
1397     int flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1398                                       mmu_idx, nonfault, phost, pfull, retaddr,
1399                                       true);
1400 
1401     /* Handle clean RAM pages.  */
1402     if (unlikely(flags & TLB_NOTDIRTY)) {
1403         int dirtysize = size == 0 ? 1 : size;
1404         notdirty_write(env_cpu(env), addr, dirtysize, *pfull, retaddr);
1405         flags &= ~TLB_NOTDIRTY;
1406     }
1407 
1408     return flags;
1409 }
1410 
1411 int probe_access_full_mmu(CPUArchState *env, vaddr addr, int size,
1412                           MMUAccessType access_type, int mmu_idx,
1413                           void **phost, CPUTLBEntryFull **pfull)
1414 {
1415     void *discard_phost;
1416     CPUTLBEntryFull *discard_tlb;
1417 
1418     /* privately handle users that don't need full results */
1419     phost = phost ? phost : &discard_phost;
1420     pfull = pfull ? pfull : &discard_tlb;
1421 
1422     int flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1423                                       mmu_idx, true, phost, pfull, 0, false);
1424 
1425     /* Handle clean RAM pages.  */
1426     if (unlikely(flags & TLB_NOTDIRTY)) {
1427         int dirtysize = size == 0 ? 1 : size;
1428         notdirty_write(env_cpu(env), addr, dirtysize, *pfull, 0);
1429         flags &= ~TLB_NOTDIRTY;
1430     }
1431 
1432     return flags;
1433 }
1434 
1435 int probe_access_flags(CPUArchState *env, vaddr addr, int size,
1436                        MMUAccessType access_type, int mmu_idx,
1437                        bool nonfault, void **phost, uintptr_t retaddr)
1438 {
1439     CPUTLBEntryFull *full;
1440     int flags;
1441 
1442     g_assert(-(addr | TARGET_PAGE_MASK) >= size);
1443 
1444     flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1445                                   mmu_idx, nonfault, phost, &full, retaddr,
1446                                   true);
1447 
1448     /* Handle clean RAM pages. */
1449     if (unlikely(flags & TLB_NOTDIRTY)) {
1450         int dirtysize = size == 0 ? 1 : size;
1451         notdirty_write(env_cpu(env), addr, dirtysize, full, retaddr);
1452         flags &= ~TLB_NOTDIRTY;
1453     }
1454 
1455     return flags;
1456 }
1457 
1458 void *probe_access(CPUArchState *env, vaddr addr, int size,
1459                    MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
1460 {
1461     CPUTLBEntryFull *full;
1462     void *host;
1463     int flags;
1464 
1465     g_assert(-(addr | TARGET_PAGE_MASK) >= size);
1466 
1467     flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1468                                   mmu_idx, false, &host, &full, retaddr,
1469                                   true);
1470 
1471     /* Per the interface, size == 0 merely faults the access. */
1472     if (size == 0) {
1473         return NULL;
1474     }
1475 
1476     if (unlikely(flags & (TLB_NOTDIRTY | TLB_WATCHPOINT))) {
1477         /* Handle watchpoints.  */
1478         if (flags & TLB_WATCHPOINT) {
1479             int wp_access = (access_type == MMU_DATA_STORE
1480                              ? BP_MEM_WRITE : BP_MEM_READ);
1481             cpu_check_watchpoint(env_cpu(env), addr, size,
1482                                  full->attrs, wp_access, retaddr);
1483         }
1484 
1485         /* Handle clean RAM pages.  */
1486         if (flags & TLB_NOTDIRTY) {
1487             notdirty_write(env_cpu(env), addr, size, full, retaddr);
1488         }
1489     }
1490 
1491     return host;
1492 }
1493 
1494 void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
1495                         MMUAccessType access_type, int mmu_idx)
1496 {
1497     CPUTLBEntryFull *full;
1498     void *host;
1499     int flags;
1500 
1501     flags = probe_access_internal(env_cpu(env), addr, 0, access_type,
1502                                   mmu_idx, true, &host, &full, 0, false);
1503 
1504     /* No combination of flags are expected by the caller. */
1505     return flags ? NULL : host;
1506 }
1507 
1508 /*
1509  * Return a ram_addr_t for the virtual address for execution.
1510  *
1511  * Return -1 if we can't translate and execute from an entire page
1512  * of RAM.  This will force us to execute by loading and translating
1513  * one insn at a time, without caching.
1514  *
1515  * NOTE: This function will trigger an exception if the page is
1516  * not executable.
1517  */
1518 tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr,
1519                                         void **hostp)
1520 {
1521     CPUTLBEntryFull *full;
1522     void *p;
1523 
1524     (void)probe_access_internal(env_cpu(env), addr, 1, MMU_INST_FETCH,
1525                                 cpu_mmu_index(env_cpu(env), true), false,
1526                                 &p, &full, 0, false);
1527     if (p == NULL) {
1528         return -1;
1529     }
1530 
1531     if (full->lg_page_size < TARGET_PAGE_BITS) {
1532         return -1;
1533     }
1534 
1535     if (hostp) {
1536         *hostp = p;
1537     }
1538     return qemu_ram_addr_from_host_nofail(p);
1539 }
1540 
1541 /* Load/store with atomicity primitives. */
1542 #include "ldst_atomicity.c.inc"
1543 
1544 #ifdef CONFIG_PLUGIN
1545 /*
1546  * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1547  * This should be a hot path as we will have just looked this path up
1548  * in the softmmu lookup code (or helper). We don't handle re-fills or
1549  * checking the victim table. This is purely informational.
1550  *
1551  * The one corner case is i/o write, which can cause changes to the
1552  * address space.  Those changes, and the corresponding tlb flush,
1553  * should be delayed until the next TB, so even then this ought not fail.
1554  * But check, Just in Case.
1555  */
1556 bool tlb_plugin_lookup(CPUState *cpu, vaddr addr, int mmu_idx,
1557                        bool is_store, struct qemu_plugin_hwaddr *data)
1558 {
1559     CPUTLBEntry *tlbe = tlb_entry(cpu, mmu_idx, addr);
1560     uintptr_t index = tlb_index(cpu, mmu_idx, addr);
1561     MMUAccessType access_type = is_store ? MMU_DATA_STORE : MMU_DATA_LOAD;
1562     uint64_t tlb_addr = tlb_read_idx(tlbe, access_type);
1563     CPUTLBEntryFull *full;
1564 
1565     if (unlikely(!tlb_hit(tlb_addr, addr))) {
1566         return false;
1567     }
1568 
1569     full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1570     data->phys_addr = full->phys_addr | (addr & ~TARGET_PAGE_MASK);
1571 
1572     /* We must have an iotlb entry for MMIO */
1573     if (tlb_addr & TLB_MMIO) {
1574         MemoryRegionSection *section =
1575             iotlb_to_section(cpu, full->xlat_section & ~TARGET_PAGE_MASK,
1576                              full->attrs);
1577         data->is_io = true;
1578         data->mr = section->mr;
1579     } else {
1580         data->is_io = false;
1581         data->mr = NULL;
1582     }
1583     return true;
1584 }
1585 #endif
1586 
1587 /*
1588  * Probe for a load/store operation.
1589  * Return the host address and into @flags.
1590  */
1591 
1592 typedef struct MMULookupPageData {
1593     CPUTLBEntryFull *full;
1594     void *haddr;
1595     vaddr addr;
1596     int flags;
1597     int size;
1598 } MMULookupPageData;
1599 
1600 typedef struct MMULookupLocals {
1601     MMULookupPageData page[2];
1602     MemOp memop;
1603     int mmu_idx;
1604 } MMULookupLocals;
1605 
1606 /**
1607  * mmu_lookup1: translate one page
1608  * @cpu: generic cpu state
1609  * @data: lookup parameters
1610  * @mmu_idx: virtual address context
1611  * @access_type: load/store/code
1612  * @ra: return address into tcg generated code, or 0
1613  *
1614  * Resolve the translation for the one page at @data.addr, filling in
1615  * the rest of @data with the results.  If the translation fails,
1616  * tlb_fill will longjmp out.  Return true if the softmmu tlb for
1617  * @mmu_idx may have resized.
1618  */
1619 static bool mmu_lookup1(CPUState *cpu, MMULookupPageData *data,
1620                         int mmu_idx, MMUAccessType access_type, uintptr_t ra)
1621 {
1622     vaddr addr = data->addr;
1623     uintptr_t index = tlb_index(cpu, mmu_idx, addr);
1624     CPUTLBEntry *entry = tlb_entry(cpu, mmu_idx, addr);
1625     uint64_t tlb_addr = tlb_read_idx(entry, access_type);
1626     bool maybe_resized = false;
1627     CPUTLBEntryFull *full;
1628     int flags;
1629 
1630     /* If the TLB entry is for a different page, reload and try again.  */
1631     if (!tlb_hit(tlb_addr, addr)) {
1632         if (!victim_tlb_hit(cpu, mmu_idx, index, access_type,
1633                             addr & TARGET_PAGE_MASK)) {
1634             tlb_fill(cpu, addr, data->size, access_type, mmu_idx, ra);
1635             maybe_resized = true;
1636             index = tlb_index(cpu, mmu_idx, addr);
1637             entry = tlb_entry(cpu, mmu_idx, addr);
1638         }
1639         tlb_addr = tlb_read_idx(entry, access_type) & ~TLB_INVALID_MASK;
1640     }
1641 
1642     full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1643     flags = tlb_addr & (TLB_FLAGS_MASK & ~TLB_FORCE_SLOW);
1644     flags |= full->slow_flags[access_type];
1645 
1646     data->full = full;
1647     data->flags = flags;
1648     /* Compute haddr speculatively; depending on flags it might be invalid. */
1649     data->haddr = (void *)((uintptr_t)addr + entry->addend);
1650 
1651     return maybe_resized;
1652 }
1653 
1654 /**
1655  * mmu_watch_or_dirty
1656  * @cpu: generic cpu state
1657  * @data: lookup parameters
1658  * @access_type: load/store/code
1659  * @ra: return address into tcg generated code, or 0
1660  *
1661  * Trigger watchpoints for @data.addr:@data.size;
1662  * record writes to protected clean pages.
1663  */
1664 static void mmu_watch_or_dirty(CPUState *cpu, MMULookupPageData *data,
1665                                MMUAccessType access_type, uintptr_t ra)
1666 {
1667     CPUTLBEntryFull *full = data->full;
1668     vaddr addr = data->addr;
1669     int flags = data->flags;
1670     int size = data->size;
1671 
1672     /* On watchpoint hit, this will longjmp out.  */
1673     if (flags & TLB_WATCHPOINT) {
1674         int wp = access_type == MMU_DATA_STORE ? BP_MEM_WRITE : BP_MEM_READ;
1675         cpu_check_watchpoint(cpu, addr, size, full->attrs, wp, ra);
1676         flags &= ~TLB_WATCHPOINT;
1677     }
1678 
1679     /* Note that notdirty is only set for writes. */
1680     if (flags & TLB_NOTDIRTY) {
1681         notdirty_write(cpu, addr, size, full, ra);
1682         flags &= ~TLB_NOTDIRTY;
1683     }
1684     data->flags = flags;
1685 }
1686 
1687 /**
1688  * mmu_lookup: translate page(s)
1689  * @cpu: generic cpu state
1690  * @addr: virtual address
1691  * @oi: combined mmu_idx and MemOp
1692  * @ra: return address into tcg generated code, or 0
1693  * @access_type: load/store/code
1694  * @l: output result
1695  *
1696  * Resolve the translation for the page(s) beginning at @addr, for MemOp.size
1697  * bytes.  Return true if the lookup crosses a page boundary.
1698  */
1699 static bool mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
1700                        uintptr_t ra, MMUAccessType type, MMULookupLocals *l)
1701 {
1702     unsigned a_bits;
1703     bool crosspage;
1704     int flags;
1705 
1706     l->memop = get_memop(oi);
1707     l->mmu_idx = get_mmuidx(oi);
1708 
1709     tcg_debug_assert(l->mmu_idx < NB_MMU_MODES);
1710 
1711     /* Handle CPU specific unaligned behaviour */
1712     a_bits = get_alignment_bits(l->memop);
1713     if (addr & ((1 << a_bits) - 1)) {
1714         cpu_unaligned_access(cpu, addr, type, l->mmu_idx, ra);
1715     }
1716 
1717     l->page[0].addr = addr;
1718     l->page[0].size = memop_size(l->memop);
1719     l->page[1].addr = (addr + l->page[0].size - 1) & TARGET_PAGE_MASK;
1720     l->page[1].size = 0;
1721     crosspage = (addr ^ l->page[1].addr) & TARGET_PAGE_MASK;
1722 
1723     if (likely(!crosspage)) {
1724         mmu_lookup1(cpu, &l->page[0], l->mmu_idx, type, ra);
1725 
1726         flags = l->page[0].flags;
1727         if (unlikely(flags & (TLB_WATCHPOINT | TLB_NOTDIRTY))) {
1728             mmu_watch_or_dirty(cpu, &l->page[0], type, ra);
1729         }
1730         if (unlikely(flags & TLB_BSWAP)) {
1731             l->memop ^= MO_BSWAP;
1732         }
1733     } else {
1734         /* Finish compute of page crossing. */
1735         int size0 = l->page[1].addr - addr;
1736         l->page[1].size = l->page[0].size - size0;
1737         l->page[0].size = size0;
1738 
1739         /*
1740          * Lookup both pages, recognizing exceptions from either.  If the
1741          * second lookup potentially resized, refresh first CPUTLBEntryFull.
1742          */
1743         mmu_lookup1(cpu, &l->page[0], l->mmu_idx, type, ra);
1744         if (mmu_lookup1(cpu, &l->page[1], l->mmu_idx, type, ra)) {
1745             uintptr_t index = tlb_index(cpu, l->mmu_idx, addr);
1746             l->page[0].full = &cpu->neg.tlb.d[l->mmu_idx].fulltlb[index];
1747         }
1748 
1749         flags = l->page[0].flags | l->page[1].flags;
1750         if (unlikely(flags & (TLB_WATCHPOINT | TLB_NOTDIRTY))) {
1751             mmu_watch_or_dirty(cpu, &l->page[0], type, ra);
1752             mmu_watch_or_dirty(cpu, &l->page[1], type, ra);
1753         }
1754 
1755         /*
1756          * Since target/sparc is the only user of TLB_BSWAP, and all
1757          * Sparc accesses are aligned, any treatment across two pages
1758          * would be arbitrary.  Refuse it until there's a use.
1759          */
1760         tcg_debug_assert((flags & TLB_BSWAP) == 0);
1761     }
1762 
1763     /*
1764      * This alignment check differs from the one above, in that this is
1765      * based on the atomicity of the operation. The intended use case is
1766      * the ARM memory type field of each PTE, where access to pages with
1767      * Device memory type require alignment.
1768      */
1769     if (unlikely(flags & TLB_CHECK_ALIGNED)) {
1770         MemOp size = l->memop & MO_SIZE;
1771 
1772         switch (l->memop & MO_ATOM_MASK) {
1773         case MO_ATOM_NONE:
1774             size = MO_8;
1775             break;
1776         case MO_ATOM_IFALIGN_PAIR:
1777         case MO_ATOM_WITHIN16_PAIR:
1778             size = size ? size - 1 : 0;
1779             break;
1780         default:
1781             break;
1782         }
1783         if (addr & ((1 << size) - 1)) {
1784             cpu_unaligned_access(cpu, addr, type, l->mmu_idx, ra);
1785         }
1786     }
1787 
1788     return crosspage;
1789 }
1790 
1791 /*
1792  * Probe for an atomic operation.  Do not allow unaligned operations,
1793  * or io operations to proceed.  Return the host address.
1794  */
1795 static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
1796                                int size, uintptr_t retaddr)
1797 {
1798     uintptr_t mmu_idx = get_mmuidx(oi);
1799     MemOp mop = get_memop(oi);
1800     int a_bits = get_alignment_bits(mop);
1801     uintptr_t index;
1802     CPUTLBEntry *tlbe;
1803     vaddr tlb_addr;
1804     void *hostaddr;
1805     CPUTLBEntryFull *full;
1806 
1807     tcg_debug_assert(mmu_idx < NB_MMU_MODES);
1808 
1809     /* Adjust the given return address.  */
1810     retaddr -= GETPC_ADJ;
1811 
1812     /* Enforce guest required alignment.  */
1813     if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
1814         /* ??? Maybe indicate atomic op to cpu_unaligned_access */
1815         cpu_unaligned_access(cpu, addr, MMU_DATA_STORE,
1816                              mmu_idx, retaddr);
1817     }
1818 
1819     /* Enforce qemu required alignment.  */
1820     if (unlikely(addr & (size - 1))) {
1821         /* We get here if guest alignment was not requested,
1822            or was not enforced by cpu_unaligned_access above.
1823            We might widen the access and emulate, but for now
1824            mark an exception and exit the cpu loop.  */
1825         goto stop_the_world;
1826     }
1827 
1828     index = tlb_index(cpu, mmu_idx, addr);
1829     tlbe = tlb_entry(cpu, mmu_idx, addr);
1830 
1831     /* Check TLB entry and enforce page permissions.  */
1832     tlb_addr = tlb_addr_write(tlbe);
1833     if (!tlb_hit(tlb_addr, addr)) {
1834         if (!victim_tlb_hit(cpu, mmu_idx, index, MMU_DATA_STORE,
1835                             addr & TARGET_PAGE_MASK)) {
1836             tlb_fill(cpu, addr, size,
1837                      MMU_DATA_STORE, mmu_idx, retaddr);
1838             index = tlb_index(cpu, mmu_idx, addr);
1839             tlbe = tlb_entry(cpu, mmu_idx, addr);
1840         }
1841         tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
1842     }
1843 
1844     /*
1845      * Let the guest notice RMW on a write-only page.
1846      * We have just verified that the page is writable.
1847      * Subpage lookups may have left TLB_INVALID_MASK set,
1848      * but addr_read will only be -1 if PAGE_READ was unset.
1849      */
1850     if (unlikely(tlbe->addr_read == -1)) {
1851         tlb_fill(cpu, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr);
1852         /*
1853          * Since we don't support reads and writes to different
1854          * addresses, and we do have the proper page loaded for
1855          * write, this shouldn't ever return.  But just in case,
1856          * handle via stop-the-world.
1857          */
1858         goto stop_the_world;
1859     }
1860     /* Collect tlb flags for read. */
1861     tlb_addr |= tlbe->addr_read;
1862 
1863     /* Notice an IO access or a needs-MMU-lookup access */
1864     if (unlikely(tlb_addr & (TLB_MMIO | TLB_DISCARD_WRITE))) {
1865         /* There's really nothing that can be done to
1866            support this apart from stop-the-world.  */
1867         goto stop_the_world;
1868     }
1869 
1870     hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
1871     full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1872 
1873     if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
1874         notdirty_write(cpu, addr, size, full, retaddr);
1875     }
1876 
1877     if (unlikely(tlb_addr & TLB_FORCE_SLOW)) {
1878         int wp_flags = 0;
1879 
1880         if (full->slow_flags[MMU_DATA_STORE] & TLB_WATCHPOINT) {
1881             wp_flags |= BP_MEM_WRITE;
1882         }
1883         if (full->slow_flags[MMU_DATA_LOAD] & TLB_WATCHPOINT) {
1884             wp_flags |= BP_MEM_READ;
1885         }
1886         if (wp_flags) {
1887             cpu_check_watchpoint(cpu, addr, size,
1888                                  full->attrs, wp_flags, retaddr);
1889         }
1890     }
1891 
1892     return hostaddr;
1893 
1894  stop_the_world:
1895     cpu_loop_exit_atomic(cpu, retaddr);
1896 }
1897 
1898 /*
1899  * Load Helpers
1900  *
1901  * We support two different access types. SOFTMMU_CODE_ACCESS is
1902  * specifically for reading instructions from system memory. It is
1903  * called by the translation loop and in some helpers where the code
1904  * is disassembled. It shouldn't be called directly by guest code.
1905  *
1906  * For the benefit of TCG generated code, we want to avoid the
1907  * complication of ABI-specific return type promotion and always
1908  * return a value extended to the register size of the host. This is
1909  * tcg_target_long, except in the case of a 32-bit host and 64-bit
1910  * data, and for that we always have uint64_t.
1911  *
1912  * We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
1913  */
1914 
1915 /**
1916  * do_ld_mmio_beN:
1917  * @cpu: generic cpu state
1918  * @full: page parameters
1919  * @ret_be: accumulated data
1920  * @addr: virtual address
1921  * @size: number of bytes
1922  * @mmu_idx: virtual address context
1923  * @ra: return address into tcg generated code, or 0
1924  * Context: BQL held
1925  *
1926  * Load @size bytes from @addr, which is memory-mapped i/o.
1927  * The bytes are concatenated in big-endian order with @ret_be.
1928  */
1929 static uint64_t int_ld_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
1930                                 uint64_t ret_be, vaddr addr, int size,
1931                                 int mmu_idx, MMUAccessType type, uintptr_t ra,
1932                                 MemoryRegion *mr, hwaddr mr_offset)
1933 {
1934     do {
1935         MemOp this_mop;
1936         unsigned this_size;
1937         uint64_t val;
1938         MemTxResult r;
1939 
1940         /* Read aligned pieces up to 8 bytes. */
1941         this_mop = ctz32(size | (int)addr | 8);
1942         this_size = 1 << this_mop;
1943         this_mop |= MO_BE;
1944 
1945         r = memory_region_dispatch_read(mr, mr_offset, &val,
1946                                         this_mop, full->attrs);
1947         if (unlikely(r != MEMTX_OK)) {
1948             io_failed(cpu, full, addr, this_size, type, mmu_idx, r, ra);
1949         }
1950         if (this_size == 8) {
1951             return val;
1952         }
1953 
1954         ret_be = (ret_be << (this_size * 8)) | val;
1955         addr += this_size;
1956         mr_offset += this_size;
1957         size -= this_size;
1958     } while (size);
1959 
1960     return ret_be;
1961 }
1962 
1963 static uint64_t do_ld_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
1964                                uint64_t ret_be, vaddr addr, int size,
1965                                int mmu_idx, MMUAccessType type, uintptr_t ra)
1966 {
1967     MemoryRegionSection *section;
1968     MemoryRegion *mr;
1969     hwaddr mr_offset;
1970     MemTxAttrs attrs;
1971 
1972     tcg_debug_assert(size > 0 && size <= 8);
1973 
1974     attrs = full->attrs;
1975     section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
1976     mr = section->mr;
1977 
1978     BQL_LOCK_GUARD();
1979     return int_ld_mmio_beN(cpu, full, ret_be, addr, size, mmu_idx,
1980                            type, ra, mr, mr_offset);
1981 }
1982 
1983 static Int128 do_ld16_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
1984                                uint64_t ret_be, vaddr addr, int size,
1985                                int mmu_idx, uintptr_t ra)
1986 {
1987     MemoryRegionSection *section;
1988     MemoryRegion *mr;
1989     hwaddr mr_offset;
1990     MemTxAttrs attrs;
1991     uint64_t a, b;
1992 
1993     tcg_debug_assert(size > 8 && size <= 16);
1994 
1995     attrs = full->attrs;
1996     section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
1997     mr = section->mr;
1998 
1999     BQL_LOCK_GUARD();
2000     a = int_ld_mmio_beN(cpu, full, ret_be, addr, size - 8, mmu_idx,
2001                         MMU_DATA_LOAD, ra, mr, mr_offset);
2002     b = int_ld_mmio_beN(cpu, full, ret_be, addr + size - 8, 8, mmu_idx,
2003                         MMU_DATA_LOAD, ra, mr, mr_offset + size - 8);
2004     return int128_make128(b, a);
2005 }
2006 
2007 /**
2008  * do_ld_bytes_beN
2009  * @p: translation parameters
2010  * @ret_be: accumulated data
2011  *
2012  * Load @p->size bytes from @p->haddr, which is RAM.
2013  * The bytes to concatenated in big-endian order with @ret_be.
2014  */
2015 static uint64_t do_ld_bytes_beN(MMULookupPageData *p, uint64_t ret_be)
2016 {
2017     uint8_t *haddr = p->haddr;
2018     int i, size = p->size;
2019 
2020     for (i = 0; i < size; i++) {
2021         ret_be = (ret_be << 8) | haddr[i];
2022     }
2023     return ret_be;
2024 }
2025 
2026 /**
2027  * do_ld_parts_beN
2028  * @p: translation parameters
2029  * @ret_be: accumulated data
2030  *
2031  * As do_ld_bytes_beN, but atomically on each aligned part.
2032  */
2033 static uint64_t do_ld_parts_beN(MMULookupPageData *p, uint64_t ret_be)
2034 {
2035     void *haddr = p->haddr;
2036     int size = p->size;
2037 
2038     do {
2039         uint64_t x;
2040         int n;
2041 
2042         /*
2043          * Find minimum of alignment and size.
2044          * This is slightly stronger than required by MO_ATOM_SUBALIGN, which
2045          * would have only checked the low bits of addr|size once at the start,
2046          * but is just as easy.
2047          */
2048         switch (((uintptr_t)haddr | size) & 7) {
2049         case 4:
2050             x = cpu_to_be32(load_atomic4(haddr));
2051             ret_be = (ret_be << 32) | x;
2052             n = 4;
2053             break;
2054         case 2:
2055         case 6:
2056             x = cpu_to_be16(load_atomic2(haddr));
2057             ret_be = (ret_be << 16) | x;
2058             n = 2;
2059             break;
2060         default:
2061             x = *(uint8_t *)haddr;
2062             ret_be = (ret_be << 8) | x;
2063             n = 1;
2064             break;
2065         case 0:
2066             g_assert_not_reached();
2067         }
2068         haddr += n;
2069         size -= n;
2070     } while (size != 0);
2071     return ret_be;
2072 }
2073 
2074 /**
2075  * do_ld_parts_be4
2076  * @p: translation parameters
2077  * @ret_be: accumulated data
2078  *
2079  * As do_ld_bytes_beN, but with one atomic load.
2080  * Four aligned bytes are guaranteed to cover the load.
2081  */
2082 static uint64_t do_ld_whole_be4(MMULookupPageData *p, uint64_t ret_be)
2083 {
2084     int o = p->addr & 3;
2085     uint32_t x = load_atomic4(p->haddr - o);
2086 
2087     x = cpu_to_be32(x);
2088     x <<= o * 8;
2089     x >>= (4 - p->size) * 8;
2090     return (ret_be << (p->size * 8)) | x;
2091 }
2092 
2093 /**
2094  * do_ld_parts_be8
2095  * @p: translation parameters
2096  * @ret_be: accumulated data
2097  *
2098  * As do_ld_bytes_beN, but with one atomic load.
2099  * Eight aligned bytes are guaranteed to cover the load.
2100  */
2101 static uint64_t do_ld_whole_be8(CPUState *cpu, uintptr_t ra,
2102                                 MMULookupPageData *p, uint64_t ret_be)
2103 {
2104     int o = p->addr & 7;
2105     uint64_t x = load_atomic8_or_exit(cpu, ra, p->haddr - o);
2106 
2107     x = cpu_to_be64(x);
2108     x <<= o * 8;
2109     x >>= (8 - p->size) * 8;
2110     return (ret_be << (p->size * 8)) | x;
2111 }
2112 
2113 /**
2114  * do_ld_parts_be16
2115  * @p: translation parameters
2116  * @ret_be: accumulated data
2117  *
2118  * As do_ld_bytes_beN, but with one atomic load.
2119  * 16 aligned bytes are guaranteed to cover the load.
2120  */
2121 static Int128 do_ld_whole_be16(CPUState *cpu, uintptr_t ra,
2122                                MMULookupPageData *p, uint64_t ret_be)
2123 {
2124     int o = p->addr & 15;
2125     Int128 x, y = load_atomic16_or_exit(cpu, ra, p->haddr - o);
2126     int size = p->size;
2127 
2128     if (!HOST_BIG_ENDIAN) {
2129         y = bswap128(y);
2130     }
2131     y = int128_lshift(y, o * 8);
2132     y = int128_urshift(y, (16 - size) * 8);
2133     x = int128_make64(ret_be);
2134     x = int128_lshift(x, size * 8);
2135     return int128_or(x, y);
2136 }
2137 
2138 /*
2139  * Wrapper for the above.
2140  */
2141 static uint64_t do_ld_beN(CPUState *cpu, MMULookupPageData *p,
2142                           uint64_t ret_be, int mmu_idx, MMUAccessType type,
2143                           MemOp mop, uintptr_t ra)
2144 {
2145     MemOp atom;
2146     unsigned tmp, half_size;
2147 
2148     if (unlikely(p->flags & TLB_MMIO)) {
2149         return do_ld_mmio_beN(cpu, p->full, ret_be, p->addr, p->size,
2150                               mmu_idx, type, ra);
2151     }
2152 
2153     /*
2154      * It is a given that we cross a page and therefore there is no
2155      * atomicity for the load as a whole, but subobjects may need attention.
2156      */
2157     atom = mop & MO_ATOM_MASK;
2158     switch (atom) {
2159     case MO_ATOM_SUBALIGN:
2160         return do_ld_parts_beN(p, ret_be);
2161 
2162     case MO_ATOM_IFALIGN_PAIR:
2163     case MO_ATOM_WITHIN16_PAIR:
2164         tmp = mop & MO_SIZE;
2165         tmp = tmp ? tmp - 1 : 0;
2166         half_size = 1 << tmp;
2167         if (atom == MO_ATOM_IFALIGN_PAIR
2168             ? p->size == half_size
2169             : p->size >= half_size) {
2170             if (!HAVE_al8_fast && p->size < 4) {
2171                 return do_ld_whole_be4(p, ret_be);
2172             } else {
2173                 return do_ld_whole_be8(cpu, ra, p, ret_be);
2174             }
2175         }
2176         /* fall through */
2177 
2178     case MO_ATOM_IFALIGN:
2179     case MO_ATOM_WITHIN16:
2180     case MO_ATOM_NONE:
2181         return do_ld_bytes_beN(p, ret_be);
2182 
2183     default:
2184         g_assert_not_reached();
2185     }
2186 }
2187 
2188 /*
2189  * Wrapper for the above, for 8 < size < 16.
2190  */
2191 static Int128 do_ld16_beN(CPUState *cpu, MMULookupPageData *p,
2192                           uint64_t a, int mmu_idx, MemOp mop, uintptr_t ra)
2193 {
2194     int size = p->size;
2195     uint64_t b;
2196     MemOp atom;
2197 
2198     if (unlikely(p->flags & TLB_MMIO)) {
2199         return do_ld16_mmio_beN(cpu, p->full, a, p->addr, size, mmu_idx, ra);
2200     }
2201 
2202     /*
2203      * It is a given that we cross a page and therefore there is no
2204      * atomicity for the load as a whole, but subobjects may need attention.
2205      */
2206     atom = mop & MO_ATOM_MASK;
2207     switch (atom) {
2208     case MO_ATOM_SUBALIGN:
2209         p->size = size - 8;
2210         a = do_ld_parts_beN(p, a);
2211         p->haddr += size - 8;
2212         p->size = 8;
2213         b = do_ld_parts_beN(p, 0);
2214         break;
2215 
2216     case MO_ATOM_WITHIN16_PAIR:
2217         /* Since size > 8, this is the half that must be atomic. */
2218         return do_ld_whole_be16(cpu, ra, p, a);
2219 
2220     case MO_ATOM_IFALIGN_PAIR:
2221         /*
2222          * Since size > 8, both halves are misaligned,
2223          * and so neither is atomic.
2224          */
2225     case MO_ATOM_IFALIGN:
2226     case MO_ATOM_WITHIN16:
2227     case MO_ATOM_NONE:
2228         p->size = size - 8;
2229         a = do_ld_bytes_beN(p, a);
2230         b = ldq_be_p(p->haddr + size - 8);
2231         break;
2232 
2233     default:
2234         g_assert_not_reached();
2235     }
2236 
2237     return int128_make128(b, a);
2238 }
2239 
2240 static uint8_t do_ld_1(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
2241                        MMUAccessType type, uintptr_t ra)
2242 {
2243     if (unlikely(p->flags & TLB_MMIO)) {
2244         return do_ld_mmio_beN(cpu, p->full, 0, p->addr, 1, mmu_idx, type, ra);
2245     } else {
2246         return *(uint8_t *)p->haddr;
2247     }
2248 }
2249 
2250 static uint16_t do_ld_2(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
2251                         MMUAccessType type, MemOp memop, uintptr_t ra)
2252 {
2253     uint16_t ret;
2254 
2255     if (unlikely(p->flags & TLB_MMIO)) {
2256         ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 2, mmu_idx, type, ra);
2257         if ((memop & MO_BSWAP) == MO_LE) {
2258             ret = bswap16(ret);
2259         }
2260     } else {
2261         /* Perform the load host endian, then swap if necessary. */
2262         ret = load_atom_2(cpu, ra, p->haddr, memop);
2263         if (memop & MO_BSWAP) {
2264             ret = bswap16(ret);
2265         }
2266     }
2267     return ret;
2268 }
2269 
2270 static uint32_t do_ld_4(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
2271                         MMUAccessType type, MemOp memop, uintptr_t ra)
2272 {
2273     uint32_t ret;
2274 
2275     if (unlikely(p->flags & TLB_MMIO)) {
2276         ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 4, mmu_idx, type, ra);
2277         if ((memop & MO_BSWAP) == MO_LE) {
2278             ret = bswap32(ret);
2279         }
2280     } else {
2281         /* Perform the load host endian. */
2282         ret = load_atom_4(cpu, ra, p->haddr, memop);
2283         if (memop & MO_BSWAP) {
2284             ret = bswap32(ret);
2285         }
2286     }
2287     return ret;
2288 }
2289 
2290 static uint64_t do_ld_8(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
2291                         MMUAccessType type, MemOp memop, uintptr_t ra)
2292 {
2293     uint64_t ret;
2294 
2295     if (unlikely(p->flags & TLB_MMIO)) {
2296         ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 8, mmu_idx, type, ra);
2297         if ((memop & MO_BSWAP) == MO_LE) {
2298             ret = bswap64(ret);
2299         }
2300     } else {
2301         /* Perform the load host endian. */
2302         ret = load_atom_8(cpu, ra, p->haddr, memop);
2303         if (memop & MO_BSWAP) {
2304             ret = bswap64(ret);
2305         }
2306     }
2307     return ret;
2308 }
2309 
2310 static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2311                           uintptr_t ra, MMUAccessType access_type)
2312 {
2313     MMULookupLocals l;
2314     bool crosspage;
2315 
2316     cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2317     crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2318     tcg_debug_assert(!crosspage);
2319 
2320     return do_ld_1(cpu, &l.page[0], l.mmu_idx, access_type, ra);
2321 }
2322 
2323 static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2324                            uintptr_t ra, MMUAccessType access_type)
2325 {
2326     MMULookupLocals l;
2327     bool crosspage;
2328     uint16_t ret;
2329     uint8_t a, b;
2330 
2331     cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2332     crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2333     if (likely(!crosspage)) {
2334         return do_ld_2(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
2335     }
2336 
2337     a = do_ld_1(cpu, &l.page[0], l.mmu_idx, access_type, ra);
2338     b = do_ld_1(cpu, &l.page[1], l.mmu_idx, access_type, ra);
2339 
2340     if ((l.memop & MO_BSWAP) == MO_LE) {
2341         ret = a | (b << 8);
2342     } else {
2343         ret = b | (a << 8);
2344     }
2345     return ret;
2346 }
2347 
2348 static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2349                            uintptr_t ra, MMUAccessType access_type)
2350 {
2351     MMULookupLocals l;
2352     bool crosspage;
2353     uint32_t ret;
2354 
2355     cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2356     crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2357     if (likely(!crosspage)) {
2358         return do_ld_4(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
2359     }
2360 
2361     ret = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
2362     ret = do_ld_beN(cpu, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
2363     if ((l.memop & MO_BSWAP) == MO_LE) {
2364         ret = bswap32(ret);
2365     }
2366     return ret;
2367 }
2368 
2369 static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2370                            uintptr_t ra, MMUAccessType access_type)
2371 {
2372     MMULookupLocals l;
2373     bool crosspage;
2374     uint64_t ret;
2375 
2376     cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2377     crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2378     if (likely(!crosspage)) {
2379         return do_ld_8(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
2380     }
2381 
2382     ret = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
2383     ret = do_ld_beN(cpu, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
2384     if ((l.memop & MO_BSWAP) == MO_LE) {
2385         ret = bswap64(ret);
2386     }
2387     return ret;
2388 }
2389 
2390 static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr,
2391                           MemOpIdx oi, uintptr_t ra)
2392 {
2393     MMULookupLocals l;
2394     bool crosspage;
2395     uint64_t a, b;
2396     Int128 ret;
2397     int first;
2398 
2399     cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2400     crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l);
2401     if (likely(!crosspage)) {
2402         if (unlikely(l.page[0].flags & TLB_MMIO)) {
2403             ret = do_ld16_mmio_beN(cpu, l.page[0].full, 0, addr, 16,
2404                                    l.mmu_idx, ra);
2405             if ((l.memop & MO_BSWAP) == MO_LE) {
2406                 ret = bswap128(ret);
2407             }
2408         } else {
2409             /* Perform the load host endian. */
2410             ret = load_atom_16(cpu, ra, l.page[0].haddr, l.memop);
2411             if (l.memop & MO_BSWAP) {
2412                 ret = bswap128(ret);
2413             }
2414         }
2415         return ret;
2416     }
2417 
2418     first = l.page[0].size;
2419     if (first == 8) {
2420         MemOp mop8 = (l.memop & ~MO_SIZE) | MO_64;
2421 
2422         a = do_ld_8(cpu, &l.page[0], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
2423         b = do_ld_8(cpu, &l.page[1], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
2424         if ((mop8 & MO_BSWAP) == MO_LE) {
2425             ret = int128_make128(a, b);
2426         } else {
2427             ret = int128_make128(b, a);
2428         }
2429         return ret;
2430     }
2431 
2432     if (first < 8) {
2433         a = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx,
2434                       MMU_DATA_LOAD, l.memop, ra);
2435         ret = do_ld16_beN(cpu, &l.page[1], a, l.mmu_idx, l.memop, ra);
2436     } else {
2437         ret = do_ld16_beN(cpu, &l.page[0], 0, l.mmu_idx, l.memop, ra);
2438         b = int128_getlo(ret);
2439         ret = int128_lshift(ret, l.page[1].size * 8);
2440         a = int128_gethi(ret);
2441         b = do_ld_beN(cpu, &l.page[1], b, l.mmu_idx,
2442                       MMU_DATA_LOAD, l.memop, ra);
2443         ret = int128_make128(b, a);
2444     }
2445     if ((l.memop & MO_BSWAP) == MO_LE) {
2446         ret = bswap128(ret);
2447     }
2448     return ret;
2449 }
2450 
2451 /*
2452  * Store Helpers
2453  */
2454 
2455 /**
2456  * do_st_mmio_leN:
2457  * @cpu: generic cpu state
2458  * @full: page parameters
2459  * @val_le: data to store
2460  * @addr: virtual address
2461  * @size: number of bytes
2462  * @mmu_idx: virtual address context
2463  * @ra: return address into tcg generated code, or 0
2464  * Context: BQL held
2465  *
2466  * Store @size bytes at @addr, which is memory-mapped i/o.
2467  * The bytes to store are extracted in little-endian order from @val_le;
2468  * return the bytes of @val_le beyond @p->size that have not been stored.
2469  */
2470 static uint64_t int_st_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
2471                                 uint64_t val_le, vaddr addr, int size,
2472                                 int mmu_idx, uintptr_t ra,
2473                                 MemoryRegion *mr, hwaddr mr_offset)
2474 {
2475     do {
2476         MemOp this_mop;
2477         unsigned this_size;
2478         MemTxResult r;
2479 
2480         /* Store aligned pieces up to 8 bytes. */
2481         this_mop = ctz32(size | (int)addr | 8);
2482         this_size = 1 << this_mop;
2483         this_mop |= MO_LE;
2484 
2485         r = memory_region_dispatch_write(mr, mr_offset, val_le,
2486                                          this_mop, full->attrs);
2487         if (unlikely(r != MEMTX_OK)) {
2488             io_failed(cpu, full, addr, this_size, MMU_DATA_STORE,
2489                       mmu_idx, r, ra);
2490         }
2491         if (this_size == 8) {
2492             return 0;
2493         }
2494 
2495         val_le >>= this_size * 8;
2496         addr += this_size;
2497         mr_offset += this_size;
2498         size -= this_size;
2499     } while (size);
2500 
2501     return val_le;
2502 }
2503 
2504 static uint64_t do_st_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
2505                                uint64_t val_le, vaddr addr, int size,
2506                                int mmu_idx, uintptr_t ra)
2507 {
2508     MemoryRegionSection *section;
2509     hwaddr mr_offset;
2510     MemoryRegion *mr;
2511     MemTxAttrs attrs;
2512 
2513     tcg_debug_assert(size > 0 && size <= 8);
2514 
2515     attrs = full->attrs;
2516     section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
2517     mr = section->mr;
2518 
2519     BQL_LOCK_GUARD();
2520     return int_st_mmio_leN(cpu, full, val_le, addr, size, mmu_idx,
2521                            ra, mr, mr_offset);
2522 }
2523 
2524 static uint64_t do_st16_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
2525                                  Int128 val_le, vaddr addr, int size,
2526                                  int mmu_idx, uintptr_t ra)
2527 {
2528     MemoryRegionSection *section;
2529     MemoryRegion *mr;
2530     hwaddr mr_offset;
2531     MemTxAttrs attrs;
2532 
2533     tcg_debug_assert(size > 8 && size <= 16);
2534 
2535     attrs = full->attrs;
2536     section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
2537     mr = section->mr;
2538 
2539     BQL_LOCK_GUARD();
2540     int_st_mmio_leN(cpu, full, int128_getlo(val_le), addr, 8,
2541                     mmu_idx, ra, mr, mr_offset);
2542     return int_st_mmio_leN(cpu, full, int128_gethi(val_le), addr + 8,
2543                            size - 8, mmu_idx, ra, mr, mr_offset + 8);
2544 }
2545 
2546 /*
2547  * Wrapper for the above.
2548  */
2549 static uint64_t do_st_leN(CPUState *cpu, MMULookupPageData *p,
2550                           uint64_t val_le, int mmu_idx,
2551                           MemOp mop, uintptr_t ra)
2552 {
2553     MemOp atom;
2554     unsigned tmp, half_size;
2555 
2556     if (unlikely(p->flags & TLB_MMIO)) {
2557         return do_st_mmio_leN(cpu, p->full, val_le, p->addr,
2558                               p->size, mmu_idx, ra);
2559     } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2560         return val_le >> (p->size * 8);
2561     }
2562 
2563     /*
2564      * It is a given that we cross a page and therefore there is no atomicity
2565      * for the store as a whole, but subobjects may need attention.
2566      */
2567     atom = mop & MO_ATOM_MASK;
2568     switch (atom) {
2569     case MO_ATOM_SUBALIGN:
2570         return store_parts_leN(p->haddr, p->size, val_le);
2571 
2572     case MO_ATOM_IFALIGN_PAIR:
2573     case MO_ATOM_WITHIN16_PAIR:
2574         tmp = mop & MO_SIZE;
2575         tmp = tmp ? tmp - 1 : 0;
2576         half_size = 1 << tmp;
2577         if (atom == MO_ATOM_IFALIGN_PAIR
2578             ? p->size == half_size
2579             : p->size >= half_size) {
2580             if (!HAVE_al8_fast && p->size <= 4) {
2581                 return store_whole_le4(p->haddr, p->size, val_le);
2582             } else if (HAVE_al8) {
2583                 return store_whole_le8(p->haddr, p->size, val_le);
2584             } else {
2585                 cpu_loop_exit_atomic(cpu, ra);
2586             }
2587         }
2588         /* fall through */
2589 
2590     case MO_ATOM_IFALIGN:
2591     case MO_ATOM_WITHIN16:
2592     case MO_ATOM_NONE:
2593         return store_bytes_leN(p->haddr, p->size, val_le);
2594 
2595     default:
2596         g_assert_not_reached();
2597     }
2598 }
2599 
2600 /*
2601  * Wrapper for the above, for 8 < size < 16.
2602  */
2603 static uint64_t do_st16_leN(CPUState *cpu, MMULookupPageData *p,
2604                             Int128 val_le, int mmu_idx,
2605                             MemOp mop, uintptr_t ra)
2606 {
2607     int size = p->size;
2608     MemOp atom;
2609 
2610     if (unlikely(p->flags & TLB_MMIO)) {
2611         return do_st16_mmio_leN(cpu, p->full, val_le, p->addr,
2612                                 size, mmu_idx, ra);
2613     } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2614         return int128_gethi(val_le) >> ((size - 8) * 8);
2615     }
2616 
2617     /*
2618      * It is a given that we cross a page and therefore there is no atomicity
2619      * for the store as a whole, but subobjects may need attention.
2620      */
2621     atom = mop & MO_ATOM_MASK;
2622     switch (atom) {
2623     case MO_ATOM_SUBALIGN:
2624         store_parts_leN(p->haddr, 8, int128_getlo(val_le));
2625         return store_parts_leN(p->haddr + 8, p->size - 8,
2626                                int128_gethi(val_le));
2627 
2628     case MO_ATOM_WITHIN16_PAIR:
2629         /* Since size > 8, this is the half that must be atomic. */
2630         if (!HAVE_CMPXCHG128) {
2631             cpu_loop_exit_atomic(cpu, ra);
2632         }
2633         return store_whole_le16(p->haddr, p->size, val_le);
2634 
2635     case MO_ATOM_IFALIGN_PAIR:
2636         /*
2637          * Since size > 8, both halves are misaligned,
2638          * and so neither is atomic.
2639          */
2640     case MO_ATOM_IFALIGN:
2641     case MO_ATOM_WITHIN16:
2642     case MO_ATOM_NONE:
2643         stq_le_p(p->haddr, int128_getlo(val_le));
2644         return store_bytes_leN(p->haddr + 8, p->size - 8,
2645                                int128_gethi(val_le));
2646 
2647     default:
2648         g_assert_not_reached();
2649     }
2650 }
2651 
2652 static void do_st_1(CPUState *cpu, MMULookupPageData *p, uint8_t val,
2653                     int mmu_idx, uintptr_t ra)
2654 {
2655     if (unlikely(p->flags & TLB_MMIO)) {
2656         do_st_mmio_leN(cpu, p->full, val, p->addr, 1, mmu_idx, ra);
2657     } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2658         /* nothing */
2659     } else {
2660         *(uint8_t *)p->haddr = val;
2661     }
2662 }
2663 
2664 static void do_st_2(CPUState *cpu, MMULookupPageData *p, uint16_t val,
2665                     int mmu_idx, MemOp memop, uintptr_t ra)
2666 {
2667     if (unlikely(p->flags & TLB_MMIO)) {
2668         if ((memop & MO_BSWAP) != MO_LE) {
2669             val = bswap16(val);
2670         }
2671         do_st_mmio_leN(cpu, p->full, val, p->addr, 2, mmu_idx, ra);
2672     } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2673         /* nothing */
2674     } else {
2675         /* Swap to host endian if necessary, then store. */
2676         if (memop & MO_BSWAP) {
2677             val = bswap16(val);
2678         }
2679         store_atom_2(cpu, ra, p->haddr, memop, val);
2680     }
2681 }
2682 
2683 static void do_st_4(CPUState *cpu, MMULookupPageData *p, uint32_t val,
2684                     int mmu_idx, MemOp memop, uintptr_t ra)
2685 {
2686     if (unlikely(p->flags & TLB_MMIO)) {
2687         if ((memop & MO_BSWAP) != MO_LE) {
2688             val = bswap32(val);
2689         }
2690         do_st_mmio_leN(cpu, p->full, val, p->addr, 4, mmu_idx, ra);
2691     } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2692         /* nothing */
2693     } else {
2694         /* Swap to host endian if necessary, then store. */
2695         if (memop & MO_BSWAP) {
2696             val = bswap32(val);
2697         }
2698         store_atom_4(cpu, ra, p->haddr, memop, val);
2699     }
2700 }
2701 
2702 static void do_st_8(CPUState *cpu, MMULookupPageData *p, uint64_t val,
2703                     int mmu_idx, MemOp memop, uintptr_t ra)
2704 {
2705     if (unlikely(p->flags & TLB_MMIO)) {
2706         if ((memop & MO_BSWAP) != MO_LE) {
2707             val = bswap64(val);
2708         }
2709         do_st_mmio_leN(cpu, p->full, val, p->addr, 8, mmu_idx, ra);
2710     } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2711         /* nothing */
2712     } else {
2713         /* Swap to host endian if necessary, then store. */
2714         if (memop & MO_BSWAP) {
2715             val = bswap64(val);
2716         }
2717         store_atom_8(cpu, ra, p->haddr, memop, val);
2718     }
2719 }
2720 
2721 static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
2722                        MemOpIdx oi, uintptr_t ra)
2723 {
2724     MMULookupLocals l;
2725     bool crosspage;
2726 
2727     cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2728     crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2729     tcg_debug_assert(!crosspage);
2730 
2731     do_st_1(cpu, &l.page[0], val, l.mmu_idx, ra);
2732 }
2733 
2734 static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val,
2735                        MemOpIdx oi, uintptr_t ra)
2736 {
2737     MMULookupLocals l;
2738     bool crosspage;
2739     uint8_t a, b;
2740 
2741     cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2742     crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2743     if (likely(!crosspage)) {
2744         do_st_2(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2745         return;
2746     }
2747 
2748     if ((l.memop & MO_BSWAP) == MO_LE) {
2749         a = val, b = val >> 8;
2750     } else {
2751         b = val, a = val >> 8;
2752     }
2753     do_st_1(cpu, &l.page[0], a, l.mmu_idx, ra);
2754     do_st_1(cpu, &l.page[1], b, l.mmu_idx, ra);
2755 }
2756 
2757 static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val,
2758                        MemOpIdx oi, uintptr_t ra)
2759 {
2760     MMULookupLocals l;
2761     bool crosspage;
2762 
2763     cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2764     crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2765     if (likely(!crosspage)) {
2766         do_st_4(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2767         return;
2768     }
2769 
2770     /* Swap to little endian for simplicity, then store by bytes. */
2771     if ((l.memop & MO_BSWAP) != MO_LE) {
2772         val = bswap32(val);
2773     }
2774     val = do_st_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2775     (void) do_st_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
2776 }
2777 
2778 static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val,
2779                        MemOpIdx oi, uintptr_t ra)
2780 {
2781     MMULookupLocals l;
2782     bool crosspage;
2783 
2784     cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2785     crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2786     if (likely(!crosspage)) {
2787         do_st_8(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2788         return;
2789     }
2790 
2791     /* Swap to little endian for simplicity, then store by bytes. */
2792     if ((l.memop & MO_BSWAP) != MO_LE) {
2793         val = bswap64(val);
2794     }
2795     val = do_st_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2796     (void) do_st_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
2797 }
2798 
2799 static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
2800                         MemOpIdx oi, uintptr_t ra)
2801 {
2802     MMULookupLocals l;
2803     bool crosspage;
2804     uint64_t a, b;
2805     int first;
2806 
2807     cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2808     crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2809     if (likely(!crosspage)) {
2810         if (unlikely(l.page[0].flags & TLB_MMIO)) {
2811             if ((l.memop & MO_BSWAP) != MO_LE) {
2812                 val = bswap128(val);
2813             }
2814             do_st16_mmio_leN(cpu, l.page[0].full, val, addr, 16, l.mmu_idx, ra);
2815         } else if (unlikely(l.page[0].flags & TLB_DISCARD_WRITE)) {
2816             /* nothing */
2817         } else {
2818             /* Swap to host endian if necessary, then store. */
2819             if (l.memop & MO_BSWAP) {
2820                 val = bswap128(val);
2821             }
2822             store_atom_16(cpu, ra, l.page[0].haddr, l.memop, val);
2823         }
2824         return;
2825     }
2826 
2827     first = l.page[0].size;
2828     if (first == 8) {
2829         MemOp mop8 = (l.memop & ~(MO_SIZE | MO_BSWAP)) | MO_64;
2830 
2831         if (l.memop & MO_BSWAP) {
2832             val = bswap128(val);
2833         }
2834         if (HOST_BIG_ENDIAN) {
2835             b = int128_getlo(val), a = int128_gethi(val);
2836         } else {
2837             a = int128_getlo(val), b = int128_gethi(val);
2838         }
2839         do_st_8(cpu, &l.page[0], a, l.mmu_idx, mop8, ra);
2840         do_st_8(cpu, &l.page[1], b, l.mmu_idx, mop8, ra);
2841         return;
2842     }
2843 
2844     if ((l.memop & MO_BSWAP) != MO_LE) {
2845         val = bswap128(val);
2846     }
2847     if (first < 8) {
2848         do_st_leN(cpu, &l.page[0], int128_getlo(val), l.mmu_idx, l.memop, ra);
2849         val = int128_urshift(val, first * 8);
2850         do_st16_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
2851     } else {
2852         b = do_st16_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2853         do_st_leN(cpu, &l.page[1], b, l.mmu_idx, l.memop, ra);
2854     }
2855 }
2856 
2857 #include "ldst_common.c.inc"
2858 
2859 /*
2860  * First set of functions passes in OI and RETADDR.
2861  * This makes them callable from other helpers.
2862  */
2863 
2864 #define ATOMIC_NAME(X) \
2865     glue(glue(glue(cpu_atomic_ ## X, SUFFIX), END), _mmu)
2866 
2867 #define ATOMIC_MMU_CLEANUP
2868 
2869 #include "atomic_common.c.inc"
2870 
2871 #define DATA_SIZE 1
2872 #include "atomic_template.h"
2873 
2874 #define DATA_SIZE 2
2875 #include "atomic_template.h"
2876 
2877 #define DATA_SIZE 4
2878 #include "atomic_template.h"
2879 
2880 #ifdef CONFIG_ATOMIC64
2881 #define DATA_SIZE 8
2882 #include "atomic_template.h"
2883 #endif
2884 
2885 #if defined(CONFIG_ATOMIC128) || HAVE_CMPXCHG128
2886 #define DATA_SIZE 16
2887 #include "atomic_template.h"
2888 #endif
2889 
2890 /* Code access functions.  */
2891 
2892 uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr)
2893 {
2894     CPUState *cs = env_cpu(env);
2895     MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(cs, true));
2896     return do_ld1_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
2897 }
2898 
2899 uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr)
2900 {
2901     CPUState *cs = env_cpu(env);
2902     MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(cs, true));
2903     return do_ld2_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
2904 }
2905 
2906 uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr)
2907 {
2908     CPUState *cs = env_cpu(env);
2909     MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(cs, true));
2910     return do_ld4_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
2911 }
2912 
2913 uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr)
2914 {
2915     CPUState *cs = env_cpu(env);
2916     MemOpIdx oi = make_memop_idx(MO_TEUQ, cpu_mmu_index(cs, true));
2917     return do_ld8_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
2918 }
2919 
2920 uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr,
2921                          MemOpIdx oi, uintptr_t retaddr)
2922 {
2923     return do_ld1_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2924 }
2925 
2926 uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr,
2927                           MemOpIdx oi, uintptr_t retaddr)
2928 {
2929     return do_ld2_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2930 }
2931 
2932 uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr,
2933                           MemOpIdx oi, uintptr_t retaddr)
2934 {
2935     return do_ld4_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2936 }
2937 
2938 uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr,
2939                           MemOpIdx oi, uintptr_t retaddr)
2940 {
2941     return do_ld8_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2942 }
2943