Lines Matching +full:i +full:- +full:tlb +full:- +full:size
2 * Common CPU TLB handling
21 #include "qemu/main-loop.h"
22 #include "hw/core/tcg-cpu-ops.h"
23 #include "exec/exec-all.h"
24 #include "exec/page-protection.h"
28 #include "exec/tb-flush.h"
29 #include "exec/memory-internal.h"
31 #include "exec/mmu-access-type.h"
32 #include "exec/tlb-common.h"
35 #include "qemu/error-report.h"
37 #include "exec/helper-proto-common.h"
40 #include "exec/translate-all.h"
42 #include "tb-hash.h"
43 #include "internal-common.h"
44 #include "internal-target.h"
46 #include "qemu/plugin-memory.h"
48 #include "tcg/tcg-ldst.h"
49 #include "tcg/oversized-guest.h"
78 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
90 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
94 return (fast->mask >> CPU_TLB_ENTRY_BITS) + 1; in tlb_n_entries()
99 return fast->mask + (1 << CPU_TLB_ENTRY_BITS); in sizeof_tlb()
115 const uint32_t *ptr = (uint32_t *)&entry->addr_idx[access_type]; in tlb_read_idx()
119 const uint64_t *ptr = &entry->addr_idx[access_type]; in tlb_read_idx()
134 /* Find the TLB index corresponding to the mmu_idx + address pair. */
138 uintptr_t size_mask = cpu->neg.tlb.f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS; in tlb_index()
143 /* Find the TLB entry corresponding to the mmu_idx + address pair. */
147 return &cpu->neg.tlb.f[mmu_idx].table[tlb_index(cpu, mmu_idx, addr)]; in tlb_entry()
153 desc->window_begin_ns = ns; in tlb_window_reset()
154 desc->window_max_entries = max_entries; in tlb_window_reset()
159 CPUJumpCache *jc = cpu->tb_jmp_cache; in tb_jmp_cache_clear_page()
160 int i, i0; in tb_jmp_cache_clear_page() local
167 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) { in tb_jmp_cache_clear_page()
168 qatomic_set(&jc->array[i0 + i].tb, NULL); in tb_jmp_cache_clear_page()
173 * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
174 * @desc: The CPUTLBDesc portion of the TLB
175 * @fast: The CPUTLBDescFast portion of the same TLB
179 * We have two main constraints when resizing a TLB: (1) we only resize it
180 * on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
186 * In general, a memory-hungry process can benefit greatly from an appropriately
187 * sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
188 * we just have to make the TLB as large as possible; while an oversized TLB
189 * results in minimal TLB miss rates, it also takes longer to be flushed
193 * To achieve near-optimal performance for all kinds of workloads, we:
195 * 1. Aggressively increase the size of the TLB when the use rate of the
196 * TLB being flushed is high, since it is likely that in the near future this
197 * memory-hungry process will execute again, and its memory hungriness will
200 * 2. Slowly reduce the size of the TLB as the use rate declines over a
202 * we have not observed a high TLB use rate, it is likely that we won't observe
204 * the TLB to match the maximum use rate observed in the window.
206 * 3. Try to keep the maximum use rate in a time window in the 30-70% range,
207 * since in that range performance is likely near-optimal. Recall that the TLB
220 bool window_expired = now > desc->window_begin_ns + window_len_ns; in tlb_mmu_resize_locked()
222 if (desc->n_used_entries > desc->window_max_entries) { in tlb_mmu_resize_locked()
223 desc->window_max_entries = desc->n_used_entries; in tlb_mmu_resize_locked()
225 rate = desc->window_max_entries * 100 / old_size; in tlb_mmu_resize_locked()
230 size_t ceil = pow2ceil(desc->window_max_entries); in tlb_mmu_resize_locked()
231 size_t expected_rate = desc->window_max_entries * 100 / ceil; in tlb_mmu_resize_locked()
237 * 1023/1024==99.9% use rate, so we'd likely end up doubling the size in tlb_mmu_resize_locked()
239 * (and since we double the size, that means the lowest rate we'd in tlb_mmu_resize_locked()
240 * expect to get is 35%, which is still in the 30-70% range where in tlb_mmu_resize_locked()
241 * we consider that the size is appropriate.) in tlb_mmu_resize_locked()
251 tlb_window_reset(desc, now, desc->n_used_entries); in tlb_mmu_resize_locked()
256 g_free(fast->table); in tlb_mmu_resize_locked()
257 g_free(desc->fulltlb); in tlb_mmu_resize_locked()
260 /* desc->n_used_entries is cleared by the caller */ in tlb_mmu_resize_locked()
261 fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; in tlb_mmu_resize_locked()
262 fast->table = g_try_new(CPUTLBEntry, new_size); in tlb_mmu_resize_locked()
263 desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size); in tlb_mmu_resize_locked()
270 * size, aborting if we cannot even allocate the smallest TLB we support. in tlb_mmu_resize_locked()
272 while (fast->table == NULL || desc->fulltlb == NULL) { in tlb_mmu_resize_locked()
278 fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; in tlb_mmu_resize_locked()
280 g_free(fast->table); in tlb_mmu_resize_locked()
281 g_free(desc->fulltlb); in tlb_mmu_resize_locked()
282 fast->table = g_try_new(CPUTLBEntry, new_size); in tlb_mmu_resize_locked()
283 desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size); in tlb_mmu_resize_locked()
289 desc->n_used_entries = 0; in tlb_mmu_flush_locked()
290 desc->large_page_addr = -1; in tlb_mmu_flush_locked()
291 desc->large_page_mask = -1; in tlb_mmu_flush_locked()
292 desc->vindex = 0; in tlb_mmu_flush_locked()
293 memset(fast->table, -1, sizeof_tlb(fast)); in tlb_mmu_flush_locked()
294 memset(desc->vtable, -1, sizeof(desc->vtable)); in tlb_mmu_flush_locked()
300 CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx]; in tlb_flush_one_mmuidx_locked()
301 CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx]; in tlb_flush_one_mmuidx_locked()
312 desc->n_used_entries = 0; in tlb_mmu_init()
313 fast->mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS; in tlb_mmu_init()
314 fast->table = g_new(CPUTLBEntry, n_entries); in tlb_mmu_init()
315 desc->fulltlb = g_new(CPUTLBEntryFull, n_entries); in tlb_mmu_init()
321 cpu->neg.tlb.d[mmu_idx].n_used_entries++; in tlb_n_used_entries_inc()
326 cpu->neg.tlb.d[mmu_idx].n_used_entries--; in tlb_n_used_entries_dec()
332 int i; in tlb_init() local
334 qemu_spin_init(&cpu->neg.tlb.c.lock); in tlb_init()
337 cpu->neg.tlb.c.dirty = 0; in tlb_init()
339 for (i = 0; i < NB_MMU_MODES; i++) { in tlb_init()
340 tlb_mmu_init(&cpu->neg.tlb.d[i], &cpu->neg.tlb.f[i], now); in tlb_init()
346 int i; in tlb_destroy() local
348 qemu_spin_destroy(&cpu->neg.tlb.c.lock); in tlb_destroy()
349 for (i = 0; i < NB_MMU_MODES; i++) { in tlb_destroy()
350 CPUTLBDesc *desc = &cpu->neg.tlb.d[i]; in tlb_destroy()
351 CPUTLBDescFast *fast = &cpu->neg.tlb.f[i]; in tlb_destroy()
353 g_free(fast->table); in tlb_destroy()
354 g_free(desc->fulltlb); in tlb_destroy()
387 qemu_spin_lock(&cpu->neg.tlb.c.lock); in tlb_flush_by_mmuidx_async_work()
389 all_dirty = cpu->neg.tlb.c.dirty; in tlb_flush_by_mmuidx_async_work()
392 cpu->neg.tlb.c.dirty = all_dirty; in tlb_flush_by_mmuidx_async_work()
394 for (work = to_clean; work != 0; work &= work - 1) { in tlb_flush_by_mmuidx_async_work()
399 qemu_spin_unlock(&cpu->neg.tlb.c.lock); in tlb_flush_by_mmuidx_async_work()
404 qatomic_set(&cpu->neg.tlb.c.full_flush_count, in tlb_flush_by_mmuidx_async_work()
405 cpu->neg.tlb.c.full_flush_count + 1); in tlb_flush_by_mmuidx_async_work()
407 qatomic_set(&cpu->neg.tlb.c.part_flush_count, in tlb_flush_by_mmuidx_async_work()
408 cpu->neg.tlb.c.part_flush_count + ctpop16(to_clean)); in tlb_flush_by_mmuidx_async_work()
410 qatomic_set(&cpu->neg.tlb.c.elide_flush_count, in tlb_flush_by_mmuidx_async_work()
411 cpu->neg.tlb.c.elide_flush_count + in tlb_flush_by_mmuidx_async_work()
452 return (page == (tlb_entry->addr_read & mask) || in tlb_hit_page_mask_anyprot()
454 page == (tlb_entry->addr_code & mask)); in tlb_hit_page_mask_anyprot()
459 return tlb_hit_page_mask_anyprot(tlb_entry, page, -1); in tlb_hit_page_anyprot()
463 * tlb_entry_is_empty - return true if the entry is not in use
468 return te->addr_read == -1 && te->addr_write == -1 && te->addr_code == -1; in tlb_entry_is_empty()
477 memset(tlb_entry, -1, sizeof(*tlb_entry)); in tlb_flush_entry_mask_locked()
485 return tlb_flush_entry_mask_locked(tlb_entry, page, -1); in tlb_flush_entry_locked()
493 CPUTLBDesc *d = &cpu->neg.tlb.d[mmu_idx]; in tlb_flush_vtlb_page_mask_locked()
498 if (tlb_flush_entry_mask_locked(&d->vtable[k], page, mask)) { in tlb_flush_vtlb_page_mask_locked()
507 tlb_flush_vtlb_page_mask_locked(cpu, mmu_idx, page, -1); in tlb_flush_vtlb_page_locked()
512 vaddr lp_addr = cpu->neg.tlb.d[midx].large_page_addr; in tlb_flush_page_locked()
513 vaddr lp_mask = cpu->neg.tlb.d[midx].large_page_mask; in tlb_flush_page_locked()
548 qemu_spin_lock(&cpu->neg.tlb.c.lock); in tlb_flush_page_by_mmuidx_async_0()
554 qemu_spin_unlock(&cpu->neg.tlb.c.lock); in tlb_flush_page_by_mmuidx_async_0()
560 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE); in tlb_flush_page_by_mmuidx_async_0()
604 tlb_flush_page_by_mmuidx_async_0(cpu, d->addr, d->idxmap); in tlb_flush_page_by_mmuidx_async_2()
651 d->addr = addr; in tlb_flush_page_by_mmuidx_all_cpus_synced()
652 d->idxmap = idxmap; in tlb_flush_page_by_mmuidx_all_cpus_synced()
659 d->addr = addr; in tlb_flush_page_by_mmuidx_all_cpus_synced()
660 d->idxmap = idxmap; in tlb_flush_page_by_mmuidx_all_cpus_synced()
675 CPUTLBDesc *d = &cpu->neg.tlb.d[midx]; in tlb_flush_range_locked()
676 CPUTLBDescFast *f = &cpu->neg.tlb.f[midx]; in tlb_flush_range_locked()
680 * If @bits is smaller than the tlb size, there may be multiple entries in tlb_flush_range_locked()
681 * within the TLB; otherwise all addresses that match under @mask hit in tlb_flush_range_locked()
682 * the same TLB entry. in tlb_flush_range_locked()
683 * TODO: Perhaps allow bits to be a few bits less than the size. in tlb_flush_range_locked()
684 * For now, just flush the entire TLB. in tlb_flush_range_locked()
686 * If @len is larger than the tlb size, then it will take longer to in tlb_flush_range_locked()
687 * test all of the entries in the TLB than it will to flush it all. in tlb_flush_range_locked()
689 if (mask < f->mask || len > f->mask) { in tlb_flush_range_locked()
702 if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) { in tlb_flush_range_locked()
705 midx, d->large_page_addr, d->large_page_mask); in tlb_flush_range_locked()
710 for (vaddr i = 0; i < len; i += TARGET_PAGE_SIZE) { in tlb_flush_range_locked() local
711 vaddr page = addr + i; in tlb_flush_range_locked()
738 qemu_spin_lock(&cpu->neg.tlb.c.lock); in tlb_flush_range_by_mmuidx_async_0()
744 qemu_spin_unlock(&cpu->neg.tlb.c.lock); in tlb_flush_range_by_mmuidx_async_0()
747 * If the length is larger than the jump cache size, then it will take in tlb_flush_range_by_mmuidx_async_0()
759 d.addr -= TARGET_PAGE_SIZE; in tlb_flush_range_by_mmuidx_async_0()
760 for (vaddr i = 0, n = d.len / TARGET_PAGE_SIZE + 1; i < n; i++) { in tlb_flush_range_by_mmuidx_async_0() local
872 /* update the TLB so that writes in physical page 'phys_addr' are no longer
884 * the TLB and uses that data to compute the final address. If any of
891 * te->addr_write with qatomic_set. We don't need to worry about this for
899 uintptr_t addr = tlb_entry->addr_write; in tlb_reset_dirty_range_locked()
904 addr += tlb_entry->addend; in tlb_reset_dirty_range_locked()
905 if ((addr - start) < length) { in tlb_reset_dirty_range_locked()
907 uint32_t *ptr_write = (uint32_t *)&tlb_entry->addr_write; in tlb_reset_dirty_range_locked()
911 tlb_entry->addr_write |= TLB_NOTDIRTY; in tlb_reset_dirty_range_locked()
913 qatomic_set(&tlb_entry->addr_write, in tlb_reset_dirty_range_locked()
914 tlb_entry->addr_write | TLB_NOTDIRTY); in tlb_reset_dirty_range_locked()
922 * Called only from the vCPU context, i.e. the TLB's owner thread.
929 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
932 * thing actually updated is the target TLB entry ->addr_write flags.
938 qemu_spin_lock(&cpu->neg.tlb.c.lock); in tlb_reset_dirty()
940 unsigned int i; in tlb_reset_dirty() local
941 unsigned int n = tlb_n_entries(&cpu->neg.tlb.f[mmu_idx]); in tlb_reset_dirty()
943 for (i = 0; i < n; i++) { in tlb_reset_dirty()
944 tlb_reset_dirty_range_locked(&cpu->neg.tlb.f[mmu_idx].table[i], in tlb_reset_dirty()
948 for (i = 0; i < CPU_VTLB_SIZE; i++) { in tlb_reset_dirty()
949 tlb_reset_dirty_range_locked(&cpu->neg.tlb.d[mmu_idx].vtable[i], in tlb_reset_dirty()
953 qemu_spin_unlock(&cpu->neg.tlb.c.lock); in tlb_reset_dirty()
960 if (tlb_entry->addr_write == (addr | TLB_NOTDIRTY)) { in tlb_set_dirty1_locked()
961 tlb_entry->addr_write = addr; in tlb_set_dirty1_locked()
965 /* update the TLB corresponding to virtual page vaddr
974 qemu_spin_lock(&cpu->neg.tlb.c.lock); in tlb_set_dirty()
982 tlb_set_dirty1_locked(&cpu->neg.tlb.d[mmu_idx].vtable[k], addr); in tlb_set_dirty()
985 qemu_spin_unlock(&cpu->neg.tlb.c.lock); in tlb_set_dirty()
988 /* Our TLB does not support large pages, so remember the area covered by
989 large pages and trigger a full TLB flush if these are invalidated. */
991 vaddr addr, uint64_t size) in tlb_add_large_page() argument
993 vaddr lp_addr = cpu->neg.tlb.d[mmu_idx].large_page_addr; in tlb_add_large_page()
994 vaddr lp_mask = ~(size - 1); in tlb_add_large_page()
996 if (lp_addr == (vaddr)-1) { in tlb_add_large_page()
1002 the cost of maintaining a full variable size TLB. */ in tlb_add_large_page()
1003 lp_mask &= cpu->neg.tlb.d[mmu_idx].large_page_mask; in tlb_add_large_page()
1008 cpu->neg.tlb.d[mmu_idx].large_page_addr = lp_addr & lp_mask; in tlb_add_large_page()
1009 cpu->neg.tlb.d[mmu_idx].large_page_mask = lp_mask; in tlb_add_large_page()
1023 address = -1; in tlb_set_compare()
1026 ent->addr_idx[access_type] = address; in tlb_set_compare()
1027 full->slow_flags[access_type] = flags; in tlb_set_compare()
1031 * Add a new TLB entry. At most one entry for a given virtual address
1033 * supplied size is only used by tlb_flush_page.
1035 * Called from TCG-generated code, which is under an RCU read-side
1041 CPUTLB *tlb = &cpu->neg.tlb; in tlb_set_page_full() local
1042 CPUTLBDesc *desc = &tlb->d[mmu_idx]; in tlb_set_page_full()
1054 if (full->lg_page_size <= TARGET_PAGE_BITS) { in tlb_set_page_full()
1057 sz = (hwaddr)1 << full->lg_page_size; in tlb_set_page_full()
1061 paddr_page = full->phys_addr & TARGET_PAGE_MASK; in tlb_set_page_full()
1063 prot = full->prot; in tlb_set_page_full()
1064 asidx = cpu_asidx_from_attrs(cpu, full->attrs); in tlb_set_page_full()
1066 &xlat, &sz, full->attrs, &prot); in tlb_set_page_full()
1071 addr, full->phys_addr, prot, mmu_idx); in tlb_set_page_full()
1073 read_flags = full->tlb_fill_flags; in tlb_set_page_full()
1074 if (full->lg_page_size < TARGET_PAGE_BITS) { in tlb_set_page_full()
1075 /* Repeat the MMU check and TLB fill on every access. */ in tlb_set_page_full()
1079 is_ram = memory_region_is_ram(section->mr); in tlb_set_page_full()
1080 is_romd = memory_region_is_romd(section->mr); in tlb_set_page_full()
1084 addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat; in tlb_set_page_full()
1086 /* I/O does not; force the host address to NULL. */ in tlb_set_page_full()
1092 iotlb = memory_region_get_ram_addr(section->mr) + xlat; in tlb_set_page_full()
1099 if (section->readonly) { in tlb_set_page_full()
1106 /* I/O or ROMD */ in tlb_set_page_full()
1111 * but of course reads to I/O must go through MMIO. in tlb_set_page_full()
1126 * Hold the TLB lock for the rest of the function. We could acquire/release in tlb_set_page_full()
1129 * a longer critical section, but this is not a concern since the TLB lock in tlb_set_page_full()
1132 qemu_spin_lock(&tlb->c.lock); in tlb_set_page_full()
1134 /* Note that the tlb is no longer clean. */ in tlb_set_page_full()
1135 tlb->c.dirty |= 1 << mmu_idx; in tlb_set_page_full()
1141 * Only evict the old entry to the victim tlb if it's for a in tlb_set_page_full()
1145 unsigned vidx = desc->vindex++ % CPU_VTLB_SIZE; in tlb_set_page_full()
1146 CPUTLBEntry *tv = &desc->vtable[vidx]; in tlb_set_page_full()
1148 /* Evict the old entry into the victim tlb. */ in tlb_set_page_full()
1150 desc->vfulltlb[vidx] = desc->fulltlb[index]; in tlb_set_page_full()
1154 /* refill the tlb */ in tlb_set_page_full()
1159 * - a physical section number in the lower TARGET_PAGE_BITS in tlb_set_page_full()
1160 * - the offset within section->mr of the page base (I/O, ROMD) with the in tlb_set_page_full()
1164 * (non-page-aligned) vaddr of the eventual memory access to get in tlb_set_page_full()
1169 desc->fulltlb[index] = *full; in tlb_set_page_full()
1170 full = &desc->fulltlb[index]; in tlb_set_page_full()
1171 full->xlat_section = iotlb - addr_page; in tlb_set_page_full()
1172 full->phys_addr = paddr_page; in tlb_set_page_full()
1175 tn.addend = addend - addr_page; in tlb_set_page_full()
1197 qemu_spin_unlock(&tlb->c.lock); in tlb_set_page_full()
1202 int mmu_idx, uint64_t size) in tlb_set_page_with_attrs() argument
1208 .lg_page_size = ctz64(size) in tlb_set_page_with_attrs()
1211 assert(is_power_of_2(size)); in tlb_set_page_with_attrs()
1217 int mmu_idx, uint64_t size) in tlb_set_page() argument
1220 prot, mmu_idx, size); in tlb_set_page()
1224 * Note: tlb_fill_align() can trigger a resize of the TLB.
1225 * This means that all of the caller's prior references to the TLB table
1230 int mmu_idx, MemOp memop, int size, in tlb_fill_align() argument
1233 const TCGCPUOps *ops = cpu->cc->tcg_ops; in tlb_fill_align()
1236 if (ops->tlb_fill_align) { in tlb_fill_align()
1237 if (ops->tlb_fill_align(cpu, &full, addr, type, mmu_idx, in tlb_fill_align()
1238 memop, size, probe, ra)) { in tlb_fill_align()
1244 if (addr & ((1u << memop_alignment_bits(memop)) - 1)) { in tlb_fill_align()
1245 ops->do_unaligned_access(cpu, addr, type, mmu_idx, ra); in tlb_fill_align()
1247 if (ops->tlb_fill(cpu, addr, size, type, mmu_idx, probe, ra)) { in tlb_fill_align()
1259 cpu->cc->tcg_ops->do_unaligned_access(cpu, addr, access_type, in cpu_unaligned_access()
1272 cpu->mem_io_pc = retaddr; in io_prepare()
1273 if (!cpu->neg.can_do_io) { in io_prepare()
1282 unsigned size, MMUAccessType access_type, int mmu_idx, in io_failed() argument
1285 if (!cpu->ignore_memory_transaction_failures in io_failed()
1286 && cpu->cc->tcg_ops->do_transaction_failed) { in io_failed()
1287 hwaddr physaddr = full->phys_addr | (addr & ~TARGET_PAGE_MASK); in io_failed()
1289 cpu->cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size, in io_failed()
1291 full->attrs, response, retaddr); in io_failed()
1295 /* Return true if ADDR is present in the victim tlb, and has been copied
1296 back to the main tlb. */
1304 CPUTLBEntry *vtlb = &cpu->neg.tlb.d[mmu_idx].vtable[vidx]; in victim_tlb_hit()
1308 /* Found entry in victim tlb, swap tlb and iotlb. */ in victim_tlb_hit()
1309 CPUTLBEntry tmptlb, *tlb = &cpu->neg.tlb.f[mmu_idx].table[index]; in victim_tlb_hit() local
1311 qemu_spin_lock(&cpu->neg.tlb.c.lock); in victim_tlb_hit()
1312 copy_tlb_helper_locked(&tmptlb, tlb); in victim_tlb_hit()
1313 copy_tlb_helper_locked(tlb, vtlb); in victim_tlb_hit()
1315 qemu_spin_unlock(&cpu->neg.tlb.c.lock); in victim_tlb_hit()
1317 CPUTLBEntryFull *f1 = &cpu->neg.tlb.d[mmu_idx].fulltlb[index]; in victim_tlb_hit()
1318 CPUTLBEntryFull *f2 = &cpu->neg.tlb.d[mmu_idx].vfulltlb[vidx]; in victim_tlb_hit()
1327 static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, in notdirty_write() argument
1330 ram_addr_t ram_addr = mem_vaddr + full->xlat_section; in notdirty_write()
1332 trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size); in notdirty_write()
1335 tb_invalidate_phys_range_fast(ram_addr, size, retaddr); in notdirty_write()
1342 cpu_physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE); in notdirty_write()
1369 /* Non-faulting page table read failed. */ in probe_access_internal()
1375 /* TLB resize via tlb_fill_align may have moved the entry. */ in probe_access_internal()
1390 *pfull = full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index]; in probe_access_internal()
1391 flags |= full->slow_flags[access_type]; in probe_access_internal()
1393 /* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */ in probe_access_internal()
1401 *phost = (void *)((uintptr_t)addr + entry->addend); in probe_access_internal()
1405 int probe_access_full(CPUArchState *env, vaddr addr, int size, in probe_access_full() argument
1410 int flags = probe_access_internal(env_cpu(env), addr, size, access_type, in probe_access_full()
1416 int dirtysize = size == 0 ? 1 : size; in probe_access_full()
1424 int probe_access_full_mmu(CPUArchState *env, vaddr addr, int size, in probe_access_full_mmu() argument
1435 int flags = probe_access_internal(env_cpu(env), addr, size, access_type, in probe_access_full_mmu()
1440 int dirtysize = size == 0 ? 1 : size; in probe_access_full_mmu()
1448 int probe_access_flags(CPUArchState *env, vaddr addr, int size, in probe_access_flags() argument
1455 g_assert(-(addr | TARGET_PAGE_MASK) >= size); in probe_access_flags()
1457 flags = probe_access_internal(env_cpu(env), addr, size, access_type, in probe_access_flags()
1463 int dirtysize = size == 0 ? 1 : size; in probe_access_flags()
1471 void *probe_access(CPUArchState *env, vaddr addr, int size, in probe_access() argument
1478 g_assert(-(addr | TARGET_PAGE_MASK) >= size); in probe_access()
1480 flags = probe_access_internal(env_cpu(env), addr, size, access_type, in probe_access()
1484 /* Per the interface, size == 0 merely faults the access. */ in probe_access()
1485 if (size == 0) { in probe_access()
1494 cpu_check_watchpoint(env_cpu(env), addr, size, in probe_access()
1495 full->attrs, wp_access, retaddr); in probe_access()
1500 notdirty_write(env_cpu(env), addr, size, full, retaddr); in probe_access()
1524 * Return -1 if we can't translate and execute from an entire page
1541 return -1; in get_page_addr_code_hostp()
1544 if (full->lg_page_size < TARGET_PAGE_BITS) { in get_page_addr_code_hostp()
1545 return -1; in get_page_addr_code_hostp()
1559 * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1561 * in the softmmu lookup code (or helper). We don't handle re-fills or
1564 * The one corner case is i/o write, which can cause changes to the
1565 * address space. Those changes, and the corresponding tlb flush,
1582 full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index]; in tlb_plugin_lookup()
1583 data->phys_addr = full->phys_addr | (addr & ~TARGET_PAGE_MASK); in tlb_plugin_lookup()
1588 iotlb_to_section(cpu, full->xlat_section & ~TARGET_PAGE_MASK, in tlb_plugin_lookup()
1589 full->attrs); in tlb_plugin_lookup()
1590 data->is_io = true; in tlb_plugin_lookup()
1591 data->mr = section->mr; in tlb_plugin_lookup()
1593 data->is_io = false; in tlb_plugin_lookup()
1594 data->mr = NULL; in tlb_plugin_lookup()
1610 int size; member
1630 * tlb_fill_align will longjmp out. Return true if the softmmu tlb for
1636 vaddr addr = data->addr; in mmu_lookup1()
1644 /* If the TLB entry is for a different page, reload and try again. */ in mmu_lookup1()
1649 memop, data->size, false, ra); in mmu_lookup1()
1657 full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index]; in mmu_lookup1()
1659 flags |= full->slow_flags[access_type]; in mmu_lookup1()
1675 if (unlikely(addr & ((1 << a_bits) - 1))) { in mmu_lookup1()
1680 data->full = full; in mmu_lookup1()
1681 data->flags = flags; in mmu_lookup1()
1683 data->haddr = (void *)((uintptr_t)addr + entry->addend); in mmu_lookup1()
1695 * Trigger watchpoints for @data.addr:@data.size;
1701 CPUTLBEntryFull *full = data->full; in mmu_watch_or_dirty()
1702 vaddr addr = data->addr; in mmu_watch_or_dirty()
1703 int flags = data->flags; in mmu_watch_or_dirty()
1704 int size = data->size; in mmu_watch_or_dirty() local
1709 cpu_check_watchpoint(cpu, addr, size, full->attrs, wp, ra); in mmu_watch_or_dirty()
1715 notdirty_write(cpu, addr, size, full, ra); in mmu_watch_or_dirty()
1718 data->flags = flags; in mmu_watch_or_dirty()
1730 * Resolve the translation for the page(s) beginning at @addr, for MemOp.size
1739 l->memop = get_memop(oi); in mmu_lookup()
1740 l->mmu_idx = get_mmuidx(oi); in mmu_lookup()
1742 tcg_debug_assert(l->mmu_idx < NB_MMU_MODES); in mmu_lookup()
1744 l->page[0].addr = addr; in mmu_lookup()
1745 l->page[0].size = memop_size(l->memop); in mmu_lookup()
1746 l->page[1].addr = (addr + l->page[0].size - 1) & TARGET_PAGE_MASK; in mmu_lookup()
1747 l->page[1].size = 0; in mmu_lookup()
1748 crosspage = (addr ^ l->page[1].addr) & TARGET_PAGE_MASK; in mmu_lookup()
1751 mmu_lookup1(cpu, &l->page[0], l->memop, l->mmu_idx, type, ra); in mmu_lookup()
1753 flags = l->page[0].flags; in mmu_lookup()
1755 mmu_watch_or_dirty(cpu, &l->page[0], type, ra); in mmu_lookup()
1758 l->memop ^= MO_BSWAP; in mmu_lookup()
1762 int size0 = l->page[1].addr - addr; in mmu_lookup()
1763 l->page[1].size = l->page[0].size - size0; in mmu_lookup()
1764 l->page[0].size = size0; in mmu_lookup()
1770 mmu_lookup1(cpu, &l->page[0], l->memop, l->mmu_idx, type, ra); in mmu_lookup()
1771 if (mmu_lookup1(cpu, &l->page[1], 0, l->mmu_idx, type, ra)) { in mmu_lookup()
1772 uintptr_t index = tlb_index(cpu, l->mmu_idx, addr); in mmu_lookup()
1773 l->page[0].full = &cpu->neg.tlb.d[l->mmu_idx].fulltlb[index]; in mmu_lookup()
1776 flags = l->page[0].flags | l->page[1].flags; in mmu_lookup()
1778 mmu_watch_or_dirty(cpu, &l->page[0], type, ra); in mmu_lookup()
1779 mmu_watch_or_dirty(cpu, &l->page[1], type, ra); in mmu_lookup()
1798 int size, uintptr_t retaddr) in atomic_mmu_lookup() argument
1812 retaddr -= GETPC_ADJ; in atomic_mmu_lookup()
1817 /* Check TLB entry and enforce page permissions. */ in atomic_mmu_lookup()
1823 mop, size, false, retaddr); in atomic_mmu_lookup()
1832 * Let the guest notice RMW on a write-only page. in atomic_mmu_lookup()
1835 * but addr_read will only be -1 if PAGE_READ was unset. in atomic_mmu_lookup()
1837 if (unlikely(tlbe->addr_read == -1)) { in atomic_mmu_lookup()
1839 0, size, false, retaddr); in atomic_mmu_lookup()
1849 if (!did_tlb_fill && (addr & ((1 << memop_alignment_bits(mop)) - 1))) { in atomic_mmu_lookup()
1854 if (unlikely(addr & (size - 1))) { in atomic_mmu_lookup()
1864 /* Collect tlb flags for read. */ in atomic_mmu_lookup()
1865 tlb_addr |= tlbe->addr_read; in atomic_mmu_lookup()
1867 /* Notice an IO access or a needs-MMU-lookup access */ in atomic_mmu_lookup()
1870 support this apart from stop-the-world. */ in atomic_mmu_lookup()
1874 hostaddr = (void *)((uintptr_t)addr + tlbe->addend); in atomic_mmu_lookup()
1875 full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index]; in atomic_mmu_lookup()
1878 notdirty_write(cpu, addr, size, full, retaddr); in atomic_mmu_lookup()
1884 if (full->slow_flags[MMU_DATA_STORE] & TLB_WATCHPOINT) { in atomic_mmu_lookup()
1887 if (full->slow_flags[MMU_DATA_LOAD] & TLB_WATCHPOINT) { in atomic_mmu_lookup()
1891 cpu_check_watchpoint(cpu, addr, size, in atomic_mmu_lookup()
1892 full->attrs, wp_flags, retaddr); in atomic_mmu_lookup()
1911 * complication of ABI-specific return type promotion and always
1912 * return a value extended to the register size of the host. This is
1913 * tcg_target_long, except in the case of a 32-bit host and 64-bit
1925 * @size: number of bytes
1930 * Load @size bytes from @addr, which is memory-mapped i/o.
1931 * The bytes are concatenated in big-endian order with @ret_be.
1934 uint64_t ret_be, vaddr addr, int size, in int_ld_mmio_beN() argument
1945 this_mop = ctz32(size | (int)addr | 8); in int_ld_mmio_beN()
1950 this_mop, full->attrs); in int_ld_mmio_beN()
1961 size -= this_size; in int_ld_mmio_beN()
1962 } while (size); in int_ld_mmio_beN()
1968 uint64_t ret_be, vaddr addr, int size, in do_ld_mmio_beN() argument
1976 tcg_debug_assert(size > 0 && size <= 8); in do_ld_mmio_beN()
1978 attrs = full->attrs; in do_ld_mmio_beN()
1979 section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra); in do_ld_mmio_beN()
1980 mr = section->mr; in do_ld_mmio_beN()
1983 return int_ld_mmio_beN(cpu, full, ret_be, addr, size, mmu_idx, in do_ld_mmio_beN()
1988 uint64_t ret_be, vaddr addr, int size, in do_ld16_mmio_beN() argument
1997 tcg_debug_assert(size > 8 && size <= 16); in do_ld16_mmio_beN()
1999 attrs = full->attrs; in do_ld16_mmio_beN()
2000 section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra); in do_ld16_mmio_beN()
2001 mr = section->mr; in do_ld16_mmio_beN()
2004 a = int_ld_mmio_beN(cpu, full, ret_be, addr, size - 8, mmu_idx, in do_ld16_mmio_beN()
2006 b = int_ld_mmio_beN(cpu, full, ret_be, addr + size - 8, 8, mmu_idx, in do_ld16_mmio_beN()
2007 MMU_DATA_LOAD, ra, mr, mr_offset + size - 8); in do_ld16_mmio_beN()
2016 * Load @p->size bytes from @p->haddr, which is RAM.
2017 * The bytes to concatenated in big-endian order with @ret_be.
2021 uint8_t *haddr = p->haddr; in do_ld_bytes_beN()
2022 int i, size = p->size; in do_ld_bytes_beN() local
2024 for (i = 0; i < size; i++) { in do_ld_bytes_beN()
2025 ret_be = (ret_be << 8) | haddr[i]; in do_ld_bytes_beN()
2039 void *haddr = p->haddr; in do_ld_parts_beN()
2040 int size = p->size; in do_ld_parts_beN() local
2047 * Find minimum of alignment and size. in do_ld_parts_beN()
2049 * would have only checked the low bits of addr|size once at the start, in do_ld_parts_beN()
2052 switch (((uintptr_t)haddr | size) & 7) { in do_ld_parts_beN()
2073 size -= n; in do_ld_parts_beN()
2074 } while (size != 0); in do_ld_parts_beN()
2088 int o = p->addr & 3; in do_ld_whole_be4()
2089 uint32_t x = load_atomic4(p->haddr - o); in do_ld_whole_be4()
2093 x >>= (4 - p->size) * 8; in do_ld_whole_be4()
2094 return (ret_be << (p->size * 8)) | x; in do_ld_whole_be4()
2108 int o = p->addr & 7; in do_ld_whole_be8()
2109 uint64_t x = load_atomic8_or_exit(cpu, ra, p->haddr - o); in do_ld_whole_be8()
2113 x >>= (8 - p->size) * 8; in do_ld_whole_be8()
2114 return (ret_be << (p->size * 8)) | x; in do_ld_whole_be8()
2128 int o = p->addr & 15; in do_ld_whole_be16()
2129 Int128 x, y = load_atomic16_or_exit(cpu, ra, p->haddr - o); in do_ld_whole_be16()
2130 int size = p->size; in do_ld_whole_be16() local
2136 y = int128_urshift(y, (16 - size) * 8); in do_ld_whole_be16()
2138 x = int128_lshift(x, size * 8); in do_ld_whole_be16()
2152 if (unlikely(p->flags & TLB_MMIO)) { in do_ld_beN()
2153 return do_ld_mmio_beN(cpu, p->full, ret_be, p->addr, p->size, in do_ld_beN()
2169 tmp = tmp ? tmp - 1 : 0; in do_ld_beN()
2172 ? p->size == half_size in do_ld_beN()
2173 : p->size >= half_size) { in do_ld_beN()
2174 if (!HAVE_al8_fast && p->size < 4) { in do_ld_beN()
2193 * Wrapper for the above, for 8 < size < 16.
2198 int size = p->size; in do_ld16_beN() local
2202 if (unlikely(p->flags & TLB_MMIO)) { in do_ld16_beN()
2203 return do_ld16_mmio_beN(cpu, p->full, a, p->addr, size, mmu_idx, ra); in do_ld16_beN()
2213 p->size = size - 8; in do_ld16_beN()
2215 p->haddr += size - 8; in do_ld16_beN()
2216 p->size = 8; in do_ld16_beN()
2221 /* Since size > 8, this is the half that must be atomic. */ in do_ld16_beN()
2226 * Since size > 8, both halves are misaligned, in do_ld16_beN()
2232 p->size = size - 8; in do_ld16_beN()
2234 b = ldq_be_p(p->haddr + size - 8); in do_ld16_beN()
2247 if (unlikely(p->flags & TLB_MMIO)) { in do_ld_1()
2248 return do_ld_mmio_beN(cpu, p->full, 0, p->addr, 1, mmu_idx, type, ra); in do_ld_1()
2250 return *(uint8_t *)p->haddr; in do_ld_1()
2259 if (unlikely(p->flags & TLB_MMIO)) { in do_ld_2()
2260 ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 2, mmu_idx, type, ra); in do_ld_2()
2266 ret = load_atom_2(cpu, ra, p->haddr, memop); in do_ld_2()
2279 if (unlikely(p->flags & TLB_MMIO)) { in do_ld_4()
2280 ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 4, mmu_idx, type, ra); in do_ld_4()
2286 ret = load_atom_4(cpu, ra, p->haddr, memop); in do_ld_4()
2299 if (unlikely(p->flags & TLB_MMIO)) { in do_ld_8()
2300 ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 8, mmu_idx, type, ra); in do_ld_8()
2306 ret = load_atom_8(cpu, ra, p->haddr, memop); in do_ld_8()
2422 first = l.page[0].size; in do_ld16_mmu()
2443 ret = int128_lshift(ret, l.page[1].size * 8); in do_ld16_mmu()
2465 * @size: number of bytes
2470 * Store @size bytes at @addr, which is memory-mapped i/o.
2471 * The bytes to store are extracted in little-endian order from @val_le;
2472 * return the bytes of @val_le beyond @p->size that have not been stored.
2475 uint64_t val_le, vaddr addr, int size, in int_st_mmio_leN() argument
2485 this_mop = ctz32(size | (int)addr | 8); in int_st_mmio_leN()
2490 this_mop, full->attrs); in int_st_mmio_leN()
2502 size -= this_size; in int_st_mmio_leN()
2503 } while (size); in int_st_mmio_leN()
2509 uint64_t val_le, vaddr addr, int size, in do_st_mmio_leN() argument
2517 tcg_debug_assert(size > 0 && size <= 8); in do_st_mmio_leN()
2519 attrs = full->attrs; in do_st_mmio_leN()
2520 section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra); in do_st_mmio_leN()
2521 mr = section->mr; in do_st_mmio_leN()
2524 return int_st_mmio_leN(cpu, full, val_le, addr, size, mmu_idx, in do_st_mmio_leN()
2529 Int128 val_le, vaddr addr, int size, in do_st16_mmio_leN() argument
2537 tcg_debug_assert(size > 8 && size <= 16); in do_st16_mmio_leN()
2539 attrs = full->attrs; in do_st16_mmio_leN()
2540 section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra); in do_st16_mmio_leN()
2541 mr = section->mr; in do_st16_mmio_leN()
2547 size - 8, mmu_idx, ra, mr, mr_offset + 8); in do_st16_mmio_leN()
2560 if (unlikely(p->flags & TLB_MMIO)) { in do_st_leN()
2561 return do_st_mmio_leN(cpu, p->full, val_le, p->addr, in do_st_leN()
2562 p->size, mmu_idx, ra); in do_st_leN()
2563 } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { in do_st_leN()
2564 return val_le >> (p->size * 8); in do_st_leN()
2574 return store_parts_leN(p->haddr, p->size, val_le); in do_st_leN()
2579 tmp = tmp ? tmp - 1 : 0; in do_st_leN()
2582 ? p->size == half_size in do_st_leN()
2583 : p->size >= half_size) { in do_st_leN()
2584 if (!HAVE_al8_fast && p->size <= 4) { in do_st_leN()
2585 return store_whole_le4(p->haddr, p->size, val_le); in do_st_leN()
2587 return store_whole_le8(p->haddr, p->size, val_le); in do_st_leN()
2597 return store_bytes_leN(p->haddr, p->size, val_le); in do_st_leN()
2605 * Wrapper for the above, for 8 < size < 16.
2611 int size = p->size; in do_st16_leN() local
2614 if (unlikely(p->flags & TLB_MMIO)) { in do_st16_leN()
2615 return do_st16_mmio_leN(cpu, p->full, val_le, p->addr, in do_st16_leN()
2616 size, mmu_idx, ra); in do_st16_leN()
2617 } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { in do_st16_leN()
2618 return int128_gethi(val_le) >> ((size - 8) * 8); in do_st16_leN()
2628 store_parts_leN(p->haddr, 8, int128_getlo(val_le)); in do_st16_leN()
2629 return store_parts_leN(p->haddr + 8, p->size - 8, in do_st16_leN()
2633 /* Since size > 8, this is the half that must be atomic. */ in do_st16_leN()
2637 return store_whole_le16(p->haddr, p->size, val_le); in do_st16_leN()
2641 * Since size > 8, both halves are misaligned, in do_st16_leN()
2647 stq_le_p(p->haddr, int128_getlo(val_le)); in do_st16_leN()
2648 return store_bytes_leN(p->haddr + 8, p->size - 8, in do_st16_leN()
2659 if (unlikely(p->flags & TLB_MMIO)) { in do_st_1()
2660 do_st_mmio_leN(cpu, p->full, val, p->addr, 1, mmu_idx, ra); in do_st_1()
2661 } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { in do_st_1()
2664 *(uint8_t *)p->haddr = val; in do_st_1()
2671 if (unlikely(p->flags & TLB_MMIO)) { in do_st_2()
2675 do_st_mmio_leN(cpu, p->full, val, p->addr, 2, mmu_idx, ra); in do_st_2()
2676 } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { in do_st_2()
2683 store_atom_2(cpu, ra, p->haddr, memop, val); in do_st_2()
2690 if (unlikely(p->flags & TLB_MMIO)) { in do_st_4()
2694 do_st_mmio_leN(cpu, p->full, val, p->addr, 4, mmu_idx, ra); in do_st_4()
2695 } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { in do_st_4()
2702 store_atom_4(cpu, ra, p->haddr, memop, val); in do_st_4()
2709 if (unlikely(p->flags & TLB_MMIO)) { in do_st_8()
2713 do_st_mmio_leN(cpu, p->full, val, p->addr, 8, mmu_idx, ra); in do_st_8()
2714 } else if (unlikely(p->flags & TLB_DISCARD_WRITE)) { in do_st_8()
2721 store_atom_8(cpu, ra, p->haddr, memop, val); in do_st_8()
2831 first = l.page[0].size; in do_st16_mmu()