1 /* 2 * Flush the host cpu caches. 3 * 4 * This work is licensed under the terms of the GNU GPL, version 2 or later. 5 * See the COPYING file in the top-level directory. 6 */ 7 8 #include "qemu/osdep.h" 9 #include "qemu/cacheflush.h" 10 #include "qemu/cacheinfo.h" 11 #include "qemu/bitops.h" 12 13 14 #if defined(__i386__) || defined(__x86_64__) || defined(__s390__) 15 16 /* Caches are coherent and do not require flushing; symbol inline. */ 17 18 #elif defined(__aarch64__) 19 20 #ifdef CONFIG_DARWIN 21 /* Apple does not expose CTR_EL0, so we must use system interfaces. */ 22 extern void sys_icache_invalidate(void *start, size_t len); 23 extern void sys_dcache_flush(void *start, size_t len); 24 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 25 { 26 sys_dcache_flush((void *)rw, len); 27 sys_icache_invalidate((void *)rx, len); 28 } 29 #else 30 31 /* 32 * TODO: unify this with cacheinfo.c. 33 * We want to save the whole contents of CTR_EL0, so that we 34 * have more than the linesize, but also IDC and DIC. 35 */ 36 static uint64_t save_ctr_el0; 37 static void __attribute__((constructor)) init_ctr_el0(void) 38 { 39 asm volatile("mrs\t%0, ctr_el0" : "=r"(save_ctr_el0)); 40 } 41 42 /* 43 * This is a copy of gcc's __aarch64_sync_cache_range, modified 44 * to fit this three-operand interface. 45 */ 46 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 47 { 48 const unsigned CTR_IDC = 1u << 28; 49 const unsigned CTR_DIC = 1u << 29; 50 const uint64_t ctr_el0 = save_ctr_el0; 51 const uintptr_t icache_lsize = 4 << extract64(ctr_el0, 0, 4); 52 const uintptr_t dcache_lsize = 4 << extract64(ctr_el0, 16, 4); 53 uintptr_t p; 54 55 /* 56 * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification 57 * is not required for instruction to data coherence. 58 */ 59 if (!(ctr_el0 & CTR_IDC)) { 60 /* 61 * Loop over the address range, clearing one cache line at once. 62 * Data cache must be flushed to unification first to make sure 63 * the instruction cache fetches the updated data. 64 */ 65 for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) { 66 asm volatile("dc\tcvau, %0" : : "r" (p) : "memory"); 67 } 68 asm volatile("dsb\tish" : : : "memory"); 69 } 70 71 /* 72 * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point 73 * of Unification is not required for instruction to data coherence. 74 */ 75 if (!(ctr_el0 & CTR_DIC)) { 76 for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) { 77 asm volatile("ic\tivau, %0" : : "r"(p) : "memory"); 78 } 79 asm volatile ("dsb\tish" : : : "memory"); 80 } 81 82 asm volatile("isb" : : : "memory"); 83 } 84 #endif /* CONFIG_DARWIN */ 85 86 #elif defined(__mips__) 87 88 #ifdef __OpenBSD__ 89 #include <machine/sysarch.h> 90 #else 91 #include <sys/cachectl.h> 92 #endif 93 94 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 95 { 96 if (rx != rw) { 97 cacheflush((void *)rw, len, DCACHE); 98 } 99 cacheflush((void *)rx, len, ICACHE); 100 } 101 102 #elif defined(__powerpc__) 103 104 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 105 { 106 uintptr_t p, b, e; 107 size_t dsize = qemu_dcache_linesize; 108 size_t isize = qemu_icache_linesize; 109 110 b = rw & ~(dsize - 1); 111 e = (rw + len + dsize - 1) & ~(dsize - 1); 112 for (p = b; p < e; p += dsize) { 113 asm volatile ("dcbst 0,%0" : : "r"(p) : "memory"); 114 } 115 asm volatile ("sync" : : : "memory"); 116 117 b = rx & ~(isize - 1); 118 e = (rx + len + isize - 1) & ~(isize - 1); 119 for (p = b; p < e; p += isize) { 120 asm volatile ("icbi 0,%0" : : "r"(p) : "memory"); 121 } 122 asm volatile ("sync" : : : "memory"); 123 asm volatile ("isync" : : : "memory"); 124 } 125 126 #elif defined(__sparc__) 127 128 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 129 { 130 /* No additional data flush to the RW virtual address required. */ 131 uintptr_t p, end = (rx + len + 7) & -8; 132 for (p = rx & -8; p < end; p += 8) { 133 __asm__ __volatile__("flush\t%0" : : "r" (p)); 134 } 135 } 136 137 #else 138 139 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 140 { 141 if (rw != rx) { 142 __builtin___clear_cache((char *)rw, (char *)rw + len); 143 } 144 __builtin___clear_cache((char *)rx, (char *)rx + len); 145 } 146 147 #endif 148