1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/tlbflush.h 4 * 5 * Copyright (C) 1999-2003 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASM_TLBFLUSH_H 9 #define __ASM_TLBFLUSH_H 10 11 #ifndef __ASSEMBLY__ 12 13 #include <linux/mm_types.h> 14 #include <linux/sched.h> 15 #include <asm/cputype.h> 16 #include <asm/mmu.h> 17 18 /* 19 * Raw TLBI operations. 20 * 21 * Where necessary, use the __tlbi() macro to avoid asm() 22 * boilerplate. Drivers and most kernel code should use the TLB 23 * management routines in preference to the macro below. 24 * 25 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending 26 * on whether a particular TLBI operation takes an argument or 27 * not. The macros handles invoking the asm with or without the 28 * register argument as appropriate. 29 */ 30 #define __TLBI_0(op, arg) asm ("tlbi " #op "\n" \ 31 ALTERNATIVE("nop\n nop", \ 32 "dsb ish\n tlbi " #op, \ 33 ARM64_WORKAROUND_REPEAT_TLBI, \ 34 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ 35 : : ) 36 37 #define __TLBI_1(op, arg) asm ("tlbi " #op ", %0\n" \ 38 ALTERNATIVE("nop\n nop", \ 39 "dsb ish\n tlbi " #op ", %0", \ 40 ARM64_WORKAROUND_REPEAT_TLBI, \ 41 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ 42 : : "r" (arg)) 43 44 #define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg) 45 46 #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0) 47 48 #define __tlbi_user(op, arg) do { \ 49 if (arm64_kernel_unmapped_at_el0()) \ 50 __tlbi(op, (arg) | USER_ASID_FLAG); \ 51 } while (0) 52 53 /* This macro creates a properly formatted VA operand for the TLBI */ 54 #define __TLBI_VADDR(addr, asid) \ 55 ({ \ 56 unsigned long __ta = (addr) >> 12; \ 57 __ta &= GENMASK_ULL(43, 0); \ 58 __ta |= (unsigned long)(asid) << 48; \ 59 __ta; \ 60 }) 61 62 /* 63 * TLB Invalidation 64 * ================ 65 * 66 * This header file implements the low-level TLB invalidation routines 67 * (sometimes referred to as "flushing" in the kernel) for arm64. 68 * 69 * Every invalidation operation uses the following template: 70 * 71 * DSB ISHST // Ensure prior page-table updates have completed 72 * TLBI ... // Invalidate the TLB 73 * DSB ISH // Ensure the TLB invalidation has completed 74 * if (invalidated kernel mappings) 75 * ISB // Discard any instructions fetched from the old mapping 76 * 77 * 78 * The following functions form part of the "core" TLB invalidation API, 79 * as documented in Documentation/core-api/cachetlb.rst: 80 * 81 * flush_tlb_all() 82 * Invalidate the entire TLB (kernel + user) on all CPUs 83 * 84 * flush_tlb_mm(mm) 85 * Invalidate an entire user address space on all CPUs. 86 * The 'mm' argument identifies the ASID to invalidate. 87 * 88 * flush_tlb_range(vma, start, end) 89 * Invalidate the virtual-address range '[start, end)' on all 90 * CPUs for the user address space corresponding to 'vma->mm'. 91 * Note that this operation also invalidates any walk-cache 92 * entries associated with translations for the specified address 93 * range. 94 * 95 * flush_tlb_kernel_range(start, end) 96 * Same as flush_tlb_range(..., start, end), but applies to 97 * kernel mappings rather than a particular user address space. 98 * Whilst not explicitly documented, this function is used when 99 * unmapping pages from vmalloc/io space. 100 * 101 * flush_tlb_page(vma, addr) 102 * Invalidate a single user mapping for address 'addr' in the 103 * address space corresponding to 'vma->mm'. Note that this 104 * operation only invalidates a single, last-level page-table 105 * entry and therefore does not affect any walk-caches. 106 * 107 * 108 * Next, we have some undocumented invalidation routines that you probably 109 * don't want to call unless you know what you're doing: 110 * 111 * local_flush_tlb_all() 112 * Same as flush_tlb_all(), but only applies to the calling CPU. 113 * 114 * __flush_tlb_kernel_pgtable(addr) 115 * Invalidate a single kernel mapping for address 'addr' on all 116 * CPUs, ensuring that any walk-cache entries associated with the 117 * translation are also invalidated. 118 * 119 * __flush_tlb_range(vma, start, end, stride, last_level) 120 * Invalidate the virtual-address range '[start, end)' on all 121 * CPUs for the user address space corresponding to 'vma->mm'. 122 * The invalidation operations are issued at a granularity 123 * determined by 'stride' and only affect any walk-cache entries 124 * if 'last_level' is equal to false. 125 * 126 * 127 * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented 128 * on top of these routines, since that is our interface to the mmu_gather 129 * API as used by munmap() and friends. 130 */ 131 static inline void local_flush_tlb_all(void) 132 { 133 dsb(nshst); 134 __tlbi(vmalle1); 135 dsb(nsh); 136 isb(); 137 } 138 139 static inline void flush_tlb_all(void) 140 { 141 dsb(ishst); 142 __tlbi(vmalle1is); 143 dsb(ish); 144 isb(); 145 } 146 147 static inline void flush_tlb_mm(struct mm_struct *mm) 148 { 149 unsigned long asid = __TLBI_VADDR(0, ASID(mm)); 150 151 dsb(ishst); 152 __tlbi(aside1is, asid); 153 __tlbi_user(aside1is, asid); 154 dsb(ish); 155 } 156 157 static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, 158 unsigned long uaddr) 159 { 160 unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm)); 161 162 dsb(ishst); 163 __tlbi(vale1is, addr); 164 __tlbi_user(vale1is, addr); 165 } 166 167 static inline void flush_tlb_page(struct vm_area_struct *vma, 168 unsigned long uaddr) 169 { 170 flush_tlb_page_nosync(vma, uaddr); 171 dsb(ish); 172 } 173 174 /* 175 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not 176 * necessarily a performance improvement. 177 */ 178 #define MAX_TLBI_OPS PTRS_PER_PTE 179 180 static inline void __flush_tlb_range(struct vm_area_struct *vma, 181 unsigned long start, unsigned long end, 182 unsigned long stride, bool last_level) 183 { 184 unsigned long asid = ASID(vma->vm_mm); 185 unsigned long addr; 186 187 start = round_down(start, stride); 188 end = round_up(end, stride); 189 190 if ((end - start) >= (MAX_TLBI_OPS * stride)) { 191 flush_tlb_mm(vma->vm_mm); 192 return; 193 } 194 195 /* Convert the stride into units of 4k */ 196 stride >>= 12; 197 198 start = __TLBI_VADDR(start, asid); 199 end = __TLBI_VADDR(end, asid); 200 201 dsb(ishst); 202 for (addr = start; addr < end; addr += stride) { 203 if (last_level) { 204 __tlbi(vale1is, addr); 205 __tlbi_user(vale1is, addr); 206 } else { 207 __tlbi(vae1is, addr); 208 __tlbi_user(vae1is, addr); 209 } 210 } 211 dsb(ish); 212 } 213 214 static inline void flush_tlb_range(struct vm_area_struct *vma, 215 unsigned long start, unsigned long end) 216 { 217 /* 218 * We cannot use leaf-only invalidation here, since we may be invalidating 219 * table entries as part of collapsing hugepages or moving page tables. 220 */ 221 __flush_tlb_range(vma, start, end, PAGE_SIZE, false); 222 } 223 224 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) 225 { 226 unsigned long addr; 227 228 if ((end - start) > (MAX_TLBI_OPS * PAGE_SIZE)) { 229 flush_tlb_all(); 230 return; 231 } 232 233 start = __TLBI_VADDR(start, 0); 234 end = __TLBI_VADDR(end, 0); 235 236 dsb(ishst); 237 for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) 238 __tlbi(vaale1is, addr); 239 dsb(ish); 240 isb(); 241 } 242 243 /* 244 * Used to invalidate the TLB (walk caches) corresponding to intermediate page 245 * table levels (pgd/pud/pmd). 246 */ 247 static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) 248 { 249 unsigned long addr = __TLBI_VADDR(kaddr, 0); 250 251 dsb(ishst); 252 __tlbi(vaae1is, addr); 253 dsb(ish); 254 } 255 #endif 256 257 #endif 258