1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Based on arch/arm/include/asm/tlbflush.h
4 *
5 * Copyright (C) 1999-2003 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8 #ifndef __ASM_TLBFLUSH_H
9 #define __ASM_TLBFLUSH_H
10
11 #ifndef __ASSEMBLY__
12
13 #include <linux/bitfield.h>
14 #include <linux/mm_types.h>
15 #include <linux/sched.h>
16 #include <linux/mmu_notifier.h>
17 #include <asm/cputype.h>
18 #include <asm/mmu.h>
19
20 /*
21 * Raw TLBI operations.
22 *
23 * Where necessary, use the __tlbi() macro to avoid asm()
24 * boilerplate. Drivers and most kernel code should use the TLB
25 * management routines in preference to the macro below.
26 *
27 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending
28 * on whether a particular TLBI operation takes an argument or
29 * not. The macros handles invoking the asm with or without the
30 * register argument as appropriate.
31 */
32 #define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \
33 "tlbi " #op "\n" \
34 ALTERNATIVE("nop\n nop", \
35 "dsb ish\n tlbi " #op, \
36 ARM64_WORKAROUND_REPEAT_TLBI, \
37 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \
38 : : )
39
40 #define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \
41 "tlbi " #op ", %0\n" \
42 ALTERNATIVE("nop\n nop", \
43 "dsb ish\n tlbi " #op ", %0", \
44 ARM64_WORKAROUND_REPEAT_TLBI, \
45 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \
46 : : "r" (arg))
47
48 #define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg)
49
50 #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0)
51
52 #define __tlbi_user(op, arg) do { \
53 if (arm64_kernel_unmapped_at_el0()) \
54 __tlbi(op, (arg) | USER_ASID_FLAG); \
55 } while (0)
56
57 /* This macro creates a properly formatted VA operand for the TLBI */
58 #define __TLBI_VADDR(addr, asid) \
59 ({ \
60 unsigned long __ta = (addr) >> 12; \
61 __ta &= GENMASK_ULL(43, 0); \
62 __ta |= (unsigned long)(asid) << 48; \
63 __ta; \
64 })
65
66 /*
67 * Get translation granule of the system, which is decided by
68 * PAGE_SIZE. Used by TTL.
69 * - 4KB : 1
70 * - 16KB : 2
71 * - 64KB : 3
72 */
73 #define TLBI_TTL_TG_4K 1
74 #define TLBI_TTL_TG_16K 2
75 #define TLBI_TTL_TG_64K 3
76
get_trans_granule(void)77 static inline unsigned long get_trans_granule(void)
78 {
79 switch (PAGE_SIZE) {
80 case SZ_4K:
81 return TLBI_TTL_TG_4K;
82 case SZ_16K:
83 return TLBI_TTL_TG_16K;
84 case SZ_64K:
85 return TLBI_TTL_TG_64K;
86 default:
87 return 0;
88 }
89 }
90
91 /*
92 * Level-based TLBI operations.
93 *
94 * When ARMv8.4-TTL exists, TLBI operations take an additional hint for
95 * the level at which the invalidation must take place. If the level is
96 * wrong, no invalidation may take place. In the case where the level
97 * cannot be easily determined, a 0 value for the level parameter will
98 * perform a non-hinted invalidation.
99 *
100 * For Stage-2 invalidation, use the level values provided to that effect
101 * in asm/stage2_pgtable.h.
102 */
103 #define TLBI_TTL_MASK GENMASK_ULL(47, 44)
104
105 #define __tlbi_level(op, addr, level) do { \
106 u64 arg = addr; \
107 \
108 if (cpus_have_const_cap(ARM64_HAS_ARMv8_4_TTL) && \
109 level) { \
110 u64 ttl = level & 3; \
111 ttl |= get_trans_granule() << 2; \
112 arg &= ~TLBI_TTL_MASK; \
113 arg |= FIELD_PREP(TLBI_TTL_MASK, ttl); \
114 } \
115 \
116 __tlbi(op, arg); \
117 } while(0)
118
119 #define __tlbi_user_level(op, arg, level) do { \
120 if (arm64_kernel_unmapped_at_el0()) \
121 __tlbi_level(op, (arg | USER_ASID_FLAG), level); \
122 } while (0)
123
124 /*
125 * This macro creates a properly formatted VA operand for the TLB RANGE.
126 * The value bit assignments are:
127 *
128 * +----------+------+-------+-------+-------+----------------------+
129 * | ASID | TG | SCALE | NUM | TTL | BADDR |
130 * +-----------------+-------+-------+-------+----------------------+
131 * |63 48|47 46|45 44|43 39|38 37|36 0|
132 *
133 * The address range is determined by below formula:
134 * [BADDR, BADDR + (NUM + 1) * 2^(5*SCALE + 1) * PAGESIZE)
135 *
136 */
137 #define __TLBI_VADDR_RANGE(addr, asid, scale, num, ttl) \
138 ({ \
139 unsigned long __ta = (addr) >> PAGE_SHIFT; \
140 __ta &= GENMASK_ULL(36, 0); \
141 __ta |= (unsigned long)(ttl) << 37; \
142 __ta |= (unsigned long)(num) << 39; \
143 __ta |= (unsigned long)(scale) << 44; \
144 __ta |= get_trans_granule() << 46; \
145 __ta |= (unsigned long)(asid) << 48; \
146 __ta; \
147 })
148
149 /* These macros are used by the TLBI RANGE feature. */
150 #define __TLBI_RANGE_PAGES(num, scale) \
151 ((unsigned long)((num) + 1) << (5 * (scale) + 1))
152 #define MAX_TLBI_RANGE_PAGES __TLBI_RANGE_PAGES(31, 3)
153
154 /*
155 * Generate 'num' values from -1 to 31 with -1 rejected by the
156 * __flush_tlb_range() loop below. Its return value is only
157 * significant for a maximum of MAX_TLBI_RANGE_PAGES pages. If
158 * 'pages' is more than that, you must iterate over the overall
159 * range.
160 */
161 #define __TLBI_RANGE_NUM(pages, scale) \
162 ({ \
163 int __pages = min((pages), \
164 __TLBI_RANGE_PAGES(31, (scale))); \
165 (__pages >> (5 * (scale) + 1)) - 1; \
166 })
167
168 /*
169 * TLB Invalidation
170 * ================
171 *
172 * This header file implements the low-level TLB invalidation routines
173 * (sometimes referred to as "flushing" in the kernel) for arm64.
174 *
175 * Every invalidation operation uses the following template:
176 *
177 * DSB ISHST // Ensure prior page-table updates have completed
178 * TLBI ... // Invalidate the TLB
179 * DSB ISH // Ensure the TLB invalidation has completed
180 * if (invalidated kernel mappings)
181 * ISB // Discard any instructions fetched from the old mapping
182 *
183 *
184 * The following functions form part of the "core" TLB invalidation API,
185 * as documented in Documentation/core-api/cachetlb.rst:
186 *
187 * flush_tlb_all()
188 * Invalidate the entire TLB (kernel + user) on all CPUs
189 *
190 * flush_tlb_mm(mm)
191 * Invalidate an entire user address space on all CPUs.
192 * The 'mm' argument identifies the ASID to invalidate.
193 *
194 * flush_tlb_range(vma, start, end)
195 * Invalidate the virtual-address range '[start, end)' on all
196 * CPUs for the user address space corresponding to 'vma->mm'.
197 * Note that this operation also invalidates any walk-cache
198 * entries associated with translations for the specified address
199 * range.
200 *
201 * flush_tlb_kernel_range(start, end)
202 * Same as flush_tlb_range(..., start, end), but applies to
203 * kernel mappings rather than a particular user address space.
204 * Whilst not explicitly documented, this function is used when
205 * unmapping pages from vmalloc/io space.
206 *
207 * flush_tlb_page(vma, addr)
208 * Invalidate a single user mapping for address 'addr' in the
209 * address space corresponding to 'vma->mm'. Note that this
210 * operation only invalidates a single, last-level page-table
211 * entry and therefore does not affect any walk-caches.
212 *
213 *
214 * Next, we have some undocumented invalidation routines that you probably
215 * don't want to call unless you know what you're doing:
216 *
217 * local_flush_tlb_all()
218 * Same as flush_tlb_all(), but only applies to the calling CPU.
219 *
220 * __flush_tlb_kernel_pgtable(addr)
221 * Invalidate a single kernel mapping for address 'addr' on all
222 * CPUs, ensuring that any walk-cache entries associated with the
223 * translation are also invalidated.
224 *
225 * __flush_tlb_range(vma, start, end, stride, last_level)
226 * Invalidate the virtual-address range '[start, end)' on all
227 * CPUs for the user address space corresponding to 'vma->mm'.
228 * The invalidation operations are issued at a granularity
229 * determined by 'stride' and only affect any walk-cache entries
230 * if 'last_level' is equal to false.
231 *
232 *
233 * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
234 * on top of these routines, since that is our interface to the mmu_gather
235 * API as used by munmap() and friends.
236 */
local_flush_tlb_all(void)237 static inline void local_flush_tlb_all(void)
238 {
239 dsb(nshst);
240 __tlbi(vmalle1);
241 dsb(nsh);
242 isb();
243 }
244
flush_tlb_all(void)245 static inline void flush_tlb_all(void)
246 {
247 dsb(ishst);
248 __tlbi(vmalle1is);
249 dsb(ish);
250 isb();
251 }
252
flush_tlb_mm(struct mm_struct * mm)253 static inline void flush_tlb_mm(struct mm_struct *mm)
254 {
255 unsigned long asid;
256
257 dsb(ishst);
258 asid = __TLBI_VADDR(0, ASID(mm));
259 __tlbi(aside1is, asid);
260 __tlbi_user(aside1is, asid);
261 dsb(ish);
262 mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
263 }
264
__flush_tlb_page_nosync(struct mm_struct * mm,unsigned long uaddr)265 static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
266 unsigned long uaddr)
267 {
268 unsigned long addr;
269
270 dsb(ishst);
271 addr = __TLBI_VADDR(uaddr, ASID(mm));
272 __tlbi(vale1is, addr);
273 __tlbi_user(vale1is, addr);
274 mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
275 (uaddr & PAGE_MASK) + PAGE_SIZE);
276 }
277
flush_tlb_page_nosync(struct vm_area_struct * vma,unsigned long uaddr)278 static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
279 unsigned long uaddr)
280 {
281 return __flush_tlb_page_nosync(vma->vm_mm, uaddr);
282 }
283
flush_tlb_page(struct vm_area_struct * vma,unsigned long uaddr)284 static inline void flush_tlb_page(struct vm_area_struct *vma,
285 unsigned long uaddr)
286 {
287 flush_tlb_page_nosync(vma, uaddr);
288 dsb(ish);
289 }
290
arch_tlbbatch_should_defer(struct mm_struct * mm)291 static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
292 {
293 #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
294 /*
295 * TLB flush deferral is not required on systems which are affected by
296 * ARM64_WORKAROUND_REPEAT_TLBI, as __tlbi()/__tlbi_user() implementation
297 * will have two consecutive TLBI instructions with a dsb(ish) in between
298 * defeating the purpose (i.e save overall 'dsb ish' cost).
299 */
300 if (unlikely(cpus_have_const_cap(ARM64_WORKAROUND_REPEAT_TLBI)))
301 return false;
302 #endif
303 return true;
304 }
305
arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch * batch,struct mm_struct * mm,unsigned long uaddr)306 static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
307 struct mm_struct *mm,
308 unsigned long uaddr)
309 {
310 __flush_tlb_page_nosync(mm, uaddr);
311 }
312
313 /*
314 * If mprotect/munmap/etc occurs during TLB batched flushing, we need to ensure
315 * all the previously issued TLBIs targeting mm have completed. But since we
316 * can be executing on a remote CPU, a DSB cannot guarantee this like it can
317 * for arch_tlbbatch_flush(). Our only option is to flush the entire mm.
318 */
arch_flush_tlb_batched_pending(struct mm_struct * mm)319 static inline void arch_flush_tlb_batched_pending(struct mm_struct *mm)
320 {
321 flush_tlb_mm(mm);
322 }
323
324 /*
325 * To support TLB batched flush for multiple pages unmapping, we only send
326 * the TLBI for each page in arch_tlbbatch_add_pending() and wait for the
327 * completion at the end in arch_tlbbatch_flush(). Since we've already issued
328 * TLBI for each page so only a DSB is needed to synchronise its effect on the
329 * other CPUs.
330 *
331 * This will save the time waiting on DSB comparing issuing a TLBI;DSB sequence
332 * for each page.
333 */
arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch * batch)334 static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
335 {
336 dsb(ish);
337 }
338
339 /*
340 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
341 * necessarily a performance improvement.
342 */
343 #define MAX_TLBI_OPS PTRS_PER_PTE
344
345 /*
346 * __flush_tlb_range_op - Perform TLBI operation upon a range
347 *
348 * @op: TLBI instruction that operates on a range (has 'r' prefix)
349 * @start: The start address of the range
350 * @pages: Range as the number of pages from 'start'
351 * @stride: Flush granularity
352 * @asid: The ASID of the task (0 for IPA instructions)
353 * @tlb_level: Translation Table level hint, if known
354 * @tlbi_user: If 'true', call an additional __tlbi_user()
355 * (typically for user ASIDs). 'flase' for IPA instructions
356 *
357 * When the CPU does not support TLB range operations, flush the TLB
358 * entries one by one at the granularity of 'stride'. If the TLB
359 * range ops are supported, then:
360 *
361 * 1. The minimum range granularity is decided by 'scale', so multiple range
362 * TLBI operations may be required. Start from scale = 3, flush the largest
363 * possible number of pages ((num+1)*2^(5*scale+1)) that fit into the
364 * requested range, then decrement scale and continue until one or zero pages
365 * are left.
366 *
367 * 2. If there is 1 page remaining, flush it through non-range operations. Range
368 * operations can only span an even number of pages.
369 */
370 #define __flush_tlb_range_op(op, start, pages, stride, \
371 asid, tlb_level, tlbi_user) \
372 do { \
373 typeof(start) __flush_start = start; \
374 typeof(pages) __flush_pages = pages; \
375 int num = 0; \
376 int scale = 3; \
377 unsigned long addr; \
378 \
379 while (__flush_pages > 0) { \
380 if (!system_supports_tlb_range() || \
381 __flush_pages == 1) { \
382 addr = __TLBI_VADDR(__flush_start, asid); \
383 __tlbi_level(op, addr, tlb_level); \
384 if (tlbi_user) \
385 __tlbi_user_level(op, addr, tlb_level); \
386 __flush_start += stride; \
387 __flush_pages -= stride >> PAGE_SHIFT; \
388 continue; \
389 } \
390 \
391 num = __TLBI_RANGE_NUM(__flush_pages, scale); \
392 if (num >= 0) { \
393 addr = __TLBI_VADDR_RANGE(__flush_start, asid, \
394 scale, num, tlb_level); \
395 __tlbi(r##op, addr); \
396 if (tlbi_user) \
397 __tlbi_user(r##op, addr); \
398 __flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \
399 __flush_pages -= __TLBI_RANGE_PAGES(num, scale);\
400 } \
401 scale--; \
402 } \
403 } while (0)
404
405 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
406 __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false)
407
__flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long stride,bool last_level,int tlb_level)408 static inline void __flush_tlb_range(struct vm_area_struct *vma,
409 unsigned long start, unsigned long end,
410 unsigned long stride, bool last_level,
411 int tlb_level)
412 {
413 unsigned long asid, pages;
414
415 start = round_down(start, stride);
416 end = round_up(end, stride);
417 pages = (end - start) >> PAGE_SHIFT;
418
419 /*
420 * When not uses TLB range ops, we can handle up to
421 * (MAX_TLBI_OPS - 1) pages;
422 * When uses TLB range ops, we can handle up to
423 * (MAX_TLBI_RANGE_PAGES - 1) pages.
424 */
425 if ((!system_supports_tlb_range() &&
426 (end - start) >= (MAX_TLBI_OPS * stride)) ||
427 pages >= MAX_TLBI_RANGE_PAGES) {
428 flush_tlb_mm(vma->vm_mm);
429 return;
430 }
431
432 dsb(ishst);
433 asid = ASID(vma->vm_mm);
434
435 if (last_level)
436 __flush_tlb_range_op(vale1is, start, pages, stride, asid, tlb_level, true);
437 else
438 __flush_tlb_range_op(vae1is, start, pages, stride, asid, tlb_level, true);
439
440 dsb(ish);
441 mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, start, end);
442 }
443
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)444 static inline void flush_tlb_range(struct vm_area_struct *vma,
445 unsigned long start, unsigned long end)
446 {
447 /*
448 * We cannot use leaf-only invalidation here, since we may be invalidating
449 * table entries as part of collapsing hugepages or moving page tables.
450 * Set the tlb_level to 0 because we can not get enough information here.
451 */
452 __flush_tlb_range(vma, start, end, PAGE_SIZE, false, 0);
453 }
454
flush_tlb_kernel_range(unsigned long start,unsigned long end)455 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
456 {
457 unsigned long addr;
458
459 if ((end - start) > (MAX_TLBI_OPS * PAGE_SIZE)) {
460 flush_tlb_all();
461 return;
462 }
463
464 start = __TLBI_VADDR(start, 0);
465 end = __TLBI_VADDR(end, 0);
466
467 dsb(ishst);
468 for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
469 __tlbi(vaale1is, addr);
470 dsb(ish);
471 isb();
472 }
473
474 /*
475 * Used to invalidate the TLB (walk caches) corresponding to intermediate page
476 * table levels (pgd/pud/pmd).
477 */
__flush_tlb_kernel_pgtable(unsigned long kaddr)478 static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
479 {
480 unsigned long addr = __TLBI_VADDR(kaddr, 0);
481
482 dsb(ishst);
483 __tlbi(vaae1is, addr);
484 dsb(ish);
485 isb();
486 }
487 #endif
488
489 #endif
490