1196d9d8bSPeter Zijlstra #include <linux/gfp.h>
2196d9d8bSPeter Zijlstra #include <linux/highmem.h>
3196d9d8bSPeter Zijlstra #include <linux/kernel.h>
4196d9d8bSPeter Zijlstra #include <linux/mmdebug.h>
5196d9d8bSPeter Zijlstra #include <linux/mm_types.h>
636090defSArnd Bergmann #include <linux/mm_inline.h>
7196d9d8bSPeter Zijlstra #include <linux/pagemap.h>
8196d9d8bSPeter Zijlstra #include <linux/rcupdate.h>
9196d9d8bSPeter Zijlstra #include <linux/smp.h>
10196d9d8bSPeter Zijlstra #include <linux/swap.h>
115df397deSLinus Torvalds #include <linux/rmap.h>
12196d9d8bSPeter Zijlstra
13196d9d8bSPeter Zijlstra #include <asm/pgalloc.h>
14196d9d8bSPeter Zijlstra #include <asm/tlb.h>
15196d9d8bSPeter Zijlstra
16580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER
17952a31c9SMartin Schwidefsky
tlb_next_batch(struct mmu_gather * tlb)18196d9d8bSPeter Zijlstra static bool tlb_next_batch(struct mmu_gather *tlb)
19196d9d8bSPeter Zijlstra {
20196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch;
21196d9d8bSPeter Zijlstra
22c4745482SLinus Torvalds /* Limit batching if we have delayed rmaps pending */
23c4745482SLinus Torvalds if (tlb->delayed_rmap && tlb->active != &tlb->local)
245df397deSLinus Torvalds return false;
255df397deSLinus Torvalds
26196d9d8bSPeter Zijlstra batch = tlb->active;
27196d9d8bSPeter Zijlstra if (batch->next) {
28196d9d8bSPeter Zijlstra tlb->active = batch->next;
29196d9d8bSPeter Zijlstra return true;
30196d9d8bSPeter Zijlstra }
31196d9d8bSPeter Zijlstra
32196d9d8bSPeter Zijlstra if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
33196d9d8bSPeter Zijlstra return false;
34196d9d8bSPeter Zijlstra
35dcc1be11SLorenzo Stoakes batch = (void *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
36196d9d8bSPeter Zijlstra if (!batch)
37196d9d8bSPeter Zijlstra return false;
38196d9d8bSPeter Zijlstra
39196d9d8bSPeter Zijlstra tlb->batch_count++;
40196d9d8bSPeter Zijlstra batch->next = NULL;
41196d9d8bSPeter Zijlstra batch->nr = 0;
42196d9d8bSPeter Zijlstra batch->max = MAX_GATHER_BATCH;
43196d9d8bSPeter Zijlstra
44196d9d8bSPeter Zijlstra tlb->active->next = batch;
45196d9d8bSPeter Zijlstra tlb->active = batch;
46196d9d8bSPeter Zijlstra
47196d9d8bSPeter Zijlstra return true;
48196d9d8bSPeter Zijlstra }
49196d9d8bSPeter Zijlstra
505df397deSLinus Torvalds #ifdef CONFIG_SMP
tlb_flush_rmap_batch(struct mmu_gather_batch * batch,struct vm_area_struct * vma)51c4745482SLinus Torvalds static void tlb_flush_rmap_batch(struct mmu_gather_batch *batch, struct vm_area_struct *vma)
525df397deSLinus Torvalds {
535df397deSLinus Torvalds for (int i = 0; i < batch->nr; i++) {
545df397deSLinus Torvalds struct encoded_page *enc = batch->encoded_pages[i];
555df397deSLinus Torvalds
565df397deSLinus Torvalds if (encoded_page_flags(enc)) {
575df397deSLinus Torvalds struct page *page = encoded_page_ptr(enc);
585df397deSLinus Torvalds page_remove_rmap(page, vma, false);
595df397deSLinus Torvalds }
605df397deSLinus Torvalds }
61c4745482SLinus Torvalds }
625df397deSLinus Torvalds
63c4745482SLinus Torvalds /**
64c4745482SLinus Torvalds * tlb_flush_rmaps - do pending rmap removals after we have flushed the TLB
65c4745482SLinus Torvalds * @tlb: the current mmu_gather
66*19134bc2SMatthew Wilcox (Oracle) * @vma: The memory area from which the pages are being removed.
67c4745482SLinus Torvalds *
68c4745482SLinus Torvalds * Note that because of how tlb_next_batch() above works, we will
69c4745482SLinus Torvalds * never start multiple new batches with pending delayed rmaps, so
70c4745482SLinus Torvalds * we only need to walk through the current active batch and the
71c4745482SLinus Torvalds * original local one.
72c4745482SLinus Torvalds */
tlb_flush_rmaps(struct mmu_gather * tlb,struct vm_area_struct * vma)73c4745482SLinus Torvalds void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma)
74c4745482SLinus Torvalds {
75c4745482SLinus Torvalds if (!tlb->delayed_rmap)
76c4745482SLinus Torvalds return;
77c4745482SLinus Torvalds
78c4745482SLinus Torvalds tlb_flush_rmap_batch(&tlb->local, vma);
79c4745482SLinus Torvalds if (tlb->active != &tlb->local)
80c4745482SLinus Torvalds tlb_flush_rmap_batch(tlb->active, vma);
815df397deSLinus Torvalds tlb->delayed_rmap = 0;
825df397deSLinus Torvalds }
835df397deSLinus Torvalds #endif
845df397deSLinus Torvalds
tlb_batch_pages_flush(struct mmu_gather * tlb)85952a31c9SMartin Schwidefsky static void tlb_batch_pages_flush(struct mmu_gather *tlb)
86196d9d8bSPeter Zijlstra {
87196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch;
88196d9d8bSPeter Zijlstra
89196d9d8bSPeter Zijlstra for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
907cc8f9c7SLinus Torvalds struct encoded_page **pages = batch->encoded_pages;
91b191c9bcSJianxing Wang
92b191c9bcSJianxing Wang do {
93b191c9bcSJianxing Wang /*
94b191c9bcSJianxing Wang * limit free batch count when PAGE_SIZE > 4K
95b191c9bcSJianxing Wang */
96b191c9bcSJianxing Wang unsigned int nr = min(512U, batch->nr);
97b191c9bcSJianxing Wang
98b191c9bcSJianxing Wang free_pages_and_swap_cache(pages, nr);
99b191c9bcSJianxing Wang pages += nr;
100b191c9bcSJianxing Wang batch->nr -= nr;
101b191c9bcSJianxing Wang
102b191c9bcSJianxing Wang cond_resched();
103b191c9bcSJianxing Wang } while (batch->nr);
104196d9d8bSPeter Zijlstra }
105196d9d8bSPeter Zijlstra tlb->active = &tlb->local;
106196d9d8bSPeter Zijlstra }
107196d9d8bSPeter Zijlstra
tlb_batch_list_free(struct mmu_gather * tlb)108952a31c9SMartin Schwidefsky static void tlb_batch_list_free(struct mmu_gather *tlb)
109196d9d8bSPeter Zijlstra {
110196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch, *next;
111196d9d8bSPeter Zijlstra
112196d9d8bSPeter Zijlstra for (batch = tlb->local.next; batch; batch = next) {
113196d9d8bSPeter Zijlstra next = batch->next;
114196d9d8bSPeter Zijlstra free_pages((unsigned long)batch, 0);
115196d9d8bSPeter Zijlstra }
116196d9d8bSPeter Zijlstra tlb->local.next = NULL;
117196d9d8bSPeter Zijlstra }
118196d9d8bSPeter Zijlstra
__tlb_remove_page_size(struct mmu_gather * tlb,struct encoded_page * page,int page_size)1197cc8f9c7SLinus Torvalds bool __tlb_remove_page_size(struct mmu_gather *tlb, struct encoded_page *page, int page_size)
120196d9d8bSPeter Zijlstra {
121196d9d8bSPeter Zijlstra struct mmu_gather_batch *batch;
122196d9d8bSPeter Zijlstra
123196d9d8bSPeter Zijlstra VM_BUG_ON(!tlb->end);
124ed6a7935SPeter Zijlstra
1253af4bd03SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
126196d9d8bSPeter Zijlstra VM_WARN_ON(tlb->page_size != page_size);
127ed6a7935SPeter Zijlstra #endif
128196d9d8bSPeter Zijlstra
129196d9d8bSPeter Zijlstra batch = tlb->active;
130196d9d8bSPeter Zijlstra /*
131196d9d8bSPeter Zijlstra * Add the page and check if we are full. If so
132196d9d8bSPeter Zijlstra * force a flush.
133196d9d8bSPeter Zijlstra */
1347cc8f9c7SLinus Torvalds batch->encoded_pages[batch->nr++] = page;
135196d9d8bSPeter Zijlstra if (batch->nr == batch->max) {
136196d9d8bSPeter Zijlstra if (!tlb_next_batch(tlb))
137196d9d8bSPeter Zijlstra return true;
138196d9d8bSPeter Zijlstra batch = tlb->active;
139196d9d8bSPeter Zijlstra }
1407cc8f9c7SLinus Torvalds VM_BUG_ON_PAGE(batch->nr > batch->max, encoded_page_ptr(page));
141196d9d8bSPeter Zijlstra
142196d9d8bSPeter Zijlstra return false;
143196d9d8bSPeter Zijlstra }
144196d9d8bSPeter Zijlstra
145580a586cSPeter Zijlstra #endif /* MMU_GATHER_NO_GATHER */
146952a31c9SMartin Schwidefsky
1470d6e24d4SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_TABLE_FREE
1480d6e24d4SPeter Zijlstra
__tlb_remove_table_free(struct mmu_table_batch * batch)1490d6e24d4SPeter Zijlstra static void __tlb_remove_table_free(struct mmu_table_batch *batch)
1500d6e24d4SPeter Zijlstra {
1510d6e24d4SPeter Zijlstra int i;
1520d6e24d4SPeter Zijlstra
1530d6e24d4SPeter Zijlstra for (i = 0; i < batch->nr; i++)
1540d6e24d4SPeter Zijlstra __tlb_remove_table(batch->tables[i]);
1550d6e24d4SPeter Zijlstra
1560d6e24d4SPeter Zijlstra free_page((unsigned long)batch);
1570d6e24d4SPeter Zijlstra }
1580d6e24d4SPeter Zijlstra
159ff2e6d72SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
160196d9d8bSPeter Zijlstra
161196d9d8bSPeter Zijlstra /*
1620d6e24d4SPeter Zijlstra * Semi RCU freeing of the page directories.
1630d6e24d4SPeter Zijlstra *
1640d6e24d4SPeter Zijlstra * This is needed by some architectures to implement software pagetable walkers.
1650d6e24d4SPeter Zijlstra *
1660d6e24d4SPeter Zijlstra * gup_fast() and other software pagetable walkers do a lockless page-table
1670d6e24d4SPeter Zijlstra * walk and therefore needs some synchronization with the freeing of the page
1680d6e24d4SPeter Zijlstra * directories. The chosen means to accomplish that is by disabling IRQs over
1690d6e24d4SPeter Zijlstra * the walk.
1700d6e24d4SPeter Zijlstra *
1710d6e24d4SPeter Zijlstra * Architectures that use IPIs to flush TLBs will then automagically DTRT,
1720d6e24d4SPeter Zijlstra * since we unlink the page, flush TLBs, free the page. Since the disabling of
1730d6e24d4SPeter Zijlstra * IRQs delays the completion of the TLB flush we can never observe an already
1740d6e24d4SPeter Zijlstra * freed page.
1750d6e24d4SPeter Zijlstra *
1760d6e24d4SPeter Zijlstra * Architectures that do not have this (PPC) need to delay the freeing by some
1770d6e24d4SPeter Zijlstra * other means, this is that means.
1780d6e24d4SPeter Zijlstra *
1790d6e24d4SPeter Zijlstra * What we do is batch the freed directory pages (tables) and RCU free them.
1800d6e24d4SPeter Zijlstra * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
1810d6e24d4SPeter Zijlstra * holds off grace periods.
1820d6e24d4SPeter Zijlstra *
1830d6e24d4SPeter Zijlstra * However, in order to batch these pages we need to allocate storage, this
1840d6e24d4SPeter Zijlstra * allocation is deep inside the MM code and can thus easily fail on memory
1850d6e24d4SPeter Zijlstra * pressure. To guarantee progress we fall back to single table freeing, see
1860d6e24d4SPeter Zijlstra * the implementation of tlb_remove_table_one().
1870d6e24d4SPeter Zijlstra *
188196d9d8bSPeter Zijlstra */
189196d9d8bSPeter Zijlstra
tlb_remove_table_smp_sync(void * arg)1900d6e24d4SPeter Zijlstra static void tlb_remove_table_smp_sync(void *arg)
1910d6e24d4SPeter Zijlstra {
1920d6e24d4SPeter Zijlstra /* Simply deliver the interrupt */
1930d6e24d4SPeter Zijlstra }
1940d6e24d4SPeter Zijlstra
tlb_remove_table_sync_one(void)1952ba99c5eSJann Horn void tlb_remove_table_sync_one(void)
1960d6e24d4SPeter Zijlstra {
1970d6e24d4SPeter Zijlstra /*
1980d6e24d4SPeter Zijlstra * This isn't an RCU grace period and hence the page-tables cannot be
1990d6e24d4SPeter Zijlstra * assumed to be actually RCU-freed.
2000d6e24d4SPeter Zijlstra *
2010d6e24d4SPeter Zijlstra * It is however sufficient for software page-table walkers that rely on
2020d6e24d4SPeter Zijlstra * IRQ disabling.
2030d6e24d4SPeter Zijlstra */
2040d6e24d4SPeter Zijlstra smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
2050d6e24d4SPeter Zijlstra }
2060d6e24d4SPeter Zijlstra
tlb_remove_table_rcu(struct rcu_head * head)2070d6e24d4SPeter Zijlstra static void tlb_remove_table_rcu(struct rcu_head *head)
2080d6e24d4SPeter Zijlstra {
2090d6e24d4SPeter Zijlstra __tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
2100d6e24d4SPeter Zijlstra }
2110d6e24d4SPeter Zijlstra
tlb_remove_table_free(struct mmu_table_batch * batch)2120d6e24d4SPeter Zijlstra static void tlb_remove_table_free(struct mmu_table_batch *batch)
2130d6e24d4SPeter Zijlstra {
2140d6e24d4SPeter Zijlstra call_rcu(&batch->rcu, tlb_remove_table_rcu);
2150d6e24d4SPeter Zijlstra }
2160d6e24d4SPeter Zijlstra
2170d6e24d4SPeter Zijlstra #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
2180d6e24d4SPeter Zijlstra
tlb_remove_table_free(struct mmu_table_batch * batch)2190d6e24d4SPeter Zijlstra static void tlb_remove_table_free(struct mmu_table_batch *batch)
2200d6e24d4SPeter Zijlstra {
2210d6e24d4SPeter Zijlstra __tlb_remove_table_free(batch);
2220d6e24d4SPeter Zijlstra }
2230d6e24d4SPeter Zijlstra
2240d6e24d4SPeter Zijlstra #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
2250d6e24d4SPeter Zijlstra
226196d9d8bSPeter Zijlstra /*
227196d9d8bSPeter Zijlstra * If we want tlb_remove_table() to imply TLB invalidates.
228196d9d8bSPeter Zijlstra */
tlb_table_invalidate(struct mmu_gather * tlb)229196d9d8bSPeter Zijlstra static inline void tlb_table_invalidate(struct mmu_gather *tlb)
230196d9d8bSPeter Zijlstra {
2310ed13259SPeter Zijlstra if (tlb_needs_table_invalidate()) {
232196d9d8bSPeter Zijlstra /*
2330ed13259SPeter Zijlstra * Invalidate page-table caches used by hardware walkers. Then
2340ed13259SPeter Zijlstra * we still need to RCU-sched wait while freeing the pages
2350ed13259SPeter Zijlstra * because software walkers can still be in-flight.
236196d9d8bSPeter Zijlstra */
237196d9d8bSPeter Zijlstra tlb_flush_mmu_tlbonly(tlb);
2380ed13259SPeter Zijlstra }
239196d9d8bSPeter Zijlstra }
240196d9d8bSPeter Zijlstra
tlb_remove_table_one(void * table)241196d9d8bSPeter Zijlstra static void tlb_remove_table_one(void *table)
242196d9d8bSPeter Zijlstra {
2430d6e24d4SPeter Zijlstra tlb_remove_table_sync_one();
244196d9d8bSPeter Zijlstra __tlb_remove_table(table);
245196d9d8bSPeter Zijlstra }
246196d9d8bSPeter Zijlstra
tlb_table_flush(struct mmu_gather * tlb)2470a8caf21SPeter Zijlstra static void tlb_table_flush(struct mmu_gather *tlb)
248196d9d8bSPeter Zijlstra {
249196d9d8bSPeter Zijlstra struct mmu_table_batch **batch = &tlb->batch;
250196d9d8bSPeter Zijlstra
251196d9d8bSPeter Zijlstra if (*batch) {
252196d9d8bSPeter Zijlstra tlb_table_invalidate(tlb);
2530d6e24d4SPeter Zijlstra tlb_remove_table_free(*batch);
254196d9d8bSPeter Zijlstra *batch = NULL;
255196d9d8bSPeter Zijlstra }
256196d9d8bSPeter Zijlstra }
257196d9d8bSPeter Zijlstra
tlb_remove_table(struct mmu_gather * tlb,void * table)258196d9d8bSPeter Zijlstra void tlb_remove_table(struct mmu_gather *tlb, void *table)
259196d9d8bSPeter Zijlstra {
260196d9d8bSPeter Zijlstra struct mmu_table_batch **batch = &tlb->batch;
261196d9d8bSPeter Zijlstra
262196d9d8bSPeter Zijlstra if (*batch == NULL) {
263196d9d8bSPeter Zijlstra *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
264196d9d8bSPeter Zijlstra if (*batch == NULL) {
265196d9d8bSPeter Zijlstra tlb_table_invalidate(tlb);
266196d9d8bSPeter Zijlstra tlb_remove_table_one(table);
267196d9d8bSPeter Zijlstra return;
268196d9d8bSPeter Zijlstra }
269196d9d8bSPeter Zijlstra (*batch)->nr = 0;
270196d9d8bSPeter Zijlstra }
271196d9d8bSPeter Zijlstra
272196d9d8bSPeter Zijlstra (*batch)->tables[(*batch)->nr++] = table;
273196d9d8bSPeter Zijlstra if ((*batch)->nr == MAX_TABLE_BATCH)
274196d9d8bSPeter Zijlstra tlb_table_flush(tlb);
275196d9d8bSPeter Zijlstra }
276196d9d8bSPeter Zijlstra
tlb_table_init(struct mmu_gather * tlb)2770d6e24d4SPeter Zijlstra static inline void tlb_table_init(struct mmu_gather *tlb)
2780d6e24d4SPeter Zijlstra {
2790d6e24d4SPeter Zijlstra tlb->batch = NULL;
2800d6e24d4SPeter Zijlstra }
2810d6e24d4SPeter Zijlstra
2820d6e24d4SPeter Zijlstra #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
2830d6e24d4SPeter Zijlstra
tlb_table_flush(struct mmu_gather * tlb)2840d6e24d4SPeter Zijlstra static inline void tlb_table_flush(struct mmu_gather *tlb) { }
tlb_table_init(struct mmu_gather * tlb)2850d6e24d4SPeter Zijlstra static inline void tlb_table_init(struct mmu_gather *tlb) { }
2860d6e24d4SPeter Zijlstra
2870d6e24d4SPeter Zijlstra #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
288196d9d8bSPeter Zijlstra
tlb_flush_mmu_free(struct mmu_gather * tlb)2890a8caf21SPeter Zijlstra static void tlb_flush_mmu_free(struct mmu_gather *tlb)
2900a8caf21SPeter Zijlstra {
2910a8caf21SPeter Zijlstra tlb_table_flush(tlb);
292580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER
2930a8caf21SPeter Zijlstra tlb_batch_pages_flush(tlb);
2940a8caf21SPeter Zijlstra #endif
2950a8caf21SPeter Zijlstra }
2960a8caf21SPeter Zijlstra
tlb_flush_mmu(struct mmu_gather * tlb)2970a8caf21SPeter Zijlstra void tlb_flush_mmu(struct mmu_gather *tlb)
2980a8caf21SPeter Zijlstra {
2990a8caf21SPeter Zijlstra tlb_flush_mmu_tlbonly(tlb);
3000a8caf21SPeter Zijlstra tlb_flush_mmu_free(tlb);
3010a8caf21SPeter Zijlstra }
3020a8caf21SPeter Zijlstra
__tlb_gather_mmu(struct mmu_gather * tlb,struct mm_struct * mm,bool fullmm)303d8b45053SWill Deacon static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
304a72afd87SWill Deacon bool fullmm)
305196d9d8bSPeter Zijlstra {
3061808d65bSPeter Zijlstra tlb->mm = mm;
307a72afd87SWill Deacon tlb->fullmm = fullmm;
3081808d65bSPeter Zijlstra
309580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER
3101808d65bSPeter Zijlstra tlb->need_flush_all = 0;
3111808d65bSPeter Zijlstra tlb->local.next = NULL;
3121808d65bSPeter Zijlstra tlb->local.nr = 0;
3131808d65bSPeter Zijlstra tlb->local.max = ARRAY_SIZE(tlb->__pages);
3141808d65bSPeter Zijlstra tlb->active = &tlb->local;
3151808d65bSPeter Zijlstra tlb->batch_count = 0;
3161808d65bSPeter Zijlstra #endif
3175df397deSLinus Torvalds tlb->delayed_rmap = 0;
3181808d65bSPeter Zijlstra
3190d6e24d4SPeter Zijlstra tlb_table_init(tlb);
3203af4bd03SPeter Zijlstra #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
3211808d65bSPeter Zijlstra tlb->page_size = 0;
3221808d65bSPeter Zijlstra #endif
3231808d65bSPeter Zijlstra
3241808d65bSPeter Zijlstra __tlb_reset_range(tlb);
325196d9d8bSPeter Zijlstra inc_tlb_flush_pending(tlb->mm);
326196d9d8bSPeter Zijlstra }
327196d9d8bSPeter Zijlstra
328845be1cdSRandy Dunlap /**
329845be1cdSRandy Dunlap * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
330845be1cdSRandy Dunlap * @tlb: the mmu_gather structure to initialize
331845be1cdSRandy Dunlap * @mm: the mm_struct of the target address space
332845be1cdSRandy Dunlap *
333845be1cdSRandy Dunlap * Called to initialize an (on-stack) mmu_gather structure for page-table
334845be1cdSRandy Dunlap * tear-down from @mm.
335845be1cdSRandy Dunlap */
tlb_gather_mmu(struct mmu_gather * tlb,struct mm_struct * mm)336a72afd87SWill Deacon void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm)
337d8b45053SWill Deacon {
338a72afd87SWill Deacon __tlb_gather_mmu(tlb, mm, false);
339d8b45053SWill Deacon }
340d8b45053SWill Deacon
341845be1cdSRandy Dunlap /**
342845be1cdSRandy Dunlap * tlb_gather_mmu_fullmm - initialize an mmu_gather structure for page-table tear-down
343845be1cdSRandy Dunlap * @tlb: the mmu_gather structure to initialize
344845be1cdSRandy Dunlap * @mm: the mm_struct of the target address space
345845be1cdSRandy Dunlap *
346845be1cdSRandy Dunlap * In this case, @mm is without users and we're going to destroy the
347845be1cdSRandy Dunlap * full address space (exit/execve).
348845be1cdSRandy Dunlap *
349845be1cdSRandy Dunlap * Called to initialize an (on-stack) mmu_gather structure for page-table
350845be1cdSRandy Dunlap * tear-down from @mm.
351845be1cdSRandy Dunlap */
tlb_gather_mmu_fullmm(struct mmu_gather * tlb,struct mm_struct * mm)352d8b45053SWill Deacon void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm)
353d8b45053SWill Deacon {
354a72afd87SWill Deacon __tlb_gather_mmu(tlb, mm, true);
355d8b45053SWill Deacon }
356d8b45053SWill Deacon
3571808d65bSPeter Zijlstra /**
3581808d65bSPeter Zijlstra * tlb_finish_mmu - finish an mmu_gather structure
3591808d65bSPeter Zijlstra * @tlb: the mmu_gather structure to finish
3601808d65bSPeter Zijlstra *
3611808d65bSPeter Zijlstra * Called at the end of the shootdown operation to free up any resources that
3621808d65bSPeter Zijlstra * were required.
3631808d65bSPeter Zijlstra */
tlb_finish_mmu(struct mmu_gather * tlb)364ae8eba8bSWill Deacon void tlb_finish_mmu(struct mmu_gather *tlb)
365196d9d8bSPeter Zijlstra {
366196d9d8bSPeter Zijlstra /*
367196d9d8bSPeter Zijlstra * If there are parallel threads are doing PTE changes on same range
368c1e8d7c6SMichel Lespinasse * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
3697a30df49SYang Shi * flush by batching, one thread may end up seeing inconsistent PTEs
3707a30df49SYang Shi * and result in having stale TLB entries. So flush TLB forcefully
3717a30df49SYang Shi * if we detect parallel PTE batching threads.
3727a30df49SYang Shi *
3737a30df49SYang Shi * However, some syscalls, e.g. munmap(), may free page tables, this
3747a30df49SYang Shi * needs force flush everything in the given range. Otherwise this
3757a30df49SYang Shi * may result in having stale TLB entries for some architectures,
3767a30df49SYang Shi * e.g. aarch64, that could specify flush what level TLB.
377196d9d8bSPeter Zijlstra */
3781808d65bSPeter Zijlstra if (mm_tlb_flush_nested(tlb->mm)) {
3797a30df49SYang Shi /*
3807a30df49SYang Shi * The aarch64 yields better performance with fullmm by
3817a30df49SYang Shi * avoiding multiple CPUs spamming TLBI messages at the
3827a30df49SYang Shi * same time.
3837a30df49SYang Shi *
3847a30df49SYang Shi * On x86 non-fullmm doesn't yield significant difference
3857a30df49SYang Shi * against fullmm.
3867a30df49SYang Shi */
3877a30df49SYang Shi tlb->fullmm = 1;
3881808d65bSPeter Zijlstra __tlb_reset_range(tlb);
3897a30df49SYang Shi tlb->freed_tables = 1;
3901808d65bSPeter Zijlstra }
391196d9d8bSPeter Zijlstra
3921808d65bSPeter Zijlstra tlb_flush_mmu(tlb);
3931808d65bSPeter Zijlstra
394580a586cSPeter Zijlstra #ifndef CONFIG_MMU_GATHER_NO_GATHER
3951808d65bSPeter Zijlstra tlb_batch_list_free(tlb);
3961808d65bSPeter Zijlstra #endif
397196d9d8bSPeter Zijlstra dec_tlb_flush_pending(tlb->mm);
398196d9d8bSPeter Zijlstra }
399