1caab277bSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */
258d0ba57SCatalin Marinas /*
358d0ba57SCatalin Marinas * Based on arch/arm/include/asm/tlb.h
458d0ba57SCatalin Marinas *
558d0ba57SCatalin Marinas * Copyright (C) 2002 Russell King
658d0ba57SCatalin Marinas * Copyright (C) 2012 ARM Ltd.
758d0ba57SCatalin Marinas */
858d0ba57SCatalin Marinas #ifndef __ASM_TLB_H
958d0ba57SCatalin Marinas #define __ASM_TLB_H
1058d0ba57SCatalin Marinas
115e5f6dc1SSteve Capper #include <linux/pagemap.h>
125e5f6dc1SSteve Capper #include <linux/swap.h>
135e5f6dc1SSteve Capper
__tlb_remove_table(void * _table)145e5f6dc1SSteve Capper static inline void __tlb_remove_table(void *_table)
155e5f6dc1SSteve Capper {
165e5f6dc1SSteve Capper free_page_and_swap_cache((struct page *)_table);
175e5f6dc1SSteve Capper }
185e5f6dc1SSteve Capper
195f307be1SPeter Zijlstra #define tlb_flush tlb_flush
20d475fac9SWill Deacon static void tlb_flush(struct mmu_gather *tlb);
21d475fac9SWill Deacon
22fb7332a9SWill Deacon #include <asm-generic/tlb.h>
23fb7332a9SWill Deacon
24c4ab2cbcSZhenyu Ye /*
25c4ab2cbcSZhenyu Ye * get the tlbi levels in arm64. Default value is 0 if more than one
26c4ab2cbcSZhenyu Ye * of cleared_* is set or neither is set.
27c4ab2cbcSZhenyu Ye * Arm64 doesn't support p4ds now.
28c4ab2cbcSZhenyu Ye */
tlb_get_level(struct mmu_gather * tlb)29c4ab2cbcSZhenyu Ye static inline int tlb_get_level(struct mmu_gather *tlb)
30c4ab2cbcSZhenyu Ye {
3152218fcdSZhenyu Ye /* The TTL field is only valid for the leaf entry. */
3252218fcdSZhenyu Ye if (tlb->freed_tables)
3352218fcdSZhenyu Ye return 0;
3452218fcdSZhenyu Ye
35c4ab2cbcSZhenyu Ye if (tlb->cleared_ptes && !(tlb->cleared_pmds ||
36c4ab2cbcSZhenyu Ye tlb->cleared_puds ||
37c4ab2cbcSZhenyu Ye tlb->cleared_p4ds))
38c4ab2cbcSZhenyu Ye return 3;
39c4ab2cbcSZhenyu Ye
40c4ab2cbcSZhenyu Ye if (tlb->cleared_pmds && !(tlb->cleared_ptes ||
41c4ab2cbcSZhenyu Ye tlb->cleared_puds ||
42c4ab2cbcSZhenyu Ye tlb->cleared_p4ds))
43c4ab2cbcSZhenyu Ye return 2;
44c4ab2cbcSZhenyu Ye
45c4ab2cbcSZhenyu Ye if (tlb->cleared_puds && !(tlb->cleared_ptes ||
46c4ab2cbcSZhenyu Ye tlb->cleared_pmds ||
47c4ab2cbcSZhenyu Ye tlb->cleared_p4ds))
48c4ab2cbcSZhenyu Ye return 1;
49c4ab2cbcSZhenyu Ye
50c4ab2cbcSZhenyu Ye return 0;
51c4ab2cbcSZhenyu Ye }
52c4ab2cbcSZhenyu Ye
tlb_flush(struct mmu_gather * tlb)5358d0ba57SCatalin Marinas static inline void tlb_flush(struct mmu_gather *tlb)
5458d0ba57SCatalin Marinas {
558b11ec1bSLinus Torvalds struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
56f270ab88SWill Deacon bool last_level = !tlb->freed_tables;
57f270ab88SWill Deacon unsigned long stride = tlb_get_unmap_size(tlb);
58c4ab2cbcSZhenyu Ye int tlb_level = tlb_get_level(tlb);
595a7862e8SWill Deacon
605a7862e8SWill Deacon /*
61f270ab88SWill Deacon * If we're tearing down the address space then we only care about
62f270ab88SWill Deacon * invalidating the walk-cache, since the ASID allocator won't
63f270ab88SWill Deacon * reallocate our ASID without invalidating the entire TLB.
645a7862e8SWill Deacon */
65f270ab88SWill Deacon if (tlb->fullmm) {
66f270ab88SWill Deacon if (!last_level)
67f270ab88SWill Deacon flush_tlb_mm(tlb->mm);
685a7862e8SWill Deacon return;
69f270ab88SWill Deacon }
705a7862e8SWill Deacon
71c4ab2cbcSZhenyu Ye __flush_tlb_range(&vma, tlb->start, tlb->end, stride,
72c4ab2cbcSZhenyu Ye last_level, tlb_level);
7358d0ba57SCatalin Marinas }
7458d0ba57SCatalin Marinas
__pte_free_tlb(struct mmu_gather * tlb,pgtable_t pte,unsigned long addr)7558d0ba57SCatalin Marinas static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
7658d0ba57SCatalin Marinas unsigned long addr)
7758d0ba57SCatalin Marinas {
78*11b4fa8bSVishal Moola (Oracle) struct ptdesc *ptdesc = page_ptdesc(pte);
79*11b4fa8bSVishal Moola (Oracle)
80*11b4fa8bSVishal Moola (Oracle) pagetable_pte_dtor(ptdesc);
81*11b4fa8bSVishal Moola (Oracle) tlb_remove_ptdesc(tlb, ptdesc);
8258d0ba57SCatalin Marinas }
8358d0ba57SCatalin Marinas
849f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 2
__pmd_free_tlb(struct mmu_gather * tlb,pmd_t * pmdp,unsigned long addr)8558d0ba57SCatalin Marinas static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
8658d0ba57SCatalin Marinas unsigned long addr)
8758d0ba57SCatalin Marinas {
88*11b4fa8bSVishal Moola (Oracle) struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);
8954c8d911SYu Zhao
90*11b4fa8bSVishal Moola (Oracle) pagetable_pmd_dtor(ptdesc);
91*11b4fa8bSVishal Moola (Oracle) tlb_remove_ptdesc(tlb, ptdesc);
9258d0ba57SCatalin Marinas }
9358d0ba57SCatalin Marinas #endif
9458d0ba57SCatalin Marinas
959f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 3
__pud_free_tlb(struct mmu_gather * tlb,pud_t * pudp,unsigned long addr)96c79b954bSJungseok Lee static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
97c79b954bSJungseok Lee unsigned long addr)
98c79b954bSJungseok Lee {
99*11b4fa8bSVishal Moola (Oracle) tlb_remove_ptdesc(tlb, virt_to_ptdesc(pudp));
100c79b954bSJungseok Lee }
101c79b954bSJungseok Lee #endif
102c79b954bSJungseok Lee
10358d0ba57SCatalin Marinas #endif
104