1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H 3 #define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H 4 5 #define MMU_NO_CONTEXT ~0UL 6 7 #include <linux/mm_types.h> 8 #include <asm/book3s/64/tlbflush-hash.h> 9 #include <asm/book3s/64/tlbflush-radix.h> 10 11 /* TLB flush actions. Used as argument to tlbiel_all() */ 12 enum { 13 TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ 14 TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */ 15 }; 16 17 static inline void tlbiel_all(void) 18 { 19 /* 20 * This is used for host machine check and bootup. 21 * 22 * This uses early_radix_enabled and implementations use 23 * early_cpu_has_feature etc because that works early in boot 24 * and this is the machine check path which is not performance 25 * critical. 26 */ 27 if (early_radix_enabled()) 28 radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); 29 else 30 hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); 31 } 32 33 static inline void tlbiel_all_lpid(bool radix) 34 { 35 /* 36 * This is used for guest machine check. 37 */ 38 if (radix) 39 radix__tlbiel_all(TLB_INVAL_SCOPE_LPID); 40 else 41 hash__tlbiel_all(TLB_INVAL_SCOPE_LPID); 42 } 43 44 45 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE 46 static inline void flush_pmd_tlb_range(struct vm_area_struct *vma, 47 unsigned long start, unsigned long end) 48 { 49 if (radix_enabled()) 50 return radix__flush_pmd_tlb_range(vma, start, end); 51 return hash__flush_tlb_range(vma, start, end); 52 } 53 54 #define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 55 static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma, 56 unsigned long start, 57 unsigned long end) 58 { 59 if (radix_enabled()) 60 return radix__flush_hugetlb_tlb_range(vma, start, end); 61 return hash__flush_tlb_range(vma, start, end); 62 } 63 64 static inline void flush_tlb_range(struct vm_area_struct *vma, 65 unsigned long start, unsigned long end) 66 { 67 if (radix_enabled()) 68 return radix__flush_tlb_range(vma, start, end); 69 return hash__flush_tlb_range(vma, start, end); 70 } 71 72 static inline void flush_tlb_kernel_range(unsigned long start, 73 unsigned long end) 74 { 75 if (radix_enabled()) 76 return radix__flush_tlb_kernel_range(start, end); 77 return hash__flush_tlb_kernel_range(start, end); 78 } 79 80 static inline void local_flush_tlb_mm(struct mm_struct *mm) 81 { 82 if (radix_enabled()) 83 return radix__local_flush_tlb_mm(mm); 84 return hash__local_flush_tlb_mm(mm); 85 } 86 87 static inline void local_flush_tlb_page(struct vm_area_struct *vma, 88 unsigned long vmaddr) 89 { 90 if (radix_enabled()) 91 return radix__local_flush_tlb_page(vma, vmaddr); 92 return hash__local_flush_tlb_page(vma, vmaddr); 93 } 94 95 static inline void local_flush_all_mm(struct mm_struct *mm) 96 { 97 if (radix_enabled()) 98 return radix__local_flush_all_mm(mm); 99 return hash__local_flush_all_mm(mm); 100 } 101 102 static inline void tlb_flush(struct mmu_gather *tlb) 103 { 104 if (radix_enabled()) 105 return radix__tlb_flush(tlb); 106 return hash__tlb_flush(tlb); 107 } 108 109 #ifdef CONFIG_SMP 110 static inline void flush_tlb_mm(struct mm_struct *mm) 111 { 112 if (radix_enabled()) 113 return radix__flush_tlb_mm(mm); 114 return hash__flush_tlb_mm(mm); 115 } 116 117 static inline void flush_tlb_page(struct vm_area_struct *vma, 118 unsigned long vmaddr) 119 { 120 if (radix_enabled()) 121 return radix__flush_tlb_page(vma, vmaddr); 122 return hash__flush_tlb_page(vma, vmaddr); 123 } 124 125 static inline void flush_all_mm(struct mm_struct *mm) 126 { 127 if (radix_enabled()) 128 return radix__flush_all_mm(mm); 129 return hash__flush_all_mm(mm); 130 } 131 #else 132 #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) 133 #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) 134 #define flush_all_mm(mm) local_flush_all_mm(mm) 135 #endif /* CONFIG_SMP */ 136 137 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault 138 static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, 139 unsigned long address) 140 { 141 /* 142 * Book3S 64 does not require spurious fault flushes because the PTE 143 * must be re-fetched in case of an access permission problem. So the 144 * only reason for a spurious fault should be concurrent modification 145 * to the PTE, in which case the PTE will eventually be re-fetched by 146 * the MMU when it attempts the access again. 147 * 148 * See: Power ISA Version 3.1B, 6.10.1.2 Modifying a Translation Table 149 * Entry, Setting a Reference or Change Bit or Upgrading Access 150 * Authority (PTE Subject to Atomic Hardware Updates): 151 * 152 * "If the only change being made to a valid PTE that is subject to 153 * atomic hardware updates is to set the Reference or Change bit to 154 * 1 or to upgrade access authority, a simpler sequence suffices 155 * because the translation hardware will refetch the PTE if an 156 * access is attempted for which the only problems were reference 157 * and/or change bits needing to be set or insufficient access 158 * authority." 159 * 160 * The nest MMU in POWER9 does not perform this PTE re-fetch, but 161 * it avoids the spurious fault problem by flushing the TLB before 162 * upgrading PTE permissions, see radix__ptep_set_access_flags. 163 */ 164 } 165 166 extern bool tlbie_capable; 167 extern bool tlbie_enabled; 168 169 static inline bool cputlb_use_tlbie(void) 170 { 171 return tlbie_enabled; 172 } 173 174 #endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */ 175