1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
3 #define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
4 
5 #define MMU_NO_CONTEXT	~0UL
6 
7 #include <linux/mm_types.h>
8 #include <asm/book3s/64/tlbflush-hash.h>
9 #include <asm/book3s/64/tlbflush-radix.h>
10 
11 /* TLB flush actions. Used as argument to tlbiel_all() */
12 enum {
13 	TLB_INVAL_SCOPE_GLOBAL = 0,	/* invalidate all TLBs */
14 	TLB_INVAL_SCOPE_LPID = 1,	/* invalidate TLBs for current LPID */
15 };
16 
17 static inline void tlbiel_all(void)
18 {
19 	/*
20 	 * This is used for host machine check and bootup.
21 	 *
22 	 * This uses early_radix_enabled and implementations use
23 	 * early_cpu_has_feature etc because that works early in boot
24 	 * and this is the machine check path which is not performance
25 	 * critical.
26 	 */
27 	if (early_radix_enabled())
28 		radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
29 	else
30 		hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
31 }
32 
33 static inline void tlbiel_all_lpid(bool radix)
34 {
35 	/*
36 	 * This is used for guest machine check.
37 	 */
38 	if (radix)
39 		radix__tlbiel_all(TLB_INVAL_SCOPE_LPID);
40 	else
41 		hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
42 }
43 
44 
45 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
46 static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
47 				       unsigned long start, unsigned long end)
48 {
49 	if (radix_enabled())
50 		return radix__flush_pmd_tlb_range(vma, start, end);
51 	return hash__flush_tlb_range(vma, start, end);
52 }
53 
54 #define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
55 static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
56 					   unsigned long start,
57 					   unsigned long end)
58 {
59 	if (radix_enabled())
60 		return radix__flush_hugetlb_tlb_range(vma, start, end);
61 	return hash__flush_tlb_range(vma, start, end);
62 }
63 
64 static inline void flush_tlb_range(struct vm_area_struct *vma,
65 				   unsigned long start, unsigned long end)
66 {
67 	if (radix_enabled())
68 		return radix__flush_tlb_range(vma, start, end);
69 	return hash__flush_tlb_range(vma, start, end);
70 }
71 
72 static inline void flush_tlb_kernel_range(unsigned long start,
73 					  unsigned long end)
74 {
75 	if (radix_enabled())
76 		return radix__flush_tlb_kernel_range(start, end);
77 	return hash__flush_tlb_kernel_range(start, end);
78 }
79 
80 static inline void local_flush_tlb_mm(struct mm_struct *mm)
81 {
82 	if (radix_enabled())
83 		return radix__local_flush_tlb_mm(mm);
84 	return hash__local_flush_tlb_mm(mm);
85 }
86 
87 static inline void local_flush_tlb_page(struct vm_area_struct *vma,
88 					unsigned long vmaddr)
89 {
90 	if (radix_enabled())
91 		return radix__local_flush_tlb_page(vma, vmaddr);
92 	return hash__local_flush_tlb_page(vma, vmaddr);
93 }
94 
95 static inline void local_flush_all_mm(struct mm_struct *mm)
96 {
97 	if (radix_enabled())
98 		return radix__local_flush_all_mm(mm);
99 	return hash__local_flush_all_mm(mm);
100 }
101 
102 static inline void tlb_flush(struct mmu_gather *tlb)
103 {
104 	if (radix_enabled())
105 		return radix__tlb_flush(tlb);
106 	return hash__tlb_flush(tlb);
107 }
108 
109 #ifdef CONFIG_SMP
110 static inline void flush_tlb_mm(struct mm_struct *mm)
111 {
112 	if (radix_enabled())
113 		return radix__flush_tlb_mm(mm);
114 	return hash__flush_tlb_mm(mm);
115 }
116 
117 static inline void flush_tlb_page(struct vm_area_struct *vma,
118 				  unsigned long vmaddr)
119 {
120 	if (radix_enabled())
121 		return radix__flush_tlb_page(vma, vmaddr);
122 	return hash__flush_tlb_page(vma, vmaddr);
123 }
124 
125 static inline void flush_all_mm(struct mm_struct *mm)
126 {
127 	if (radix_enabled())
128 		return radix__flush_all_mm(mm);
129 	return hash__flush_all_mm(mm);
130 }
131 #else
132 #define flush_tlb_mm(mm)		local_flush_tlb_mm(mm)
133 #define flush_tlb_page(vma, addr)	local_flush_tlb_page(vma, addr)
134 #define flush_all_mm(mm)		local_flush_all_mm(mm)
135 #endif /* CONFIG_SMP */
136 
137 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
138 static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
139 						unsigned long address)
140 {
141 	/*
142 	 * Book3S 64 does not require spurious fault flushes because the PTE
143 	 * must be re-fetched in case of an access permission problem. So the
144 	 * only reason for a spurious fault should be concurrent modification
145 	 * to the PTE, in which case the PTE will eventually be re-fetched by
146 	 * the MMU when it attempts the access again.
147 	 *
148 	 * See: Power ISA Version 3.1B, 6.10.1.2 Modifying a Translation Table
149 	 * Entry, Setting a Reference or Change Bit or Upgrading Access
150 	 * Authority (PTE Subject to Atomic Hardware Updates):
151 	 *
152 	 * "If the only change being made to a valid PTE that is subject to
153 	 *  atomic hardware updates is to set the Reference or Change bit to
154 	 *  1 or to upgrade access authority, a simpler sequence suffices
155 	 *  because the translation hardware will refetch the PTE if an
156 	 *  access is attempted for which the only problems were reference
157 	 *  and/or change bits needing to be set or insufficient access
158 	 *  authority."
159 	 *
160 	 * The nest MMU in POWER9 does not perform this PTE re-fetch, but
161 	 * it avoids the spurious fault problem by flushing the TLB before
162 	 * upgrading PTE permissions, see radix__ptep_set_access_flags.
163 	 */
164 }
165 
166 static inline bool __pte_flags_need_flush(unsigned long oldval,
167 					  unsigned long newval)
168 {
169 	unsigned long delta = oldval ^ newval;
170 
171 	/*
172 	 * The return value of this function doesn't matter for hash,
173 	 * ptep_modify_prot_start() does a pte_update() which does or schedules
174 	 * any necessary hash table update and flush.
175 	 */
176 	if (!radix_enabled())
177 		return true;
178 
179 	/*
180 	 * We do not expect kernel mappings or non-PTEs or not-present PTEs.
181 	 */
182 	VM_WARN_ON_ONCE(oldval & _PAGE_PRIVILEGED);
183 	VM_WARN_ON_ONCE(newval & _PAGE_PRIVILEGED);
184 	VM_WARN_ON_ONCE(!(oldval & _PAGE_PTE));
185 	VM_WARN_ON_ONCE(!(newval & _PAGE_PTE));
186 	VM_WARN_ON_ONCE(!(oldval & _PAGE_PRESENT));
187 	VM_WARN_ON_ONCE(!(newval & _PAGE_PRESENT));
188 
189 	/*
190 	*  Must flush on any change except READ, WRITE, EXEC, DIRTY, ACCESSED.
191 	*
192 	 * In theory, some changed software bits could be tolerated, in
193 	 * practice those should rarely if ever matter.
194 	 */
195 
196 	if (delta & ~(_PAGE_RWX | _PAGE_DIRTY | _PAGE_ACCESSED))
197 		return true;
198 
199 	/*
200 	 * If any of the above was present in old but cleared in new, flush.
201 	 * With the exception of _PAGE_ACCESSED, don't worry about flushing
202 	 * if that was cleared (see the comment in ptep_clear_flush_young()).
203 	 */
204 	if ((delta & ~_PAGE_ACCESSED) & oldval)
205 		return true;
206 
207 	return false;
208 }
209 
210 static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)
211 {
212 	return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte));
213 }
214 #define pte_needs_flush pte_needs_flush
215 
216 static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
217 {
218 	return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd));
219 }
220 #define huge_pmd_needs_flush huge_pmd_needs_flush
221 
222 extern bool tlbie_capable;
223 extern bool tlbie_enabled;
224 
225 static inline bool cputlb_use_tlbie(void)
226 {
227 	return tlbie_enabled;
228 }
229 
230 #endif /*  _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */
231