1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_POWERPC_BOOK3S_64_KUP_H
3 #define _ASM_POWERPC_BOOK3S_64_KUP_H
4 
5 #include <linux/const.h>
6 #include <asm/reg.h>
7 
8 #define AMR_KUAP_BLOCK_READ	UL(0x5455555555555555)
9 #define AMR_KUAP_BLOCK_WRITE	UL(0xa8aaaaaaaaaaaaaa)
10 #define AMR_KUEP_BLOCKED	UL(0x5455555555555555)
11 #define AMR_KUAP_BLOCKED	(AMR_KUAP_BLOCK_READ | AMR_KUAP_BLOCK_WRITE)
12 
13 #ifdef __ASSEMBLY__
14 
15 .macro kuap_user_restore gpr1, gpr2
16 #if defined(CONFIG_PPC_PKEY)
17 	BEGIN_MMU_FTR_SECTION_NESTED(67)
18 	b	100f  // skip_restore_amr
19 	END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67)
20 	/*
21 	 * AMR and IAMR are going to be different when
22 	 * returning to userspace.
23 	 */
24 	ld	\gpr1, STACK_REGS_AMR(r1)
25 
26 	/*
27 	 * If kuap feature is not enabled, do the mtspr
28 	 * only if AMR value is different.
29 	 */
30 	BEGIN_MMU_FTR_SECTION_NESTED(68)
31 	mfspr	\gpr2, SPRN_AMR
32 	cmpd	\gpr1, \gpr2
33 	beq	99f
34 	END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUAP, 68)
35 
36 	isync
37 	mtspr	SPRN_AMR, \gpr1
38 99:
39 	/*
40 	 * Restore IAMR only when returning to userspace
41 	 */
42 	ld	\gpr1, STACK_REGS_IAMR(r1)
43 
44 	/*
45 	 * If kuep feature is not enabled, do the mtspr
46 	 * only if IAMR value is different.
47 	 */
48 	BEGIN_MMU_FTR_SECTION_NESTED(69)
49 	mfspr	\gpr2, SPRN_IAMR
50 	cmpd	\gpr1, \gpr2
51 	beq	100f
52 	END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69)
53 
54 	isync
55 	mtspr	SPRN_IAMR, \gpr1
56 
57 100: //skip_restore_amr
58 	/* No isync required, see kuap_user_restore() */
59 #endif
60 .endm
61 
62 .macro kuap_kernel_restore gpr1, gpr2
63 #if defined(CONFIG_PPC_PKEY)
64 
65 	BEGIN_MMU_FTR_SECTION_NESTED(67)
66 	/*
67 	 * AMR is going to be mostly the same since we are
68 	 * returning to the kernel. Compare and do a mtspr.
69 	 */
70 	ld	\gpr2, STACK_REGS_AMR(r1)
71 	mfspr	\gpr1, SPRN_AMR
72 	cmpd	\gpr1, \gpr2
73 	beq	100f
74 	isync
75 	mtspr	SPRN_AMR, \gpr2
76 	/*
77 	 * No isync required, see kuap_restore_amr()
78 	 * No need to restore IAMR when returning to kernel space.
79 	 */
80 100:
81 	END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67)
82 #endif
83 .endm
84 
85 #ifdef CONFIG_PPC_KUAP
86 .macro kuap_check_amr gpr1, gpr2
87 #ifdef CONFIG_PPC_KUAP_DEBUG
88 	BEGIN_MMU_FTR_SECTION_NESTED(67)
89 	mfspr	\gpr1, SPRN_AMR
90 	/* Prevent access to userspace using any key values */
91 	LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED)
92 999:	tdne	\gpr1, \gpr2
93 	EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE)
94 	END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67)
95 #endif
96 .endm
97 #endif
98 
99 /*
100  *	if (pkey) {
101  *
102  *		save AMR -> stack;
103  *		if (kuap) {
104  *			if (AMR != BLOCKED)
105  *				KUAP_BLOCKED -> AMR;
106  *		}
107  *		if (from_user) {
108  *			save IAMR -> stack;
109  *			if (kuep) {
110  *				KUEP_BLOCKED ->IAMR
111  *			}
112  *		}
113  *		return;
114  *	}
115  *
116  *	if (kuap) {
117  *		if (from_kernel) {
118  *			save AMR -> stack;
119  *			if (AMR != BLOCKED)
120  *				KUAP_BLOCKED -> AMR;
121  *		}
122  *
123  *	}
124  */
125 .macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr
126 #if defined(CONFIG_PPC_PKEY)
127 
128 	/*
129 	 * if both pkey and kuap is disabled, nothing to do
130 	 */
131 	BEGIN_MMU_FTR_SECTION_NESTED(68)
132 	b	100f  // skip_save_amr
133 	END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_BOOK3S_KUAP, 68)
134 
135 	/*
136 	 * if pkey is disabled and we are entering from userspace
137 	 * don't do anything.
138 	 */
139 	BEGIN_MMU_FTR_SECTION_NESTED(67)
140 	.ifnb \msr_pr_cr
141 	/*
142 	 * Without pkey we are not changing AMR outside the kernel
143 	 * hence skip this completely.
144 	 */
145 	bne	\msr_pr_cr, 100f  // from userspace
146 	.endif
147         END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67)
148 
149 	/*
150 	 * pkey is enabled or pkey is disabled but entering from kernel
151 	 */
152 	mfspr	\gpr1, SPRN_AMR
153 	std	\gpr1, STACK_REGS_AMR(r1)
154 
155 	/*
156 	 * update kernel AMR with AMR_KUAP_BLOCKED only
157 	 * if KUAP feature is enabled
158 	 */
159 	BEGIN_MMU_FTR_SECTION_NESTED(69)
160 	LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED)
161 	cmpd	\use_cr, \gpr1, \gpr2
162 	beq	\use_cr, 102f
163 	/*
164 	 * We don't isync here because we very recently entered via an interrupt
165 	 */
166 	mtspr	SPRN_AMR, \gpr2
167 	isync
168 102:
169 	END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 69)
170 
171 	/*
172 	 * if entering from kernel we don't need save IAMR
173 	 */
174 	.ifnb \msr_pr_cr
175 	beq	\msr_pr_cr, 100f // from kernel space
176 	mfspr	\gpr1, SPRN_IAMR
177 	std	\gpr1, STACK_REGS_IAMR(r1)
178 
179 	/*
180 	 * update kernel IAMR with AMR_KUEP_BLOCKED only
181 	 * if KUEP feature is enabled
182 	 */
183 	BEGIN_MMU_FTR_SECTION_NESTED(70)
184 	LOAD_REG_IMMEDIATE(\gpr2, AMR_KUEP_BLOCKED)
185 	mtspr	SPRN_IAMR, \gpr2
186 	isync
187 	END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUEP, 70)
188 	.endif
189 
190 100: // skip_save_amr
191 #endif
192 .endm
193 
194 #else /* !__ASSEMBLY__ */
195 
196 #include <linux/jump_label.h>
197 
198 DECLARE_STATIC_KEY_FALSE(uaccess_flush_key);
199 
200 #ifdef CONFIG_PPC_PKEY
201 
202 extern u64 __ro_after_init default_uamor;
203 extern u64 __ro_after_init default_amr;
204 extern u64 __ro_after_init default_iamr;
205 
206 #include <asm/mmu.h>
207 #include <asm/ptrace.h>
208 
209 /* usage of kthread_use_mm() should inherit the
210  * AMR value of the operating address space. But, the AMR value is
211  * thread-specific and we inherit the address space and not thread
212  * access restrictions. Because of this ignore AMR value when accessing
213  * userspace via kernel thread.
214  */
215 static inline u64 current_thread_amr(void)
216 {
217 	if (current->thread.regs)
218 		return current->thread.regs->amr;
219 	return default_amr;
220 }
221 
222 static inline u64 current_thread_iamr(void)
223 {
224 	if (current->thread.regs)
225 		return current->thread.regs->iamr;
226 	return default_iamr;
227 }
228 #endif /* CONFIG_PPC_PKEY */
229 
230 #ifdef CONFIG_PPC_KUAP
231 
232 static inline void kuap_user_restore(struct pt_regs *regs)
233 {
234 	bool restore_amr = false, restore_iamr = false;
235 	unsigned long amr, iamr;
236 
237 	if (!mmu_has_feature(MMU_FTR_PKEY))
238 		return;
239 
240 	if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
241 		amr = mfspr(SPRN_AMR);
242 		if (amr != regs->amr)
243 			restore_amr = true;
244 	} else {
245 		restore_amr = true;
246 	}
247 
248 	if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
249 		iamr = mfspr(SPRN_IAMR);
250 		if (iamr != regs->iamr)
251 			restore_iamr = true;
252 	} else {
253 		restore_iamr = true;
254 	}
255 
256 
257 	if (restore_amr || restore_iamr) {
258 		isync();
259 		if (restore_amr)
260 			mtspr(SPRN_AMR, regs->amr);
261 		if (restore_iamr)
262 			mtspr(SPRN_IAMR, regs->iamr);
263 	}
264 	/*
265 	 * No isync required here because we are about to rfi
266 	 * back to previous context before any user accesses
267 	 * would be made, which is a CSI.
268 	 */
269 }
270 
271 static inline void kuap_kernel_restore(struct pt_regs *regs,
272 					   unsigned long amr)
273 {
274 	if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
275 		if (unlikely(regs->amr != amr)) {
276 			isync();
277 			mtspr(SPRN_AMR, regs->amr);
278 			/*
279 			 * No isync required here because we are about to rfi
280 			 * back to previous context before any user accesses
281 			 * would be made, which is a CSI.
282 			 */
283 		}
284 	}
285 	/*
286 	 * No need to restore IAMR when returning to kernel space.
287 	 */
288 }
289 
290 static inline unsigned long kuap_get_and_assert_locked(void)
291 {
292 	if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
293 		unsigned long amr = mfspr(SPRN_AMR);
294 		if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) /* kuap_check_amr() */
295 			WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED);
296 		return amr;
297 	}
298 	return 0;
299 }
300 
301 static inline void kuap_assert_locked(void)
302 {
303 	if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
304 		WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);
305 }
306 
307 /*
308  * We support individually allowing read or write, but we don't support nesting
309  * because that would require an expensive read/modify write of the AMR.
310  */
311 
312 static inline unsigned long get_kuap(void)
313 {
314 	/*
315 	 * We return AMR_KUAP_BLOCKED when we don't support KUAP because
316 	 * prevent_user_access_return needs to return AMR_KUAP_BLOCKED to
317 	 * cause restore_user_access to do a flush.
318 	 *
319 	 * This has no effect in terms of actually blocking things on hash,
320 	 * so it doesn't break anything.
321 	 */
322 	if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
323 		return AMR_KUAP_BLOCKED;
324 
325 	return mfspr(SPRN_AMR);
326 }
327 
328 static inline void set_kuap(unsigned long value)
329 {
330 	if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
331 		return;
332 
333 	/*
334 	 * ISA v3.0B says we need a CSI (Context Synchronising Instruction) both
335 	 * before and after the move to AMR. See table 6 on page 1134.
336 	 */
337 	isync();
338 	mtspr(SPRN_AMR, value);
339 	isync();
340 }
341 
342 static inline bool bad_kuap_fault(struct pt_regs *regs, unsigned long address,
343 				  bool is_write)
344 {
345 	if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
346 		return false;
347 	/*
348 	 * For radix this will be a storage protection fault (DSISR_PROTFAULT).
349 	 * For hash this will be a key fault (DSISR_KEYFAULT)
350 	 */
351 	/*
352 	 * We do have exception table entry, but accessing the
353 	 * userspace results in fault.  This could be because we
354 	 * didn't unlock the AMR or access is denied by userspace
355 	 * using a key value that blocks access. We are only interested
356 	 * in catching the use case of accessing without unlocking
357 	 * the AMR. Hence check for BLOCK_WRITE/READ against AMR.
358 	 */
359 	if (is_write) {
360 		return (regs->amr & AMR_KUAP_BLOCK_WRITE) == AMR_KUAP_BLOCK_WRITE;
361 	}
362 	return (regs->amr & AMR_KUAP_BLOCK_READ) == AMR_KUAP_BLOCK_READ;
363 }
364 
365 static __always_inline void allow_user_access(void __user *to, const void __user *from,
366 					      unsigned long size, unsigned long dir)
367 {
368 	unsigned long thread_amr = 0;
369 
370 	// This is written so we can resolve to a single case at build time
371 	BUILD_BUG_ON(!__builtin_constant_p(dir));
372 
373 	if (mmu_has_feature(MMU_FTR_PKEY))
374 		thread_amr = current_thread_amr();
375 
376 	if (dir == KUAP_READ)
377 		set_kuap(thread_amr | AMR_KUAP_BLOCK_WRITE);
378 	else if (dir == KUAP_WRITE)
379 		set_kuap(thread_amr | AMR_KUAP_BLOCK_READ);
380 	else if (dir == KUAP_READ_WRITE)
381 		set_kuap(thread_amr);
382 	else
383 		BUILD_BUG();
384 }
385 
386 #else /* CONFIG_PPC_KUAP */
387 
388 static inline unsigned long get_kuap(void)
389 {
390 	return AMR_KUAP_BLOCKED;
391 }
392 
393 static inline void set_kuap(unsigned long value) { }
394 
395 static __always_inline void allow_user_access(void __user *to, const void __user *from,
396 					      unsigned long size, unsigned long dir)
397 { }
398 
399 #endif /* !CONFIG_PPC_KUAP */
400 
401 static inline void prevent_user_access(unsigned long dir)
402 {
403 	set_kuap(AMR_KUAP_BLOCKED);
404 	if (static_branch_unlikely(&uaccess_flush_key))
405 		do_uaccess_flush();
406 }
407 
408 static inline unsigned long prevent_user_access_return(void)
409 {
410 	unsigned long flags = get_kuap();
411 
412 	set_kuap(AMR_KUAP_BLOCKED);
413 	if (static_branch_unlikely(&uaccess_flush_key))
414 		do_uaccess_flush();
415 
416 	return flags;
417 }
418 
419 static inline void restore_user_access(unsigned long flags)
420 {
421 	set_kuap(flags);
422 	if (static_branch_unlikely(&uaccess_flush_key) && flags == AMR_KUAP_BLOCKED)
423 		do_uaccess_flush();
424 }
425 #endif /* __ASSEMBLY__ */
426 
427 #endif /* _ASM_POWERPC_BOOK3S_64_KUP_H */
428