xref: /openbmc/linux/arch/x86/kernel/shstk.c (revision 046b212a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * shstk.c - Intel shadow stack support
4  *
5  * Copyright (c) 2021, Intel Corporation.
6  * Yu-cheng Yu <yu-cheng.yu@intel.com>
7  */
8 
9 #include <linux/sched.h>
10 #include <linux/bitops.h>
11 #include <linux/types.h>
12 #include <linux/mm.h>
13 #include <linux/mman.h>
14 #include <linux/slab.h>
15 #include <linux/uaccess.h>
16 #include <linux/sched/signal.h>
17 #include <linux/compat.h>
18 #include <linux/sizes.h>
19 #include <linux/user.h>
20 #include <linux/syscalls.h>
21 #include <asm/msr.h>
22 #include <asm/fpu/xstate.h>
23 #include <asm/fpu/types.h>
24 #include <asm/shstk.h>
25 #include <asm/special_insns.h>
26 #include <asm/fpu/api.h>
27 #include <asm/prctl.h>
28 
29 #define SS_FRAME_SIZE 8
30 
31 static bool features_enabled(unsigned long features)
32 {
33 	return current->thread.features & features;
34 }
35 
36 static void features_set(unsigned long features)
37 {
38 	current->thread.features |= features;
39 }
40 
41 static void features_clr(unsigned long features)
42 {
43 	current->thread.features &= ~features;
44 }
45 
46 /*
47  * Create a restore token on the shadow stack.  A token is always 8-byte
48  * and aligned to 8.
49  */
50 static int create_rstor_token(unsigned long ssp, unsigned long *token_addr)
51 {
52 	unsigned long addr;
53 
54 	/* Token must be aligned */
55 	if (!IS_ALIGNED(ssp, 8))
56 		return -EINVAL;
57 
58 	addr = ssp - SS_FRAME_SIZE;
59 
60 	/*
61 	 * SSP is aligned, so reserved bits and mode bit are a zero, just mark
62 	 * the token 64-bit.
63 	 */
64 	ssp |= BIT(0);
65 
66 	if (write_user_shstk_64((u64 __user *)addr, (u64)ssp))
67 		return -EFAULT;
68 
69 	if (token_addr)
70 		*token_addr = addr;
71 
72 	return 0;
73 }
74 
75 /*
76  * VM_SHADOW_STACK will have a guard page. This helps userspace protect
77  * itself from attacks. The reasoning is as follows:
78  *
79  * The shadow stack pointer(SSP) is moved by CALL, RET, and INCSSPQ. The
80  * INCSSP instruction can increment the shadow stack pointer. It is the
81  * shadow stack analog of an instruction like:
82  *
83  *   addq $0x80, %rsp
84  *
85  * However, there is one important difference between an ADD on %rsp
86  * and INCSSP. In addition to modifying SSP, INCSSP also reads from the
87  * memory of the first and last elements that were "popped". It can be
88  * thought of as acting like this:
89  *
90  * READ_ONCE(ssp);       // read+discard top element on stack
91  * ssp += nr_to_pop * 8; // move the shadow stack
92  * READ_ONCE(ssp-8);     // read+discard last popped stack element
93  *
94  * The maximum distance INCSSP can move the SSP is 2040 bytes, before
95  * it would read the memory. Therefore a single page gap will be enough
96  * to prevent any operation from shifting the SSP to an adjacent stack,
97  * since it would have to land in the gap at least once, causing a
98  * fault.
99  */
100 static unsigned long alloc_shstk(unsigned long addr, unsigned long size,
101 				 unsigned long token_offset, bool set_res_tok)
102 {
103 	int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_ABOVE4G;
104 	struct mm_struct *mm = current->mm;
105 	unsigned long mapped_addr, unused;
106 
107 	if (addr)
108 		flags |= MAP_FIXED_NOREPLACE;
109 
110 	mmap_write_lock(mm);
111 	mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags,
112 			      VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL);
113 	mmap_write_unlock(mm);
114 
115 	if (!set_res_tok || IS_ERR_VALUE(mapped_addr))
116 		goto out;
117 
118 	if (create_rstor_token(mapped_addr + token_offset, NULL)) {
119 		vm_munmap(mapped_addr, size);
120 		return -EINVAL;
121 	}
122 
123 out:
124 	return mapped_addr;
125 }
126 
127 static unsigned long adjust_shstk_size(unsigned long size)
128 {
129 	if (size)
130 		return PAGE_ALIGN(size);
131 
132 	return PAGE_ALIGN(min_t(unsigned long long, rlimit(RLIMIT_STACK), SZ_4G));
133 }
134 
135 static void unmap_shadow_stack(u64 base, u64 size)
136 {
137 	int r;
138 
139 	r = vm_munmap(base, size);
140 
141 	/*
142 	 * mmap_write_lock_killable() failed with -EINTR. This means
143 	 * the process is about to die and have it's MM cleaned up.
144 	 * This task shouldn't ever make it back to userspace. In this
145 	 * case it is ok to leak a shadow stack, so just exit out.
146 	 */
147 	if (r == -EINTR)
148 		return;
149 
150 	/*
151 	 * For all other types of vm_munmap() failure, either the
152 	 * system is out of memory or there is bug.
153 	 */
154 	WARN_ON_ONCE(r);
155 }
156 
157 static int shstk_setup(void)
158 {
159 	struct thread_shstk *shstk = &current->thread.shstk;
160 	unsigned long addr, size;
161 
162 	/* Already enabled */
163 	if (features_enabled(ARCH_SHSTK_SHSTK))
164 		return 0;
165 
166 	/* Also not supported for 32 bit and x32 */
167 	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || in_32bit_syscall())
168 		return -EOPNOTSUPP;
169 
170 	size = adjust_shstk_size(0);
171 	addr = alloc_shstk(0, size, 0, false);
172 	if (IS_ERR_VALUE(addr))
173 		return PTR_ERR((void *)addr);
174 
175 	fpregs_lock_and_load();
176 	wrmsrl(MSR_IA32_PL3_SSP, addr + size);
177 	wrmsrl(MSR_IA32_U_CET, CET_SHSTK_EN);
178 	fpregs_unlock();
179 
180 	shstk->base = addr;
181 	shstk->size = size;
182 	features_set(ARCH_SHSTK_SHSTK);
183 
184 	return 0;
185 }
186 
187 void reset_thread_features(void)
188 {
189 	memset(&current->thread.shstk, 0, sizeof(struct thread_shstk));
190 	current->thread.features = 0;
191 	current->thread.features_locked = 0;
192 }
193 
194 unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, unsigned long clone_flags,
195 				       unsigned long stack_size)
196 {
197 	struct thread_shstk *shstk = &tsk->thread.shstk;
198 	unsigned long addr, size;
199 
200 	/*
201 	 * If shadow stack is not enabled on the new thread, skip any
202 	 * switch to a new shadow stack.
203 	 */
204 	if (!features_enabled(ARCH_SHSTK_SHSTK))
205 		return 0;
206 
207 	/*
208 	 * For CLONE_VM, except vfork, the child needs a separate shadow
209 	 * stack.
210 	 */
211 	if ((clone_flags & (CLONE_VFORK | CLONE_VM)) != CLONE_VM)
212 		return 0;
213 
214 	size = adjust_shstk_size(stack_size);
215 	addr = alloc_shstk(0, size, 0, false);
216 	if (IS_ERR_VALUE(addr))
217 		return addr;
218 
219 	shstk->base = addr;
220 	shstk->size = size;
221 
222 	return addr + size;
223 }
224 
225 static unsigned long get_user_shstk_addr(void)
226 {
227 	unsigned long long ssp;
228 
229 	fpregs_lock_and_load();
230 
231 	rdmsrl(MSR_IA32_PL3_SSP, ssp);
232 
233 	fpregs_unlock();
234 
235 	return ssp;
236 }
237 
238 #define SHSTK_DATA_BIT BIT(63)
239 
240 static int put_shstk_data(u64 __user *addr, u64 data)
241 {
242 	if (WARN_ON_ONCE(data & SHSTK_DATA_BIT))
243 		return -EINVAL;
244 
245 	/*
246 	 * Mark the high bit so that the sigframe can't be processed as a
247 	 * return address.
248 	 */
249 	if (write_user_shstk_64(addr, data | SHSTK_DATA_BIT))
250 		return -EFAULT;
251 	return 0;
252 }
253 
254 static int get_shstk_data(unsigned long *data, unsigned long __user *addr)
255 {
256 	unsigned long ldata;
257 
258 	if (unlikely(get_user(ldata, addr)))
259 		return -EFAULT;
260 
261 	if (!(ldata & SHSTK_DATA_BIT))
262 		return -EINVAL;
263 
264 	*data = ldata & ~SHSTK_DATA_BIT;
265 
266 	return 0;
267 }
268 
269 static int shstk_push_sigframe(unsigned long *ssp)
270 {
271 	unsigned long target_ssp = *ssp;
272 
273 	/* Token must be aligned */
274 	if (!IS_ALIGNED(target_ssp, 8))
275 		return -EINVAL;
276 
277 	*ssp -= SS_FRAME_SIZE;
278 	if (put_shstk_data((void __user *)*ssp, target_ssp))
279 		return -EFAULT;
280 
281 	return 0;
282 }
283 
284 static int shstk_pop_sigframe(unsigned long *ssp)
285 {
286 	struct vm_area_struct *vma;
287 	unsigned long token_addr;
288 	bool need_to_check_vma;
289 	int err = 1;
290 
291 	/*
292 	 * It is possible for the SSP to be off the end of a shadow stack by 4
293 	 * or 8 bytes. If the shadow stack is at the start of a page or 4 bytes
294 	 * before it, it might be this case, so check that the address being
295 	 * read is actually shadow stack.
296 	 */
297 	if (!IS_ALIGNED(*ssp, 8))
298 		return -EINVAL;
299 
300 	need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp;
301 
302 	if (need_to_check_vma)
303 		mmap_read_lock_killable(current->mm);
304 
305 	err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp);
306 	if (unlikely(err))
307 		goto out_err;
308 
309 	if (need_to_check_vma) {
310 		vma = find_vma(current->mm, *ssp);
311 		if (!vma || !(vma->vm_flags & VM_SHADOW_STACK)) {
312 			err = -EFAULT;
313 			goto out_err;
314 		}
315 
316 		mmap_read_unlock(current->mm);
317 	}
318 
319 	/* Restore SSP aligned? */
320 	if (unlikely(!IS_ALIGNED(token_addr, 8)))
321 		return -EINVAL;
322 
323 	/* SSP in userspace? */
324 	if (unlikely(token_addr >= TASK_SIZE_MAX))
325 		return -EINVAL;
326 
327 	*ssp = token_addr;
328 
329 	return 0;
330 out_err:
331 	if (need_to_check_vma)
332 		mmap_read_unlock(current->mm);
333 	return err;
334 }
335 
336 int setup_signal_shadow_stack(struct ksignal *ksig)
337 {
338 	void __user *restorer = ksig->ka.sa.sa_restorer;
339 	unsigned long ssp;
340 	int err;
341 
342 	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
343 	    !features_enabled(ARCH_SHSTK_SHSTK))
344 		return 0;
345 
346 	if (!restorer)
347 		return -EINVAL;
348 
349 	ssp = get_user_shstk_addr();
350 	if (unlikely(!ssp))
351 		return -EINVAL;
352 
353 	err = shstk_push_sigframe(&ssp);
354 	if (unlikely(err))
355 		return err;
356 
357 	/* Push restorer address */
358 	ssp -= SS_FRAME_SIZE;
359 	err = write_user_shstk_64((u64 __user *)ssp, (u64)restorer);
360 	if (unlikely(err))
361 		return -EFAULT;
362 
363 	fpregs_lock_and_load();
364 	wrmsrl(MSR_IA32_PL3_SSP, ssp);
365 	fpregs_unlock();
366 
367 	return 0;
368 }
369 
370 int restore_signal_shadow_stack(void)
371 {
372 	unsigned long ssp;
373 	int err;
374 
375 	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
376 	    !features_enabled(ARCH_SHSTK_SHSTK))
377 		return 0;
378 
379 	ssp = get_user_shstk_addr();
380 	if (unlikely(!ssp))
381 		return -EINVAL;
382 
383 	err = shstk_pop_sigframe(&ssp);
384 	if (unlikely(err))
385 		return err;
386 
387 	fpregs_lock_and_load();
388 	wrmsrl(MSR_IA32_PL3_SSP, ssp);
389 	fpregs_unlock();
390 
391 	return 0;
392 }
393 
394 void shstk_free(struct task_struct *tsk)
395 {
396 	struct thread_shstk *shstk = &tsk->thread.shstk;
397 
398 	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
399 	    !features_enabled(ARCH_SHSTK_SHSTK))
400 		return;
401 
402 	/*
403 	 * When fork() with CLONE_VM fails, the child (tsk) already has a
404 	 * shadow stack allocated, and exit_thread() calls this function to
405 	 * free it.  In this case the parent (current) and the child share
406 	 * the same mm struct.
407 	 */
408 	if (!tsk->mm || tsk->mm != current->mm)
409 		return;
410 
411 	unmap_shadow_stack(shstk->base, shstk->size);
412 }
413 
414 static int wrss_control(bool enable)
415 {
416 	u64 msrval;
417 
418 	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
419 		return -EOPNOTSUPP;
420 
421 	/*
422 	 * Only enable WRSS if shadow stack is enabled. If shadow stack is not
423 	 * enabled, WRSS will already be disabled, so don't bother clearing it
424 	 * when disabling.
425 	 */
426 	if (!features_enabled(ARCH_SHSTK_SHSTK))
427 		return -EPERM;
428 
429 	/* Already enabled/disabled? */
430 	if (features_enabled(ARCH_SHSTK_WRSS) == enable)
431 		return 0;
432 
433 	fpregs_lock_and_load();
434 	rdmsrl(MSR_IA32_U_CET, msrval);
435 
436 	if (enable) {
437 		features_set(ARCH_SHSTK_WRSS);
438 		msrval |= CET_WRSS_EN;
439 	} else {
440 		features_clr(ARCH_SHSTK_WRSS);
441 		if (!(msrval & CET_WRSS_EN))
442 			goto unlock;
443 
444 		msrval &= ~CET_WRSS_EN;
445 	}
446 
447 	wrmsrl(MSR_IA32_U_CET, msrval);
448 
449 unlock:
450 	fpregs_unlock();
451 
452 	return 0;
453 }
454 
455 static int shstk_disable(void)
456 {
457 	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
458 		return -EOPNOTSUPP;
459 
460 	/* Already disabled? */
461 	if (!features_enabled(ARCH_SHSTK_SHSTK))
462 		return 0;
463 
464 	fpregs_lock_and_load();
465 	/* Disable WRSS too when disabling shadow stack */
466 	wrmsrl(MSR_IA32_U_CET, 0);
467 	wrmsrl(MSR_IA32_PL3_SSP, 0);
468 	fpregs_unlock();
469 
470 	shstk_free(current);
471 	features_clr(ARCH_SHSTK_SHSTK | ARCH_SHSTK_WRSS);
472 
473 	return 0;
474 }
475 
476 SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags)
477 {
478 	bool set_tok = flags & SHADOW_STACK_SET_TOKEN;
479 	unsigned long aligned_size;
480 
481 	if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
482 		return -EOPNOTSUPP;
483 
484 	if (flags & ~SHADOW_STACK_SET_TOKEN)
485 		return -EINVAL;
486 
487 	/* If there isn't space for a token */
488 	if (set_tok && size < 8)
489 		return -ENOSPC;
490 
491 	if (addr && addr < SZ_4G)
492 		return -ERANGE;
493 
494 	/*
495 	 * An overflow would result in attempting to write the restore token
496 	 * to the wrong location. Not catastrophic, but just return the right
497 	 * error code and block it.
498 	 */
499 	aligned_size = PAGE_ALIGN(size);
500 	if (aligned_size < size)
501 		return -EOVERFLOW;
502 
503 	return alloc_shstk(addr, aligned_size, size, set_tok);
504 }
505 
506 long shstk_prctl(struct task_struct *task, int option, unsigned long arg2)
507 {
508 	unsigned long features = arg2;
509 
510 	if (option == ARCH_SHSTK_STATUS) {
511 		return put_user(task->thread.features, (unsigned long __user *)arg2);
512 	}
513 
514 	if (option == ARCH_SHSTK_LOCK) {
515 		task->thread.features_locked |= features;
516 		return 0;
517 	}
518 
519 	/* Only allow via ptrace */
520 	if (task != current) {
521 		if (option == ARCH_SHSTK_UNLOCK && IS_ENABLED(CONFIG_CHECKPOINT_RESTORE)) {
522 			task->thread.features_locked &= ~features;
523 			return 0;
524 		}
525 		return -EINVAL;
526 	}
527 
528 	/* Do not allow to change locked features */
529 	if (features & task->thread.features_locked)
530 		return -EPERM;
531 
532 	/* Only support enabling/disabling one feature at a time. */
533 	if (hweight_long(features) > 1)
534 		return -EINVAL;
535 
536 	if (option == ARCH_SHSTK_DISABLE) {
537 		if (features & ARCH_SHSTK_WRSS)
538 			return wrss_control(false);
539 		if (features & ARCH_SHSTK_SHSTK)
540 			return shstk_disable();
541 		return -EINVAL;
542 	}
543 
544 	/* Handle ARCH_SHSTK_ENABLE */
545 	if (features & ARCH_SHSTK_SHSTK)
546 		return shstk_setup();
547 	if (features & ARCH_SHSTK_WRSS)
548 		return wrss_control(true);
549 	return -EINVAL;
550 }
551