1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shadow Call Stack support. 4 * 5 * Copyright (C) 2019 Google LLC 6 */ 7 8 #include <linux/kasan.h> 9 #include <linux/mm.h> 10 #include <linux/scs.h> 11 #include <linux/slab.h> 12 #include <linux/vmstat.h> 13 #include <asm/scs.h> 14 15 static struct kmem_cache *scs_cache; 16 17 static void *scs_alloc(int node) 18 { 19 void *s; 20 21 s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node); 22 if (s) { 23 *__scs_magic(s) = SCS_END_MAGIC; 24 /* 25 * Poison the allocation to catch unintentional accesses to 26 * the shadow stack when KASAN is enabled. 27 */ 28 kasan_poison_object_data(scs_cache, s); 29 } 30 31 return s; 32 } 33 34 static void scs_free(void *s) 35 { 36 kasan_unpoison_object_data(scs_cache, s); 37 kmem_cache_free(scs_cache, s); 38 } 39 40 void __init scs_init(void) 41 { 42 scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL); 43 } 44 45 static struct page *__scs_page(struct task_struct *tsk) 46 { 47 return virt_to_page(task_scs(tsk)); 48 } 49 50 static void scs_account(struct task_struct *tsk, int account) 51 { 52 mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_KB, 53 account * (SCS_SIZE / 1024)); 54 } 55 56 int scs_prepare(struct task_struct *tsk, int node) 57 { 58 void *s = scs_alloc(node); 59 60 if (!s) 61 return -ENOMEM; 62 63 task_scs(tsk) = task_scs_sp(tsk) = s; 64 scs_account(tsk, 1); 65 return 0; 66 } 67 68 static void scs_check_usage(struct task_struct *tsk) 69 { 70 static unsigned long highest; 71 72 unsigned long *p, prev, curr = highest, used = 0; 73 74 if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) 75 return; 76 77 for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { 78 if (!READ_ONCE_NOCHECK(*p)) 79 break; 80 used++; 81 } 82 83 while (used > curr) { 84 prev = cmpxchg_relaxed(&highest, curr, used); 85 86 if (prev == curr) { 87 pr_info("%s (%d): highest shadow stack usage: %lu bytes\n", 88 tsk->comm, task_pid_nr(tsk), used); 89 break; 90 } 91 92 curr = prev; 93 } 94 } 95 96 void scs_release(struct task_struct *tsk) 97 { 98 void *s = task_scs(tsk); 99 100 if (!s) 101 return; 102 103 WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n"); 104 scs_check_usage(tsk); 105 scs_account(tsk, -1); 106 scs_free(s); 107 } 108