1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This file contains core hardware tag-based KASAN code. 4 * 5 * Copyright (c) 2020 Google, Inc. 6 * Author: Andrey Konovalov <andreyknvl@google.com> 7 */ 8 9 #define pr_fmt(fmt) "kasan: " fmt 10 11 #include <linux/init.h> 12 #include <linux/kasan.h> 13 #include <linux/kernel.h> 14 #include <linux/memory.h> 15 #include <linux/mm.h> 16 #include <linux/static_key.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 20 #include "kasan.h" 21 22 enum kasan_arg { 23 KASAN_ARG_DEFAULT, 24 KASAN_ARG_OFF, 25 KASAN_ARG_ON, 26 }; 27 28 enum kasan_arg_mode { 29 KASAN_ARG_MODE_DEFAULT, 30 KASAN_ARG_MODE_SYNC, 31 KASAN_ARG_MODE_ASYNC, 32 }; 33 34 enum kasan_arg_stacktrace { 35 KASAN_ARG_STACKTRACE_DEFAULT, 36 KASAN_ARG_STACKTRACE_OFF, 37 KASAN_ARG_STACKTRACE_ON, 38 }; 39 40 static enum kasan_arg kasan_arg __ro_after_init; 41 static enum kasan_arg_mode kasan_arg_mode __ro_after_init; 42 static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init; 43 44 /* Whether KASAN is enabled at all. */ 45 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); 46 EXPORT_SYMBOL(kasan_flag_enabled); 47 48 /* Whether the asynchronous mode is enabled. */ 49 bool kasan_flag_async __ro_after_init; 50 EXPORT_SYMBOL_GPL(kasan_flag_async); 51 52 /* Whether to collect alloc/free stack traces. */ 53 DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace); 54 55 /* kasan=off/on */ 56 static int __init early_kasan_flag(char *arg) 57 { 58 if (!arg) 59 return -EINVAL; 60 61 if (!strcmp(arg, "off")) 62 kasan_arg = KASAN_ARG_OFF; 63 else if (!strcmp(arg, "on")) 64 kasan_arg = KASAN_ARG_ON; 65 else 66 return -EINVAL; 67 68 return 0; 69 } 70 early_param("kasan", early_kasan_flag); 71 72 /* kasan.mode=sync/async */ 73 static int __init early_kasan_mode(char *arg) 74 { 75 if (!arg) 76 return -EINVAL; 77 78 if (!strcmp(arg, "sync")) 79 kasan_arg_mode = KASAN_ARG_MODE_SYNC; 80 else if (!strcmp(arg, "async")) 81 kasan_arg_mode = KASAN_ARG_MODE_ASYNC; 82 else 83 return -EINVAL; 84 85 return 0; 86 } 87 early_param("kasan.mode", early_kasan_mode); 88 89 /* kasan.stacktrace=off/on */ 90 static int __init early_kasan_flag_stacktrace(char *arg) 91 { 92 if (!arg) 93 return -EINVAL; 94 95 if (!strcmp(arg, "off")) 96 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; 97 else if (!strcmp(arg, "on")) 98 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; 99 else 100 return -EINVAL; 101 102 return 0; 103 } 104 early_param("kasan.stacktrace", early_kasan_flag_stacktrace); 105 106 /* kasan_init_hw_tags_cpu() is called for each CPU. */ 107 void kasan_init_hw_tags_cpu(void) 108 { 109 /* 110 * There's no need to check that the hardware is MTE-capable here, 111 * as this function is only called for MTE-capable hardware. 112 */ 113 114 /* If KASAN is disabled via command line, don't initialize it. */ 115 if (kasan_arg == KASAN_ARG_OFF) 116 return; 117 118 /* 119 * Enable async mode only when explicitly requested through 120 * the command line. 121 */ 122 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) 123 hw_enable_tagging_async(); 124 else 125 hw_enable_tagging_sync(); 126 } 127 128 /* kasan_init_hw_tags() is called once on boot CPU. */ 129 void __init kasan_init_hw_tags(void) 130 { 131 /* If hardware doesn't support MTE, don't initialize KASAN. */ 132 if (!system_supports_mte()) 133 return; 134 135 /* If KASAN is disabled via command line, don't initialize it. */ 136 if (kasan_arg == KASAN_ARG_OFF) 137 return; 138 139 /* Enable KASAN. */ 140 static_branch_enable(&kasan_flag_enabled); 141 142 switch (kasan_arg_mode) { 143 case KASAN_ARG_MODE_DEFAULT: 144 /* 145 * Default to sync mode. 146 * Do nothing, kasan_flag_async keeps its default value. 147 */ 148 break; 149 case KASAN_ARG_MODE_SYNC: 150 /* Do nothing, kasan_flag_async keeps its default value. */ 151 break; 152 case KASAN_ARG_MODE_ASYNC: 153 /* Async mode enabled. */ 154 kasan_flag_async = true; 155 break; 156 } 157 158 switch (kasan_arg_stacktrace) { 159 case KASAN_ARG_STACKTRACE_DEFAULT: 160 /* Default to enabling stack trace collection. */ 161 static_branch_enable(&kasan_flag_stacktrace); 162 break; 163 case KASAN_ARG_STACKTRACE_OFF: 164 /* Do nothing, kasan_flag_stacktrace keeps its default value. */ 165 break; 166 case KASAN_ARG_STACKTRACE_ON: 167 static_branch_enable(&kasan_flag_stacktrace); 168 break; 169 } 170 171 pr_info("KernelAddressSanitizer initialized\n"); 172 } 173 174 void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags) 175 { 176 /* 177 * This condition should match the one in post_alloc_hook() in 178 * page_alloc.c. 179 */ 180 bool init = !want_init_on_free() && want_init_on_alloc(flags); 181 182 if (flags & __GFP_SKIP_KASAN_POISON) 183 SetPageSkipKASanPoison(page); 184 185 if (flags & __GFP_ZEROTAGS) { 186 int i; 187 188 for (i = 0; i != 1 << order; ++i) 189 tag_clear_highpage(page + i); 190 } else { 191 kasan_unpoison_pages(page, order, init); 192 } 193 } 194 195 void kasan_free_pages(struct page *page, unsigned int order) 196 { 197 /* 198 * This condition should match the one in free_pages_prepare() in 199 * page_alloc.c. 200 */ 201 bool init = want_init_on_free(); 202 203 kasan_poison_pages(page, order, init); 204 } 205 206 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) 207 208 void kasan_enable_tagging_sync(void) 209 { 210 hw_enable_tagging_sync(); 211 } 212 EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync); 213 214 void kasan_force_async_fault(void) 215 { 216 hw_force_async_tag_fault(); 217 } 218 EXPORT_SYMBOL_GPL(kasan_force_async_fault); 219 220 #endif 221