1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This file contains core hardware tag-based KASAN code. 4 * 5 * Copyright (c) 2020 Google, Inc. 6 * Author: Andrey Konovalov <andreyknvl@google.com> 7 */ 8 9 #define pr_fmt(fmt) "kasan: " fmt 10 11 #include <linux/init.h> 12 #include <linux/kasan.h> 13 #include <linux/kernel.h> 14 #include <linux/memory.h> 15 #include <linux/mm.h> 16 #include <linux/static_key.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 20 #include "kasan.h" 21 22 enum kasan_arg { 23 KASAN_ARG_DEFAULT, 24 KASAN_ARG_OFF, 25 KASAN_ARG_ON, 26 }; 27 28 enum kasan_arg_mode { 29 KASAN_ARG_MODE_DEFAULT, 30 KASAN_ARG_MODE_SYNC, 31 KASAN_ARG_MODE_ASYNC, 32 KASAN_ARG_MODE_ASYMM, 33 }; 34 35 enum kasan_arg_stacktrace { 36 KASAN_ARG_STACKTRACE_DEFAULT, 37 KASAN_ARG_STACKTRACE_OFF, 38 KASAN_ARG_STACKTRACE_ON, 39 }; 40 41 static enum kasan_arg kasan_arg __ro_after_init; 42 static enum kasan_arg_mode kasan_arg_mode __ro_after_init; 43 static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init; 44 45 /* Whether KASAN is enabled at all. */ 46 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); 47 EXPORT_SYMBOL(kasan_flag_enabled); 48 49 /* Whether the selected mode is synchronous/asynchronous/asymmetric.*/ 50 enum kasan_mode kasan_mode __ro_after_init; 51 EXPORT_SYMBOL_GPL(kasan_mode); 52 53 /* Whether to collect alloc/free stack traces. */ 54 DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace); 55 56 /* kasan=off/on */ 57 static int __init early_kasan_flag(char *arg) 58 { 59 if (!arg) 60 return -EINVAL; 61 62 if (!strcmp(arg, "off")) 63 kasan_arg = KASAN_ARG_OFF; 64 else if (!strcmp(arg, "on")) 65 kasan_arg = KASAN_ARG_ON; 66 else 67 return -EINVAL; 68 69 return 0; 70 } 71 early_param("kasan", early_kasan_flag); 72 73 /* kasan.mode=sync/async/asymm */ 74 static int __init early_kasan_mode(char *arg) 75 { 76 if (!arg) 77 return -EINVAL; 78 79 if (!strcmp(arg, "sync")) 80 kasan_arg_mode = KASAN_ARG_MODE_SYNC; 81 else if (!strcmp(arg, "async")) 82 kasan_arg_mode = KASAN_ARG_MODE_ASYNC; 83 else if (!strcmp(arg, "asymm")) 84 kasan_arg_mode = KASAN_ARG_MODE_ASYMM; 85 else 86 return -EINVAL; 87 88 return 0; 89 } 90 early_param("kasan.mode", early_kasan_mode); 91 92 /* kasan.stacktrace=off/on */ 93 static int __init early_kasan_flag_stacktrace(char *arg) 94 { 95 if (!arg) 96 return -EINVAL; 97 98 if (!strcmp(arg, "off")) 99 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; 100 else if (!strcmp(arg, "on")) 101 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; 102 else 103 return -EINVAL; 104 105 return 0; 106 } 107 early_param("kasan.stacktrace", early_kasan_flag_stacktrace); 108 109 static inline const char *kasan_mode_info(void) 110 { 111 if (kasan_mode == KASAN_MODE_ASYNC) 112 return "async"; 113 else if (kasan_mode == KASAN_MODE_ASYMM) 114 return "asymm"; 115 else 116 return "sync"; 117 } 118 119 /* kasan_init_hw_tags_cpu() is called for each CPU. */ 120 void kasan_init_hw_tags_cpu(void) 121 { 122 /* 123 * There's no need to check that the hardware is MTE-capable here, 124 * as this function is only called for MTE-capable hardware. 125 */ 126 127 /* If KASAN is disabled via command line, don't initialize it. */ 128 if (kasan_arg == KASAN_ARG_OFF) 129 return; 130 131 /* 132 * Enable async or asymm modes only when explicitly requested 133 * through the command line. 134 */ 135 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) 136 hw_enable_tagging_async(); 137 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM) 138 hw_enable_tagging_asymm(); 139 else 140 hw_enable_tagging_sync(); 141 } 142 143 /* kasan_init_hw_tags() is called once on boot CPU. */ 144 void __init kasan_init_hw_tags(void) 145 { 146 /* If hardware doesn't support MTE, don't initialize KASAN. */ 147 if (!system_supports_mte()) 148 return; 149 150 /* If KASAN is disabled via command line, don't initialize it. */ 151 if (kasan_arg == KASAN_ARG_OFF) 152 return; 153 154 /* Enable KASAN. */ 155 static_branch_enable(&kasan_flag_enabled); 156 157 switch (kasan_arg_mode) { 158 case KASAN_ARG_MODE_DEFAULT: 159 /* 160 * Default to sync mode. 161 */ 162 fallthrough; 163 case KASAN_ARG_MODE_SYNC: 164 /* Sync mode enabled. */ 165 kasan_mode = KASAN_MODE_SYNC; 166 break; 167 case KASAN_ARG_MODE_ASYNC: 168 /* Async mode enabled. */ 169 kasan_mode = KASAN_MODE_ASYNC; 170 break; 171 case KASAN_ARG_MODE_ASYMM: 172 /* Asymm mode enabled. */ 173 kasan_mode = KASAN_MODE_ASYMM; 174 break; 175 } 176 177 switch (kasan_arg_stacktrace) { 178 case KASAN_ARG_STACKTRACE_DEFAULT: 179 /* Default to enabling stack trace collection. */ 180 static_branch_enable(&kasan_flag_stacktrace); 181 break; 182 case KASAN_ARG_STACKTRACE_OFF: 183 /* Do nothing, kasan_flag_stacktrace keeps its default value. */ 184 break; 185 case KASAN_ARG_STACKTRACE_ON: 186 static_branch_enable(&kasan_flag_stacktrace); 187 break; 188 } 189 190 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, stacktrace=%s)\n", 191 kasan_mode_info(), 192 kasan_stack_collection_enabled() ? "on" : "off"); 193 } 194 195 void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags) 196 { 197 /* 198 * This condition should match the one in post_alloc_hook() in 199 * page_alloc.c. 200 */ 201 bool init = !want_init_on_free() && want_init_on_alloc(flags); 202 203 if (flags & __GFP_SKIP_KASAN_POISON) 204 SetPageSkipKASanPoison(page); 205 206 if (flags & __GFP_ZEROTAGS) { 207 int i; 208 209 for (i = 0; i != 1 << order; ++i) 210 tag_clear_highpage(page + i); 211 } else { 212 kasan_unpoison_pages(page, order, init); 213 } 214 } 215 216 void kasan_free_pages(struct page *page, unsigned int order) 217 { 218 /* 219 * This condition should match the one in free_pages_prepare() in 220 * page_alloc.c. 221 */ 222 bool init = want_init_on_free(); 223 224 kasan_poison_pages(page, order, init); 225 } 226 227 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) 228 229 void kasan_enable_tagging_sync(void) 230 { 231 hw_enable_tagging_sync(); 232 } 233 EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync); 234 235 void kasan_force_async_fault(void) 236 { 237 hw_force_async_tag_fault(); 238 } 239 EXPORT_SYMBOL_GPL(kasan_force_async_fault); 240 241 #endif 242