1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This file contains core hardware tag-based KASAN code. 4 * 5 * Copyright (c) 2020 Google, Inc. 6 * Author: Andrey Konovalov <andreyknvl@google.com> 7 */ 8 9 #define pr_fmt(fmt) "kasan: " fmt 10 11 #include <linux/init.h> 12 #include <linux/kasan.h> 13 #include <linux/kernel.h> 14 #include <linux/memory.h> 15 #include <linux/mm.h> 16 #include <linux/static_key.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 20 #include "kasan.h" 21 22 enum kasan_arg { 23 KASAN_ARG_DEFAULT, 24 KASAN_ARG_OFF, 25 KASAN_ARG_ON, 26 }; 27 28 enum kasan_arg_mode { 29 KASAN_ARG_MODE_DEFAULT, 30 KASAN_ARG_MODE_SYNC, 31 KASAN_ARG_MODE_ASYNC, 32 KASAN_ARG_MODE_ASYMM, 33 }; 34 35 enum kasan_arg_vmalloc { 36 KASAN_ARG_VMALLOC_DEFAULT, 37 KASAN_ARG_VMALLOC_OFF, 38 KASAN_ARG_VMALLOC_ON, 39 }; 40 41 enum kasan_arg_stacktrace { 42 KASAN_ARG_STACKTRACE_DEFAULT, 43 KASAN_ARG_STACKTRACE_OFF, 44 KASAN_ARG_STACKTRACE_ON, 45 }; 46 47 static enum kasan_arg kasan_arg __ro_after_init; 48 static enum kasan_arg_mode kasan_arg_mode __ro_after_init; 49 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata; 50 static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata; 51 52 /* 53 * Whether KASAN is enabled at all. 54 * The value remains false until KASAN is initialized by kasan_init_hw_tags(). 55 */ 56 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); 57 EXPORT_SYMBOL(kasan_flag_enabled); 58 59 /* 60 * Whether the selected mode is synchronous, asynchronous, or asymmetric. 61 * Defaults to KASAN_MODE_SYNC. 62 */ 63 enum kasan_mode kasan_mode __ro_after_init; 64 EXPORT_SYMBOL_GPL(kasan_mode); 65 66 /* Whether to enable vmalloc tagging. */ 67 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc); 68 69 /* Whether to collect alloc/free stack traces. */ 70 DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace); 71 72 /* kasan=off/on */ 73 static int __init early_kasan_flag(char *arg) 74 { 75 if (!arg) 76 return -EINVAL; 77 78 if (!strcmp(arg, "off")) 79 kasan_arg = KASAN_ARG_OFF; 80 else if (!strcmp(arg, "on")) 81 kasan_arg = KASAN_ARG_ON; 82 else 83 return -EINVAL; 84 85 return 0; 86 } 87 early_param("kasan", early_kasan_flag); 88 89 /* kasan.mode=sync/async/asymm */ 90 static int __init early_kasan_mode(char *arg) 91 { 92 if (!arg) 93 return -EINVAL; 94 95 if (!strcmp(arg, "sync")) 96 kasan_arg_mode = KASAN_ARG_MODE_SYNC; 97 else if (!strcmp(arg, "async")) 98 kasan_arg_mode = KASAN_ARG_MODE_ASYNC; 99 else if (!strcmp(arg, "asymm")) 100 kasan_arg_mode = KASAN_ARG_MODE_ASYMM; 101 else 102 return -EINVAL; 103 104 return 0; 105 } 106 early_param("kasan.mode", early_kasan_mode); 107 108 /* kasan.vmalloc=off/on */ 109 static int __init early_kasan_flag_vmalloc(char *arg) 110 { 111 if (!arg) 112 return -EINVAL; 113 114 if (!strcmp(arg, "off")) 115 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF; 116 else if (!strcmp(arg, "on")) 117 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON; 118 else 119 return -EINVAL; 120 121 return 0; 122 } 123 early_param("kasan.vmalloc", early_kasan_flag_vmalloc); 124 125 /* kasan.stacktrace=off/on */ 126 static int __init early_kasan_flag_stacktrace(char *arg) 127 { 128 if (!arg) 129 return -EINVAL; 130 131 if (!strcmp(arg, "off")) 132 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; 133 else if (!strcmp(arg, "on")) 134 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; 135 else 136 return -EINVAL; 137 138 return 0; 139 } 140 early_param("kasan.stacktrace", early_kasan_flag_stacktrace); 141 142 static inline const char *kasan_mode_info(void) 143 { 144 if (kasan_mode == KASAN_MODE_ASYNC) 145 return "async"; 146 else if (kasan_mode == KASAN_MODE_ASYMM) 147 return "asymm"; 148 else 149 return "sync"; 150 } 151 152 /* 153 * kasan_init_hw_tags_cpu() is called for each CPU. 154 * Not marked as __init as a CPU can be hot-plugged after boot. 155 */ 156 void kasan_init_hw_tags_cpu(void) 157 { 158 /* 159 * There's no need to check that the hardware is MTE-capable here, 160 * as this function is only called for MTE-capable hardware. 161 */ 162 163 /* 164 * If KASAN is disabled via command line, don't initialize it. 165 * When this function is called, kasan_flag_enabled is not yet 166 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead. 167 */ 168 if (kasan_arg == KASAN_ARG_OFF) 169 return; 170 171 /* 172 * Enable async or asymm modes only when explicitly requested 173 * through the command line. 174 */ 175 kasan_enable_tagging(); 176 } 177 178 /* kasan_init_hw_tags() is called once on boot CPU. */ 179 void __init kasan_init_hw_tags(void) 180 { 181 /* If hardware doesn't support MTE, don't initialize KASAN. */ 182 if (!system_supports_mte()) 183 return; 184 185 /* If KASAN is disabled via command line, don't initialize it. */ 186 if (kasan_arg == KASAN_ARG_OFF) 187 return; 188 189 switch (kasan_arg_mode) { 190 case KASAN_ARG_MODE_DEFAULT: 191 /* Default is specified by kasan_mode definition. */ 192 break; 193 case KASAN_ARG_MODE_SYNC: 194 kasan_mode = KASAN_MODE_SYNC; 195 break; 196 case KASAN_ARG_MODE_ASYNC: 197 kasan_mode = KASAN_MODE_ASYNC; 198 break; 199 case KASAN_ARG_MODE_ASYMM: 200 kasan_mode = KASAN_MODE_ASYMM; 201 break; 202 } 203 204 switch (kasan_arg_vmalloc) { 205 case KASAN_ARG_VMALLOC_DEFAULT: 206 /* Default is specified by kasan_flag_vmalloc definition. */ 207 break; 208 case KASAN_ARG_VMALLOC_OFF: 209 static_branch_disable(&kasan_flag_vmalloc); 210 break; 211 case KASAN_ARG_VMALLOC_ON: 212 static_branch_enable(&kasan_flag_vmalloc); 213 break; 214 } 215 216 switch (kasan_arg_stacktrace) { 217 case KASAN_ARG_STACKTRACE_DEFAULT: 218 /* Default is specified by kasan_flag_stacktrace definition. */ 219 break; 220 case KASAN_ARG_STACKTRACE_OFF: 221 static_branch_disable(&kasan_flag_stacktrace); 222 break; 223 case KASAN_ARG_STACKTRACE_ON: 224 static_branch_enable(&kasan_flag_stacktrace); 225 break; 226 } 227 228 /* KASAN is now initialized, enable it. */ 229 static_branch_enable(&kasan_flag_enabled); 230 231 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n", 232 kasan_mode_info(), 233 kasan_vmalloc_enabled() ? "on" : "off", 234 kasan_stack_collection_enabled() ? "on" : "off"); 235 } 236 237 #ifdef CONFIG_KASAN_VMALLOC 238 239 static void unpoison_vmalloc_pages(const void *addr, u8 tag) 240 { 241 struct vm_struct *area; 242 int i; 243 244 /* 245 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations 246 * (see the comment in __kasan_unpoison_vmalloc), all of the pages 247 * should belong to a single area. 248 */ 249 area = find_vm_area((void *)addr); 250 if (WARN_ON(!area)) 251 return; 252 253 for (i = 0; i < area->nr_pages; i++) { 254 struct page *page = area->pages[i]; 255 256 page_kasan_tag_set(page, tag); 257 } 258 } 259 260 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, 261 kasan_vmalloc_flags_t flags) 262 { 263 u8 tag; 264 unsigned long redzone_start, redzone_size; 265 266 if (!kasan_vmalloc_enabled()) 267 return (void *)start; 268 269 if (!is_vmalloc_or_module_addr(start)) 270 return (void *)start; 271 272 /* 273 * Skip unpoisoning and assigning a pointer tag for non-VM_ALLOC 274 * mappings as: 275 * 276 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only 277 * supports tagging physical memory. Therefore, it can only tag a 278 * single mapping of normal physical pages. 279 * 2. Hardware tag-based KASAN can only tag memory mapped with special 280 * mapping protection bits, see arch_vmalloc_pgprot_modify(). 281 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code, 282 * providing these bits would require tracking all non-VM_ALLOC 283 * mappers. 284 * 285 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags 286 * the first virtual mapping, which is created by vmalloc(). 287 * Tagging the page_alloc memory backing that vmalloc() allocation is 288 * skipped, see ___GFP_SKIP_KASAN_UNPOISON. 289 * 290 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual. 291 */ 292 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) 293 return (void *)start; 294 295 /* 296 * Don't tag executable memory. 297 * The kernel doesn't tolerate having the PC register tagged. 298 */ 299 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) 300 return (void *)start; 301 302 tag = kasan_random_tag(); 303 start = set_tag(start, tag); 304 305 /* Unpoison and initialize memory up to size. */ 306 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT); 307 308 /* 309 * Explicitly poison and initialize the in-page vmalloc() redzone. 310 * Unlike software KASAN modes, hardware tag-based KASAN doesn't 311 * unpoison memory when populating shadow for vmalloc() space. 312 */ 313 redzone_start = round_up((unsigned long)start + size, 314 KASAN_GRANULE_SIZE); 315 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start; 316 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID, 317 flags & KASAN_VMALLOC_INIT); 318 319 /* 320 * Set per-page tag flags to allow accessing physical memory for the 321 * vmalloc() mapping through page_address(vmalloc_to_page()). 322 */ 323 unpoison_vmalloc_pages(start, tag); 324 325 return (void *)start; 326 } 327 328 void __kasan_poison_vmalloc(const void *start, unsigned long size) 329 { 330 /* 331 * No tagging here. 332 * The physical pages backing the vmalloc() allocation are poisoned 333 * through the usual page_alloc paths. 334 */ 335 } 336 337 #endif 338 339 void kasan_enable_tagging(void) 340 { 341 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) 342 hw_enable_tagging_async(); 343 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM) 344 hw_enable_tagging_asymm(); 345 else 346 hw_enable_tagging_sync(); 347 } 348 349 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) 350 351 EXPORT_SYMBOL_GPL(kasan_enable_tagging); 352 353 void kasan_force_async_fault(void) 354 { 355 hw_force_async_tag_fault(); 356 } 357 EXPORT_SYMBOL_GPL(kasan_force_async_fault); 358 359 #endif 360