1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This file contains core hardware tag-based KASAN code. 4 * 5 * Copyright (c) 2020 Google, Inc. 6 * Author: Andrey Konovalov <andreyknvl@google.com> 7 */ 8 9 #define pr_fmt(fmt) "kasan: " fmt 10 11 #include <linux/init.h> 12 #include <linux/kasan.h> 13 #include <linux/kernel.h> 14 #include <linux/memory.h> 15 #include <linux/mm.h> 16 #include <linux/static_key.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 20 #include "kasan.h" 21 22 enum kasan_arg { 23 KASAN_ARG_DEFAULT, 24 KASAN_ARG_OFF, 25 KASAN_ARG_ON, 26 }; 27 28 enum kasan_arg_mode { 29 KASAN_ARG_MODE_DEFAULT, 30 KASAN_ARG_MODE_SYNC, 31 KASAN_ARG_MODE_ASYNC, 32 KASAN_ARG_MODE_ASYMM, 33 }; 34 35 enum kasan_arg_vmalloc { 36 KASAN_ARG_VMALLOC_DEFAULT, 37 KASAN_ARG_VMALLOC_OFF, 38 KASAN_ARG_VMALLOC_ON, 39 }; 40 41 enum kasan_arg_stacktrace { 42 KASAN_ARG_STACKTRACE_DEFAULT, 43 KASAN_ARG_STACKTRACE_OFF, 44 KASAN_ARG_STACKTRACE_ON, 45 }; 46 47 static enum kasan_arg kasan_arg __ro_after_init; 48 static enum kasan_arg_mode kasan_arg_mode __ro_after_init; 49 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata; 50 static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata; 51 52 /* 53 * Whether KASAN is enabled at all. 54 * The value remains false until KASAN is initialized by kasan_init_hw_tags(). 55 */ 56 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); 57 EXPORT_SYMBOL(kasan_flag_enabled); 58 59 /* 60 * Whether the selected mode is synchronous, asynchronous, or asymmetric. 61 * Defaults to KASAN_MODE_SYNC. 62 */ 63 enum kasan_mode kasan_mode __ro_after_init; 64 EXPORT_SYMBOL_GPL(kasan_mode); 65 66 /* Whether to enable vmalloc tagging. */ 67 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc); 68 69 /* Whether to collect alloc/free stack traces. */ 70 DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace); 71 72 /* kasan=off/on */ 73 static int __init early_kasan_flag(char *arg) 74 { 75 if (!arg) 76 return -EINVAL; 77 78 if (!strcmp(arg, "off")) 79 kasan_arg = KASAN_ARG_OFF; 80 else if (!strcmp(arg, "on")) 81 kasan_arg = KASAN_ARG_ON; 82 else 83 return -EINVAL; 84 85 return 0; 86 } 87 early_param("kasan", early_kasan_flag); 88 89 /* kasan.mode=sync/async/asymm */ 90 static int __init early_kasan_mode(char *arg) 91 { 92 if (!arg) 93 return -EINVAL; 94 95 if (!strcmp(arg, "sync")) 96 kasan_arg_mode = KASAN_ARG_MODE_SYNC; 97 else if (!strcmp(arg, "async")) 98 kasan_arg_mode = KASAN_ARG_MODE_ASYNC; 99 else if (!strcmp(arg, "asymm")) 100 kasan_arg_mode = KASAN_ARG_MODE_ASYMM; 101 else 102 return -EINVAL; 103 104 return 0; 105 } 106 early_param("kasan.mode", early_kasan_mode); 107 108 /* kasan.vmalloc=off/on */ 109 static int __init early_kasan_flag_vmalloc(char *arg) 110 { 111 if (!arg) 112 return -EINVAL; 113 114 if (!strcmp(arg, "off")) 115 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF; 116 else if (!strcmp(arg, "on")) 117 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON; 118 else 119 return -EINVAL; 120 121 return 0; 122 } 123 early_param("kasan.vmalloc", early_kasan_flag_vmalloc); 124 125 /* kasan.stacktrace=off/on */ 126 static int __init early_kasan_flag_stacktrace(char *arg) 127 { 128 if (!arg) 129 return -EINVAL; 130 131 if (!strcmp(arg, "off")) 132 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; 133 else if (!strcmp(arg, "on")) 134 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; 135 else 136 return -EINVAL; 137 138 return 0; 139 } 140 early_param("kasan.stacktrace", early_kasan_flag_stacktrace); 141 142 static inline const char *kasan_mode_info(void) 143 { 144 if (kasan_mode == KASAN_MODE_ASYNC) 145 return "async"; 146 else if (kasan_mode == KASAN_MODE_ASYMM) 147 return "asymm"; 148 else 149 return "sync"; 150 } 151 152 /* 153 * kasan_init_hw_tags_cpu() is called for each CPU. 154 * Not marked as __init as a CPU can be hot-plugged after boot. 155 */ 156 void kasan_init_hw_tags_cpu(void) 157 { 158 /* 159 * There's no need to check that the hardware is MTE-capable here, 160 * as this function is only called for MTE-capable hardware. 161 */ 162 163 /* 164 * If KASAN is disabled via command line, don't initialize it. 165 * When this function is called, kasan_flag_enabled is not yet 166 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead. 167 */ 168 if (kasan_arg == KASAN_ARG_OFF) 169 return; 170 171 /* 172 * Enable async or asymm modes only when explicitly requested 173 * through the command line. 174 */ 175 kasan_enable_tagging(); 176 } 177 178 /* kasan_init_hw_tags() is called once on boot CPU. */ 179 void __init kasan_init_hw_tags(void) 180 { 181 /* If hardware doesn't support MTE, don't initialize KASAN. */ 182 if (!system_supports_mte()) 183 return; 184 185 /* If KASAN is disabled via command line, don't initialize it. */ 186 if (kasan_arg == KASAN_ARG_OFF) 187 return; 188 189 switch (kasan_arg_mode) { 190 case KASAN_ARG_MODE_DEFAULT: 191 /* Default is specified by kasan_mode definition. */ 192 break; 193 case KASAN_ARG_MODE_SYNC: 194 kasan_mode = KASAN_MODE_SYNC; 195 break; 196 case KASAN_ARG_MODE_ASYNC: 197 kasan_mode = KASAN_MODE_ASYNC; 198 break; 199 case KASAN_ARG_MODE_ASYMM: 200 kasan_mode = KASAN_MODE_ASYMM; 201 break; 202 } 203 204 switch (kasan_arg_vmalloc) { 205 case KASAN_ARG_VMALLOC_DEFAULT: 206 /* Default is specified by kasan_flag_vmalloc definition. */ 207 break; 208 case KASAN_ARG_VMALLOC_OFF: 209 static_branch_disable(&kasan_flag_vmalloc); 210 break; 211 case KASAN_ARG_VMALLOC_ON: 212 static_branch_enable(&kasan_flag_vmalloc); 213 break; 214 } 215 216 switch (kasan_arg_stacktrace) { 217 case KASAN_ARG_STACKTRACE_DEFAULT: 218 /* Default is specified by kasan_flag_stacktrace definition. */ 219 break; 220 case KASAN_ARG_STACKTRACE_OFF: 221 static_branch_disable(&kasan_flag_stacktrace); 222 break; 223 case KASAN_ARG_STACKTRACE_ON: 224 static_branch_enable(&kasan_flag_stacktrace); 225 break; 226 } 227 228 /* KASAN is now initialized, enable it. */ 229 static_branch_enable(&kasan_flag_enabled); 230 231 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n", 232 kasan_mode_info(), 233 kasan_vmalloc_enabled() ? "on" : "off", 234 kasan_stack_collection_enabled() ? "on" : "off"); 235 } 236 237 #ifdef CONFIG_KASAN_VMALLOC 238 239 static void unpoison_vmalloc_pages(const void *addr, u8 tag) 240 { 241 struct vm_struct *area; 242 int i; 243 244 /* 245 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations 246 * (see the comment in __kasan_unpoison_vmalloc), all of the pages 247 * should belong to a single area. 248 */ 249 area = find_vm_area((void *)addr); 250 if (WARN_ON(!area)) 251 return; 252 253 for (i = 0; i < area->nr_pages; i++) { 254 struct page *page = area->pages[i]; 255 256 page_kasan_tag_set(page, tag); 257 } 258 } 259 260 static void init_vmalloc_pages(const void *start, unsigned long size) 261 { 262 const void *addr; 263 264 for (addr = start; addr < start + size; addr += PAGE_SIZE) { 265 struct page *page = virt_to_page(addr); 266 267 clear_highpage_kasan_tagged(page); 268 } 269 } 270 271 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, 272 kasan_vmalloc_flags_t flags) 273 { 274 u8 tag; 275 unsigned long redzone_start, redzone_size; 276 277 if (!kasan_vmalloc_enabled() || !is_vmalloc_or_module_addr(start)) { 278 if (flags & KASAN_VMALLOC_INIT) 279 init_vmalloc_pages(start, size); 280 return (void *)start; 281 } 282 283 /* 284 * Don't tag non-VM_ALLOC mappings, as: 285 * 286 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only 287 * supports tagging physical memory. Therefore, it can only tag a 288 * single mapping of normal physical pages. 289 * 2. Hardware tag-based KASAN can only tag memory mapped with special 290 * mapping protection bits, see arch_vmap_pgprot_tagged(). 291 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code, 292 * providing these bits would require tracking all non-VM_ALLOC 293 * mappers. 294 * 295 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags 296 * the first virtual mapping, which is created by vmalloc(). 297 * Tagging the page_alloc memory backing that vmalloc() allocation is 298 * skipped, see ___GFP_SKIP_KASAN_UNPOISON. 299 * 300 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual. 301 */ 302 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) { 303 WARN_ON(flags & KASAN_VMALLOC_INIT); 304 return (void *)start; 305 } 306 307 /* 308 * Don't tag executable memory. 309 * The kernel doesn't tolerate having the PC register tagged. 310 */ 311 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) { 312 WARN_ON(flags & KASAN_VMALLOC_INIT); 313 return (void *)start; 314 } 315 316 tag = kasan_random_tag(); 317 start = set_tag(start, tag); 318 319 /* Unpoison and initialize memory up to size. */ 320 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT); 321 322 /* 323 * Explicitly poison and initialize the in-page vmalloc() redzone. 324 * Unlike software KASAN modes, hardware tag-based KASAN doesn't 325 * unpoison memory when populating shadow for vmalloc() space. 326 */ 327 redzone_start = round_up((unsigned long)start + size, 328 KASAN_GRANULE_SIZE); 329 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start; 330 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID, 331 flags & KASAN_VMALLOC_INIT); 332 333 /* 334 * Set per-page tag flags to allow accessing physical memory for the 335 * vmalloc() mapping through page_address(vmalloc_to_page()). 336 */ 337 unpoison_vmalloc_pages(start, tag); 338 339 return (void *)start; 340 } 341 342 void __kasan_poison_vmalloc(const void *start, unsigned long size) 343 { 344 /* 345 * No tagging here. 346 * The physical pages backing the vmalloc() allocation are poisoned 347 * through the usual page_alloc paths. 348 */ 349 } 350 351 #endif 352 353 void kasan_enable_tagging(void) 354 { 355 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) 356 hw_enable_tagging_async(); 357 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM) 358 hw_enable_tagging_asymm(); 359 else 360 hw_enable_tagging_sync(); 361 } 362 363 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) 364 365 EXPORT_SYMBOL_GPL(kasan_enable_tagging); 366 367 void kasan_force_async_fault(void) 368 { 369 hw_force_async_tag_fault(); 370 } 371 EXPORT_SYMBOL_GPL(kasan_force_async_fault); 372 373 #endif 374