1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This file contains core hardware tag-based KASAN code. 4 * 5 * Copyright (c) 2020 Google, Inc. 6 * Author: Andrey Konovalov <andreyknvl@google.com> 7 */ 8 9 #define pr_fmt(fmt) "kasan: " fmt 10 11 #include <linux/init.h> 12 #include <linux/kasan.h> 13 #include <linux/kernel.h> 14 #include <linux/memory.h> 15 #include <linux/mm.h> 16 #include <linux/static_key.h> 17 #include <linux/string.h> 18 #include <linux/types.h> 19 20 #include "kasan.h" 21 22 enum kasan_arg { 23 KASAN_ARG_DEFAULT, 24 KASAN_ARG_OFF, 25 KASAN_ARG_ON, 26 }; 27 28 enum kasan_arg_mode { 29 KASAN_ARG_MODE_DEFAULT, 30 KASAN_ARG_MODE_SYNC, 31 KASAN_ARG_MODE_ASYNC, 32 KASAN_ARG_MODE_ASYMM, 33 }; 34 35 enum kasan_arg_vmalloc { 36 KASAN_ARG_VMALLOC_DEFAULT, 37 KASAN_ARG_VMALLOC_OFF, 38 KASAN_ARG_VMALLOC_ON, 39 }; 40 41 static enum kasan_arg kasan_arg __ro_after_init; 42 static enum kasan_arg_mode kasan_arg_mode __ro_after_init; 43 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata; 44 45 /* 46 * Whether KASAN is enabled at all. 47 * The value remains false until KASAN is initialized by kasan_init_hw_tags(). 48 */ 49 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); 50 EXPORT_SYMBOL(kasan_flag_enabled); 51 52 /* 53 * Whether the selected mode is synchronous, asynchronous, or asymmetric. 54 * Defaults to KASAN_MODE_SYNC. 55 */ 56 enum kasan_mode kasan_mode __ro_after_init; 57 EXPORT_SYMBOL_GPL(kasan_mode); 58 59 /* Whether to enable vmalloc tagging. */ 60 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc); 61 62 /* kasan=off/on */ 63 static int __init early_kasan_flag(char *arg) 64 { 65 if (!arg) 66 return -EINVAL; 67 68 if (!strcmp(arg, "off")) 69 kasan_arg = KASAN_ARG_OFF; 70 else if (!strcmp(arg, "on")) 71 kasan_arg = KASAN_ARG_ON; 72 else 73 return -EINVAL; 74 75 return 0; 76 } 77 early_param("kasan", early_kasan_flag); 78 79 /* kasan.mode=sync/async/asymm */ 80 static int __init early_kasan_mode(char *arg) 81 { 82 if (!arg) 83 return -EINVAL; 84 85 if (!strcmp(arg, "sync")) 86 kasan_arg_mode = KASAN_ARG_MODE_SYNC; 87 else if (!strcmp(arg, "async")) 88 kasan_arg_mode = KASAN_ARG_MODE_ASYNC; 89 else if (!strcmp(arg, "asymm")) 90 kasan_arg_mode = KASAN_ARG_MODE_ASYMM; 91 else 92 return -EINVAL; 93 94 return 0; 95 } 96 early_param("kasan.mode", early_kasan_mode); 97 98 /* kasan.vmalloc=off/on */ 99 static int __init early_kasan_flag_vmalloc(char *arg) 100 { 101 if (!arg) 102 return -EINVAL; 103 104 if (!strcmp(arg, "off")) 105 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF; 106 else if (!strcmp(arg, "on")) 107 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON; 108 else 109 return -EINVAL; 110 111 return 0; 112 } 113 early_param("kasan.vmalloc", early_kasan_flag_vmalloc); 114 115 static inline const char *kasan_mode_info(void) 116 { 117 if (kasan_mode == KASAN_MODE_ASYNC) 118 return "async"; 119 else if (kasan_mode == KASAN_MODE_ASYMM) 120 return "asymm"; 121 else 122 return "sync"; 123 } 124 125 /* 126 * kasan_init_hw_tags_cpu() is called for each CPU. 127 * Not marked as __init as a CPU can be hot-plugged after boot. 128 */ 129 void kasan_init_hw_tags_cpu(void) 130 { 131 /* 132 * There's no need to check that the hardware is MTE-capable here, 133 * as this function is only called for MTE-capable hardware. 134 */ 135 136 /* 137 * If KASAN is disabled via command line, don't initialize it. 138 * When this function is called, kasan_flag_enabled is not yet 139 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead. 140 */ 141 if (kasan_arg == KASAN_ARG_OFF) 142 return; 143 144 /* 145 * Enable async or asymm modes only when explicitly requested 146 * through the command line. 147 */ 148 kasan_enable_tagging(); 149 } 150 151 /* kasan_init_hw_tags() is called once on boot CPU. */ 152 void __init kasan_init_hw_tags(void) 153 { 154 /* If hardware doesn't support MTE, don't initialize KASAN. */ 155 if (!system_supports_mte()) 156 return; 157 158 /* If KASAN is disabled via command line, don't initialize it. */ 159 if (kasan_arg == KASAN_ARG_OFF) 160 return; 161 162 switch (kasan_arg_mode) { 163 case KASAN_ARG_MODE_DEFAULT: 164 /* Default is specified by kasan_mode definition. */ 165 break; 166 case KASAN_ARG_MODE_SYNC: 167 kasan_mode = KASAN_MODE_SYNC; 168 break; 169 case KASAN_ARG_MODE_ASYNC: 170 kasan_mode = KASAN_MODE_ASYNC; 171 break; 172 case KASAN_ARG_MODE_ASYMM: 173 kasan_mode = KASAN_MODE_ASYMM; 174 break; 175 } 176 177 switch (kasan_arg_vmalloc) { 178 case KASAN_ARG_VMALLOC_DEFAULT: 179 /* Default is specified by kasan_flag_vmalloc definition. */ 180 break; 181 case KASAN_ARG_VMALLOC_OFF: 182 static_branch_disable(&kasan_flag_vmalloc); 183 break; 184 case KASAN_ARG_VMALLOC_ON: 185 static_branch_enable(&kasan_flag_vmalloc); 186 break; 187 } 188 189 kasan_init_tags(); 190 191 /* KASAN is now initialized, enable it. */ 192 static_branch_enable(&kasan_flag_enabled); 193 194 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n", 195 kasan_mode_info(), 196 kasan_vmalloc_enabled() ? "on" : "off", 197 kasan_stack_collection_enabled() ? "on" : "off"); 198 } 199 200 #ifdef CONFIG_KASAN_VMALLOC 201 202 static void unpoison_vmalloc_pages(const void *addr, u8 tag) 203 { 204 struct vm_struct *area; 205 int i; 206 207 /* 208 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations 209 * (see the comment in __kasan_unpoison_vmalloc), all of the pages 210 * should belong to a single area. 211 */ 212 area = find_vm_area((void *)addr); 213 if (WARN_ON(!area)) 214 return; 215 216 for (i = 0; i < area->nr_pages; i++) { 217 struct page *page = area->pages[i]; 218 219 page_kasan_tag_set(page, tag); 220 } 221 } 222 223 static void init_vmalloc_pages(const void *start, unsigned long size) 224 { 225 const void *addr; 226 227 for (addr = start; addr < start + size; addr += PAGE_SIZE) { 228 struct page *page = virt_to_page(addr); 229 230 clear_highpage_kasan_tagged(page); 231 } 232 } 233 234 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, 235 kasan_vmalloc_flags_t flags) 236 { 237 u8 tag; 238 unsigned long redzone_start, redzone_size; 239 240 if (!kasan_vmalloc_enabled() || !is_vmalloc_or_module_addr(start)) { 241 if (flags & KASAN_VMALLOC_INIT) 242 init_vmalloc_pages(start, size); 243 return (void *)start; 244 } 245 246 /* 247 * Don't tag non-VM_ALLOC mappings, as: 248 * 249 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only 250 * supports tagging physical memory. Therefore, it can only tag a 251 * single mapping of normal physical pages. 252 * 2. Hardware tag-based KASAN can only tag memory mapped with special 253 * mapping protection bits, see arch_vmap_pgprot_tagged(). 254 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code, 255 * providing these bits would require tracking all non-VM_ALLOC 256 * mappers. 257 * 258 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags 259 * the first virtual mapping, which is created by vmalloc(). 260 * Tagging the page_alloc memory backing that vmalloc() allocation is 261 * skipped, see ___GFP_SKIP_KASAN_UNPOISON. 262 * 263 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual. 264 */ 265 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) { 266 WARN_ON(flags & KASAN_VMALLOC_INIT); 267 return (void *)start; 268 } 269 270 /* 271 * Don't tag executable memory. 272 * The kernel doesn't tolerate having the PC register tagged. 273 */ 274 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) { 275 WARN_ON(flags & KASAN_VMALLOC_INIT); 276 return (void *)start; 277 } 278 279 tag = kasan_random_tag(); 280 start = set_tag(start, tag); 281 282 /* Unpoison and initialize memory up to size. */ 283 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT); 284 285 /* 286 * Explicitly poison and initialize the in-page vmalloc() redzone. 287 * Unlike software KASAN modes, hardware tag-based KASAN doesn't 288 * unpoison memory when populating shadow for vmalloc() space. 289 */ 290 redzone_start = round_up((unsigned long)start + size, 291 KASAN_GRANULE_SIZE); 292 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start; 293 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID, 294 flags & KASAN_VMALLOC_INIT); 295 296 /* 297 * Set per-page tag flags to allow accessing physical memory for the 298 * vmalloc() mapping through page_address(vmalloc_to_page()). 299 */ 300 unpoison_vmalloc_pages(start, tag); 301 302 return (void *)start; 303 } 304 305 void __kasan_poison_vmalloc(const void *start, unsigned long size) 306 { 307 /* 308 * No tagging here. 309 * The physical pages backing the vmalloc() allocation are poisoned 310 * through the usual page_alloc paths. 311 */ 312 } 313 314 #endif 315 316 void kasan_enable_tagging(void) 317 { 318 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) 319 hw_enable_tagging_async(); 320 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM) 321 hw_enable_tagging_asymm(); 322 else 323 hw_enable_tagging_sync(); 324 } 325 326 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) 327 328 EXPORT_SYMBOL_GPL(kasan_enable_tagging); 329 330 void kasan_force_async_fault(void) 331 { 332 hw_force_async_tag_fault(); 333 } 334 EXPORT_SYMBOL_GPL(kasan_force_async_fault); 335 336 #endif 337