1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __MM_KASAN_KASAN_H 3 #define __MM_KASAN_KASAN_H 4 5 #include <linux/kasan.h> 6 #include <linux/kfence.h> 7 #include <linux/stackdepot.h> 8 9 #ifdef CONFIG_KASAN_HW_TAGS 10 11 #include <linux/static_key.h> 12 13 DECLARE_STATIC_KEY_FALSE(kasan_flag_stacktrace); 14 extern bool kasan_flag_async __ro_after_init; 15 16 static inline bool kasan_stack_collection_enabled(void) 17 { 18 return static_branch_unlikely(&kasan_flag_stacktrace); 19 } 20 21 static inline bool kasan_async_mode_enabled(void) 22 { 23 return kasan_flag_async; 24 } 25 #else 26 27 static inline bool kasan_stack_collection_enabled(void) 28 { 29 return true; 30 } 31 32 static inline bool kasan_async_mode_enabled(void) 33 { 34 return false; 35 } 36 37 #endif 38 39 extern bool kasan_flag_panic __ro_after_init; 40 extern bool kasan_flag_async __ro_after_init; 41 42 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 43 #define KASAN_GRANULE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) 44 #else 45 #include <asm/mte-kasan.h> 46 #define KASAN_GRANULE_SIZE MTE_GRANULE_SIZE 47 #endif 48 49 #define KASAN_GRANULE_MASK (KASAN_GRANULE_SIZE - 1) 50 51 #define KASAN_MEMORY_PER_SHADOW_PAGE (KASAN_GRANULE_SIZE << PAGE_SHIFT) 52 53 #define KASAN_TAG_KERNEL 0xFF /* native kernel pointers tag */ 54 #define KASAN_TAG_INVALID 0xFE /* inaccessible memory tag */ 55 #define KASAN_TAG_MAX 0xFD /* maximum value for random tags */ 56 57 #ifdef CONFIG_KASAN_HW_TAGS 58 #define KASAN_TAG_MIN 0xF0 /* minimum value for random tags */ 59 #else 60 #define KASAN_TAG_MIN 0x00 /* minimum value for random tags */ 61 #endif 62 63 #ifdef CONFIG_KASAN_GENERIC 64 #define KASAN_FREE_PAGE 0xFF /* page was freed */ 65 #define KASAN_PAGE_REDZONE 0xFE /* redzone for kmalloc_large allocations */ 66 #define KASAN_KMALLOC_REDZONE 0xFC /* redzone inside slub object */ 67 #define KASAN_KMALLOC_FREE 0xFB /* object was freed (kmem_cache_free/kfree) */ 68 #define KASAN_KMALLOC_FREETRACK 0xFA /* object was freed and has free track set */ 69 #else 70 #define KASAN_FREE_PAGE KASAN_TAG_INVALID 71 #define KASAN_PAGE_REDZONE KASAN_TAG_INVALID 72 #define KASAN_KMALLOC_REDZONE KASAN_TAG_INVALID 73 #define KASAN_KMALLOC_FREE KASAN_TAG_INVALID 74 #define KASAN_KMALLOC_FREETRACK KASAN_TAG_INVALID 75 #endif 76 77 #define KASAN_GLOBAL_REDZONE 0xF9 /* redzone for global variable */ 78 #define KASAN_VMALLOC_INVALID 0xF8 /* unallocated space in vmapped page */ 79 80 /* 81 * Stack redzone shadow values 82 * (Those are compiler's ABI, don't change them) 83 */ 84 #define KASAN_STACK_LEFT 0xF1 85 #define KASAN_STACK_MID 0xF2 86 #define KASAN_STACK_RIGHT 0xF3 87 #define KASAN_STACK_PARTIAL 0xF4 88 89 /* 90 * alloca redzone shadow values 91 */ 92 #define KASAN_ALLOCA_LEFT 0xCA 93 #define KASAN_ALLOCA_RIGHT 0xCB 94 95 #define KASAN_ALLOCA_REDZONE_SIZE 32 96 97 /* 98 * Stack frame marker (compiler ABI). 99 */ 100 #define KASAN_CURRENT_STACK_FRAME_MAGIC 0x41B58AB3 101 102 /* Don't break randconfig/all*config builds */ 103 #ifndef KASAN_ABI_VERSION 104 #define KASAN_ABI_VERSION 1 105 #endif 106 107 /* Metadata layout customization. */ 108 #define META_BYTES_PER_BLOCK 1 109 #define META_BLOCKS_PER_ROW 16 110 #define META_BYTES_PER_ROW (META_BLOCKS_PER_ROW * META_BYTES_PER_BLOCK) 111 #define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE) 112 #define META_ROWS_AROUND_ADDR 2 113 114 struct kasan_access_info { 115 const void *access_addr; 116 const void *first_bad_addr; 117 size_t access_size; 118 bool is_write; 119 unsigned long ip; 120 }; 121 122 /* The layout of struct dictated by compiler */ 123 struct kasan_source_location { 124 const char *filename; 125 int line_no; 126 int column_no; 127 }; 128 129 /* The layout of struct dictated by compiler */ 130 struct kasan_global { 131 const void *beg; /* Address of the beginning of the global variable. */ 132 size_t size; /* Size of the global variable. */ 133 size_t size_with_redzone; /* Size of the variable + size of the red zone. 32 bytes aligned */ 134 const void *name; 135 const void *module_name; /* Name of the module where the global variable is declared. */ 136 unsigned long has_dynamic_init; /* This needed for C++ */ 137 #if KASAN_ABI_VERSION >= 4 138 struct kasan_source_location *location; 139 #endif 140 #if KASAN_ABI_VERSION >= 5 141 char *odr_indicator; 142 #endif 143 }; 144 145 /** 146 * Structures to keep alloc and free tracks * 147 */ 148 149 #define KASAN_STACK_DEPTH 64 150 151 struct kasan_track { 152 u32 pid; 153 depot_stack_handle_t stack; 154 }; 155 156 #if defined(CONFIG_KASAN_TAGS_IDENTIFY) && defined(CONFIG_KASAN_SW_TAGS) 157 #define KASAN_NR_FREE_STACKS 5 158 #else 159 #define KASAN_NR_FREE_STACKS 1 160 #endif 161 162 struct kasan_alloc_meta { 163 struct kasan_track alloc_track; 164 #ifdef CONFIG_KASAN_GENERIC 165 /* 166 * The auxiliary stack is stored into struct kasan_alloc_meta. 167 * The free stack is stored into struct kasan_free_meta. 168 */ 169 depot_stack_handle_t aux_stack[2]; 170 #else 171 struct kasan_track free_track[KASAN_NR_FREE_STACKS]; 172 #endif 173 #ifdef CONFIG_KASAN_TAGS_IDENTIFY 174 u8 free_pointer_tag[KASAN_NR_FREE_STACKS]; 175 u8 free_track_idx; 176 #endif 177 }; 178 179 struct qlist_node { 180 struct qlist_node *next; 181 }; 182 183 /* 184 * Generic mode either stores free meta in the object itself or in the redzone 185 * after the object. In the former case free meta offset is 0, in the latter 186 * case it has some sane value smaller than INT_MAX. Use INT_MAX as free meta 187 * offset when free meta isn't present. 188 */ 189 #define KASAN_NO_FREE_META INT_MAX 190 191 struct kasan_free_meta { 192 #ifdef CONFIG_KASAN_GENERIC 193 /* This field is used while the object is in the quarantine. 194 * Otherwise it might be used for the allocator freelist. 195 */ 196 struct qlist_node quarantine_link; 197 struct kasan_track free_track; 198 #endif 199 }; 200 201 struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, 202 const void *object); 203 #ifdef CONFIG_KASAN_GENERIC 204 struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, 205 const void *object); 206 #endif 207 208 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 209 210 static inline const void *kasan_shadow_to_mem(const void *shadow_addr) 211 { 212 return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET) 213 << KASAN_SHADOW_SCALE_SHIFT); 214 } 215 216 static inline bool addr_has_metadata(const void *addr) 217 { 218 return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START)); 219 } 220 221 /** 222 * kasan_check_range - Check memory region, and report if invalid access. 223 * @addr: the accessed address 224 * @size: the accessed size 225 * @write: true if access is a write access 226 * @ret_ip: return address 227 * @return: true if access was valid, false if invalid 228 */ 229 bool kasan_check_range(unsigned long addr, size_t size, bool write, 230 unsigned long ret_ip); 231 232 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ 233 234 static inline bool addr_has_metadata(const void *addr) 235 { 236 return (is_vmalloc_addr(addr) || virt_addr_valid(addr)); 237 } 238 239 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ 240 241 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) 242 void kasan_print_tags(u8 addr_tag, const void *addr); 243 #else 244 static inline void kasan_print_tags(u8 addr_tag, const void *addr) { } 245 #endif 246 247 void *kasan_find_first_bad_addr(void *addr, size_t size); 248 const char *kasan_get_bug_type(struct kasan_access_info *info); 249 void kasan_metadata_fetch_row(char *buffer, void *row); 250 251 #if defined(CONFIG_KASAN_GENERIC) && defined(CONFIG_KASAN_STACK) 252 void kasan_print_address_stack_frame(const void *addr); 253 #else 254 static inline void kasan_print_address_stack_frame(const void *addr) { } 255 #endif 256 257 bool kasan_report(unsigned long addr, size_t size, 258 bool is_write, unsigned long ip); 259 void kasan_report_invalid_free(void *object, unsigned long ip); 260 261 struct page *kasan_addr_to_page(const void *addr); 262 263 depot_stack_handle_t kasan_save_stack(gfp_t flags); 264 void kasan_set_track(struct kasan_track *track, gfp_t flags); 265 void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag); 266 struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, 267 void *object, u8 tag); 268 269 #if defined(CONFIG_KASAN_GENERIC) && \ 270 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB)) 271 bool kasan_quarantine_put(struct kmem_cache *cache, void *object); 272 void kasan_quarantine_reduce(void); 273 void kasan_quarantine_remove_cache(struct kmem_cache *cache); 274 #else 275 static inline bool kasan_quarantine_put(struct kmem_cache *cache, void *object) { return false; } 276 static inline void kasan_quarantine_reduce(void) { } 277 static inline void kasan_quarantine_remove_cache(struct kmem_cache *cache) { } 278 #endif 279 280 #ifndef arch_kasan_set_tag 281 static inline const void *arch_kasan_set_tag(const void *addr, u8 tag) 282 { 283 return addr; 284 } 285 #endif 286 #ifndef arch_kasan_get_tag 287 #define arch_kasan_get_tag(addr) 0 288 #endif 289 290 #define set_tag(addr, tag) ((void *)arch_kasan_set_tag((addr), (tag))) 291 #define get_tag(addr) arch_kasan_get_tag(addr) 292 293 #ifdef CONFIG_KASAN_HW_TAGS 294 295 #ifndef arch_enable_tagging_sync 296 #define arch_enable_tagging_sync() 297 #endif 298 #ifndef arch_enable_tagging_async 299 #define arch_enable_tagging_async() 300 #endif 301 #ifndef arch_init_tags 302 #define arch_init_tags(max_tag) 303 #endif 304 #ifndef arch_set_tagging_report_once 305 #define arch_set_tagging_report_once(state) 306 #endif 307 #ifndef arch_force_async_tag_fault 308 #define arch_force_async_tag_fault() 309 #endif 310 #ifndef arch_get_random_tag 311 #define arch_get_random_tag() (0xFF) 312 #endif 313 #ifndef arch_get_mem_tag 314 #define arch_get_mem_tag(addr) (0xFF) 315 #endif 316 #ifndef arch_set_mem_tag_range 317 #define arch_set_mem_tag_range(addr, size, tag, init) ((void *)(addr)) 318 #endif 319 320 #define hw_enable_tagging_sync() arch_enable_tagging_sync() 321 #define hw_enable_tagging_async() arch_enable_tagging_async() 322 #define hw_init_tags(max_tag) arch_init_tags(max_tag) 323 #define hw_set_tagging_report_once(state) arch_set_tagging_report_once(state) 324 #define hw_force_async_tag_fault() arch_force_async_tag_fault() 325 #define hw_get_random_tag() arch_get_random_tag() 326 #define hw_get_mem_tag(addr) arch_get_mem_tag(addr) 327 #define hw_set_mem_tag_range(addr, size, tag, init) \ 328 arch_set_mem_tag_range((addr), (size), (tag), (init)) 329 330 #else /* CONFIG_KASAN_HW_TAGS */ 331 332 #define hw_enable_tagging_sync() 333 #define hw_enable_tagging_async() 334 #define hw_set_tagging_report_once(state) 335 336 #endif /* CONFIG_KASAN_HW_TAGS */ 337 338 #if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) 339 340 void kasan_set_tagging_report_once(bool state); 341 void kasan_enable_tagging_sync(void); 342 void kasan_force_async_fault(void); 343 344 #else /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */ 345 346 static inline void kasan_set_tagging_report_once(bool state) { } 347 static inline void kasan_enable_tagging_sync(void) { } 348 static inline void kasan_force_async_fault(void) { } 349 350 #endif /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */ 351 352 #ifdef CONFIG_KASAN_SW_TAGS 353 u8 kasan_random_tag(void); 354 #elif defined(CONFIG_KASAN_HW_TAGS) 355 static inline u8 kasan_random_tag(void) { return hw_get_random_tag(); } 356 #else 357 static inline u8 kasan_random_tag(void) { return 0; } 358 #endif 359 360 #ifdef CONFIG_KASAN_HW_TAGS 361 362 static inline void kasan_poison(const void *addr, size_t size, u8 value, bool init) 363 { 364 addr = kasan_reset_tag(addr); 365 366 /* Skip KFENCE memory if called explicitly outside of sl*b. */ 367 if (is_kfence_address(addr)) 368 return; 369 370 if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK)) 371 return; 372 if (WARN_ON(size & KASAN_GRANULE_MASK)) 373 return; 374 375 hw_set_mem_tag_range((void *)addr, size, value, init); 376 } 377 378 static inline void kasan_unpoison(const void *addr, size_t size, bool init) 379 { 380 u8 tag = get_tag(addr); 381 382 addr = kasan_reset_tag(addr); 383 384 /* Skip KFENCE memory if called explicitly outside of sl*b. */ 385 if (is_kfence_address(addr)) 386 return; 387 388 if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK)) 389 return; 390 size = round_up(size, KASAN_GRANULE_SIZE); 391 392 hw_set_mem_tag_range((void *)addr, size, tag, init); 393 } 394 395 static inline bool kasan_byte_accessible(const void *addr) 396 { 397 u8 ptr_tag = get_tag(addr); 398 u8 mem_tag = hw_get_mem_tag((void *)addr); 399 400 return ptr_tag == KASAN_TAG_KERNEL || ptr_tag == mem_tag; 401 } 402 403 #else /* CONFIG_KASAN_HW_TAGS */ 404 405 /** 406 * kasan_poison - mark the memory range as inaccessible 407 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE 408 * @size - range size, must be aligned to KASAN_GRANULE_SIZE 409 * @value - value that's written to metadata for the range 410 * @init - whether to initialize the memory range (only for hardware tag-based) 411 * 412 * The size gets aligned to KASAN_GRANULE_SIZE before marking the range. 413 */ 414 void kasan_poison(const void *addr, size_t size, u8 value, bool init); 415 416 /** 417 * kasan_unpoison - mark the memory range as accessible 418 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE 419 * @size - range size, can be unaligned 420 * @init - whether to initialize the memory range (only for hardware tag-based) 421 * 422 * For the tag-based modes, the @size gets aligned to KASAN_GRANULE_SIZE before 423 * marking the range. 424 * For the generic mode, the last granule of the memory range gets partially 425 * unpoisoned based on the @size. 426 */ 427 void kasan_unpoison(const void *addr, size_t size, bool init); 428 429 bool kasan_byte_accessible(const void *addr); 430 431 #endif /* CONFIG_KASAN_HW_TAGS */ 432 433 #ifdef CONFIG_KASAN_GENERIC 434 435 /** 436 * kasan_poison_last_granule - mark the last granule of the memory range as 437 * inaccessible 438 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE 439 * @size - range size 440 * 441 * This function is only available for the generic mode, as it's the only mode 442 * that has partially poisoned memory granules. 443 */ 444 void kasan_poison_last_granule(const void *address, size_t size); 445 446 #else /* CONFIG_KASAN_GENERIC */ 447 448 static inline void kasan_poison_last_granule(const void *address, size_t size) { } 449 450 #endif /* CONFIG_KASAN_GENERIC */ 451 452 #ifndef kasan_arch_is_ready 453 static inline bool kasan_arch_is_ready(void) { return true; } 454 #elif !defined(CONFIG_KASAN_GENERIC) || !defined(CONFIG_KASAN_OUTLINE) 455 #error kasan_arch_is_ready only works in KASAN generic outline mode! 456 #endif 457 458 /* 459 * Exported functions for interfaces called from assembly or from generated 460 * code. Declarations here to avoid warning about missing declarations. 461 */ 462 asmlinkage void kasan_unpoison_task_stack_below(const void *watermark); 463 void __asan_register_globals(struct kasan_global *globals, size_t size); 464 void __asan_unregister_globals(struct kasan_global *globals, size_t size); 465 void __asan_handle_no_return(void); 466 void __asan_alloca_poison(unsigned long addr, size_t size); 467 void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom); 468 469 void __asan_load1(unsigned long addr); 470 void __asan_store1(unsigned long addr); 471 void __asan_load2(unsigned long addr); 472 void __asan_store2(unsigned long addr); 473 void __asan_load4(unsigned long addr); 474 void __asan_store4(unsigned long addr); 475 void __asan_load8(unsigned long addr); 476 void __asan_store8(unsigned long addr); 477 void __asan_load16(unsigned long addr); 478 void __asan_store16(unsigned long addr); 479 void __asan_loadN(unsigned long addr, size_t size); 480 void __asan_storeN(unsigned long addr, size_t size); 481 482 void __asan_load1_noabort(unsigned long addr); 483 void __asan_store1_noabort(unsigned long addr); 484 void __asan_load2_noabort(unsigned long addr); 485 void __asan_store2_noabort(unsigned long addr); 486 void __asan_load4_noabort(unsigned long addr); 487 void __asan_store4_noabort(unsigned long addr); 488 void __asan_load8_noabort(unsigned long addr); 489 void __asan_store8_noabort(unsigned long addr); 490 void __asan_load16_noabort(unsigned long addr); 491 void __asan_store16_noabort(unsigned long addr); 492 void __asan_loadN_noabort(unsigned long addr, size_t size); 493 void __asan_storeN_noabort(unsigned long addr, size_t size); 494 495 void __asan_report_load1_noabort(unsigned long addr); 496 void __asan_report_store1_noabort(unsigned long addr); 497 void __asan_report_load2_noabort(unsigned long addr); 498 void __asan_report_store2_noabort(unsigned long addr); 499 void __asan_report_load4_noabort(unsigned long addr); 500 void __asan_report_store4_noabort(unsigned long addr); 501 void __asan_report_load8_noabort(unsigned long addr); 502 void __asan_report_store8_noabort(unsigned long addr); 503 void __asan_report_load16_noabort(unsigned long addr); 504 void __asan_report_store16_noabort(unsigned long addr); 505 void __asan_report_load_n_noabort(unsigned long addr, size_t size); 506 void __asan_report_store_n_noabort(unsigned long addr, size_t size); 507 508 void __asan_set_shadow_00(const void *addr, size_t size); 509 void __asan_set_shadow_f1(const void *addr, size_t size); 510 void __asan_set_shadow_f2(const void *addr, size_t size); 511 void __asan_set_shadow_f3(const void *addr, size_t size); 512 void __asan_set_shadow_f5(const void *addr, size_t size); 513 void __asan_set_shadow_f8(const void *addr, size_t size); 514 515 void __hwasan_load1_noabort(unsigned long addr); 516 void __hwasan_store1_noabort(unsigned long addr); 517 void __hwasan_load2_noabort(unsigned long addr); 518 void __hwasan_store2_noabort(unsigned long addr); 519 void __hwasan_load4_noabort(unsigned long addr); 520 void __hwasan_store4_noabort(unsigned long addr); 521 void __hwasan_load8_noabort(unsigned long addr); 522 void __hwasan_store8_noabort(unsigned long addr); 523 void __hwasan_load16_noabort(unsigned long addr); 524 void __hwasan_store16_noabort(unsigned long addr); 525 void __hwasan_loadN_noabort(unsigned long addr, size_t size); 526 void __hwasan_storeN_noabort(unsigned long addr, size_t size); 527 528 void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size); 529 530 #endif 531