1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Functions used by the KMSAN runtime. 4 * 5 * Copyright (C) 2017-2022 Google LLC 6 * Author: Alexander Potapenko <glider@google.com> 7 * 8 */ 9 10 #ifndef __MM_KMSAN_KMSAN_H 11 #define __MM_KMSAN_KMSAN_H 12 13 #include <asm/pgtable_64_types.h> 14 #include <linux/irqflags.h> 15 #include <linux/sched.h> 16 #include <linux/stackdepot.h> 17 #include <linux/stacktrace.h> 18 #include <linux/nmi.h> 19 #include <linux/mm.h> 20 #include <linux/printk.h> 21 22 #define KMSAN_ALLOCA_MAGIC_ORIGIN 0xabcd0100 23 #define KMSAN_CHAIN_MAGIC_ORIGIN 0xabcd0200 24 25 #define KMSAN_POISON_NOCHECK 0x0 26 #define KMSAN_POISON_CHECK 0x1 27 #define KMSAN_POISON_FREE 0x2 28 29 #define KMSAN_ORIGIN_SIZE 4 30 #define KMSAN_MAX_ORIGIN_DEPTH 7 31 32 #define KMSAN_STACK_DEPTH 64 33 34 #define KMSAN_META_SHADOW (false) 35 #define KMSAN_META_ORIGIN (true) 36 37 extern bool kmsan_enabled; 38 extern int panic_on_kmsan; 39 40 /* 41 * KMSAN performs a lot of consistency checks that are currently enabled by 42 * default. BUG_ON is normally discouraged in the kernel, unless used for 43 * debugging, but KMSAN itself is a debugging tool, so it makes little sense to 44 * recover if something goes wrong. 45 */ 46 #define KMSAN_WARN_ON(cond) \ 47 ({ \ 48 const bool __cond = WARN_ON(cond); \ 49 if (unlikely(__cond)) { \ 50 WRITE_ONCE(kmsan_enabled, false); \ 51 if (panic_on_kmsan) { \ 52 /* Can't call panic() here because */ \ 53 /* of uaccess checks. */ \ 54 BUG(); \ 55 } \ 56 } \ 57 __cond; \ 58 }) 59 60 /* 61 * A pair of metadata pointers to be returned by the instrumentation functions. 62 */ 63 struct shadow_origin_ptr { 64 void *shadow, *origin; 65 }; 66 67 struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *addr, u64 size, 68 bool store); 69 void *kmsan_get_metadata(void *addr, bool is_origin); 70 71 enum kmsan_bug_reason { 72 REASON_ANY, 73 REASON_COPY_TO_USER, 74 REASON_SUBMIT_URB, 75 }; 76 77 void kmsan_print_origin(depot_stack_handle_t origin); 78 79 /** 80 * kmsan_report() - Report a use of uninitialized value. 81 * @origin: Stack ID of the uninitialized value. 82 * @address: Address at which the memory access happens. 83 * @size: Memory access size. 84 * @off_first: Offset (from @address) of the first byte to be reported. 85 * @off_last: Offset (from @address) of the last byte to be reported. 86 * @user_addr: When non-NULL, denotes the userspace address to which the kernel 87 * is leaking data. 88 * @reason: Error type from enum kmsan_bug_reason. 89 * 90 * kmsan_report() prints an error message for a consequent group of bytes 91 * sharing the same origin. If an uninitialized value is used in a comparison, 92 * this function is called once without specifying the addresses. When checking 93 * a memory range, KMSAN may call kmsan_report() multiple times with the same 94 * @address, @size, @user_addr and @reason, but different @off_first and 95 * @off_last corresponding to different @origin values. 96 */ 97 void kmsan_report(depot_stack_handle_t origin, void *address, int size, 98 int off_first, int off_last, const void *user_addr, 99 enum kmsan_bug_reason reason); 100 101 DECLARE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx); 102 103 static __always_inline struct kmsan_ctx *kmsan_get_context(void) 104 { 105 return in_task() ? ¤t->kmsan_ctx : raw_cpu_ptr(&kmsan_percpu_ctx); 106 } 107 108 /* 109 * When a compiler hook or KMSAN runtime function is invoked, it may make a 110 * call to instrumented code and eventually call itself recursively. To avoid 111 * that, we guard the runtime entry regions with 112 * kmsan_enter_runtime()/kmsan_leave_runtime() and exit the hook if 113 * kmsan_in_runtime() is true. 114 * 115 * Non-runtime code may occasionally get executed in nested IRQs from the 116 * runtime code (e.g. when called via smp_call_function_single()). Because some 117 * KMSAN routines may take locks (e.g. for memory allocation), we conservatively 118 * bail out instead of calling them. To minimize the effect of this (potentially 119 * missing initialization events) kmsan_in_runtime() is not checked in 120 * non-blocking runtime functions. 121 */ 122 static __always_inline bool kmsan_in_runtime(void) 123 { 124 if ((hardirq_count() >> HARDIRQ_SHIFT) > 1) 125 return true; 126 return kmsan_get_context()->kmsan_in_runtime; 127 } 128 129 static __always_inline void kmsan_enter_runtime(void) 130 { 131 struct kmsan_ctx *ctx; 132 133 ctx = kmsan_get_context(); 134 KMSAN_WARN_ON(ctx->kmsan_in_runtime++); 135 } 136 137 static __always_inline void kmsan_leave_runtime(void) 138 { 139 struct kmsan_ctx *ctx = kmsan_get_context(); 140 141 KMSAN_WARN_ON(--ctx->kmsan_in_runtime); 142 } 143 144 depot_stack_handle_t kmsan_save_stack(void); 145 depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags, 146 unsigned int extra_bits); 147 148 /* 149 * Pack and unpack the origin chain depth and UAF flag to/from the extra bits 150 * provided by the stack depot. 151 * The UAF flag is stored in the lowest bit, followed by the depth in the upper 152 * bits. 153 * set_dsh_extra_bits() is responsible for clamping the value. 154 */ 155 static __always_inline unsigned int kmsan_extra_bits(unsigned int depth, 156 bool uaf) 157 { 158 return (depth << 1) | uaf; 159 } 160 161 static __always_inline bool kmsan_uaf_from_eb(unsigned int extra_bits) 162 { 163 return extra_bits & 1; 164 } 165 166 static __always_inline unsigned int kmsan_depth_from_eb(unsigned int extra_bits) 167 { 168 return extra_bits >> 1; 169 } 170 171 /* 172 * kmsan_internal_ functions are supposed to be very simple and not require the 173 * kmsan_in_runtime() checks. 174 */ 175 void kmsan_internal_memmove_metadata(void *dst, void *src, size_t n); 176 void kmsan_internal_poison_memory(void *address, size_t size, gfp_t flags, 177 unsigned int poison_flags); 178 void kmsan_internal_unpoison_memory(void *address, size_t size, bool checked); 179 void kmsan_internal_set_shadow_origin(void *address, size_t size, int b, 180 u32 origin, bool checked); 181 depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id); 182 183 bool kmsan_metadata_is_contiguous(void *addr, size_t size); 184 void kmsan_internal_check_memory(void *addr, size_t size, const void *user_addr, 185 int reason); 186 187 struct page *kmsan_vmalloc_to_page_or_null(void *vaddr); 188 189 /* 190 * kmsan_internal_is_module_addr() and kmsan_internal_is_vmalloc_addr() are 191 * non-instrumented versions of is_module_address() and is_vmalloc_addr() that 192 * are safe to call from KMSAN runtime without recursion. 193 */ 194 static inline bool kmsan_internal_is_module_addr(void *vaddr) 195 { 196 return ((u64)vaddr >= MODULES_VADDR) && ((u64)vaddr < MODULES_END); 197 } 198 199 static inline bool kmsan_internal_is_vmalloc_addr(void *addr) 200 { 201 return ((u64)addr >= VMALLOC_START) && ((u64)addr < VMALLOC_END); 202 } 203 204 #endif /* __MM_KMSAN_KMSAN_H */ 205