xref: /openbmc/linux/mm/kmsan/instrumentation.c (revision 726ccdba)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * KMSAN compiler API.
4  *
5  * This file implements __msan_XXX hooks that Clang inserts into the code
6  * compiled with -fsanitize=kernel-memory.
7  * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN
8  * instrumentation works.
9  *
10  * Copyright (C) 2017-2022 Google LLC
11  * Author: Alexander Potapenko <glider@google.com>
12  *
13  */
14 
15 #include "kmsan.h"
16 #include <linux/gfp.h>
17 #include <linux/kmsan_string.h>
18 #include <linux/mm.h>
19 #include <linux/uaccess.h>
20 
is_bad_asm_addr(void * addr,uintptr_t size,bool is_store)21 static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store)
22 {
23 	if ((u64)addr < TASK_SIZE)
24 		return true;
25 	if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW))
26 		return true;
27 	return false;
28 }
29 
30 static inline struct shadow_origin_ptr
get_shadow_origin_ptr(void * addr,u64 size,bool store)31 get_shadow_origin_ptr(void *addr, u64 size, bool store)
32 {
33 	unsigned long ua_flags = user_access_save();
34 	struct shadow_origin_ptr ret;
35 
36 	ret = kmsan_get_shadow_origin_ptr(addr, size, store);
37 	user_access_restore(ua_flags);
38 	return ret;
39 }
40 
41 /*
42  * KMSAN instrumentation functions follow. They are not declared elsewhere in
43  * the kernel code, so they are preceded by prototypes, to silence
44  * -Wmissing-prototypes warnings.
45  */
46 
47 /* Get shadow and origin pointers for a memory load with non-standard size. */
48 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
49 							uintptr_t size);
__msan_metadata_ptr_for_load_n(void * addr,uintptr_t size)50 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
51 							uintptr_t size)
52 {
53 	return get_shadow_origin_ptr(addr, size, /*store*/ false);
54 }
55 EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n);
56 
57 /* Get shadow and origin pointers for a memory store with non-standard size. */
58 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
59 							 uintptr_t size);
__msan_metadata_ptr_for_store_n(void * addr,uintptr_t size)60 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
61 							 uintptr_t size)
62 {
63 	return get_shadow_origin_ptr(addr, size, /*store*/ true);
64 }
65 EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n);
66 
67 /*
68  * Declare functions that obtain shadow/origin pointers for loads and stores
69  * with fixed size.
70  */
71 #define DECLARE_METADATA_PTR_GETTER(size)                                  \
72 	struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size(      \
73 		void *addr);                                               \
74 	struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size(      \
75 		void *addr)                                                \
76 	{                                                                  \
77 		return get_shadow_origin_ptr(addr, size, /*store*/ false); \
78 	}                                                                  \
79 	EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size);                \
80 	struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size(     \
81 		void *addr);                                               \
82 	struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size(     \
83 		void *addr)                                                \
84 	{                                                                  \
85 		return get_shadow_origin_ptr(addr, size, /*store*/ true);  \
86 	}                                                                  \
87 	EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size)
88 
89 DECLARE_METADATA_PTR_GETTER(1);
90 DECLARE_METADATA_PTR_GETTER(2);
91 DECLARE_METADATA_PTR_GETTER(4);
92 DECLARE_METADATA_PTR_GETTER(8);
93 
94 /*
95  * Handle a memory store performed by inline assembly. KMSAN conservatively
96  * attempts to unpoison the outputs of asm() directives to prevent false
97  * positives caused by missed stores.
98  *
99  * __msan_instrument_asm_store() may be called for inline assembly code when
100  * entering or leaving IRQ. We omit the check for kmsan_in_runtime() to ensure
101  * the memory written to in these cases is also marked as initialized.
102  */
103 void __msan_instrument_asm_store(void *addr, uintptr_t size);
__msan_instrument_asm_store(void * addr,uintptr_t size)104 void __msan_instrument_asm_store(void *addr, uintptr_t size)
105 {
106 	unsigned long ua_flags;
107 
108 	if (!kmsan_enabled)
109 		return;
110 
111 	ua_flags = user_access_save();
112 	/*
113 	 * Most of the accesses are below 32 bytes. The two exceptions so far
114 	 * are clwb() (64 bytes) and FPU state (512 bytes).
115 	 * It's unlikely that the assembly will touch more than 512 bytes.
116 	 */
117 	if (size > 512) {
118 		WARN_ONCE(1, "assembly store size too big: %ld\n", size);
119 		size = 8;
120 	}
121 	if (is_bad_asm_addr(addr, size, /*is_store*/ true)) {
122 		user_access_restore(ua_flags);
123 		return;
124 	}
125 	/* Unpoisoning the memory on best effort. */
126 	kmsan_internal_unpoison_memory(addr, size, /*checked*/ false);
127 	user_access_restore(ua_flags);
128 }
129 EXPORT_SYMBOL(__msan_instrument_asm_store);
130 
131 /*
132  * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset
133  * intrinsics with calls to respective __msan_ functions. We use
134  * get_param0_metadata() and set_retval_metadata() to store the shadow/origin
135  * values for the destination argument of these functions and use them for the
136  * functions' return values.
137  */
get_param0_metadata(u64 * shadow,depot_stack_handle_t * origin)138 static inline void get_param0_metadata(u64 *shadow,
139 				       depot_stack_handle_t *origin)
140 {
141 	struct kmsan_ctx *ctx = kmsan_get_context();
142 
143 	*shadow = *(u64 *)(ctx->cstate.param_tls);
144 	*origin = ctx->cstate.param_origin_tls[0];
145 }
146 
set_retval_metadata(u64 shadow,depot_stack_handle_t origin)147 static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin)
148 {
149 	struct kmsan_ctx *ctx = kmsan_get_context();
150 
151 	*(u64 *)(ctx->cstate.retval_tls) = shadow;
152 	ctx->cstate.retval_origin_tls = origin;
153 }
154 
155 /* Handle llvm.memmove intrinsic. */
156 void *__msan_memmove(void *dst, const void *src, uintptr_t n);
__msan_memmove(void * dst,const void * src,uintptr_t n)157 void *__msan_memmove(void *dst, const void *src, uintptr_t n)
158 {
159 	depot_stack_handle_t origin;
160 	void *result;
161 	u64 shadow;
162 
163 	get_param0_metadata(&shadow, &origin);
164 	result = __memmove(dst, src, n);
165 	if (!n)
166 		/* Some people call memmove() with zero length. */
167 		return result;
168 	if (!kmsan_enabled || kmsan_in_runtime())
169 		return result;
170 
171 	kmsan_enter_runtime();
172 	kmsan_internal_memmove_metadata(dst, (void *)src, n);
173 	kmsan_leave_runtime();
174 
175 	set_retval_metadata(shadow, origin);
176 	return result;
177 }
178 EXPORT_SYMBOL(__msan_memmove);
179 
180 /* Handle llvm.memcpy intrinsic. */
181 void *__msan_memcpy(void *dst, const void *src, uintptr_t n);
__msan_memcpy(void * dst,const void * src,uintptr_t n)182 void *__msan_memcpy(void *dst, const void *src, uintptr_t n)
183 {
184 	depot_stack_handle_t origin;
185 	void *result;
186 	u64 shadow;
187 
188 	get_param0_metadata(&shadow, &origin);
189 	result = __memcpy(dst, src, n);
190 	if (!n)
191 		/* Some people call memcpy() with zero length. */
192 		return result;
193 
194 	if (!kmsan_enabled || kmsan_in_runtime())
195 		return result;
196 
197 	kmsan_enter_runtime();
198 	/* Using memmove instead of memcpy doesn't affect correctness. */
199 	kmsan_internal_memmove_metadata(dst, (void *)src, n);
200 	kmsan_leave_runtime();
201 
202 	set_retval_metadata(shadow, origin);
203 	return result;
204 }
205 EXPORT_SYMBOL(__msan_memcpy);
206 
207 /* Handle llvm.memset intrinsic. */
208 void *__msan_memset(void *dst, int c, uintptr_t n);
__msan_memset(void * dst,int c,uintptr_t n)209 void *__msan_memset(void *dst, int c, uintptr_t n)
210 {
211 	depot_stack_handle_t origin;
212 	void *result;
213 	u64 shadow;
214 
215 	get_param0_metadata(&shadow, &origin);
216 	result = __memset(dst, c, n);
217 	if (!kmsan_enabled || kmsan_in_runtime())
218 		return result;
219 
220 	kmsan_enter_runtime();
221 	/*
222 	 * Clang doesn't pass parameter metadata here, so it is impossible to
223 	 * use shadow of @c to set up the shadow for @dst.
224 	 */
225 	kmsan_internal_unpoison_memory(dst, n, /*checked*/ false);
226 	kmsan_leave_runtime();
227 
228 	set_retval_metadata(shadow, origin);
229 	return result;
230 }
231 EXPORT_SYMBOL(__msan_memset);
232 
233 /*
234  * Create a new origin from an old one. This is done when storing an
235  * uninitialized value to memory. When reporting an error, KMSAN unrolls and
236  * prints the whole chain of stores that preceded the use of this value.
237  */
238 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin);
__msan_chain_origin(depot_stack_handle_t origin)239 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin)
240 {
241 	depot_stack_handle_t ret = 0;
242 	unsigned long ua_flags;
243 
244 	if (!kmsan_enabled || kmsan_in_runtime())
245 		return ret;
246 
247 	ua_flags = user_access_save();
248 
249 	/* Creating new origins may allocate memory. */
250 	kmsan_enter_runtime();
251 	ret = kmsan_internal_chain_origin(origin);
252 	kmsan_leave_runtime();
253 	user_access_restore(ua_flags);
254 	return ret;
255 }
256 EXPORT_SYMBOL(__msan_chain_origin);
257 
258 /* Poison a local variable when entering a function. */
259 void __msan_poison_alloca(void *address, uintptr_t size, char *descr);
__msan_poison_alloca(void * address,uintptr_t size,char * descr)260 void __msan_poison_alloca(void *address, uintptr_t size, char *descr)
261 {
262 	depot_stack_handle_t handle;
263 	unsigned long entries[4];
264 	unsigned long ua_flags;
265 
266 	if (!kmsan_enabled || kmsan_in_runtime())
267 		return;
268 
269 	ua_flags = user_access_save();
270 	entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN;
271 	entries[1] = (u64)descr;
272 	entries[2] = (u64)__builtin_return_address(0);
273 	/*
274 	 * With frame pointers enabled, it is possible to quickly fetch the
275 	 * second frame of the caller stack without calling the unwinder.
276 	 * Without them, simply do not bother.
277 	 */
278 	if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER))
279 		entries[3] = (u64)__builtin_return_address(1);
280 	else
281 		entries[3] = 0;
282 
283 	/* stack_depot_save() may allocate memory. */
284 	kmsan_enter_runtime();
285 	handle = stack_depot_save(entries, ARRAY_SIZE(entries), __GFP_HIGH);
286 	kmsan_leave_runtime();
287 
288 	kmsan_internal_set_shadow_origin(address, size, -1, handle,
289 					 /*checked*/ true);
290 	user_access_restore(ua_flags);
291 }
292 EXPORT_SYMBOL(__msan_poison_alloca);
293 
294 /* Unpoison a local variable. */
295 void __msan_unpoison_alloca(void *address, uintptr_t size);
__msan_unpoison_alloca(void * address,uintptr_t size)296 void __msan_unpoison_alloca(void *address, uintptr_t size)
297 {
298 	if (!kmsan_enabled || kmsan_in_runtime())
299 		return;
300 
301 	kmsan_enter_runtime();
302 	kmsan_internal_unpoison_memory(address, size, /*checked*/ true);
303 	kmsan_leave_runtime();
304 }
305 EXPORT_SYMBOL(__msan_unpoison_alloca);
306 
307 /*
308  * Report that an uninitialized value with the given origin was used in a way
309  * that constituted undefined behavior.
310  */
311 void __msan_warning(u32 origin);
__msan_warning(u32 origin)312 void __msan_warning(u32 origin)
313 {
314 	if (!kmsan_enabled || kmsan_in_runtime())
315 		return;
316 	kmsan_enter_runtime();
317 	kmsan_report(origin, /*address*/ 0, /*size*/ 0,
318 		     /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ 0,
319 		     REASON_ANY);
320 	kmsan_leave_runtime();
321 }
322 EXPORT_SYMBOL(__msan_warning);
323 
324 /*
325  * At the beginning of an instrumented function, obtain the pointer to
326  * `struct kmsan_context_state` holding the metadata for function parameters.
327  */
328 struct kmsan_context_state *__msan_get_context_state(void);
__msan_get_context_state(void)329 struct kmsan_context_state *__msan_get_context_state(void)
330 {
331 	return &kmsan_get_context()->cstate;
332 }
333 EXPORT_SYMBOL(__msan_get_context_state);
334