1f80be457SAlexander Potapenko // SPDX-License-Identifier: GPL-2.0 2f80be457SAlexander Potapenko /* 3f80be457SAlexander Potapenko * KMSAN compiler API. 4f80be457SAlexander Potapenko * 5f80be457SAlexander Potapenko * This file implements __msan_XXX hooks that Clang inserts into the code 6f80be457SAlexander Potapenko * compiled with -fsanitize=kernel-memory. 7f80be457SAlexander Potapenko * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN 8f80be457SAlexander Potapenko * instrumentation works. 9f80be457SAlexander Potapenko * 10f80be457SAlexander Potapenko * Copyright (C) 2017-2022 Google LLC 11f80be457SAlexander Potapenko * Author: Alexander Potapenko <glider@google.com> 12f80be457SAlexander Potapenko * 13f80be457SAlexander Potapenko */ 14f80be457SAlexander Potapenko 15f80be457SAlexander Potapenko #include "kmsan.h" 16f80be457SAlexander Potapenko #include <linux/gfp.h> 1778a498c3SAlexander Potapenko #include <linux/kmsan_string.h> 18f80be457SAlexander Potapenko #include <linux/mm.h> 19f80be457SAlexander Potapenko #include <linux/uaccess.h> 20f80be457SAlexander Potapenko 21f80be457SAlexander Potapenko static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store) 22f80be457SAlexander Potapenko { 23f80be457SAlexander Potapenko if ((u64)addr < TASK_SIZE) 24f80be457SAlexander Potapenko return true; 25f80be457SAlexander Potapenko if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW)) 26f80be457SAlexander Potapenko return true; 27f80be457SAlexander Potapenko return false; 28f80be457SAlexander Potapenko } 29f80be457SAlexander Potapenko 30f80be457SAlexander Potapenko static inline struct shadow_origin_ptr 31f80be457SAlexander Potapenko get_shadow_origin_ptr(void *addr, u64 size, bool store) 32f80be457SAlexander Potapenko { 33f80be457SAlexander Potapenko unsigned long ua_flags = user_access_save(); 34f80be457SAlexander Potapenko struct shadow_origin_ptr ret; 35f80be457SAlexander Potapenko 36f80be457SAlexander Potapenko ret = kmsan_get_shadow_origin_ptr(addr, size, store); 37f80be457SAlexander Potapenko user_access_restore(ua_flags); 38f80be457SAlexander Potapenko return ret; 39f80be457SAlexander Potapenko } 40f80be457SAlexander Potapenko 41*62a9bbf2SAlexander Potapenko /* 42*62a9bbf2SAlexander Potapenko * KMSAN instrumentation functions follow. They are not declared elsewhere in 43*62a9bbf2SAlexander Potapenko * the kernel code, so they are preceded by prototypes, to silence 44*62a9bbf2SAlexander Potapenko * -Wmissing-prototypes warnings. 45*62a9bbf2SAlexander Potapenko */ 46*62a9bbf2SAlexander Potapenko 47f80be457SAlexander Potapenko /* Get shadow and origin pointers for a memory load with non-standard size. */ 48f80be457SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr, 49*62a9bbf2SAlexander Potapenko uintptr_t size); 50*62a9bbf2SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr, 51f80be457SAlexander Potapenko uintptr_t size) 52f80be457SAlexander Potapenko { 53f80be457SAlexander Potapenko return get_shadow_origin_ptr(addr, size, /*store*/ false); 54f80be457SAlexander Potapenko } 55f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n); 56f80be457SAlexander Potapenko 57f80be457SAlexander Potapenko /* Get shadow and origin pointers for a memory store with non-standard size. */ 58f80be457SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr, 59*62a9bbf2SAlexander Potapenko uintptr_t size); 60*62a9bbf2SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr, 61f80be457SAlexander Potapenko uintptr_t size) 62f80be457SAlexander Potapenko { 63f80be457SAlexander Potapenko return get_shadow_origin_ptr(addr, size, /*store*/ true); 64f80be457SAlexander Potapenko } 65f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n); 66f80be457SAlexander Potapenko 67f80be457SAlexander Potapenko /* 68f80be457SAlexander Potapenko * Declare functions that obtain shadow/origin pointers for loads and stores 69f80be457SAlexander Potapenko * with fixed size. 70f80be457SAlexander Potapenko */ 71f80be457SAlexander Potapenko #define DECLARE_METADATA_PTR_GETTER(size) \ 72f80be457SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \ 73*62a9bbf2SAlexander Potapenko void *addr); \ 74*62a9bbf2SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \ 75f80be457SAlexander Potapenko void *addr) \ 76f80be457SAlexander Potapenko { \ 77f80be457SAlexander Potapenko return get_shadow_origin_ptr(addr, size, /*store*/ false); \ 78f80be457SAlexander Potapenko } \ 79f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size); \ 80f80be457SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \ 81*62a9bbf2SAlexander Potapenko void *addr); \ 82*62a9bbf2SAlexander Potapenko struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \ 83f80be457SAlexander Potapenko void *addr) \ 84f80be457SAlexander Potapenko { \ 85f80be457SAlexander Potapenko return get_shadow_origin_ptr(addr, size, /*store*/ true); \ 86f80be457SAlexander Potapenko } \ 87f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size) 88f80be457SAlexander Potapenko 89f80be457SAlexander Potapenko DECLARE_METADATA_PTR_GETTER(1); 90f80be457SAlexander Potapenko DECLARE_METADATA_PTR_GETTER(2); 91f80be457SAlexander Potapenko DECLARE_METADATA_PTR_GETTER(4); 92f80be457SAlexander Potapenko DECLARE_METADATA_PTR_GETTER(8); 93f80be457SAlexander Potapenko 94f80be457SAlexander Potapenko /* 95f80be457SAlexander Potapenko * Handle a memory store performed by inline assembly. KMSAN conservatively 96f80be457SAlexander Potapenko * attempts to unpoison the outputs of asm() directives to prevent false 97f80be457SAlexander Potapenko * positives caused by missed stores. 9885716a80SAlexander Potapenko * 9985716a80SAlexander Potapenko * __msan_instrument_asm_store() may be called for inline assembly code when 10085716a80SAlexander Potapenko * entering or leaving IRQ. We omit the check for kmsan_in_runtime() to ensure 10185716a80SAlexander Potapenko * the memory written to in these cases is also marked as initialized. 102f80be457SAlexander Potapenko */ 103*62a9bbf2SAlexander Potapenko void __msan_instrument_asm_store(void *addr, uintptr_t size); 104f80be457SAlexander Potapenko void __msan_instrument_asm_store(void *addr, uintptr_t size) 105f80be457SAlexander Potapenko { 106f80be457SAlexander Potapenko unsigned long ua_flags; 107f80be457SAlexander Potapenko 10885716a80SAlexander Potapenko if (!kmsan_enabled) 109f80be457SAlexander Potapenko return; 110f80be457SAlexander Potapenko 111f80be457SAlexander Potapenko ua_flags = user_access_save(); 112f80be457SAlexander Potapenko /* 113f80be457SAlexander Potapenko * Most of the accesses are below 32 bytes. The two exceptions so far 114f80be457SAlexander Potapenko * are clwb() (64 bytes) and FPU state (512 bytes). 115f80be457SAlexander Potapenko * It's unlikely that the assembly will touch more than 512 bytes. 116f80be457SAlexander Potapenko */ 117f80be457SAlexander Potapenko if (size > 512) { 118f80be457SAlexander Potapenko WARN_ONCE(1, "assembly store size too big: %ld\n", size); 119f80be457SAlexander Potapenko size = 8; 120f80be457SAlexander Potapenko } 121f80be457SAlexander Potapenko if (is_bad_asm_addr(addr, size, /*is_store*/ true)) { 122f80be457SAlexander Potapenko user_access_restore(ua_flags); 123f80be457SAlexander Potapenko return; 124f80be457SAlexander Potapenko } 125f80be457SAlexander Potapenko /* Unpoisoning the memory on best effort. */ 126f80be457SAlexander Potapenko kmsan_internal_unpoison_memory(addr, size, /*checked*/ false); 127f80be457SAlexander Potapenko user_access_restore(ua_flags); 128f80be457SAlexander Potapenko } 129f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_instrument_asm_store); 130f80be457SAlexander Potapenko 131f80be457SAlexander Potapenko /* 132f80be457SAlexander Potapenko * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset 133f80be457SAlexander Potapenko * intrinsics with calls to respective __msan_ functions. We use 134f80be457SAlexander Potapenko * get_param0_metadata() and set_retval_metadata() to store the shadow/origin 135f80be457SAlexander Potapenko * values for the destination argument of these functions and use them for the 136f80be457SAlexander Potapenko * functions' return values. 137f80be457SAlexander Potapenko */ 138f80be457SAlexander Potapenko static inline void get_param0_metadata(u64 *shadow, 139f80be457SAlexander Potapenko depot_stack_handle_t *origin) 140f80be457SAlexander Potapenko { 141f80be457SAlexander Potapenko struct kmsan_ctx *ctx = kmsan_get_context(); 142f80be457SAlexander Potapenko 143f80be457SAlexander Potapenko *shadow = *(u64 *)(ctx->cstate.param_tls); 144f80be457SAlexander Potapenko *origin = ctx->cstate.param_origin_tls[0]; 145f80be457SAlexander Potapenko } 146f80be457SAlexander Potapenko 147f80be457SAlexander Potapenko static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin) 148f80be457SAlexander Potapenko { 149f80be457SAlexander Potapenko struct kmsan_ctx *ctx = kmsan_get_context(); 150f80be457SAlexander Potapenko 151f80be457SAlexander Potapenko *(u64 *)(ctx->cstate.retval_tls) = shadow; 152f80be457SAlexander Potapenko ctx->cstate.retval_origin_tls = origin; 153f80be457SAlexander Potapenko } 154f80be457SAlexander Potapenko 155f80be457SAlexander Potapenko /* Handle llvm.memmove intrinsic. */ 156*62a9bbf2SAlexander Potapenko void *__msan_memmove(void *dst, const void *src, uintptr_t n); 157f80be457SAlexander Potapenko void *__msan_memmove(void *dst, const void *src, uintptr_t n) 158f80be457SAlexander Potapenko { 159f80be457SAlexander Potapenko depot_stack_handle_t origin; 160f80be457SAlexander Potapenko void *result; 161f80be457SAlexander Potapenko u64 shadow; 162f80be457SAlexander Potapenko 163f80be457SAlexander Potapenko get_param0_metadata(&shadow, &origin); 164f80be457SAlexander Potapenko result = __memmove(dst, src, n); 165f80be457SAlexander Potapenko if (!n) 166f80be457SAlexander Potapenko /* Some people call memmove() with zero length. */ 167f80be457SAlexander Potapenko return result; 168f80be457SAlexander Potapenko if (!kmsan_enabled || kmsan_in_runtime()) 169f80be457SAlexander Potapenko return result; 170f80be457SAlexander Potapenko 171f80be457SAlexander Potapenko kmsan_enter_runtime(); 172f80be457SAlexander Potapenko kmsan_internal_memmove_metadata(dst, (void *)src, n); 173f80be457SAlexander Potapenko kmsan_leave_runtime(); 174f80be457SAlexander Potapenko 175f80be457SAlexander Potapenko set_retval_metadata(shadow, origin); 176f80be457SAlexander Potapenko return result; 177f80be457SAlexander Potapenko } 178f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_memmove); 179f80be457SAlexander Potapenko 180f80be457SAlexander Potapenko /* Handle llvm.memcpy intrinsic. */ 181*62a9bbf2SAlexander Potapenko void *__msan_memcpy(void *dst, const void *src, uintptr_t n); 182f80be457SAlexander Potapenko void *__msan_memcpy(void *dst, const void *src, uintptr_t n) 183f80be457SAlexander Potapenko { 184f80be457SAlexander Potapenko depot_stack_handle_t origin; 185f80be457SAlexander Potapenko void *result; 186f80be457SAlexander Potapenko u64 shadow; 187f80be457SAlexander Potapenko 188f80be457SAlexander Potapenko get_param0_metadata(&shadow, &origin); 189f80be457SAlexander Potapenko result = __memcpy(dst, src, n); 190f80be457SAlexander Potapenko if (!n) 191f80be457SAlexander Potapenko /* Some people call memcpy() with zero length. */ 192f80be457SAlexander Potapenko return result; 193f80be457SAlexander Potapenko 194f80be457SAlexander Potapenko if (!kmsan_enabled || kmsan_in_runtime()) 195f80be457SAlexander Potapenko return result; 196f80be457SAlexander Potapenko 197f80be457SAlexander Potapenko kmsan_enter_runtime(); 198f80be457SAlexander Potapenko /* Using memmove instead of memcpy doesn't affect correctness. */ 199f80be457SAlexander Potapenko kmsan_internal_memmove_metadata(dst, (void *)src, n); 200f80be457SAlexander Potapenko kmsan_leave_runtime(); 201f80be457SAlexander Potapenko 202f80be457SAlexander Potapenko set_retval_metadata(shadow, origin); 203f80be457SAlexander Potapenko return result; 204f80be457SAlexander Potapenko } 205f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_memcpy); 206f80be457SAlexander Potapenko 207f80be457SAlexander Potapenko /* Handle llvm.memset intrinsic. */ 208*62a9bbf2SAlexander Potapenko void *__msan_memset(void *dst, int c, uintptr_t n); 209f80be457SAlexander Potapenko void *__msan_memset(void *dst, int c, uintptr_t n) 210f80be457SAlexander Potapenko { 211f80be457SAlexander Potapenko depot_stack_handle_t origin; 212f80be457SAlexander Potapenko void *result; 213f80be457SAlexander Potapenko u64 shadow; 214f80be457SAlexander Potapenko 215f80be457SAlexander Potapenko get_param0_metadata(&shadow, &origin); 216f80be457SAlexander Potapenko result = __memset(dst, c, n); 217f80be457SAlexander Potapenko if (!kmsan_enabled || kmsan_in_runtime()) 218f80be457SAlexander Potapenko return result; 219f80be457SAlexander Potapenko 220f80be457SAlexander Potapenko kmsan_enter_runtime(); 221f80be457SAlexander Potapenko /* 222f80be457SAlexander Potapenko * Clang doesn't pass parameter metadata here, so it is impossible to 223f80be457SAlexander Potapenko * use shadow of @c to set up the shadow for @dst. 224f80be457SAlexander Potapenko */ 225f80be457SAlexander Potapenko kmsan_internal_unpoison_memory(dst, n, /*checked*/ false); 226f80be457SAlexander Potapenko kmsan_leave_runtime(); 227f80be457SAlexander Potapenko 228f80be457SAlexander Potapenko set_retval_metadata(shadow, origin); 229f80be457SAlexander Potapenko return result; 230f80be457SAlexander Potapenko } 231f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_memset); 232f80be457SAlexander Potapenko 233f80be457SAlexander Potapenko /* 234f80be457SAlexander Potapenko * Create a new origin from an old one. This is done when storing an 235f80be457SAlexander Potapenko * uninitialized value to memory. When reporting an error, KMSAN unrolls and 236f80be457SAlexander Potapenko * prints the whole chain of stores that preceded the use of this value. 237f80be457SAlexander Potapenko */ 238*62a9bbf2SAlexander Potapenko depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin); 239f80be457SAlexander Potapenko depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin) 240f80be457SAlexander Potapenko { 241f80be457SAlexander Potapenko depot_stack_handle_t ret = 0; 242f80be457SAlexander Potapenko unsigned long ua_flags; 243f80be457SAlexander Potapenko 244f80be457SAlexander Potapenko if (!kmsan_enabled || kmsan_in_runtime()) 245f80be457SAlexander Potapenko return ret; 246f80be457SAlexander Potapenko 247f80be457SAlexander Potapenko ua_flags = user_access_save(); 248f80be457SAlexander Potapenko 249f80be457SAlexander Potapenko /* Creating new origins may allocate memory. */ 250f80be457SAlexander Potapenko kmsan_enter_runtime(); 251f80be457SAlexander Potapenko ret = kmsan_internal_chain_origin(origin); 252f80be457SAlexander Potapenko kmsan_leave_runtime(); 253f80be457SAlexander Potapenko user_access_restore(ua_flags); 254f80be457SAlexander Potapenko return ret; 255f80be457SAlexander Potapenko } 256f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_chain_origin); 257f80be457SAlexander Potapenko 258f80be457SAlexander Potapenko /* Poison a local variable when entering a function. */ 259*62a9bbf2SAlexander Potapenko void __msan_poison_alloca(void *address, uintptr_t size, char *descr); 260f80be457SAlexander Potapenko void __msan_poison_alloca(void *address, uintptr_t size, char *descr) 261f80be457SAlexander Potapenko { 262f80be457SAlexander Potapenko depot_stack_handle_t handle; 263f80be457SAlexander Potapenko unsigned long entries[4]; 264f80be457SAlexander Potapenko unsigned long ua_flags; 265f80be457SAlexander Potapenko 266f80be457SAlexander Potapenko if (!kmsan_enabled || kmsan_in_runtime()) 267f80be457SAlexander Potapenko return; 268f80be457SAlexander Potapenko 269f80be457SAlexander Potapenko ua_flags = user_access_save(); 270f80be457SAlexander Potapenko entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN; 271f80be457SAlexander Potapenko entries[1] = (u64)descr; 272f80be457SAlexander Potapenko entries[2] = (u64)__builtin_return_address(0); 273f80be457SAlexander Potapenko /* 274f80be457SAlexander Potapenko * With frame pointers enabled, it is possible to quickly fetch the 275f80be457SAlexander Potapenko * second frame of the caller stack without calling the unwinder. 276f80be457SAlexander Potapenko * Without them, simply do not bother. 277f80be457SAlexander Potapenko */ 278f80be457SAlexander Potapenko if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) 279f80be457SAlexander Potapenko entries[3] = (u64)__builtin_return_address(1); 280f80be457SAlexander Potapenko else 281f80be457SAlexander Potapenko entries[3] = 0; 282f80be457SAlexander Potapenko 283f80be457SAlexander Potapenko /* stack_depot_save() may allocate memory. */ 284f80be457SAlexander Potapenko kmsan_enter_runtime(); 285f80be457SAlexander Potapenko handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_ATOMIC); 286f80be457SAlexander Potapenko kmsan_leave_runtime(); 287f80be457SAlexander Potapenko 288f80be457SAlexander Potapenko kmsan_internal_set_shadow_origin(address, size, -1, handle, 289f80be457SAlexander Potapenko /*checked*/ true); 290f80be457SAlexander Potapenko user_access_restore(ua_flags); 291f80be457SAlexander Potapenko } 292f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_poison_alloca); 293f80be457SAlexander Potapenko 294f80be457SAlexander Potapenko /* Unpoison a local variable. */ 295*62a9bbf2SAlexander Potapenko void __msan_unpoison_alloca(void *address, uintptr_t size); 296f80be457SAlexander Potapenko void __msan_unpoison_alloca(void *address, uintptr_t size) 297f80be457SAlexander Potapenko { 298f80be457SAlexander Potapenko if (!kmsan_enabled || kmsan_in_runtime()) 299f80be457SAlexander Potapenko return; 300f80be457SAlexander Potapenko 301f80be457SAlexander Potapenko kmsan_enter_runtime(); 302f80be457SAlexander Potapenko kmsan_internal_unpoison_memory(address, size, /*checked*/ true); 303f80be457SAlexander Potapenko kmsan_leave_runtime(); 304f80be457SAlexander Potapenko } 305f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_unpoison_alloca); 306f80be457SAlexander Potapenko 307f80be457SAlexander Potapenko /* 308f80be457SAlexander Potapenko * Report that an uninitialized value with the given origin was used in a way 309f80be457SAlexander Potapenko * that constituted undefined behavior. 310f80be457SAlexander Potapenko */ 311*62a9bbf2SAlexander Potapenko void __msan_warning(u32 origin); 312f80be457SAlexander Potapenko void __msan_warning(u32 origin) 313f80be457SAlexander Potapenko { 314f80be457SAlexander Potapenko if (!kmsan_enabled || kmsan_in_runtime()) 315f80be457SAlexander Potapenko return; 316f80be457SAlexander Potapenko kmsan_enter_runtime(); 317f80be457SAlexander Potapenko kmsan_report(origin, /*address*/ 0, /*size*/ 0, 318f80be457SAlexander Potapenko /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ 0, 319f80be457SAlexander Potapenko REASON_ANY); 320f80be457SAlexander Potapenko kmsan_leave_runtime(); 321f80be457SAlexander Potapenko } 322f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_warning); 323f80be457SAlexander Potapenko 324f80be457SAlexander Potapenko /* 325f80be457SAlexander Potapenko * At the beginning of an instrumented function, obtain the pointer to 326f80be457SAlexander Potapenko * `struct kmsan_context_state` holding the metadata for function parameters. 327f80be457SAlexander Potapenko */ 328*62a9bbf2SAlexander Potapenko struct kmsan_context_state *__msan_get_context_state(void); 329f80be457SAlexander Potapenko struct kmsan_context_state *__msan_get_context_state(void) 330f80be457SAlexander Potapenko { 331f80be457SAlexander Potapenko return &kmsan_get_context()->cstate; 332f80be457SAlexander Potapenko } 333f80be457SAlexander Potapenko EXPORT_SYMBOL(__msan_get_context_state); 334