1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ 3 4 #include <linux/jump_label.h> 5 #include <linux/uaccess.h> 6 #include <linux/export.h> 7 #include <linux/string.h> 8 #include <linux/types.h> 9 10 #include <asm/mce.h> 11 12 #ifdef CONFIG_X86_MCE 13 static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key); 14 15 void enable_copy_mc_fragile(void) 16 { 17 static_branch_inc(©_mc_fragile_key); 18 } 19 #define copy_mc_fragile_enabled (static_branch_unlikely(©_mc_fragile_key)) 20 21 /* 22 * Similar to copy_user_handle_tail, probe for the write fault point, or 23 * source exception point. 24 */ 25 __visible notrace unsigned long 26 copy_mc_fragile_handle_tail(char *to, char *from, unsigned len) 27 { 28 for (; len; --len, to++, from++) 29 if (copy_mc_fragile(to, from, 1)) 30 break; 31 return len; 32 } 33 #else 34 /* 35 * No point in doing careful copying, or consulting a static key when 36 * there is no #MC handler in the CONFIG_X86_MCE=n case. 37 */ 38 void enable_copy_mc_fragile(void) 39 { 40 } 41 #define copy_mc_fragile_enabled (0) 42 #endif 43 44 unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len); 45 46 /** 47 * copy_mc_to_kernel - memory copy that handles source exceptions 48 * 49 * @dst: destination address 50 * @src: source address 51 * @len: number of bytes to copy 52 * 53 * Call into the 'fragile' version on systems that benefit from avoiding 54 * corner case poison consumption scenarios, For example, accessing 55 * poison across 2 cachelines with a single instruction. Almost all 56 * other uses case can use copy_mc_enhanced_fast_string() for a fast 57 * recoverable copy, or fallback to plain memcpy. 58 * 59 * Return 0 for success, or number of bytes not copied if there was an 60 * exception. 61 */ 62 unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len) 63 { 64 if (copy_mc_fragile_enabled) 65 return copy_mc_fragile(dst, src, len); 66 if (static_cpu_has(X86_FEATURE_ERMS)) 67 return copy_mc_enhanced_fast_string(dst, src, len); 68 memcpy(dst, src, len); 69 return 0; 70 } 71 EXPORT_SYMBOL_GPL(copy_mc_to_kernel); 72 73 unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, unsigned len) 74 { 75 unsigned long ret; 76 77 if (copy_mc_fragile_enabled) { 78 __uaccess_begin(); 79 ret = copy_mc_fragile((__force void *)dst, src, len); 80 __uaccess_end(); 81 return ret; 82 } 83 84 if (static_cpu_has(X86_FEATURE_ERMS)) { 85 __uaccess_begin(); 86 ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len); 87 __uaccess_end(); 88 return ret; 89 } 90 91 return copy_user_generic((__force void *)dst, src, len); 92 } 93