1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_UACCESS_64_H 3 #define _ASM_X86_UACCESS_64_H 4 5 /* 6 * User space memory access functions 7 */ 8 #include <linux/compiler.h> 9 #include <linux/lockdep.h> 10 #include <linux/kasan-checks.h> 11 #include <asm/alternative.h> 12 #include <asm/cpufeatures.h> 13 #include <asm/page.h> 14 15 /* 16 * Copy To/From Userspace 17 */ 18 19 /* Handles exceptions in both to and from, but doesn't do access_ok */ 20 __must_check unsigned long 21 copy_user_enhanced_fast_string(void *to, const void *from, unsigned len); 22 __must_check unsigned long 23 copy_user_generic_string(void *to, const void *from, unsigned len); 24 __must_check unsigned long 25 copy_user_generic_unrolled(void *to, const void *from, unsigned len); 26 27 static __always_inline __must_check unsigned long 28 copy_user_generic(void *to, const void *from, unsigned len) 29 { 30 unsigned ret; 31 32 /* 33 * If CPU has ERMS feature, use copy_user_enhanced_fast_string. 34 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string. 35 * Otherwise, use copy_user_generic_unrolled. 36 */ 37 alternative_call_2(copy_user_generic_unrolled, 38 copy_user_generic_string, 39 X86_FEATURE_REP_GOOD, 40 copy_user_enhanced_fast_string, 41 X86_FEATURE_ERMS, 42 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from), 43 "=d" (len)), 44 "1" (to), "2" (from), "3" (len) 45 : "memory", "rcx", "r8", "r9", "r10", "r11"); 46 return ret; 47 } 48 49 static __always_inline __must_check unsigned long 50 copy_to_user_mcsafe(void *to, const void *from, unsigned len) 51 { 52 unsigned long ret; 53 54 __uaccess_begin(); 55 ret = memcpy_mcsafe(to, from, len); 56 __uaccess_end(); 57 return ret; 58 } 59 60 static __always_inline __must_check unsigned long 61 raw_copy_from_user(void *dst, const void __user *src, unsigned long size) 62 { 63 int ret = 0; 64 65 if (!__builtin_constant_p(size)) 66 return copy_user_generic(dst, (__force void *)src, size); 67 switch (size) { 68 case 1: 69 __uaccess_begin_nospec(); 70 __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src, 71 ret, "b", "b", "=q", 1); 72 __uaccess_end(); 73 return ret; 74 case 2: 75 __uaccess_begin_nospec(); 76 __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src, 77 ret, "w", "w", "=r", 2); 78 __uaccess_end(); 79 return ret; 80 case 4: 81 __uaccess_begin_nospec(); 82 __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, 83 ret, "l", "k", "=r", 4); 84 __uaccess_end(); 85 return ret; 86 case 8: 87 __uaccess_begin_nospec(); 88 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, 89 ret, "q", "", "=r", 8); 90 __uaccess_end(); 91 return ret; 92 case 10: 93 __uaccess_begin_nospec(); 94 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, 95 ret, "q", "", "=r", 10); 96 if (likely(!ret)) 97 __get_user_asm_nozero(*(u16 *)(8 + (char *)dst), 98 (u16 __user *)(8 + (char __user *)src), 99 ret, "w", "w", "=r", 2); 100 __uaccess_end(); 101 return ret; 102 case 16: 103 __uaccess_begin_nospec(); 104 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, 105 ret, "q", "", "=r", 16); 106 if (likely(!ret)) 107 __get_user_asm_nozero(*(u64 *)(8 + (char *)dst), 108 (u64 __user *)(8 + (char __user *)src), 109 ret, "q", "", "=r", 8); 110 __uaccess_end(); 111 return ret; 112 default: 113 return copy_user_generic(dst, (__force void *)src, size); 114 } 115 } 116 117 static __always_inline __must_check unsigned long 118 raw_copy_to_user(void __user *dst, const void *src, unsigned long size) 119 { 120 int ret = 0; 121 122 if (!__builtin_constant_p(size)) 123 return copy_user_generic((__force void *)dst, src, size); 124 switch (size) { 125 case 1: 126 __uaccess_begin(); 127 __put_user_asm(*(u8 *)src, (u8 __user *)dst, 128 ret, "b", "b", "iq", 1); 129 __uaccess_end(); 130 return ret; 131 case 2: 132 __uaccess_begin(); 133 __put_user_asm(*(u16 *)src, (u16 __user *)dst, 134 ret, "w", "w", "ir", 2); 135 __uaccess_end(); 136 return ret; 137 case 4: 138 __uaccess_begin(); 139 __put_user_asm(*(u32 *)src, (u32 __user *)dst, 140 ret, "l", "k", "ir", 4); 141 __uaccess_end(); 142 return ret; 143 case 8: 144 __uaccess_begin(); 145 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 146 ret, "q", "", "er", 8); 147 __uaccess_end(); 148 return ret; 149 case 10: 150 __uaccess_begin(); 151 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 152 ret, "q", "", "er", 10); 153 if (likely(!ret)) { 154 asm("":::"memory"); 155 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, 156 ret, "w", "w", "ir", 2); 157 } 158 __uaccess_end(); 159 return ret; 160 case 16: 161 __uaccess_begin(); 162 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 163 ret, "q", "", "er", 16); 164 if (likely(!ret)) { 165 asm("":::"memory"); 166 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, 167 ret, "q", "", "er", 8); 168 } 169 __uaccess_end(); 170 return ret; 171 default: 172 return copy_user_generic((__force void *)dst, src, size); 173 } 174 } 175 176 static __always_inline __must_check 177 unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long size) 178 { 179 return copy_user_generic((__force void *)dst, 180 (__force void *)src, size); 181 } 182 183 extern long __copy_user_nocache(void *dst, const void __user *src, 184 unsigned size, int zerorest); 185 186 extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size); 187 extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset, 188 size_t len); 189 190 static inline int 191 __copy_from_user_inatomic_nocache(void *dst, const void __user *src, 192 unsigned size) 193 { 194 kasan_check_write(dst, size); 195 return __copy_user_nocache(dst, src, size, 0); 196 } 197 198 static inline int 199 __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) 200 { 201 kasan_check_write(dst, size); 202 return __copy_user_flushcache(dst, src, size); 203 } 204 205 unsigned long 206 copy_user_handle_tail(char *to, char *from, unsigned len); 207 208 unsigned long 209 mcsafe_handle_tail(char *to, char *from, unsigned len); 210 211 #endif /* _ASM_X86_UACCESS_64_H */ 212