1 #ifndef _ASM_X86_UACCESS_64_H 2 #define _ASM_X86_UACCESS_64_H 3 4 /* 5 * User space memory access functions 6 */ 7 #include <linux/compiler.h> 8 #include <linux/errno.h> 9 #include <linux/lockdep.h> 10 #include <linux/kasan-checks.h> 11 #include <asm/alternative.h> 12 #include <asm/cpufeatures.h> 13 #include <asm/page.h> 14 15 /* 16 * Copy To/From Userspace 17 */ 18 19 /* Handles exceptions in both to and from, but doesn't do access_ok */ 20 __must_check unsigned long 21 copy_user_enhanced_fast_string(void *to, const void *from, unsigned len); 22 __must_check unsigned long 23 copy_user_generic_string(void *to, const void *from, unsigned len); 24 __must_check unsigned long 25 copy_user_generic_unrolled(void *to, const void *from, unsigned len); 26 27 static __always_inline __must_check unsigned long 28 copy_user_generic(void *to, const void *from, unsigned len) 29 { 30 unsigned ret; 31 32 /* 33 * If CPU has ERMS feature, use copy_user_enhanced_fast_string. 34 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string. 35 * Otherwise, use copy_user_generic_unrolled. 36 */ 37 alternative_call_2(copy_user_generic_unrolled, 38 copy_user_generic_string, 39 X86_FEATURE_REP_GOOD, 40 copy_user_enhanced_fast_string, 41 X86_FEATURE_ERMS, 42 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from), 43 "=d" (len)), 44 "1" (to), "2" (from), "3" (len) 45 : "memory", "rcx", "r8", "r9", "r10", "r11"); 46 return ret; 47 } 48 49 __must_check unsigned long 50 copy_in_user(void __user *to, const void __user *from, unsigned len); 51 52 static __always_inline __must_check 53 int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size) 54 { 55 int ret = 0; 56 57 if (!__builtin_constant_p(size)) 58 return copy_user_generic(dst, (__force void *)src, size); 59 switch (size) { 60 case 1: 61 __uaccess_begin(); 62 __get_user_asm(*(u8 *)dst, (u8 __user *)src, 63 ret, "b", "b", "=q", 1); 64 __uaccess_end(); 65 return ret; 66 case 2: 67 __uaccess_begin(); 68 __get_user_asm(*(u16 *)dst, (u16 __user *)src, 69 ret, "w", "w", "=r", 2); 70 __uaccess_end(); 71 return ret; 72 case 4: 73 __uaccess_begin(); 74 __get_user_asm(*(u32 *)dst, (u32 __user *)src, 75 ret, "l", "k", "=r", 4); 76 __uaccess_end(); 77 return ret; 78 case 8: 79 __uaccess_begin(); 80 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 81 ret, "q", "", "=r", 8); 82 __uaccess_end(); 83 return ret; 84 case 10: 85 __uaccess_begin(); 86 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 87 ret, "q", "", "=r", 10); 88 if (likely(!ret)) 89 __get_user_asm(*(u16 *)(8 + (char *)dst), 90 (u16 __user *)(8 + (char __user *)src), 91 ret, "w", "w", "=r", 2); 92 __uaccess_end(); 93 return ret; 94 case 16: 95 __uaccess_begin(); 96 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 97 ret, "q", "", "=r", 16); 98 if (likely(!ret)) 99 __get_user_asm(*(u64 *)(8 + (char *)dst), 100 (u64 __user *)(8 + (char __user *)src), 101 ret, "q", "", "=r", 8); 102 __uaccess_end(); 103 return ret; 104 default: 105 return copy_user_generic(dst, (__force void *)src, size); 106 } 107 } 108 109 static __always_inline __must_check 110 int __copy_from_user(void *dst, const void __user *src, unsigned size) 111 { 112 might_fault(); 113 kasan_check_write(dst, size); 114 return __copy_from_user_nocheck(dst, src, size); 115 } 116 117 static __always_inline __must_check 118 int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size) 119 { 120 int ret = 0; 121 122 if (!__builtin_constant_p(size)) 123 return copy_user_generic((__force void *)dst, src, size); 124 switch (size) { 125 case 1: 126 __uaccess_begin(); 127 __put_user_asm(*(u8 *)src, (u8 __user *)dst, 128 ret, "b", "b", "iq", 1); 129 __uaccess_end(); 130 return ret; 131 case 2: 132 __uaccess_begin(); 133 __put_user_asm(*(u16 *)src, (u16 __user *)dst, 134 ret, "w", "w", "ir", 2); 135 __uaccess_end(); 136 return ret; 137 case 4: 138 __uaccess_begin(); 139 __put_user_asm(*(u32 *)src, (u32 __user *)dst, 140 ret, "l", "k", "ir", 4); 141 __uaccess_end(); 142 return ret; 143 case 8: 144 __uaccess_begin(); 145 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 146 ret, "q", "", "er", 8); 147 __uaccess_end(); 148 return ret; 149 case 10: 150 __uaccess_begin(); 151 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 152 ret, "q", "", "er", 10); 153 if (likely(!ret)) { 154 asm("":::"memory"); 155 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, 156 ret, "w", "w", "ir", 2); 157 } 158 __uaccess_end(); 159 return ret; 160 case 16: 161 __uaccess_begin(); 162 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 163 ret, "q", "", "er", 16); 164 if (likely(!ret)) { 165 asm("":::"memory"); 166 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, 167 ret, "q", "", "er", 8); 168 } 169 __uaccess_end(); 170 return ret; 171 default: 172 return copy_user_generic((__force void *)dst, src, size); 173 } 174 } 175 176 static __always_inline __must_check 177 int __copy_to_user(void __user *dst, const void *src, unsigned size) 178 { 179 might_fault(); 180 kasan_check_read(src, size); 181 return __copy_to_user_nocheck(dst, src, size); 182 } 183 184 static __always_inline __must_check 185 int __copy_in_user(void __user *dst, const void __user *src, unsigned size) 186 { 187 int ret = 0; 188 189 might_fault(); 190 if (!__builtin_constant_p(size)) 191 return copy_user_generic((__force void *)dst, 192 (__force void *)src, size); 193 switch (size) { 194 case 1: { 195 u8 tmp; 196 __uaccess_begin(); 197 __get_user_asm(tmp, (u8 __user *)src, 198 ret, "b", "b", "=q", 1); 199 if (likely(!ret)) 200 __put_user_asm(tmp, (u8 __user *)dst, 201 ret, "b", "b", "iq", 1); 202 __uaccess_end(); 203 return ret; 204 } 205 case 2: { 206 u16 tmp; 207 __uaccess_begin(); 208 __get_user_asm(tmp, (u16 __user *)src, 209 ret, "w", "w", "=r", 2); 210 if (likely(!ret)) 211 __put_user_asm(tmp, (u16 __user *)dst, 212 ret, "w", "w", "ir", 2); 213 __uaccess_end(); 214 return ret; 215 } 216 217 case 4: { 218 u32 tmp; 219 __uaccess_begin(); 220 __get_user_asm(tmp, (u32 __user *)src, 221 ret, "l", "k", "=r", 4); 222 if (likely(!ret)) 223 __put_user_asm(tmp, (u32 __user *)dst, 224 ret, "l", "k", "ir", 4); 225 __uaccess_end(); 226 return ret; 227 } 228 case 8: { 229 u64 tmp; 230 __uaccess_begin(); 231 __get_user_asm(tmp, (u64 __user *)src, 232 ret, "q", "", "=r", 8); 233 if (likely(!ret)) 234 __put_user_asm(tmp, (u64 __user *)dst, 235 ret, "q", "", "er", 8); 236 __uaccess_end(); 237 return ret; 238 } 239 default: 240 return copy_user_generic((__force void *)dst, 241 (__force void *)src, size); 242 } 243 } 244 245 static __must_check __always_inline int 246 __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size) 247 { 248 kasan_check_write(dst, size); 249 return __copy_from_user_nocheck(dst, src, size); 250 } 251 252 static __must_check __always_inline int 253 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) 254 { 255 kasan_check_read(src, size); 256 return __copy_to_user_nocheck(dst, src, size); 257 } 258 259 extern long __copy_user_nocache(void *dst, const void __user *src, 260 unsigned size, int zerorest); 261 262 static inline int 263 __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) 264 { 265 might_fault(); 266 kasan_check_write(dst, size); 267 return __copy_user_nocache(dst, src, size, 1); 268 } 269 270 static inline int 271 __copy_from_user_inatomic_nocache(void *dst, const void __user *src, 272 unsigned size) 273 { 274 kasan_check_write(dst, size); 275 return __copy_user_nocache(dst, src, size, 0); 276 } 277 278 unsigned long 279 copy_user_handle_tail(char *to, char *from, unsigned len); 280 281 #endif /* _ASM_X86_UACCESS_64_H */ 282