1 #ifndef _ASM_X86_UACCESS_64_H 2 #define _ASM_X86_UACCESS_64_H 3 4 /* 5 * User space memory access functions 6 */ 7 #include <linux/compiler.h> 8 #include <linux/errno.h> 9 #include <linux/lockdep.h> 10 #include <linux/kasan-checks.h> 11 #include <asm/alternative.h> 12 #include <asm/cpufeatures.h> 13 #include <asm/page.h> 14 15 /* 16 * Copy To/From Userspace 17 */ 18 19 /* Handles exceptions in both to and from, but doesn't do access_ok */ 20 __must_check unsigned long 21 copy_user_enhanced_fast_string(void *to, const void *from, unsigned len); 22 __must_check unsigned long 23 copy_user_generic_string(void *to, const void *from, unsigned len); 24 __must_check unsigned long 25 copy_user_generic_unrolled(void *to, const void *from, unsigned len); 26 27 static __always_inline __must_check unsigned long 28 copy_user_generic(void *to, const void *from, unsigned len) 29 { 30 unsigned ret; 31 32 /* 33 * If CPU has ERMS feature, use copy_user_enhanced_fast_string. 34 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string. 35 * Otherwise, use copy_user_generic_unrolled. 36 */ 37 alternative_call_2(copy_user_generic_unrolled, 38 copy_user_generic_string, 39 X86_FEATURE_REP_GOOD, 40 copy_user_enhanced_fast_string, 41 X86_FEATURE_ERMS, 42 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from), 43 "=d" (len)), 44 "1" (to), "2" (from), "3" (len) 45 : "memory", "rcx", "r8", "r9", "r10", "r11"); 46 return ret; 47 } 48 49 __must_check unsigned long 50 copy_in_user(void __user *to, const void __user *from, unsigned len); 51 52 static __always_inline __must_check 53 int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size) 54 { 55 int ret = 0; 56 57 check_object_size(dst, size, false); 58 if (!__builtin_constant_p(size)) 59 return copy_user_generic(dst, (__force void *)src, size); 60 switch (size) { 61 case 1: 62 __uaccess_begin(); 63 __get_user_asm(*(u8 *)dst, (u8 __user *)src, 64 ret, "b", "b", "=q", 1); 65 __uaccess_end(); 66 return ret; 67 case 2: 68 __uaccess_begin(); 69 __get_user_asm(*(u16 *)dst, (u16 __user *)src, 70 ret, "w", "w", "=r", 2); 71 __uaccess_end(); 72 return ret; 73 case 4: 74 __uaccess_begin(); 75 __get_user_asm(*(u32 *)dst, (u32 __user *)src, 76 ret, "l", "k", "=r", 4); 77 __uaccess_end(); 78 return ret; 79 case 8: 80 __uaccess_begin(); 81 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 82 ret, "q", "", "=r", 8); 83 __uaccess_end(); 84 return ret; 85 case 10: 86 __uaccess_begin(); 87 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 88 ret, "q", "", "=r", 10); 89 if (likely(!ret)) 90 __get_user_asm(*(u16 *)(8 + (char *)dst), 91 (u16 __user *)(8 + (char __user *)src), 92 ret, "w", "w", "=r", 2); 93 __uaccess_end(); 94 return ret; 95 case 16: 96 __uaccess_begin(); 97 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 98 ret, "q", "", "=r", 16); 99 if (likely(!ret)) 100 __get_user_asm(*(u64 *)(8 + (char *)dst), 101 (u64 __user *)(8 + (char __user *)src), 102 ret, "q", "", "=r", 8); 103 __uaccess_end(); 104 return ret; 105 default: 106 return copy_user_generic(dst, (__force void *)src, size); 107 } 108 } 109 110 static __always_inline __must_check 111 int __copy_from_user(void *dst, const void __user *src, unsigned size) 112 { 113 might_fault(); 114 kasan_check_write(dst, size); 115 return __copy_from_user_nocheck(dst, src, size); 116 } 117 118 static __always_inline __must_check 119 int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size) 120 { 121 int ret = 0; 122 123 check_object_size(src, size, true); 124 if (!__builtin_constant_p(size)) 125 return copy_user_generic((__force void *)dst, src, size); 126 switch (size) { 127 case 1: 128 __uaccess_begin(); 129 __put_user_asm(*(u8 *)src, (u8 __user *)dst, 130 ret, "b", "b", "iq", 1); 131 __uaccess_end(); 132 return ret; 133 case 2: 134 __uaccess_begin(); 135 __put_user_asm(*(u16 *)src, (u16 __user *)dst, 136 ret, "w", "w", "ir", 2); 137 __uaccess_end(); 138 return ret; 139 case 4: 140 __uaccess_begin(); 141 __put_user_asm(*(u32 *)src, (u32 __user *)dst, 142 ret, "l", "k", "ir", 4); 143 __uaccess_end(); 144 return ret; 145 case 8: 146 __uaccess_begin(); 147 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 148 ret, "q", "", "er", 8); 149 __uaccess_end(); 150 return ret; 151 case 10: 152 __uaccess_begin(); 153 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 154 ret, "q", "", "er", 10); 155 if (likely(!ret)) { 156 asm("":::"memory"); 157 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, 158 ret, "w", "w", "ir", 2); 159 } 160 __uaccess_end(); 161 return ret; 162 case 16: 163 __uaccess_begin(); 164 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 165 ret, "q", "", "er", 16); 166 if (likely(!ret)) { 167 asm("":::"memory"); 168 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, 169 ret, "q", "", "er", 8); 170 } 171 __uaccess_end(); 172 return ret; 173 default: 174 return copy_user_generic((__force void *)dst, src, size); 175 } 176 } 177 178 static __always_inline __must_check 179 int __copy_to_user(void __user *dst, const void *src, unsigned size) 180 { 181 might_fault(); 182 kasan_check_read(src, size); 183 return __copy_to_user_nocheck(dst, src, size); 184 } 185 186 static __always_inline __must_check 187 int __copy_in_user(void __user *dst, const void __user *src, unsigned size) 188 { 189 int ret = 0; 190 191 might_fault(); 192 if (!__builtin_constant_p(size)) 193 return copy_user_generic((__force void *)dst, 194 (__force void *)src, size); 195 switch (size) { 196 case 1: { 197 u8 tmp; 198 __uaccess_begin(); 199 __get_user_asm(tmp, (u8 __user *)src, 200 ret, "b", "b", "=q", 1); 201 if (likely(!ret)) 202 __put_user_asm(tmp, (u8 __user *)dst, 203 ret, "b", "b", "iq", 1); 204 __uaccess_end(); 205 return ret; 206 } 207 case 2: { 208 u16 tmp; 209 __uaccess_begin(); 210 __get_user_asm(tmp, (u16 __user *)src, 211 ret, "w", "w", "=r", 2); 212 if (likely(!ret)) 213 __put_user_asm(tmp, (u16 __user *)dst, 214 ret, "w", "w", "ir", 2); 215 __uaccess_end(); 216 return ret; 217 } 218 219 case 4: { 220 u32 tmp; 221 __uaccess_begin(); 222 __get_user_asm(tmp, (u32 __user *)src, 223 ret, "l", "k", "=r", 4); 224 if (likely(!ret)) 225 __put_user_asm(tmp, (u32 __user *)dst, 226 ret, "l", "k", "ir", 4); 227 __uaccess_end(); 228 return ret; 229 } 230 case 8: { 231 u64 tmp; 232 __uaccess_begin(); 233 __get_user_asm(tmp, (u64 __user *)src, 234 ret, "q", "", "=r", 8); 235 if (likely(!ret)) 236 __put_user_asm(tmp, (u64 __user *)dst, 237 ret, "q", "", "er", 8); 238 __uaccess_end(); 239 return ret; 240 } 241 default: 242 return copy_user_generic((__force void *)dst, 243 (__force void *)src, size); 244 } 245 } 246 247 static __must_check __always_inline int 248 __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size) 249 { 250 kasan_check_write(dst, size); 251 return __copy_from_user_nocheck(dst, src, size); 252 } 253 254 static __must_check __always_inline int 255 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) 256 { 257 kasan_check_read(src, size); 258 return __copy_to_user_nocheck(dst, src, size); 259 } 260 261 extern long __copy_user_nocache(void *dst, const void __user *src, 262 unsigned size, int zerorest); 263 264 static inline int 265 __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) 266 { 267 might_fault(); 268 kasan_check_write(dst, size); 269 return __copy_user_nocache(dst, src, size, 1); 270 } 271 272 static inline int 273 __copy_from_user_inatomic_nocache(void *dst, const void __user *src, 274 unsigned size) 275 { 276 kasan_check_write(dst, size); 277 return __copy_user_nocache(dst, src, size, 0); 278 } 279 280 unsigned long 281 copy_user_handle_tail(char *to, char *from, unsigned len); 282 283 #endif /* _ASM_X86_UACCESS_64_H */ 284