1 #ifndef _ASM_X86_UACCESS_64_H 2 #define _ASM_X86_UACCESS_64_H 3 4 /* 5 * User space memory access functions 6 */ 7 #include <linux/compiler.h> 8 #include <linux/errno.h> 9 #include <linux/prefetch.h> 10 #include <linux/lockdep.h> 11 #include <asm/alternative.h> 12 #include <asm/cpufeature.h> 13 #include <asm/page.h> 14 15 /* 16 * Copy To/From Userspace 17 */ 18 19 /* Handles exceptions in both to and from, but doesn't do access_ok */ 20 __must_check unsigned long 21 copy_user_generic_string(void *to, const void *from, unsigned len); 22 __must_check unsigned long 23 copy_user_generic_unrolled(void *to, const void *from, unsigned len); 24 25 static __always_inline __must_check unsigned long 26 copy_user_generic(void *to, const void *from, unsigned len) 27 { 28 unsigned ret; 29 30 alternative_call(copy_user_generic_unrolled, 31 copy_user_generic_string, 32 X86_FEATURE_REP_GOOD, 33 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from), 34 "=d" (len)), 35 "1" (to), "2" (from), "3" (len) 36 : "memory", "rcx", "r8", "r9", "r10", "r11"); 37 return ret; 38 } 39 40 __must_check unsigned long 41 _copy_to_user(void __user *to, const void *from, unsigned len); 42 __must_check unsigned long 43 _copy_from_user(void *to, const void __user *from, unsigned len); 44 __must_check unsigned long 45 copy_in_user(void __user *to, const void __user *from, unsigned len); 46 47 static inline unsigned long __must_check copy_from_user(void *to, 48 const void __user *from, 49 unsigned long n) 50 { 51 int sz = __compiletime_object_size(to); 52 53 might_fault(); 54 if (likely(sz == -1 || sz >= n)) 55 n = _copy_from_user(to, from, n); 56 #ifdef CONFIG_DEBUG_VM 57 else 58 WARN(1, "Buffer overflow detected!\n"); 59 #endif 60 return n; 61 } 62 63 static __always_inline __must_check 64 int copy_to_user(void __user *dst, const void *src, unsigned size) 65 { 66 might_fault(); 67 68 return _copy_to_user(dst, src, size); 69 } 70 71 static __always_inline __must_check 72 int __copy_from_user(void *dst, const void __user *src, unsigned size) 73 { 74 int ret = 0; 75 76 might_fault(); 77 if (!__builtin_constant_p(size)) 78 return copy_user_generic(dst, (__force void *)src, size); 79 switch (size) { 80 case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src, 81 ret, "b", "b", "=q", 1); 82 return ret; 83 case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src, 84 ret, "w", "w", "=r", 2); 85 return ret; 86 case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src, 87 ret, "l", "k", "=r", 4); 88 return ret; 89 case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src, 90 ret, "q", "", "=r", 8); 91 return ret; 92 case 10: 93 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 94 ret, "q", "", "=r", 10); 95 if (unlikely(ret)) 96 return ret; 97 __get_user_asm(*(u16 *)(8 + (char *)dst), 98 (u16 __user *)(8 + (char __user *)src), 99 ret, "w", "w", "=r", 2); 100 return ret; 101 case 16: 102 __get_user_asm(*(u64 *)dst, (u64 __user *)src, 103 ret, "q", "", "=r", 16); 104 if (unlikely(ret)) 105 return ret; 106 __get_user_asm(*(u64 *)(8 + (char *)dst), 107 (u64 __user *)(8 + (char __user *)src), 108 ret, "q", "", "=r", 8); 109 return ret; 110 default: 111 return copy_user_generic(dst, (__force void *)src, size); 112 } 113 } 114 115 static __always_inline __must_check 116 int __copy_to_user(void __user *dst, const void *src, unsigned size) 117 { 118 int ret = 0; 119 120 might_fault(); 121 if (!__builtin_constant_p(size)) 122 return copy_user_generic((__force void *)dst, src, size); 123 switch (size) { 124 case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst, 125 ret, "b", "b", "iq", 1); 126 return ret; 127 case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst, 128 ret, "w", "w", "ir", 2); 129 return ret; 130 case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst, 131 ret, "l", "k", "ir", 4); 132 return ret; 133 case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst, 134 ret, "q", "", "er", 8); 135 return ret; 136 case 10: 137 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 138 ret, "q", "", "er", 10); 139 if (unlikely(ret)) 140 return ret; 141 asm("":::"memory"); 142 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, 143 ret, "w", "w", "ir", 2); 144 return ret; 145 case 16: 146 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 147 ret, "q", "", "er", 16); 148 if (unlikely(ret)) 149 return ret; 150 asm("":::"memory"); 151 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, 152 ret, "q", "", "er", 8); 153 return ret; 154 default: 155 return copy_user_generic((__force void *)dst, src, size); 156 } 157 } 158 159 static __always_inline __must_check 160 int __copy_in_user(void __user *dst, const void __user *src, unsigned size) 161 { 162 int ret = 0; 163 164 might_fault(); 165 if (!__builtin_constant_p(size)) 166 return copy_user_generic((__force void *)dst, 167 (__force void *)src, size); 168 switch (size) { 169 case 1: { 170 u8 tmp; 171 __get_user_asm(tmp, (u8 __user *)src, 172 ret, "b", "b", "=q", 1); 173 if (likely(!ret)) 174 __put_user_asm(tmp, (u8 __user *)dst, 175 ret, "b", "b", "iq", 1); 176 return ret; 177 } 178 case 2: { 179 u16 tmp; 180 __get_user_asm(tmp, (u16 __user *)src, 181 ret, "w", "w", "=r", 2); 182 if (likely(!ret)) 183 __put_user_asm(tmp, (u16 __user *)dst, 184 ret, "w", "w", "ir", 2); 185 return ret; 186 } 187 188 case 4: { 189 u32 tmp; 190 __get_user_asm(tmp, (u32 __user *)src, 191 ret, "l", "k", "=r", 4); 192 if (likely(!ret)) 193 __put_user_asm(tmp, (u32 __user *)dst, 194 ret, "l", "k", "ir", 4); 195 return ret; 196 } 197 case 8: { 198 u64 tmp; 199 __get_user_asm(tmp, (u64 __user *)src, 200 ret, "q", "", "=r", 8); 201 if (likely(!ret)) 202 __put_user_asm(tmp, (u64 __user *)dst, 203 ret, "q", "", "er", 8); 204 return ret; 205 } 206 default: 207 return copy_user_generic((__force void *)dst, 208 (__force void *)src, size); 209 } 210 } 211 212 __must_check long 213 strncpy_from_user(char *dst, const char __user *src, long count); 214 __must_check long 215 __strncpy_from_user(char *dst, const char __user *src, long count); 216 __must_check long strnlen_user(const char __user *str, long n); 217 __must_check long __strnlen_user(const char __user *str, long n); 218 __must_check long strlen_user(const char __user *str); 219 __must_check unsigned long clear_user(void __user *mem, unsigned long len); 220 __must_check unsigned long __clear_user(void __user *mem, unsigned long len); 221 222 static __must_check __always_inline int 223 __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size) 224 { 225 return copy_user_generic(dst, (__force const void *)src, size); 226 } 227 228 static __must_check __always_inline int 229 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) 230 { 231 return copy_user_generic((__force void *)dst, src, size); 232 } 233 234 extern long __copy_user_nocache(void *dst, const void __user *src, 235 unsigned size, int zerorest); 236 237 static inline int 238 __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) 239 { 240 might_sleep(); 241 return __copy_user_nocache(dst, src, size, 1); 242 } 243 244 static inline int 245 __copy_from_user_inatomic_nocache(void *dst, const void __user *src, 246 unsigned size) 247 { 248 return __copy_user_nocache(dst, src, size, 0); 249 } 250 251 unsigned long 252 copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest); 253 254 #endif /* _ASM_X86_UACCESS_64_H */ 255