xref: /openbmc/linux/arch/x86/include/asm/uaccess_64.h (revision 4f205687)
1 #ifndef _ASM_X86_UACCESS_64_H
2 #define _ASM_X86_UACCESS_64_H
3 
4 /*
5  * User space memory access functions
6  */
7 #include <linux/compiler.h>
8 #include <linux/errno.h>
9 #include <linux/lockdep.h>
10 #include <linux/kasan-checks.h>
11 #include <asm/alternative.h>
12 #include <asm/cpufeatures.h>
13 #include <asm/page.h>
14 
15 /*
16  * Copy To/From Userspace
17  */
18 
19 /* Handles exceptions in both to and from, but doesn't do access_ok */
20 __must_check unsigned long
21 copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
22 __must_check unsigned long
23 copy_user_generic_string(void *to, const void *from, unsigned len);
24 __must_check unsigned long
25 copy_user_generic_unrolled(void *to, const void *from, unsigned len);
26 
27 static __always_inline __must_check unsigned long
28 copy_user_generic(void *to, const void *from, unsigned len)
29 {
30 	unsigned ret;
31 
32 	/*
33 	 * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
34 	 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
35 	 * Otherwise, use copy_user_generic_unrolled.
36 	 */
37 	alternative_call_2(copy_user_generic_unrolled,
38 			 copy_user_generic_string,
39 			 X86_FEATURE_REP_GOOD,
40 			 copy_user_enhanced_fast_string,
41 			 X86_FEATURE_ERMS,
42 			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
43 				     "=d" (len)),
44 			 "1" (to), "2" (from), "3" (len)
45 			 : "memory", "rcx", "r8", "r9", "r10", "r11");
46 	return ret;
47 }
48 
49 __must_check unsigned long
50 copy_in_user(void __user *to, const void __user *from, unsigned len);
51 
52 static __always_inline __must_check
53 int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
54 {
55 	int ret = 0;
56 
57 	if (!__builtin_constant_p(size))
58 		return copy_user_generic(dst, (__force void *)src, size);
59 	switch (size) {
60 	case 1:
61 		__uaccess_begin();
62 		__get_user_asm(*(u8 *)dst, (u8 __user *)src,
63 			      ret, "b", "b", "=q", 1);
64 		__uaccess_end();
65 		return ret;
66 	case 2:
67 		__uaccess_begin();
68 		__get_user_asm(*(u16 *)dst, (u16 __user *)src,
69 			      ret, "w", "w", "=r", 2);
70 		__uaccess_end();
71 		return ret;
72 	case 4:
73 		__uaccess_begin();
74 		__get_user_asm(*(u32 *)dst, (u32 __user *)src,
75 			      ret, "l", "k", "=r", 4);
76 		__uaccess_end();
77 		return ret;
78 	case 8:
79 		__uaccess_begin();
80 		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
81 			      ret, "q", "", "=r", 8);
82 		__uaccess_end();
83 		return ret;
84 	case 10:
85 		__uaccess_begin();
86 		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
87 			       ret, "q", "", "=r", 10);
88 		if (likely(!ret))
89 			__get_user_asm(*(u16 *)(8 + (char *)dst),
90 				       (u16 __user *)(8 + (char __user *)src),
91 				       ret, "w", "w", "=r", 2);
92 		__uaccess_end();
93 		return ret;
94 	case 16:
95 		__uaccess_begin();
96 		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
97 			       ret, "q", "", "=r", 16);
98 		if (likely(!ret))
99 			__get_user_asm(*(u64 *)(8 + (char *)dst),
100 				       (u64 __user *)(8 + (char __user *)src),
101 				       ret, "q", "", "=r", 8);
102 		__uaccess_end();
103 		return ret;
104 	default:
105 		return copy_user_generic(dst, (__force void *)src, size);
106 	}
107 }
108 
109 static __always_inline __must_check
110 int __copy_from_user(void *dst, const void __user *src, unsigned size)
111 {
112 	might_fault();
113 	kasan_check_write(dst, size);
114 	return __copy_from_user_nocheck(dst, src, size);
115 }
116 
117 static __always_inline __must_check
118 int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
119 {
120 	int ret = 0;
121 
122 	if (!__builtin_constant_p(size))
123 		return copy_user_generic((__force void *)dst, src, size);
124 	switch (size) {
125 	case 1:
126 		__uaccess_begin();
127 		__put_user_asm(*(u8 *)src, (u8 __user *)dst,
128 			      ret, "b", "b", "iq", 1);
129 		__uaccess_end();
130 		return ret;
131 	case 2:
132 		__uaccess_begin();
133 		__put_user_asm(*(u16 *)src, (u16 __user *)dst,
134 			      ret, "w", "w", "ir", 2);
135 		__uaccess_end();
136 		return ret;
137 	case 4:
138 		__uaccess_begin();
139 		__put_user_asm(*(u32 *)src, (u32 __user *)dst,
140 			      ret, "l", "k", "ir", 4);
141 		__uaccess_end();
142 		return ret;
143 	case 8:
144 		__uaccess_begin();
145 		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
146 			      ret, "q", "", "er", 8);
147 		__uaccess_end();
148 		return ret;
149 	case 10:
150 		__uaccess_begin();
151 		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
152 			       ret, "q", "", "er", 10);
153 		if (likely(!ret)) {
154 			asm("":::"memory");
155 			__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
156 				       ret, "w", "w", "ir", 2);
157 		}
158 		__uaccess_end();
159 		return ret;
160 	case 16:
161 		__uaccess_begin();
162 		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
163 			       ret, "q", "", "er", 16);
164 		if (likely(!ret)) {
165 			asm("":::"memory");
166 			__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
167 				       ret, "q", "", "er", 8);
168 		}
169 		__uaccess_end();
170 		return ret;
171 	default:
172 		return copy_user_generic((__force void *)dst, src, size);
173 	}
174 }
175 
176 static __always_inline __must_check
177 int __copy_to_user(void __user *dst, const void *src, unsigned size)
178 {
179 	might_fault();
180 	kasan_check_read(src, size);
181 	return __copy_to_user_nocheck(dst, src, size);
182 }
183 
184 static __always_inline __must_check
185 int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
186 {
187 	int ret = 0;
188 
189 	might_fault();
190 	if (!__builtin_constant_p(size))
191 		return copy_user_generic((__force void *)dst,
192 					 (__force void *)src, size);
193 	switch (size) {
194 	case 1: {
195 		u8 tmp;
196 		__uaccess_begin();
197 		__get_user_asm(tmp, (u8 __user *)src,
198 			       ret, "b", "b", "=q", 1);
199 		if (likely(!ret))
200 			__put_user_asm(tmp, (u8 __user *)dst,
201 				       ret, "b", "b", "iq", 1);
202 		__uaccess_end();
203 		return ret;
204 	}
205 	case 2: {
206 		u16 tmp;
207 		__uaccess_begin();
208 		__get_user_asm(tmp, (u16 __user *)src,
209 			       ret, "w", "w", "=r", 2);
210 		if (likely(!ret))
211 			__put_user_asm(tmp, (u16 __user *)dst,
212 				       ret, "w", "w", "ir", 2);
213 		__uaccess_end();
214 		return ret;
215 	}
216 
217 	case 4: {
218 		u32 tmp;
219 		__uaccess_begin();
220 		__get_user_asm(tmp, (u32 __user *)src,
221 			       ret, "l", "k", "=r", 4);
222 		if (likely(!ret))
223 			__put_user_asm(tmp, (u32 __user *)dst,
224 				       ret, "l", "k", "ir", 4);
225 		__uaccess_end();
226 		return ret;
227 	}
228 	case 8: {
229 		u64 tmp;
230 		__uaccess_begin();
231 		__get_user_asm(tmp, (u64 __user *)src,
232 			       ret, "q", "", "=r", 8);
233 		if (likely(!ret))
234 			__put_user_asm(tmp, (u64 __user *)dst,
235 				       ret, "q", "", "er", 8);
236 		__uaccess_end();
237 		return ret;
238 	}
239 	default:
240 		return copy_user_generic((__force void *)dst,
241 					 (__force void *)src, size);
242 	}
243 }
244 
245 static __must_check __always_inline int
246 __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
247 {
248 	kasan_check_write(dst, size);
249 	return __copy_from_user_nocheck(dst, src, size);
250 }
251 
252 static __must_check __always_inline int
253 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
254 {
255 	kasan_check_read(src, size);
256 	return __copy_to_user_nocheck(dst, src, size);
257 }
258 
259 extern long __copy_user_nocache(void *dst, const void __user *src,
260 				unsigned size, int zerorest);
261 
262 static inline int
263 __copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
264 {
265 	might_fault();
266 	kasan_check_write(dst, size);
267 	return __copy_user_nocache(dst, src, size, 1);
268 }
269 
270 static inline int
271 __copy_from_user_inatomic_nocache(void *dst, const void __user *src,
272 				  unsigned size)
273 {
274 	kasan_check_write(dst, size);
275 	return __copy_user_nocache(dst, src, size, 0);
276 }
277 
278 unsigned long
279 copy_user_handle_tail(char *to, char *from, unsigned len);
280 
281 #endif /* _ASM_X86_UACCESS_64_H */
282