1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * User address space access functions. 4 * The non inlined parts of asm-i386/uaccess.h are here. 5 * 6 * Copyright 1997 Andi Kleen <ak@muc.de> 7 * Copyright 1997 Linus Torvalds 8 */ 9 #include <linux/export.h> 10 #include <linux/uaccess.h> 11 #include <asm/mmx.h> 12 #include <asm/asm.h> 13 14 #ifdef CONFIG_X86_INTEL_USERCOPY 15 /* 16 * Alignment at which movsl is preferred for bulk memory copies. 17 */ 18 struct movsl_mask movsl_mask __read_mostly; 19 #endif 20 21 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n) 22 { 23 #ifdef CONFIG_X86_INTEL_USERCOPY 24 if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask)) 25 return 0; 26 #endif 27 return 1; 28 } 29 #define movsl_is_ok(a1, a2, n) \ 30 __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n)) 31 32 /* 33 * Zero Userspace 34 */ 35 36 #define __do_clear_user(addr,size) \ 37 do { \ 38 int __d0; \ 39 might_fault(); \ 40 __asm__ __volatile__( \ 41 ASM_STAC "\n" \ 42 "0: rep; stosl\n" \ 43 " movl %2,%0\n" \ 44 "1: rep; stosb\n" \ 45 "2: " ASM_CLAC "\n" \ 46 ".section .fixup,\"ax\"\n" \ 47 "3: lea 0(%2,%0,4),%0\n" \ 48 " jmp 2b\n" \ 49 ".previous\n" \ 50 _ASM_EXTABLE_UA(0b, 3b) \ 51 _ASM_EXTABLE_UA(1b, 2b) \ 52 : "=&c"(size), "=&D" (__d0) \ 53 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ 54 } while (0) 55 56 /** 57 * clear_user: - Zero a block of memory in user space. 58 * @to: Destination address, in user space. 59 * @n: Number of bytes to zero. 60 * 61 * Zero a block of memory in user space. 62 * 63 * Returns number of bytes that could not be cleared. 64 * On success, this will be zero. 65 */ 66 unsigned long 67 clear_user(void __user *to, unsigned long n) 68 { 69 might_fault(); 70 if (access_ok(to, n)) 71 __do_clear_user(to, n); 72 return n; 73 } 74 EXPORT_SYMBOL(clear_user); 75 76 /** 77 * __clear_user: - Zero a block of memory in user space, with less checking. 78 * @to: Destination address, in user space. 79 * @n: Number of bytes to zero. 80 * 81 * Zero a block of memory in user space. Caller must check 82 * the specified block with access_ok() before calling this function. 83 * 84 * Returns number of bytes that could not be cleared. 85 * On success, this will be zero. 86 */ 87 unsigned long 88 __clear_user(void __user *to, unsigned long n) 89 { 90 __do_clear_user(to, n); 91 return n; 92 } 93 EXPORT_SYMBOL(__clear_user); 94 95 #ifdef CONFIG_X86_INTEL_USERCOPY 96 static unsigned long 97 __copy_user_intel(void __user *to, const void *from, unsigned long size) 98 { 99 int d0, d1; 100 __asm__ __volatile__( 101 " .align 2,0x90\n" 102 "1: movl 32(%4), %%eax\n" 103 " cmpl $67, %0\n" 104 " jbe 3f\n" 105 "2: movl 64(%4), %%eax\n" 106 " .align 2,0x90\n" 107 "3: movl 0(%4), %%eax\n" 108 "4: movl 4(%4), %%edx\n" 109 "5: movl %%eax, 0(%3)\n" 110 "6: movl %%edx, 4(%3)\n" 111 "7: movl 8(%4), %%eax\n" 112 "8: movl 12(%4),%%edx\n" 113 "9: movl %%eax, 8(%3)\n" 114 "10: movl %%edx, 12(%3)\n" 115 "11: movl 16(%4), %%eax\n" 116 "12: movl 20(%4), %%edx\n" 117 "13: movl %%eax, 16(%3)\n" 118 "14: movl %%edx, 20(%3)\n" 119 "15: movl 24(%4), %%eax\n" 120 "16: movl 28(%4), %%edx\n" 121 "17: movl %%eax, 24(%3)\n" 122 "18: movl %%edx, 28(%3)\n" 123 "19: movl 32(%4), %%eax\n" 124 "20: movl 36(%4), %%edx\n" 125 "21: movl %%eax, 32(%3)\n" 126 "22: movl %%edx, 36(%3)\n" 127 "23: movl 40(%4), %%eax\n" 128 "24: movl 44(%4), %%edx\n" 129 "25: movl %%eax, 40(%3)\n" 130 "26: movl %%edx, 44(%3)\n" 131 "27: movl 48(%4), %%eax\n" 132 "28: movl 52(%4), %%edx\n" 133 "29: movl %%eax, 48(%3)\n" 134 "30: movl %%edx, 52(%3)\n" 135 "31: movl 56(%4), %%eax\n" 136 "32: movl 60(%4), %%edx\n" 137 "33: movl %%eax, 56(%3)\n" 138 "34: movl %%edx, 60(%3)\n" 139 " addl $-64, %0\n" 140 " addl $64, %4\n" 141 " addl $64, %3\n" 142 " cmpl $63, %0\n" 143 " ja 1b\n" 144 "35: movl %0, %%eax\n" 145 " shrl $2, %0\n" 146 " andl $3, %%eax\n" 147 " cld\n" 148 "99: rep; movsl\n" 149 "36: movl %%eax, %0\n" 150 "37: rep; movsb\n" 151 "100:\n" 152 ".section .fixup,\"ax\"\n" 153 "101: lea 0(%%eax,%0,4),%0\n" 154 " jmp 100b\n" 155 ".previous\n" 156 _ASM_EXTABLE_UA(1b, 100b) 157 _ASM_EXTABLE_UA(2b, 100b) 158 _ASM_EXTABLE_UA(3b, 100b) 159 _ASM_EXTABLE_UA(4b, 100b) 160 _ASM_EXTABLE_UA(5b, 100b) 161 _ASM_EXTABLE_UA(6b, 100b) 162 _ASM_EXTABLE_UA(7b, 100b) 163 _ASM_EXTABLE_UA(8b, 100b) 164 _ASM_EXTABLE_UA(9b, 100b) 165 _ASM_EXTABLE_UA(10b, 100b) 166 _ASM_EXTABLE_UA(11b, 100b) 167 _ASM_EXTABLE_UA(12b, 100b) 168 _ASM_EXTABLE_UA(13b, 100b) 169 _ASM_EXTABLE_UA(14b, 100b) 170 _ASM_EXTABLE_UA(15b, 100b) 171 _ASM_EXTABLE_UA(16b, 100b) 172 _ASM_EXTABLE_UA(17b, 100b) 173 _ASM_EXTABLE_UA(18b, 100b) 174 _ASM_EXTABLE_UA(19b, 100b) 175 _ASM_EXTABLE_UA(20b, 100b) 176 _ASM_EXTABLE_UA(21b, 100b) 177 _ASM_EXTABLE_UA(22b, 100b) 178 _ASM_EXTABLE_UA(23b, 100b) 179 _ASM_EXTABLE_UA(24b, 100b) 180 _ASM_EXTABLE_UA(25b, 100b) 181 _ASM_EXTABLE_UA(26b, 100b) 182 _ASM_EXTABLE_UA(27b, 100b) 183 _ASM_EXTABLE_UA(28b, 100b) 184 _ASM_EXTABLE_UA(29b, 100b) 185 _ASM_EXTABLE_UA(30b, 100b) 186 _ASM_EXTABLE_UA(31b, 100b) 187 _ASM_EXTABLE_UA(32b, 100b) 188 _ASM_EXTABLE_UA(33b, 100b) 189 _ASM_EXTABLE_UA(34b, 100b) 190 _ASM_EXTABLE_UA(35b, 100b) 191 _ASM_EXTABLE_UA(36b, 100b) 192 _ASM_EXTABLE_UA(37b, 100b) 193 _ASM_EXTABLE_UA(99b, 101b) 194 : "=&c"(size), "=&D" (d0), "=&S" (d1) 195 : "1"(to), "2"(from), "0"(size) 196 : "eax", "edx", "memory"); 197 return size; 198 } 199 200 static unsigned long __copy_user_intel_nocache(void *to, 201 const void __user *from, unsigned long size) 202 { 203 int d0, d1; 204 205 __asm__ __volatile__( 206 " .align 2,0x90\n" 207 "0: movl 32(%4), %%eax\n" 208 " cmpl $67, %0\n" 209 " jbe 2f\n" 210 "1: movl 64(%4), %%eax\n" 211 " .align 2,0x90\n" 212 "2: movl 0(%4), %%eax\n" 213 "21: movl 4(%4), %%edx\n" 214 " movnti %%eax, 0(%3)\n" 215 " movnti %%edx, 4(%3)\n" 216 "3: movl 8(%4), %%eax\n" 217 "31: movl 12(%4),%%edx\n" 218 " movnti %%eax, 8(%3)\n" 219 " movnti %%edx, 12(%3)\n" 220 "4: movl 16(%4), %%eax\n" 221 "41: movl 20(%4), %%edx\n" 222 " movnti %%eax, 16(%3)\n" 223 " movnti %%edx, 20(%3)\n" 224 "10: movl 24(%4), %%eax\n" 225 "51: movl 28(%4), %%edx\n" 226 " movnti %%eax, 24(%3)\n" 227 " movnti %%edx, 28(%3)\n" 228 "11: movl 32(%4), %%eax\n" 229 "61: movl 36(%4), %%edx\n" 230 " movnti %%eax, 32(%3)\n" 231 " movnti %%edx, 36(%3)\n" 232 "12: movl 40(%4), %%eax\n" 233 "71: movl 44(%4), %%edx\n" 234 " movnti %%eax, 40(%3)\n" 235 " movnti %%edx, 44(%3)\n" 236 "13: movl 48(%4), %%eax\n" 237 "81: movl 52(%4), %%edx\n" 238 " movnti %%eax, 48(%3)\n" 239 " movnti %%edx, 52(%3)\n" 240 "14: movl 56(%4), %%eax\n" 241 "91: movl 60(%4), %%edx\n" 242 " movnti %%eax, 56(%3)\n" 243 " movnti %%edx, 60(%3)\n" 244 " addl $-64, %0\n" 245 " addl $64, %4\n" 246 " addl $64, %3\n" 247 " cmpl $63, %0\n" 248 " ja 0b\n" 249 " sfence \n" 250 "5: movl %0, %%eax\n" 251 " shrl $2, %0\n" 252 " andl $3, %%eax\n" 253 " cld\n" 254 "6: rep; movsl\n" 255 " movl %%eax,%0\n" 256 "7: rep; movsb\n" 257 "8:\n" 258 ".section .fixup,\"ax\"\n" 259 "9: lea 0(%%eax,%0,4),%0\n" 260 "16: jmp 8b\n" 261 ".previous\n" 262 _ASM_EXTABLE_UA(0b, 16b) 263 _ASM_EXTABLE_UA(1b, 16b) 264 _ASM_EXTABLE_UA(2b, 16b) 265 _ASM_EXTABLE_UA(21b, 16b) 266 _ASM_EXTABLE_UA(3b, 16b) 267 _ASM_EXTABLE_UA(31b, 16b) 268 _ASM_EXTABLE_UA(4b, 16b) 269 _ASM_EXTABLE_UA(41b, 16b) 270 _ASM_EXTABLE_UA(10b, 16b) 271 _ASM_EXTABLE_UA(51b, 16b) 272 _ASM_EXTABLE_UA(11b, 16b) 273 _ASM_EXTABLE_UA(61b, 16b) 274 _ASM_EXTABLE_UA(12b, 16b) 275 _ASM_EXTABLE_UA(71b, 16b) 276 _ASM_EXTABLE_UA(13b, 16b) 277 _ASM_EXTABLE_UA(81b, 16b) 278 _ASM_EXTABLE_UA(14b, 16b) 279 _ASM_EXTABLE_UA(91b, 16b) 280 _ASM_EXTABLE_UA(6b, 9b) 281 _ASM_EXTABLE_UA(7b, 16b) 282 : "=&c"(size), "=&D" (d0), "=&S" (d1) 283 : "1"(to), "2"(from), "0"(size) 284 : "eax", "edx", "memory"); 285 return size; 286 } 287 288 #else 289 290 /* 291 * Leave these declared but undefined. They should not be any references to 292 * them 293 */ 294 unsigned long __copy_user_intel(void __user *to, const void *from, 295 unsigned long size); 296 #endif /* CONFIG_X86_INTEL_USERCOPY */ 297 298 /* Generic arbitrary sized copy. */ 299 #define __copy_user(to, from, size) \ 300 do { \ 301 int __d0, __d1, __d2; \ 302 __asm__ __volatile__( \ 303 " cmp $7,%0\n" \ 304 " jbe 1f\n" \ 305 " movl %1,%0\n" \ 306 " negl %0\n" \ 307 " andl $7,%0\n" \ 308 " subl %0,%3\n" \ 309 "4: rep; movsb\n" \ 310 " movl %3,%0\n" \ 311 " shrl $2,%0\n" \ 312 " andl $3,%3\n" \ 313 " .align 2,0x90\n" \ 314 "0: rep; movsl\n" \ 315 " movl %3,%0\n" \ 316 "1: rep; movsb\n" \ 317 "2:\n" \ 318 ".section .fixup,\"ax\"\n" \ 319 "5: addl %3,%0\n" \ 320 " jmp 2b\n" \ 321 "3: lea 0(%3,%0,4),%0\n" \ 322 " jmp 2b\n" \ 323 ".previous\n" \ 324 _ASM_EXTABLE_UA(4b, 5b) \ 325 _ASM_EXTABLE_UA(0b, 3b) \ 326 _ASM_EXTABLE_UA(1b, 2b) \ 327 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ 328 : "3"(size), "0"(size), "1"(to), "2"(from) \ 329 : "memory"); \ 330 } while (0) 331 332 unsigned long __copy_user_ll(void *to, const void *from, unsigned long n) 333 { 334 __uaccess_begin_nospec(); 335 if (movsl_is_ok(to, from, n)) 336 __copy_user(to, from, n); 337 else 338 n = __copy_user_intel(to, from, n); 339 __uaccess_end(); 340 return n; 341 } 342 EXPORT_SYMBOL(__copy_user_ll); 343 344 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, 345 unsigned long n) 346 { 347 __uaccess_begin_nospec(); 348 #ifdef CONFIG_X86_INTEL_USERCOPY 349 if (n > 64 && static_cpu_has(X86_FEATURE_XMM2)) 350 n = __copy_user_intel_nocache(to, from, n); 351 else 352 __copy_user(to, from, n); 353 #else 354 __copy_user(to, from, n); 355 #endif 356 __uaccess_end(); 357 return n; 358 } 359 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); 360