1 /* 2 * arch/arm/include/asm/uaccess.h 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 #ifndef _ASMARM_UACCESS_H 9 #define _ASMARM_UACCESS_H 10 11 /* 12 * User space memory access functions 13 */ 14 #include <linux/string.h> 15 #include <asm/memory.h> 16 #include <asm/domain.h> 17 #include <asm/unified.h> 18 #include <asm/compiler.h> 19 20 #include <asm/extable.h> 21 22 /* 23 * These two functions allow hooking accesses to userspace to increase 24 * system integrity by ensuring that the kernel can not inadvertantly 25 * perform such accesses (eg, via list poison values) which could then 26 * be exploited for priviledge escalation. 27 */ 28 static inline unsigned int uaccess_save_and_enable(void) 29 { 30 #ifdef CONFIG_CPU_SW_DOMAIN_PAN 31 unsigned int old_domain = get_domain(); 32 33 /* Set the current domain access to permit user accesses */ 34 set_domain((old_domain & ~domain_mask(DOMAIN_USER)) | 35 domain_val(DOMAIN_USER, DOMAIN_CLIENT)); 36 37 return old_domain; 38 #else 39 return 0; 40 #endif 41 } 42 43 static inline void uaccess_restore(unsigned int flags) 44 { 45 #ifdef CONFIG_CPU_SW_DOMAIN_PAN 46 /* Restore the user access mask */ 47 set_domain(flags); 48 #endif 49 } 50 51 /* 52 * These two are intentionally not defined anywhere - if the kernel 53 * code generates any references to them, that's a bug. 54 */ 55 extern int __get_user_bad(void); 56 extern int __put_user_bad(void); 57 58 /* 59 * Note that this is actually 0x1,0000,0000 60 */ 61 #define KERNEL_DS 0x00000000 62 63 #ifdef CONFIG_MMU 64 65 #define USER_DS TASK_SIZE 66 #define get_fs() (current_thread_info()->addr_limit) 67 68 static inline void set_fs(mm_segment_t fs) 69 { 70 current_thread_info()->addr_limit = fs; 71 72 /* 73 * Prevent a mispredicted conditional call to set_fs from forwarding 74 * the wrong address limit to access_ok under speculation. 75 */ 76 dsb(nsh); 77 isb(); 78 79 modify_domain(DOMAIN_KERNEL, fs ? DOMAIN_CLIENT : DOMAIN_MANAGER); 80 } 81 82 #define segment_eq(a, b) ((a) == (b)) 83 84 /* We use 33-bit arithmetic here... */ 85 #define __range_ok(addr, size) ({ \ 86 unsigned long flag, roksum; \ 87 __chk_user_ptr(addr); \ 88 __asm__(".syntax unified\n" \ 89 "adds %1, %2, %3; sbcscc %1, %1, %0; movcc %0, #0" \ 90 : "=&r" (flag), "=&r" (roksum) \ 91 : "r" (addr), "Ir" (size), "0" (current_thread_info()->addr_limit) \ 92 : "cc"); \ 93 flag; }) 94 95 /* 96 * This is a type: either unsigned long, if the argument fits into 97 * that type, or otherwise unsigned long long. 98 */ 99 #define __inttype(x) \ 100 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) 101 102 /* 103 * Sanitise a uaccess pointer such that it becomes NULL if addr+size 104 * is above the current addr_limit. 105 */ 106 #define uaccess_mask_range_ptr(ptr, size) \ 107 ((__typeof__(ptr))__uaccess_mask_range_ptr(ptr, size)) 108 static inline void __user *__uaccess_mask_range_ptr(const void __user *ptr, 109 size_t size) 110 { 111 void __user *safe_ptr = (void __user *)ptr; 112 unsigned long tmp; 113 114 asm volatile( 115 " .syntax unified\n" 116 " sub %1, %3, #1\n" 117 " subs %1, %1, %0\n" 118 " addhs %1, %1, #1\n" 119 " subshs %1, %1, %2\n" 120 " movlo %0, #0\n" 121 : "+r" (safe_ptr), "=&r" (tmp) 122 : "r" (size), "r" (current_thread_info()->addr_limit) 123 : "cc"); 124 125 csdb(); 126 return safe_ptr; 127 } 128 129 /* 130 * Single-value transfer routines. They automatically use the right 131 * size if we just have the right pointer type. Note that the functions 132 * which read from user space (*get_*) need to take care not to leak 133 * kernel data even if the calling code is buggy and fails to check 134 * the return value. This means zeroing out the destination variable 135 * or buffer on error. Normally this is done out of line by the 136 * fixup code, but there are a few places where it intrudes on the 137 * main code path. When we only write to user space, there is no 138 * problem. 139 */ 140 extern int __get_user_1(void *); 141 extern int __get_user_2(void *); 142 extern int __get_user_4(void *); 143 extern int __get_user_32t_8(void *); 144 extern int __get_user_8(void *); 145 extern int __get_user_64t_1(void *); 146 extern int __get_user_64t_2(void *); 147 extern int __get_user_64t_4(void *); 148 149 #define __GUP_CLOBBER_1 "lr", "cc" 150 #ifdef CONFIG_CPU_USE_DOMAINS 151 #define __GUP_CLOBBER_2 "ip", "lr", "cc" 152 #else 153 #define __GUP_CLOBBER_2 "lr", "cc" 154 #endif 155 #define __GUP_CLOBBER_4 "lr", "cc" 156 #define __GUP_CLOBBER_32t_8 "lr", "cc" 157 #define __GUP_CLOBBER_8 "lr", "cc" 158 159 #define __get_user_x(__r2, __p, __e, __l, __s) \ 160 __asm__ __volatile__ ( \ 161 __asmeq("%0", "r0") __asmeq("%1", "r2") \ 162 __asmeq("%3", "r1") \ 163 "bl __get_user_" #__s \ 164 : "=&r" (__e), "=r" (__r2) \ 165 : "0" (__p), "r" (__l) \ 166 : __GUP_CLOBBER_##__s) 167 168 /* narrowing a double-word get into a single 32bit word register: */ 169 #ifdef __ARMEB__ 170 #define __get_user_x_32t(__r2, __p, __e, __l, __s) \ 171 __get_user_x(__r2, __p, __e, __l, 32t_8) 172 #else 173 #define __get_user_x_32t __get_user_x 174 #endif 175 176 /* 177 * storing result into proper least significant word of 64bit target var, 178 * different only for big endian case where 64 bit __r2 lsw is r3: 179 */ 180 #ifdef __ARMEB__ 181 #define __get_user_x_64t(__r2, __p, __e, __l, __s) \ 182 __asm__ __volatile__ ( \ 183 __asmeq("%0", "r0") __asmeq("%1", "r2") \ 184 __asmeq("%3", "r1") \ 185 "bl __get_user_64t_" #__s \ 186 : "=&r" (__e), "=r" (__r2) \ 187 : "0" (__p), "r" (__l) \ 188 : __GUP_CLOBBER_##__s) 189 #else 190 #define __get_user_x_64t __get_user_x 191 #endif 192 193 194 #define __get_user_check(x, p) \ 195 ({ \ 196 unsigned long __limit = current_thread_info()->addr_limit - 1; \ 197 register typeof(*(p)) __user *__p asm("r0") = (p); \ 198 register __inttype(x) __r2 asm("r2"); \ 199 register unsigned long __l asm("r1") = __limit; \ 200 register int __e asm("r0"); \ 201 unsigned int __ua_flags = uaccess_save_and_enable(); \ 202 switch (sizeof(*(__p))) { \ 203 case 1: \ 204 if (sizeof((x)) >= 8) \ 205 __get_user_x_64t(__r2, __p, __e, __l, 1); \ 206 else \ 207 __get_user_x(__r2, __p, __e, __l, 1); \ 208 break; \ 209 case 2: \ 210 if (sizeof((x)) >= 8) \ 211 __get_user_x_64t(__r2, __p, __e, __l, 2); \ 212 else \ 213 __get_user_x(__r2, __p, __e, __l, 2); \ 214 break; \ 215 case 4: \ 216 if (sizeof((x)) >= 8) \ 217 __get_user_x_64t(__r2, __p, __e, __l, 4); \ 218 else \ 219 __get_user_x(__r2, __p, __e, __l, 4); \ 220 break; \ 221 case 8: \ 222 if (sizeof((x)) < 8) \ 223 __get_user_x_32t(__r2, __p, __e, __l, 4); \ 224 else \ 225 __get_user_x(__r2, __p, __e, __l, 8); \ 226 break; \ 227 default: __e = __get_user_bad(); break; \ 228 } \ 229 uaccess_restore(__ua_flags); \ 230 x = (typeof(*(p))) __r2; \ 231 __e; \ 232 }) 233 234 #define get_user(x, p) \ 235 ({ \ 236 might_fault(); \ 237 __get_user_check(x, p); \ 238 }) 239 240 extern int __put_user_1(void *, unsigned int); 241 extern int __put_user_2(void *, unsigned int); 242 extern int __put_user_4(void *, unsigned int); 243 extern int __put_user_8(void *, unsigned long long); 244 245 #define __put_user_check(__pu_val, __ptr, __err, __s) \ 246 ({ \ 247 unsigned long __limit = current_thread_info()->addr_limit - 1; \ 248 register typeof(__pu_val) __r2 asm("r2") = __pu_val; \ 249 register const void __user *__p asm("r0") = __ptr; \ 250 register unsigned long __l asm("r1") = __limit; \ 251 register int __e asm("r0"); \ 252 __asm__ __volatile__ ( \ 253 __asmeq("%0", "r0") __asmeq("%2", "r2") \ 254 __asmeq("%3", "r1") \ 255 "bl __put_user_" #__s \ 256 : "=&r" (__e) \ 257 : "0" (__p), "r" (__r2), "r" (__l) \ 258 : "ip", "lr", "cc"); \ 259 __err = __e; \ 260 }) 261 262 #else /* CONFIG_MMU */ 263 264 /* 265 * uClinux has only one addr space, so has simplified address limits. 266 */ 267 #define USER_DS KERNEL_DS 268 269 #define segment_eq(a, b) (1) 270 #define __addr_ok(addr) ((void)(addr), 1) 271 #define __range_ok(addr, size) ((void)(addr), 0) 272 #define get_fs() (KERNEL_DS) 273 274 static inline void set_fs(mm_segment_t fs) 275 { 276 } 277 278 #define get_user(x, p) __get_user(x, p) 279 #define __put_user_check __put_user_nocheck 280 281 #endif /* CONFIG_MMU */ 282 283 #define access_ok(addr, size) (__range_ok(addr, size) == 0) 284 285 #define user_addr_max() \ 286 (uaccess_kernel() ? ~0UL : get_fs()) 287 288 #ifdef CONFIG_CPU_SPECTRE 289 /* 290 * When mitigating Spectre variant 1, it is not worth fixing the non- 291 * verifying accessors, because we need to add verification of the 292 * address space there. Force these to use the standard get_user() 293 * version instead. 294 */ 295 #define __get_user(x, ptr) get_user(x, ptr) 296 #else 297 298 /* 299 * The "__xxx" versions of the user access functions do not verify the 300 * address space - it must have been done previously with a separate 301 * "access_ok()" call. 302 * 303 * The "xxx_error" versions set the third argument to EFAULT if an 304 * error occurs, and leave it unchanged on success. Note that these 305 * versions are void (ie, don't return a value as such). 306 */ 307 #define __get_user(x, ptr) \ 308 ({ \ 309 long __gu_err = 0; \ 310 __get_user_err((x), (ptr), __gu_err); \ 311 __gu_err; \ 312 }) 313 314 #define __get_user_err(x, ptr, err) \ 315 do { \ 316 unsigned long __gu_addr = (unsigned long)(ptr); \ 317 unsigned long __gu_val; \ 318 unsigned int __ua_flags; \ 319 __chk_user_ptr(ptr); \ 320 might_fault(); \ 321 __ua_flags = uaccess_save_and_enable(); \ 322 switch (sizeof(*(ptr))) { \ 323 case 1: __get_user_asm_byte(__gu_val, __gu_addr, err); break; \ 324 case 2: __get_user_asm_half(__gu_val, __gu_addr, err); break; \ 325 case 4: __get_user_asm_word(__gu_val, __gu_addr, err); break; \ 326 default: (__gu_val) = __get_user_bad(); \ 327 } \ 328 uaccess_restore(__ua_flags); \ 329 (x) = (__typeof__(*(ptr)))__gu_val; \ 330 } while (0) 331 332 #define __get_user_asm(x, addr, err, instr) \ 333 __asm__ __volatile__( \ 334 "1: " TUSER(instr) " %1, [%2], #0\n" \ 335 "2:\n" \ 336 " .pushsection .text.fixup,\"ax\"\n" \ 337 " .align 2\n" \ 338 "3: mov %0, %3\n" \ 339 " mov %1, #0\n" \ 340 " b 2b\n" \ 341 " .popsection\n" \ 342 " .pushsection __ex_table,\"a\"\n" \ 343 " .align 3\n" \ 344 " .long 1b, 3b\n" \ 345 " .popsection" \ 346 : "+r" (err), "=&r" (x) \ 347 : "r" (addr), "i" (-EFAULT) \ 348 : "cc") 349 350 #define __get_user_asm_byte(x, addr, err) \ 351 __get_user_asm(x, addr, err, ldrb) 352 353 #if __LINUX_ARM_ARCH__ >= 6 354 355 #define __get_user_asm_half(x, addr, err) \ 356 __get_user_asm(x, addr, err, ldrh) 357 358 #else 359 360 #ifndef __ARMEB__ 361 #define __get_user_asm_half(x, __gu_addr, err) \ 362 ({ \ 363 unsigned long __b1, __b2; \ 364 __get_user_asm_byte(__b1, __gu_addr, err); \ 365 __get_user_asm_byte(__b2, __gu_addr + 1, err); \ 366 (x) = __b1 | (__b2 << 8); \ 367 }) 368 #else 369 #define __get_user_asm_half(x, __gu_addr, err) \ 370 ({ \ 371 unsigned long __b1, __b2; \ 372 __get_user_asm_byte(__b1, __gu_addr, err); \ 373 __get_user_asm_byte(__b2, __gu_addr + 1, err); \ 374 (x) = (__b1 << 8) | __b2; \ 375 }) 376 #endif 377 378 #endif /* __LINUX_ARM_ARCH__ >= 6 */ 379 380 #define __get_user_asm_word(x, addr, err) \ 381 __get_user_asm(x, addr, err, ldr) 382 #endif 383 384 385 #define __put_user_switch(x, ptr, __err, __fn) \ 386 do { \ 387 const __typeof__(*(ptr)) __user *__pu_ptr = (ptr); \ 388 __typeof__(*(ptr)) __pu_val = (x); \ 389 unsigned int __ua_flags; \ 390 might_fault(); \ 391 __ua_flags = uaccess_save_and_enable(); \ 392 switch (sizeof(*(ptr))) { \ 393 case 1: __fn(__pu_val, __pu_ptr, __err, 1); break; \ 394 case 2: __fn(__pu_val, __pu_ptr, __err, 2); break; \ 395 case 4: __fn(__pu_val, __pu_ptr, __err, 4); break; \ 396 case 8: __fn(__pu_val, __pu_ptr, __err, 8); break; \ 397 default: __err = __put_user_bad(); break; \ 398 } \ 399 uaccess_restore(__ua_flags); \ 400 } while (0) 401 402 #define put_user(x, ptr) \ 403 ({ \ 404 int __pu_err = 0; \ 405 __put_user_switch((x), (ptr), __pu_err, __put_user_check); \ 406 __pu_err; \ 407 }) 408 409 #ifdef CONFIG_CPU_SPECTRE 410 /* 411 * When mitigating Spectre variant 1.1, all accessors need to include 412 * verification of the address space. 413 */ 414 #define __put_user(x, ptr) put_user(x, ptr) 415 416 #else 417 #define __put_user(x, ptr) \ 418 ({ \ 419 long __pu_err = 0; \ 420 __put_user_switch((x), (ptr), __pu_err, __put_user_nocheck); \ 421 __pu_err; \ 422 }) 423 424 #define __put_user_nocheck(x, __pu_ptr, __err, __size) \ 425 do { \ 426 unsigned long __pu_addr = (unsigned long)__pu_ptr; \ 427 __put_user_nocheck_##__size(x, __pu_addr, __err); \ 428 } while (0) 429 430 #define __put_user_nocheck_1 __put_user_asm_byte 431 #define __put_user_nocheck_2 __put_user_asm_half 432 #define __put_user_nocheck_4 __put_user_asm_word 433 #define __put_user_nocheck_8 __put_user_asm_dword 434 435 #define __put_user_asm(x, __pu_addr, err, instr) \ 436 __asm__ __volatile__( \ 437 "1: " TUSER(instr) " %1, [%2], #0\n" \ 438 "2:\n" \ 439 " .pushsection .text.fixup,\"ax\"\n" \ 440 " .align 2\n" \ 441 "3: mov %0, %3\n" \ 442 " b 2b\n" \ 443 " .popsection\n" \ 444 " .pushsection __ex_table,\"a\"\n" \ 445 " .align 3\n" \ 446 " .long 1b, 3b\n" \ 447 " .popsection" \ 448 : "+r" (err) \ 449 : "r" (x), "r" (__pu_addr), "i" (-EFAULT) \ 450 : "cc") 451 452 #define __put_user_asm_byte(x, __pu_addr, err) \ 453 __put_user_asm(x, __pu_addr, err, strb) 454 455 #if __LINUX_ARM_ARCH__ >= 6 456 457 #define __put_user_asm_half(x, __pu_addr, err) \ 458 __put_user_asm(x, __pu_addr, err, strh) 459 460 #else 461 462 #ifndef __ARMEB__ 463 #define __put_user_asm_half(x, __pu_addr, err) \ 464 ({ \ 465 unsigned long __temp = (__force unsigned long)(x); \ 466 __put_user_asm_byte(__temp, __pu_addr, err); \ 467 __put_user_asm_byte(__temp >> 8, __pu_addr + 1, err); \ 468 }) 469 #else 470 #define __put_user_asm_half(x, __pu_addr, err) \ 471 ({ \ 472 unsigned long __temp = (__force unsigned long)(x); \ 473 __put_user_asm_byte(__temp >> 8, __pu_addr, err); \ 474 __put_user_asm_byte(__temp, __pu_addr + 1, err); \ 475 }) 476 #endif 477 478 #endif /* __LINUX_ARM_ARCH__ >= 6 */ 479 480 #define __put_user_asm_word(x, __pu_addr, err) \ 481 __put_user_asm(x, __pu_addr, err, str) 482 483 #ifndef __ARMEB__ 484 #define __reg_oper0 "%R2" 485 #define __reg_oper1 "%Q2" 486 #else 487 #define __reg_oper0 "%Q2" 488 #define __reg_oper1 "%R2" 489 #endif 490 491 #define __put_user_asm_dword(x, __pu_addr, err) \ 492 __asm__ __volatile__( \ 493 ARM( "1: " TUSER(str) " " __reg_oper1 ", [%1], #4\n" ) \ 494 ARM( "2: " TUSER(str) " " __reg_oper0 ", [%1]\n" ) \ 495 THUMB( "1: " TUSER(str) " " __reg_oper1 ", [%1]\n" ) \ 496 THUMB( "2: " TUSER(str) " " __reg_oper0 ", [%1, #4]\n" ) \ 497 "3:\n" \ 498 " .pushsection .text.fixup,\"ax\"\n" \ 499 " .align 2\n" \ 500 "4: mov %0, %3\n" \ 501 " b 3b\n" \ 502 " .popsection\n" \ 503 " .pushsection __ex_table,\"a\"\n" \ 504 " .align 3\n" \ 505 " .long 1b, 4b\n" \ 506 " .long 2b, 4b\n" \ 507 " .popsection" \ 508 : "+r" (err), "+r" (__pu_addr) \ 509 : "r" (x), "i" (-EFAULT) \ 510 : "cc") 511 512 #endif /* !CONFIG_CPU_SPECTRE */ 513 514 #ifdef CONFIG_MMU 515 extern unsigned long __must_check 516 arm_copy_from_user(void *to, const void __user *from, unsigned long n); 517 518 static inline unsigned long __must_check 519 raw_copy_from_user(void *to, const void __user *from, unsigned long n) 520 { 521 unsigned int __ua_flags; 522 523 __ua_flags = uaccess_save_and_enable(); 524 n = arm_copy_from_user(to, from, n); 525 uaccess_restore(__ua_flags); 526 return n; 527 } 528 529 extern unsigned long __must_check 530 arm_copy_to_user(void __user *to, const void *from, unsigned long n); 531 extern unsigned long __must_check 532 __copy_to_user_std(void __user *to, const void *from, unsigned long n); 533 534 static inline unsigned long __must_check 535 raw_copy_to_user(void __user *to, const void *from, unsigned long n) 536 { 537 #ifndef CONFIG_UACCESS_WITH_MEMCPY 538 unsigned int __ua_flags; 539 __ua_flags = uaccess_save_and_enable(); 540 n = arm_copy_to_user(to, from, n); 541 uaccess_restore(__ua_flags); 542 return n; 543 #else 544 return arm_copy_to_user(to, from, n); 545 #endif 546 } 547 548 extern unsigned long __must_check 549 arm_clear_user(void __user *addr, unsigned long n); 550 extern unsigned long __must_check 551 __clear_user_std(void __user *addr, unsigned long n); 552 553 static inline unsigned long __must_check 554 __clear_user(void __user *addr, unsigned long n) 555 { 556 unsigned int __ua_flags = uaccess_save_and_enable(); 557 n = arm_clear_user(addr, n); 558 uaccess_restore(__ua_flags); 559 return n; 560 } 561 562 #else 563 static inline unsigned long 564 raw_copy_from_user(void *to, const void __user *from, unsigned long n) 565 { 566 memcpy(to, (const void __force *)from, n); 567 return 0; 568 } 569 static inline unsigned long 570 raw_copy_to_user(void __user *to, const void *from, unsigned long n) 571 { 572 memcpy((void __force *)to, from, n); 573 return 0; 574 } 575 #define __clear_user(addr, n) (memset((void __force *)addr, 0, n), 0) 576 #endif 577 #define INLINE_COPY_TO_USER 578 #define INLINE_COPY_FROM_USER 579 580 static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) 581 { 582 if (access_ok(to, n)) 583 n = __clear_user(to, n); 584 return n; 585 } 586 587 /* These are from lib/ code, and use __get_user() and friends */ 588 extern long strncpy_from_user(char *dest, const char __user *src, long count); 589 590 extern __must_check long strnlen_user(const char __user *str, long n); 591 592 #endif /* _ASMARM_UACCESS_H */ 593