1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/uaccess.h 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 */ 7 #ifndef __ASM_UACCESS_H 8 #define __ASM_UACCESS_H 9 10 #include <asm/alternative.h> 11 #include <asm/kernel-pgtable.h> 12 #include <asm/sysreg.h> 13 14 /* 15 * User space memory access functions 16 */ 17 #include <linux/bitops.h> 18 #include <linux/kasan-checks.h> 19 #include <linux/string.h> 20 21 #include <asm/asm-extable.h> 22 #include <asm/cpufeature.h> 23 #include <asm/mmu.h> 24 #include <asm/mte.h> 25 #include <asm/ptrace.h> 26 #include <asm/memory.h> 27 #include <asm/extable.h> 28 29 #define HAVE_GET_KERNEL_NOFAULT 30 31 /* 32 * Test whether a block of memory is a valid user space address. 33 * Returns 1 if the range is valid, 0 otherwise. 34 * 35 * This is equivalent to the following test: 36 * (u65)addr + (u65)size <= (u65)TASK_SIZE_MAX 37 */ 38 static inline unsigned long __range_ok(const void __user *addr, unsigned long size) 39 { 40 unsigned long ret, limit = TASK_SIZE_MAX - 1; 41 42 /* 43 * Asynchronous I/O running in a kernel thread does not have the 44 * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag 45 * the user address before checking. 46 */ 47 if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 48 (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 49 addr = untagged_addr(addr); 50 51 __chk_user_ptr(addr); 52 asm volatile( 53 // A + B <= C + 1 for all A,B,C, in four easy steps: 54 // 1: X = A + B; X' = X % 2^64 55 " adds %0, %3, %2\n" 56 // 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4 57 " csel %1, xzr, %1, hi\n" 58 // 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X' 59 // to compensate for the carry flag being set in step 4. For 60 // X > 2^64, X' merely has to remain nonzero, which it does. 61 " csinv %0, %0, xzr, cc\n" 62 // 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1 63 // comes from the carry in being clear. Otherwise, we are 64 // testing X' - C == 0, subject to the previous adjustments. 65 " sbcs xzr, %0, %1\n" 66 " cset %0, ls\n" 67 : "=&r" (ret), "+r" (limit) : "Ir" (size), "0" (addr) : "cc"); 68 69 return ret; 70 } 71 72 #define access_ok(addr, size) __range_ok(addr, size) 73 74 /* 75 * User access enabling/disabling. 76 */ 77 #ifdef CONFIG_ARM64_SW_TTBR0_PAN 78 static inline void __uaccess_ttbr0_disable(void) 79 { 80 unsigned long flags, ttbr; 81 82 local_irq_save(flags); 83 ttbr = read_sysreg(ttbr1_el1); 84 ttbr &= ~TTBR_ASID_MASK; 85 /* reserved_pg_dir placed before swapper_pg_dir */ 86 write_sysreg(ttbr - RESERVED_SWAPPER_OFFSET, ttbr0_el1); 87 isb(); 88 /* Set reserved ASID */ 89 write_sysreg(ttbr, ttbr1_el1); 90 isb(); 91 local_irq_restore(flags); 92 } 93 94 static inline void __uaccess_ttbr0_enable(void) 95 { 96 unsigned long flags, ttbr0, ttbr1; 97 98 /* 99 * Disable interrupts to avoid preemption between reading the 'ttbr0' 100 * variable and the MSR. A context switch could trigger an ASID 101 * roll-over and an update of 'ttbr0'. 102 */ 103 local_irq_save(flags); 104 ttbr0 = READ_ONCE(current_thread_info()->ttbr0); 105 106 /* Restore active ASID */ 107 ttbr1 = read_sysreg(ttbr1_el1); 108 ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */ 109 ttbr1 |= ttbr0 & TTBR_ASID_MASK; 110 write_sysreg(ttbr1, ttbr1_el1); 111 isb(); 112 113 /* Restore user page table */ 114 write_sysreg(ttbr0, ttbr0_el1); 115 isb(); 116 local_irq_restore(flags); 117 } 118 119 static inline bool uaccess_ttbr0_disable(void) 120 { 121 if (!system_uses_ttbr0_pan()) 122 return false; 123 __uaccess_ttbr0_disable(); 124 return true; 125 } 126 127 static inline bool uaccess_ttbr0_enable(void) 128 { 129 if (!system_uses_ttbr0_pan()) 130 return false; 131 __uaccess_ttbr0_enable(); 132 return true; 133 } 134 #else 135 static inline bool uaccess_ttbr0_disable(void) 136 { 137 return false; 138 } 139 140 static inline bool uaccess_ttbr0_enable(void) 141 { 142 return false; 143 } 144 #endif 145 146 static inline void __uaccess_disable_hw_pan(void) 147 { 148 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, 149 CONFIG_ARM64_PAN)); 150 } 151 152 static inline void __uaccess_enable_hw_pan(void) 153 { 154 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, 155 CONFIG_ARM64_PAN)); 156 } 157 158 /* 159 * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0 160 * affects EL0 and TCF affects EL1 irrespective of which TTBR is 161 * used. 162 * The kernel accesses TTBR0 usually with LDTR/STTR instructions 163 * when UAO is available, so these would act as EL0 accesses using 164 * TCF0. 165 * However futex.h code uses exclusives which would be executed as 166 * EL1, this can potentially cause a tag check fault even if the 167 * user disables TCF0. 168 * 169 * To address the problem we set the PSTATE.TCO bit in uaccess_enable() 170 * and reset it in uaccess_disable(). 171 * 172 * The Tag check override (TCO) bit disables temporarily the tag checking 173 * preventing the issue. 174 */ 175 static inline void __uaccess_disable_tco(void) 176 { 177 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0), 178 ARM64_MTE, CONFIG_KASAN_HW_TAGS)); 179 } 180 181 static inline void __uaccess_enable_tco(void) 182 { 183 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1), 184 ARM64_MTE, CONFIG_KASAN_HW_TAGS)); 185 } 186 187 /* 188 * These functions disable tag checking only if in MTE async mode 189 * since the sync mode generates exceptions synchronously and the 190 * nofault or load_unaligned_zeropad can handle them. 191 */ 192 static inline void __uaccess_disable_tco_async(void) 193 { 194 if (system_uses_mte_async_or_asymm_mode()) 195 __uaccess_disable_tco(); 196 } 197 198 static inline void __uaccess_enable_tco_async(void) 199 { 200 if (system_uses_mte_async_or_asymm_mode()) 201 __uaccess_enable_tco(); 202 } 203 204 static inline void uaccess_disable_privileged(void) 205 { 206 __uaccess_disable_tco(); 207 208 if (uaccess_ttbr0_disable()) 209 return; 210 211 __uaccess_enable_hw_pan(); 212 } 213 214 static inline void uaccess_enable_privileged(void) 215 { 216 __uaccess_enable_tco(); 217 218 if (uaccess_ttbr0_enable()) 219 return; 220 221 __uaccess_disable_hw_pan(); 222 } 223 224 /* 225 * Sanitise a uaccess pointer such that it becomes NULL if above the maximum 226 * user address. In case the pointer is tagged (has the top byte set), untag 227 * the pointer before checking. 228 */ 229 #define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr) 230 static inline void __user *__uaccess_mask_ptr(const void __user *ptr) 231 { 232 void __user *safe_ptr; 233 234 asm volatile( 235 " bics xzr, %3, %2\n" 236 " csel %0, %1, xzr, eq\n" 237 : "=&r" (safe_ptr) 238 : "r" (ptr), "r" (TASK_SIZE_MAX - 1), 239 "r" (untagged_addr(ptr)) 240 : "cc"); 241 242 csdb(); 243 return safe_ptr; 244 } 245 246 /* 247 * The "__xxx" versions of the user access functions do not verify the address 248 * space - it must have been done previously with a separate "access_ok()" 249 * call. 250 * 251 * The "__xxx_error" versions set the third argument to -EFAULT if an error 252 * occurs, and leave it unchanged on success. 253 */ 254 #define __get_mem_asm(load, reg, x, addr, err) \ 255 asm volatile( \ 256 "1: " load " " reg "1, [%2]\n" \ 257 "2:\n" \ 258 _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %w0, %w1) \ 259 : "+r" (err), "=&r" (x) \ 260 : "r" (addr)) 261 262 #define __raw_get_mem(ldr, x, ptr, err) \ 263 do { \ 264 unsigned long __gu_val; \ 265 switch (sizeof(*(ptr))) { \ 266 case 1: \ 267 __get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err)); \ 268 break; \ 269 case 2: \ 270 __get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err)); \ 271 break; \ 272 case 4: \ 273 __get_mem_asm(ldr, "%w", __gu_val, (ptr), (err)); \ 274 break; \ 275 case 8: \ 276 __get_mem_asm(ldr, "%x", __gu_val, (ptr), (err)); \ 277 break; \ 278 default: \ 279 BUILD_BUG(); \ 280 } \ 281 (x) = (__force __typeof__(*(ptr)))__gu_val; \ 282 } while (0) 283 284 #define __raw_get_user(x, ptr, err) \ 285 do { \ 286 __chk_user_ptr(ptr); \ 287 uaccess_ttbr0_enable(); \ 288 __raw_get_mem("ldtr", x, ptr, err); \ 289 uaccess_ttbr0_disable(); \ 290 } while (0) 291 292 #define __get_user_error(x, ptr, err) \ 293 do { \ 294 __typeof__(*(ptr)) __user *__p = (ptr); \ 295 might_fault(); \ 296 if (access_ok(__p, sizeof(*__p))) { \ 297 __p = uaccess_mask_ptr(__p); \ 298 __raw_get_user((x), __p, (err)); \ 299 } else { \ 300 (x) = (__force __typeof__(x))0; (err) = -EFAULT; \ 301 } \ 302 } while (0) 303 304 #define __get_user(x, ptr) \ 305 ({ \ 306 int __gu_err = 0; \ 307 __get_user_error((x), (ptr), __gu_err); \ 308 __gu_err; \ 309 }) 310 311 #define get_user __get_user 312 313 #define __get_kernel_nofault(dst, src, type, err_label) \ 314 do { \ 315 int __gkn_err = 0; \ 316 \ 317 __uaccess_enable_tco_async(); \ 318 __raw_get_mem("ldr", *((type *)(dst)), \ 319 (__force type *)(src), __gkn_err); \ 320 __uaccess_disable_tco_async(); \ 321 if (unlikely(__gkn_err)) \ 322 goto err_label; \ 323 } while (0) 324 325 #define __put_mem_asm(store, reg, x, addr, err) \ 326 asm volatile( \ 327 "1: " store " " reg "1, [%2]\n" \ 328 "2:\n" \ 329 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \ 330 : "+r" (err) \ 331 : "r" (x), "r" (addr)) 332 333 #define __raw_put_mem(str, x, ptr, err) \ 334 do { \ 335 __typeof__(*(ptr)) __pu_val = (x); \ 336 switch (sizeof(*(ptr))) { \ 337 case 1: \ 338 __put_mem_asm(str "b", "%w", __pu_val, (ptr), (err)); \ 339 break; \ 340 case 2: \ 341 __put_mem_asm(str "h", "%w", __pu_val, (ptr), (err)); \ 342 break; \ 343 case 4: \ 344 __put_mem_asm(str, "%w", __pu_val, (ptr), (err)); \ 345 break; \ 346 case 8: \ 347 __put_mem_asm(str, "%x", __pu_val, (ptr), (err)); \ 348 break; \ 349 default: \ 350 BUILD_BUG(); \ 351 } \ 352 } while (0) 353 354 #define __raw_put_user(x, ptr, err) \ 355 do { \ 356 __chk_user_ptr(ptr); \ 357 uaccess_ttbr0_enable(); \ 358 __raw_put_mem("sttr", x, ptr, err); \ 359 uaccess_ttbr0_disable(); \ 360 } while (0) 361 362 #define __put_user_error(x, ptr, err) \ 363 do { \ 364 __typeof__(*(ptr)) __user *__p = (ptr); \ 365 might_fault(); \ 366 if (access_ok(__p, sizeof(*__p))) { \ 367 __p = uaccess_mask_ptr(__p); \ 368 __raw_put_user((x), __p, (err)); \ 369 } else { \ 370 (err) = -EFAULT; \ 371 } \ 372 } while (0) 373 374 #define __put_user(x, ptr) \ 375 ({ \ 376 int __pu_err = 0; \ 377 __put_user_error((x), (ptr), __pu_err); \ 378 __pu_err; \ 379 }) 380 381 #define put_user __put_user 382 383 #define __put_kernel_nofault(dst, src, type, err_label) \ 384 do { \ 385 int __pkn_err = 0; \ 386 \ 387 __uaccess_enable_tco_async(); \ 388 __raw_put_mem("str", *((type *)(src)), \ 389 (__force type *)(dst), __pkn_err); \ 390 __uaccess_disable_tco_async(); \ 391 if (unlikely(__pkn_err)) \ 392 goto err_label; \ 393 } while(0) 394 395 extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); 396 #define raw_copy_from_user(to, from, n) \ 397 ({ \ 398 unsigned long __acfu_ret; \ 399 uaccess_ttbr0_enable(); \ 400 __acfu_ret = __arch_copy_from_user((to), \ 401 __uaccess_mask_ptr(from), (n)); \ 402 uaccess_ttbr0_disable(); \ 403 __acfu_ret; \ 404 }) 405 406 extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); 407 #define raw_copy_to_user(to, from, n) \ 408 ({ \ 409 unsigned long __actu_ret; \ 410 uaccess_ttbr0_enable(); \ 411 __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to), \ 412 (from), (n)); \ 413 uaccess_ttbr0_disable(); \ 414 __actu_ret; \ 415 }) 416 417 #define INLINE_COPY_TO_USER 418 #define INLINE_COPY_FROM_USER 419 420 extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n); 421 static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n) 422 { 423 if (access_ok(to, n)) { 424 uaccess_ttbr0_enable(); 425 n = __arch_clear_user(__uaccess_mask_ptr(to), n); 426 uaccess_ttbr0_disable(); 427 } 428 return n; 429 } 430 #define clear_user __clear_user 431 432 extern long strncpy_from_user(char *dest, const char __user *src, long count); 433 434 extern __must_check long strnlen_user(const char __user *str, long n); 435 436 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 437 struct page; 438 void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len); 439 extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); 440 441 static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) 442 { 443 kasan_check_write(dst, size); 444 return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size); 445 } 446 #endif 447 448 #endif /* __ASM_UACCESS_H */ 449