1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_UACCESS_64_H 3 #define _ASM_X86_UACCESS_64_H 4 5 /* 6 * User space memory access functions 7 */ 8 #include <linux/compiler.h> 9 #include <linux/lockdep.h> 10 #include <linux/kasan-checks.h> 11 #include <asm/alternative.h> 12 #include <asm/cpufeatures.h> 13 #include <asm/page.h> 14 15 /* 16 * Copy To/From Userspace 17 */ 18 19 /* Handles exceptions in both to and from, but doesn't do access_ok */ 20 __must_check unsigned long 21 copy_user_enhanced_fast_string(void *to, const void *from, unsigned len); 22 __must_check unsigned long 23 copy_user_generic_string(void *to, const void *from, unsigned len); 24 __must_check unsigned long 25 copy_user_generic_unrolled(void *to, const void *from, unsigned len); 26 27 static __always_inline __must_check unsigned long 28 copy_user_generic(void *to, const void *from, unsigned len) 29 { 30 unsigned ret; 31 32 /* 33 * If CPU has ERMS feature, use copy_user_enhanced_fast_string. 34 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string. 35 * Otherwise, use copy_user_generic_unrolled. 36 */ 37 alternative_call_2(copy_user_generic_unrolled, 38 copy_user_generic_string, 39 X86_FEATURE_REP_GOOD, 40 copy_user_enhanced_fast_string, 41 X86_FEATURE_ERMS, 42 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from), 43 "=d" (len)), 44 "1" (to), "2" (from), "3" (len) 45 : "memory", "rcx", "r8", "r9", "r10", "r11"); 46 return ret; 47 } 48 49 static __always_inline __must_check unsigned long 50 raw_copy_from_user(void *dst, const void __user *src, unsigned long size) 51 { 52 int ret = 0; 53 54 if (!__builtin_constant_p(size)) 55 return copy_user_generic(dst, (__force void *)src, size); 56 switch (size) { 57 case 1: 58 __uaccess_begin_nospec(); 59 __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src, 60 ret, "b", "b", "=q", 1); 61 __uaccess_end(); 62 return ret; 63 case 2: 64 __uaccess_begin_nospec(); 65 __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src, 66 ret, "w", "w", "=r", 2); 67 __uaccess_end(); 68 return ret; 69 case 4: 70 __uaccess_begin_nospec(); 71 __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, 72 ret, "l", "k", "=r", 4); 73 __uaccess_end(); 74 return ret; 75 case 8: 76 __uaccess_begin_nospec(); 77 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, 78 ret, "q", "", "=r", 8); 79 __uaccess_end(); 80 return ret; 81 case 10: 82 __uaccess_begin_nospec(); 83 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, 84 ret, "q", "", "=r", 10); 85 if (likely(!ret)) 86 __get_user_asm_nozero(*(u16 *)(8 + (char *)dst), 87 (u16 __user *)(8 + (char __user *)src), 88 ret, "w", "w", "=r", 2); 89 __uaccess_end(); 90 return ret; 91 case 16: 92 __uaccess_begin_nospec(); 93 __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, 94 ret, "q", "", "=r", 16); 95 if (likely(!ret)) 96 __get_user_asm_nozero(*(u64 *)(8 + (char *)dst), 97 (u64 __user *)(8 + (char __user *)src), 98 ret, "q", "", "=r", 8); 99 __uaccess_end(); 100 return ret; 101 default: 102 return copy_user_generic(dst, (__force void *)src, size); 103 } 104 } 105 106 static __always_inline __must_check unsigned long 107 raw_copy_to_user(void __user *dst, const void *src, unsigned long size) 108 { 109 int ret = 0; 110 111 if (!__builtin_constant_p(size)) 112 return copy_user_generic((__force void *)dst, src, size); 113 switch (size) { 114 case 1: 115 __uaccess_begin(); 116 __put_user_asm(*(u8 *)src, (u8 __user *)dst, 117 ret, "b", "b", "iq", 1); 118 __uaccess_end(); 119 return ret; 120 case 2: 121 __uaccess_begin(); 122 __put_user_asm(*(u16 *)src, (u16 __user *)dst, 123 ret, "w", "w", "ir", 2); 124 __uaccess_end(); 125 return ret; 126 case 4: 127 __uaccess_begin(); 128 __put_user_asm(*(u32 *)src, (u32 __user *)dst, 129 ret, "l", "k", "ir", 4); 130 __uaccess_end(); 131 return ret; 132 case 8: 133 __uaccess_begin(); 134 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 135 ret, "q", "", "er", 8); 136 __uaccess_end(); 137 return ret; 138 case 10: 139 __uaccess_begin(); 140 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 141 ret, "q", "", "er", 10); 142 if (likely(!ret)) { 143 asm("":::"memory"); 144 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, 145 ret, "w", "w", "ir", 2); 146 } 147 __uaccess_end(); 148 return ret; 149 case 16: 150 __uaccess_begin(); 151 __put_user_asm(*(u64 *)src, (u64 __user *)dst, 152 ret, "q", "", "er", 16); 153 if (likely(!ret)) { 154 asm("":::"memory"); 155 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, 156 ret, "q", "", "er", 8); 157 } 158 __uaccess_end(); 159 return ret; 160 default: 161 return copy_user_generic((__force void *)dst, src, size); 162 } 163 } 164 165 static __always_inline __must_check 166 unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long size) 167 { 168 return copy_user_generic((__force void *)dst, 169 (__force void *)src, size); 170 } 171 172 extern long __copy_user_nocache(void *dst, const void __user *src, 173 unsigned size, int zerorest); 174 175 extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size); 176 extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset, 177 size_t len); 178 179 static inline int 180 __copy_from_user_inatomic_nocache(void *dst, const void __user *src, 181 unsigned size) 182 { 183 kasan_check_write(dst, size); 184 return __copy_user_nocache(dst, src, size, 0); 185 } 186 187 static inline int 188 __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) 189 { 190 kasan_check_write(dst, size); 191 return __copy_user_flushcache(dst, src, size); 192 } 193 194 unsigned long 195 copy_user_handle_tail(char *to, char *from, unsigned len); 196 197 #endif /* _ASM_X86_UACCESS_64_H */ 198