1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * __get_user functions. 4 * 5 * (C) Copyright 1998 Linus Torvalds 6 * (C) Copyright 2005 Andi Kleen 7 * (C) Copyright 2008 Glauber Costa 8 * 9 * These functions have a non-standard call interface 10 * to make them more efficient, especially as they 11 * return an error value in addition to the "real" 12 * return value. 13 */ 14 15/* 16 * __get_user_X 17 * 18 * Inputs: %[r|e]ax contains the address. 19 * 20 * Outputs: %[r|e]ax is error code (0 or -EFAULT) 21 * %[r|e]dx contains zero-extended value 22 * %ecx contains the high half for 32-bit __get_user_8 23 * 24 * 25 * These functions should not modify any other registers, 26 * as they get called from within inline assembly. 27 */ 28 29#include <linux/linkage.h> 30#include <asm/page_types.h> 31#include <asm/errno.h> 32#include <asm/asm-offsets.h> 33#include <asm/thread_info.h> 34#include <asm/asm.h> 35#include <asm/smap.h> 36#include <asm/export.h> 37 38#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC 39 40.macro check_range size:req 41.if IS_ENABLED(CONFIG_X86_64) 42 mov %rax, %rdx 43 sar $63, %rdx 44 or %rdx, %rax 45.else 46 cmp $TASK_SIZE_MAX-\size+1, %eax 47.if \size != 8 48 jae .Lbad_get_user 49.else 50 jae .Lbad_get_user_8 51.endif 52 sbb %edx, %edx /* array_index_mask_nospec() */ 53 and %edx, %eax 54.endif 55.endm 56 57 .text 58SYM_FUNC_START(__get_user_1) 59 check_range size=1 60 ASM_STAC 611: movzbl (%_ASM_AX),%edx 62 xor %eax,%eax 63 ASM_CLAC 64 RET 65SYM_FUNC_END(__get_user_1) 66EXPORT_SYMBOL(__get_user_1) 67 68SYM_FUNC_START(__get_user_2) 69 check_range size=2 70 ASM_STAC 712: movzwl (%_ASM_AX),%edx 72 xor %eax,%eax 73 ASM_CLAC 74 RET 75SYM_FUNC_END(__get_user_2) 76EXPORT_SYMBOL(__get_user_2) 77 78SYM_FUNC_START(__get_user_4) 79 check_range size=4 80 ASM_STAC 813: movl (%_ASM_AX),%edx 82 xor %eax,%eax 83 ASM_CLAC 84 RET 85SYM_FUNC_END(__get_user_4) 86EXPORT_SYMBOL(__get_user_4) 87 88SYM_FUNC_START(__get_user_8) 89 check_range size=8 90 ASM_STAC 91#ifdef CONFIG_X86_64 924: movq (%_ASM_AX),%rdx 93#else 944: movl (%_ASM_AX),%edx 955: movl 4(%_ASM_AX),%ecx 96#endif 97 xor %eax,%eax 98 ASM_CLAC 99 RET 100SYM_FUNC_END(__get_user_8) 101EXPORT_SYMBOL(__get_user_8) 102 103/* .. and the same for __get_user, just without the range checks */ 104SYM_FUNC_START(__get_user_nocheck_1) 105 ASM_STAC 106 ASM_BARRIER_NOSPEC 1076: movzbl (%_ASM_AX),%edx 108 xor %eax,%eax 109 ASM_CLAC 110 RET 111SYM_FUNC_END(__get_user_nocheck_1) 112EXPORT_SYMBOL(__get_user_nocheck_1) 113 114SYM_FUNC_START(__get_user_nocheck_2) 115 ASM_STAC 116 ASM_BARRIER_NOSPEC 1177: movzwl (%_ASM_AX),%edx 118 xor %eax,%eax 119 ASM_CLAC 120 RET 121SYM_FUNC_END(__get_user_nocheck_2) 122EXPORT_SYMBOL(__get_user_nocheck_2) 123 124SYM_FUNC_START(__get_user_nocheck_4) 125 ASM_STAC 126 ASM_BARRIER_NOSPEC 1278: movl (%_ASM_AX),%edx 128 xor %eax,%eax 129 ASM_CLAC 130 RET 131SYM_FUNC_END(__get_user_nocheck_4) 132EXPORT_SYMBOL(__get_user_nocheck_4) 133 134SYM_FUNC_START(__get_user_nocheck_8) 135 ASM_STAC 136 ASM_BARRIER_NOSPEC 137#ifdef CONFIG_X86_64 1389: movq (%_ASM_AX),%rdx 139#else 1409: movl (%_ASM_AX),%edx 14110: movl 4(%_ASM_AX),%ecx 142#endif 143 xor %eax,%eax 144 ASM_CLAC 145 RET 146SYM_FUNC_END(__get_user_nocheck_8) 147EXPORT_SYMBOL(__get_user_nocheck_8) 148 149 150SYM_CODE_START_LOCAL(__get_user_handle_exception) 151 ASM_CLAC 152.Lbad_get_user: 153 xor %edx,%edx 154 mov $(-EFAULT),%_ASM_AX 155 RET 156SYM_CODE_END(__get_user_handle_exception) 157 158#ifdef CONFIG_X86_32 159SYM_CODE_START_LOCAL(__get_user_8_handle_exception) 160 ASM_CLAC 161.Lbad_get_user_8: 162 xor %edx,%edx 163 xor %ecx,%ecx 164 mov $(-EFAULT),%_ASM_AX 165 RET 166SYM_CODE_END(__get_user_8_handle_exception) 167#endif 168 169/* get_user */ 170 _ASM_EXTABLE_UA(1b, __get_user_handle_exception) 171 _ASM_EXTABLE_UA(2b, __get_user_handle_exception) 172 _ASM_EXTABLE_UA(3b, __get_user_handle_exception) 173#ifdef CONFIG_X86_64 174 _ASM_EXTABLE_UA(4b, __get_user_handle_exception) 175#else 176 _ASM_EXTABLE_UA(4b, __get_user_8_handle_exception) 177 _ASM_EXTABLE_UA(5b, __get_user_8_handle_exception) 178#endif 179 180/* __get_user */ 181 _ASM_EXTABLE_UA(6b, __get_user_handle_exception) 182 _ASM_EXTABLE_UA(7b, __get_user_handle_exception) 183 _ASM_EXTABLE_UA(8b, __get_user_handle_exception) 184#ifdef CONFIG_X86_64 185 _ASM_EXTABLE_UA(9b, __get_user_handle_exception) 186#else 187 _ASM_EXTABLE_UA(9b, __get_user_8_handle_exception) 188 _ASM_EXTABLE_UA(10b, __get_user_8_handle_exception) 189#endif 190