1/* 2 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> 3 * Copyright 2002 Andi Kleen, SuSE Labs. 4 * Subject to the GNU Public License v2. 5 * 6 * Functions to copy from and to user space. 7 */ 8 9#include <linux/linkage.h> 10#include <asm/dwarf2.h> 11#include <asm/current.h> 12#include <asm/asm-offsets.h> 13#include <asm/thread_info.h> 14#include <asm/cpufeature.h> 15#include <asm/alternative-asm.h> 16#include <asm/asm.h> 17#include <asm/smap.h> 18 19/* Standard copy_to_user with segment limit checking */ 20ENTRY(_copy_to_user) 21 CFI_STARTPROC 22 GET_THREAD_INFO(%rax) 23 movq %rdi,%rcx 24 addq %rdx,%rcx 25 jc bad_to_user 26 cmpq TI_addr_limit(%rax),%rcx 27 ja bad_to_user 28 ALTERNATIVE_2 "jmp copy_user_generic_unrolled", \ 29 "jmp copy_user_generic_string", \ 30 X86_FEATURE_REP_GOOD, \ 31 "jmp copy_user_enhanced_fast_string", \ 32 X86_FEATURE_ERMS 33 CFI_ENDPROC 34ENDPROC(_copy_to_user) 35 36/* Standard copy_from_user with segment limit checking */ 37ENTRY(_copy_from_user) 38 CFI_STARTPROC 39 GET_THREAD_INFO(%rax) 40 movq %rsi,%rcx 41 addq %rdx,%rcx 42 jc bad_from_user 43 cmpq TI_addr_limit(%rax),%rcx 44 ja bad_from_user 45 ALTERNATIVE_2 "jmp copy_user_generic_unrolled", \ 46 "jmp copy_user_generic_string", \ 47 X86_FEATURE_REP_GOOD, \ 48 "jmp copy_user_enhanced_fast_string", \ 49 X86_FEATURE_ERMS 50 CFI_ENDPROC 51ENDPROC(_copy_from_user) 52 53 .section .fixup,"ax" 54 /* must zero dest */ 55ENTRY(bad_from_user) 56bad_from_user: 57 CFI_STARTPROC 58 movl %edx,%ecx 59 xorl %eax,%eax 60 rep 61 stosb 62bad_to_user: 63 movl %edx,%eax 64 ret 65 CFI_ENDPROC 66ENDPROC(bad_from_user) 67 .previous 68 69/* 70 * copy_user_generic_unrolled - memory copy with exception handling. 71 * This version is for CPUs like P4 that don't have efficient micro 72 * code for rep movsq 73 * 74 * Input: 75 * rdi destination 76 * rsi source 77 * rdx count 78 * 79 * Output: 80 * eax uncopied bytes or 0 if successful. 81 */ 82ENTRY(copy_user_generic_unrolled) 83 CFI_STARTPROC 84 ASM_STAC 85 cmpl $8,%edx 86 jb 20f /* less then 8 bytes, go to byte copy loop */ 87 ALIGN_DESTINATION 88 movl %edx,%ecx 89 andl $63,%edx 90 shrl $6,%ecx 91 jz 17f 921: movq (%rsi),%r8 932: movq 1*8(%rsi),%r9 943: movq 2*8(%rsi),%r10 954: movq 3*8(%rsi),%r11 965: movq %r8,(%rdi) 976: movq %r9,1*8(%rdi) 987: movq %r10,2*8(%rdi) 998: movq %r11,3*8(%rdi) 1009: movq 4*8(%rsi),%r8 10110: movq 5*8(%rsi),%r9 10211: movq 6*8(%rsi),%r10 10312: movq 7*8(%rsi),%r11 10413: movq %r8,4*8(%rdi) 10514: movq %r9,5*8(%rdi) 10615: movq %r10,6*8(%rdi) 10716: movq %r11,7*8(%rdi) 108 leaq 64(%rsi),%rsi 109 leaq 64(%rdi),%rdi 110 decl %ecx 111 jnz 1b 11217: movl %edx,%ecx 113 andl $7,%edx 114 shrl $3,%ecx 115 jz 20f 11618: movq (%rsi),%r8 11719: movq %r8,(%rdi) 118 leaq 8(%rsi),%rsi 119 leaq 8(%rdi),%rdi 120 decl %ecx 121 jnz 18b 12220: andl %edx,%edx 123 jz 23f 124 movl %edx,%ecx 12521: movb (%rsi),%al 12622: movb %al,(%rdi) 127 incq %rsi 128 incq %rdi 129 decl %ecx 130 jnz 21b 13123: xor %eax,%eax 132 ASM_CLAC 133 ret 134 135 .section .fixup,"ax" 13630: shll $6,%ecx 137 addl %ecx,%edx 138 jmp 60f 13940: leal (%rdx,%rcx,8),%edx 140 jmp 60f 14150: movl %ecx,%edx 14260: jmp copy_user_handle_tail /* ecx is zerorest also */ 143 .previous 144 145 _ASM_EXTABLE(1b,30b) 146 _ASM_EXTABLE(2b,30b) 147 _ASM_EXTABLE(3b,30b) 148 _ASM_EXTABLE(4b,30b) 149 _ASM_EXTABLE(5b,30b) 150 _ASM_EXTABLE(6b,30b) 151 _ASM_EXTABLE(7b,30b) 152 _ASM_EXTABLE(8b,30b) 153 _ASM_EXTABLE(9b,30b) 154 _ASM_EXTABLE(10b,30b) 155 _ASM_EXTABLE(11b,30b) 156 _ASM_EXTABLE(12b,30b) 157 _ASM_EXTABLE(13b,30b) 158 _ASM_EXTABLE(14b,30b) 159 _ASM_EXTABLE(15b,30b) 160 _ASM_EXTABLE(16b,30b) 161 _ASM_EXTABLE(18b,40b) 162 _ASM_EXTABLE(19b,40b) 163 _ASM_EXTABLE(21b,50b) 164 _ASM_EXTABLE(22b,50b) 165 CFI_ENDPROC 166ENDPROC(copy_user_generic_unrolled) 167 168/* Some CPUs run faster using the string copy instructions. 169 * This is also a lot simpler. Use them when possible. 170 * 171 * Only 4GB of copy is supported. This shouldn't be a problem 172 * because the kernel normally only writes from/to page sized chunks 173 * even if user space passed a longer buffer. 174 * And more would be dangerous because both Intel and AMD have 175 * errata with rep movsq > 4GB. If someone feels the need to fix 176 * this please consider this. 177 * 178 * Input: 179 * rdi destination 180 * rsi source 181 * rdx count 182 * 183 * Output: 184 * eax uncopied bytes or 0 if successful. 185 */ 186ENTRY(copy_user_generic_string) 187 CFI_STARTPROC 188 ASM_STAC 189 cmpl $8,%edx 190 jb 2f /* less than 8 bytes, go to byte copy loop */ 191 ALIGN_DESTINATION 192 movl %edx,%ecx 193 shrl $3,%ecx 194 andl $7,%edx 1951: rep 196 movsq 1972: movl %edx,%ecx 1983: rep 199 movsb 200 xorl %eax,%eax 201 ASM_CLAC 202 ret 203 204 .section .fixup,"ax" 20511: leal (%rdx,%rcx,8),%ecx 20612: movl %ecx,%edx /* ecx is zerorest also */ 207 jmp copy_user_handle_tail 208 .previous 209 210 _ASM_EXTABLE(1b,11b) 211 _ASM_EXTABLE(3b,12b) 212 CFI_ENDPROC 213ENDPROC(copy_user_generic_string) 214 215/* 216 * Some CPUs are adding enhanced REP MOVSB/STOSB instructions. 217 * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled. 218 * 219 * Input: 220 * rdi destination 221 * rsi source 222 * rdx count 223 * 224 * Output: 225 * eax uncopied bytes or 0 if successful. 226 */ 227ENTRY(copy_user_enhanced_fast_string) 228 CFI_STARTPROC 229 ASM_STAC 230 movl %edx,%ecx 2311: rep 232 movsb 233 xorl %eax,%eax 234 ASM_CLAC 235 ret 236 237 .section .fixup,"ax" 23812: movl %ecx,%edx /* ecx is zerorest also */ 239 jmp copy_user_handle_tail 240 .previous 241 242 _ASM_EXTABLE(1b,12b) 243 CFI_ENDPROC 244ENDPROC(copy_user_enhanced_fast_string) 245