1/* 2 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> 3 * Copyright 2002 Andi Kleen, SuSE Labs. 4 * Subject to the GNU Public License v2. 5 * 6 * Functions to copy from and to user space. 7 */ 8 9#include <linux/linkage.h> 10#include <asm/dwarf2.h> 11 12#define FIX_ALIGNMENT 1 13 14#include <asm/current.h> 15#include <asm/asm-offsets.h> 16#include <asm/thread_info.h> 17#include <asm/cpufeature.h> 18 19 .macro ALTERNATIVE_JUMP feature,orig,alt 200: 21 .byte 0xe9 /* 32bit jump */ 22 .long \orig-1f /* by default jump to orig */ 231: 24 .section .altinstr_replacement,"ax" 252: .byte 0xe9 /* near jump with 32bit immediate */ 26 .long \alt-1b /* offset */ /* or alternatively to alt */ 27 .previous 28 .section .altinstructions,"a" 29 .align 8 30 .quad 0b 31 .quad 2b 32 .byte \feature /* when feature is set */ 33 .byte 5 34 .byte 5 35 .previous 36 .endm 37 38 .macro ALIGN_DESTINATION 39#ifdef FIX_ALIGNMENT 40 /* check for bad alignment of destination */ 41 movl %edi,%ecx 42 andl $7,%ecx 43 jz 102f /* already aligned */ 44 subl $8,%ecx 45 negl %ecx 46 subl %ecx,%edx 47100: movb (%rsi),%al 48101: movb %al,(%rdi) 49 incq %rsi 50 incq %rdi 51 decl %ecx 52 jnz 100b 53102: 54 .section .fixup,"ax" 55103: addl %ecx,%edx /* ecx is zerorest also */ 56 jmp copy_user_handle_tail 57 .previous 58 59 .section __ex_table,"a" 60 .align 8 61 .quad 100b,103b 62 .quad 101b,103b 63 .previous 64#endif 65 .endm 66 67/* Standard copy_to_user with segment limit checking */ 68ENTRY(copy_to_user) 69 CFI_STARTPROC 70 GET_THREAD_INFO(%rax) 71 movq %rdi,%rcx 72 addq %rdx,%rcx 73 jc bad_to_user 74 cmpq TI_addr_limit(%rax),%rcx 75 jae bad_to_user 76 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string 77 CFI_ENDPROC 78ENDPROC(copy_to_user) 79 80/* Standard copy_from_user with segment limit checking */ 81ENTRY(_copy_from_user) 82 CFI_STARTPROC 83 GET_THREAD_INFO(%rax) 84 movq %rsi,%rcx 85 addq %rdx,%rcx 86 jc bad_from_user 87 cmpq TI_addr_limit(%rax),%rcx 88 jae bad_from_user 89 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string 90 CFI_ENDPROC 91ENDPROC(_copy_from_user) 92 93ENTRY(copy_user_generic) 94 CFI_STARTPROC 95 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string 96 CFI_ENDPROC 97ENDPROC(copy_user_generic) 98 99ENTRY(__copy_from_user_inatomic) 100 CFI_STARTPROC 101 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string 102 CFI_ENDPROC 103ENDPROC(__copy_from_user_inatomic) 104 105 .section .fixup,"ax" 106 /* must zero dest */ 107ENTRY(bad_from_user) 108bad_from_user: 109 CFI_STARTPROC 110 movl %edx,%ecx 111 xorl %eax,%eax 112 rep 113 stosb 114bad_to_user: 115 movl %edx,%eax 116 ret 117 CFI_ENDPROC 118ENDPROC(bad_from_user) 119 .previous 120 121/* 122 * copy_user_generic_unrolled - memory copy with exception handling. 123 * This version is for CPUs like P4 that don't have efficient micro 124 * code for rep movsq 125 * 126 * Input: 127 * rdi destination 128 * rsi source 129 * rdx count 130 * 131 * Output: 132 * eax uncopied bytes or 0 if successfull. 133 */ 134ENTRY(copy_user_generic_unrolled) 135 CFI_STARTPROC 136 cmpl $8,%edx 137 jb 20f /* less then 8 bytes, go to byte copy loop */ 138 ALIGN_DESTINATION 139 movl %edx,%ecx 140 andl $63,%edx 141 shrl $6,%ecx 142 jz 17f 1431: movq (%rsi),%r8 1442: movq 1*8(%rsi),%r9 1453: movq 2*8(%rsi),%r10 1464: movq 3*8(%rsi),%r11 1475: movq %r8,(%rdi) 1486: movq %r9,1*8(%rdi) 1497: movq %r10,2*8(%rdi) 1508: movq %r11,3*8(%rdi) 1519: movq 4*8(%rsi),%r8 15210: movq 5*8(%rsi),%r9 15311: movq 6*8(%rsi),%r10 15412: movq 7*8(%rsi),%r11 15513: movq %r8,4*8(%rdi) 15614: movq %r9,5*8(%rdi) 15715: movq %r10,6*8(%rdi) 15816: movq %r11,7*8(%rdi) 159 leaq 64(%rsi),%rsi 160 leaq 64(%rdi),%rdi 161 decl %ecx 162 jnz 1b 16317: movl %edx,%ecx 164 andl $7,%edx 165 shrl $3,%ecx 166 jz 20f 16718: movq (%rsi),%r8 16819: movq %r8,(%rdi) 169 leaq 8(%rsi),%rsi 170 leaq 8(%rdi),%rdi 171 decl %ecx 172 jnz 18b 17320: andl %edx,%edx 174 jz 23f 175 movl %edx,%ecx 17621: movb (%rsi),%al 17722: movb %al,(%rdi) 178 incq %rsi 179 incq %rdi 180 decl %ecx 181 jnz 21b 18223: xor %eax,%eax 183 ret 184 185 .section .fixup,"ax" 18630: shll $6,%ecx 187 addl %ecx,%edx 188 jmp 60f 18940: lea (%rdx,%rcx,8),%rdx 190 jmp 60f 19150: movl %ecx,%edx 19260: jmp copy_user_handle_tail /* ecx is zerorest also */ 193 .previous 194 195 .section __ex_table,"a" 196 .align 8 197 .quad 1b,30b 198 .quad 2b,30b 199 .quad 3b,30b 200 .quad 4b,30b 201 .quad 5b,30b 202 .quad 6b,30b 203 .quad 7b,30b 204 .quad 8b,30b 205 .quad 9b,30b 206 .quad 10b,30b 207 .quad 11b,30b 208 .quad 12b,30b 209 .quad 13b,30b 210 .quad 14b,30b 211 .quad 15b,30b 212 .quad 16b,30b 213 .quad 18b,40b 214 .quad 19b,40b 215 .quad 21b,50b 216 .quad 22b,50b 217 .previous 218 CFI_ENDPROC 219ENDPROC(copy_user_generic_unrolled) 220 221/* Some CPUs run faster using the string copy instructions. 222 * This is also a lot simpler. Use them when possible. 223 * 224 * Only 4GB of copy is supported. This shouldn't be a problem 225 * because the kernel normally only writes from/to page sized chunks 226 * even if user space passed a longer buffer. 227 * And more would be dangerous because both Intel and AMD have 228 * errata with rep movsq > 4GB. If someone feels the need to fix 229 * this please consider this. 230 * 231 * Input: 232 * rdi destination 233 * rsi source 234 * rdx count 235 * 236 * Output: 237 * eax uncopied bytes or 0 if successful. 238 */ 239ENTRY(copy_user_generic_string) 240 CFI_STARTPROC 241 andl %edx,%edx 242 jz 4f 243 cmpl $8,%edx 244 jb 2f /* less than 8 bytes, go to byte copy loop */ 245 ALIGN_DESTINATION 246 movl %edx,%ecx 247 shrl $3,%ecx 248 andl $7,%edx 2491: rep 250 movsq 2512: movl %edx,%ecx 2523: rep 253 movsb 2544: xorl %eax,%eax 255 ret 256 257 .section .fixup,"ax" 25811: lea (%rdx,%rcx,8),%rcx 25912: movl %ecx,%edx /* ecx is zerorest also */ 260 jmp copy_user_handle_tail 261 .previous 262 263 .section __ex_table,"a" 264 .align 8 265 .quad 1b,11b 266 .quad 3b,12b 267 .previous 268 CFI_ENDPROC 269ENDPROC(copy_user_generic_string) 270