1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> 4 * Copyright 2002 Andi Kleen, SuSE Labs. 5 * 6 * Functions to copy from and to user space. 7 */ 8 9#include <linux/linkage.h> 10#include <asm/cpufeatures.h> 11#include <asm/alternative.h> 12#include <asm/asm.h> 13#include <asm/export.h> 14 15/* 16 * rep_movs_alternative - memory copy with exception handling. 17 * This version is for CPUs that don't have FSRM (Fast Short Rep Movs) 18 * 19 * Input: 20 * rdi destination 21 * rsi source 22 * rcx count 23 * 24 * Output: 25 * rcx uncopied bytes or 0 if successful. 26 * 27 * NOTE! The calling convention is very intentionally the same as 28 * for 'rep movs', so that we can rewrite the function call with 29 * just a plain 'rep movs' on machines that have FSRM. But to make 30 * it simpler for us, we can clobber rsi/rdi and rax/r8-r11 freely. 31 */ 32SYM_FUNC_START(rep_movs_alternative) 33 cmpq $64,%rcx 34 jae .Llarge 35 36 cmp $8,%ecx 37 jae .Lword 38 39 testl %ecx,%ecx 40 je .Lexit 41 42.Lcopy_user_tail: 430: movb (%rsi),%al 441: movb %al,(%rdi) 45 inc %rdi 46 inc %rsi 47 dec %rcx 48 jne .Lcopy_user_tail 49.Lexit: 50 RET 51 52 _ASM_EXTABLE_UA( 0b, .Lexit) 53 _ASM_EXTABLE_UA( 1b, .Lexit) 54 55 .p2align 4 56.Lword: 572: movq (%rsi),%rax 583: movq %rax,(%rdi) 59 addq $8,%rsi 60 addq $8,%rdi 61 sub $8,%ecx 62 je .Lexit 63 cmp $8,%ecx 64 jae .Lword 65 jmp .Lcopy_user_tail 66 67 _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail) 68 _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail) 69 70.Llarge: 710: ALTERNATIVE "jmp .Lunrolled", "rep movsb", X86_FEATURE_ERMS 721: RET 73 74 _ASM_EXTABLE_UA( 0b, 1b) 75 76 .p2align 4 77.Lunrolled: 7810: movq (%rsi),%r8 7911: movq 8(%rsi),%r9 8012: movq 16(%rsi),%r10 8113: movq 24(%rsi),%r11 8214: movq %r8,(%rdi) 8315: movq %r9,8(%rdi) 8416: movq %r10,16(%rdi) 8517: movq %r11,24(%rdi) 8620: movq 32(%rsi),%r8 8721: movq 40(%rsi),%r9 8822: movq 48(%rsi),%r10 8923: movq 56(%rsi),%r11 9024: movq %r8,32(%rdi) 9125: movq %r9,40(%rdi) 9226: movq %r10,48(%rdi) 9327: movq %r11,56(%rdi) 94 addq $64,%rsi 95 addq $64,%rdi 96 subq $64,%rcx 97 cmpq $64,%rcx 98 jae .Lunrolled 99 cmpl $8,%ecx 100 jae .Lword 101 testl %ecx,%ecx 102 jne .Lcopy_user_tail 103 RET 104 105 _ASM_EXTABLE_UA(10b, .Lcopy_user_tail) 106 _ASM_EXTABLE_UA(11b, .Lcopy_user_tail) 107 _ASM_EXTABLE_UA(12b, .Lcopy_user_tail) 108 _ASM_EXTABLE_UA(13b, .Lcopy_user_tail) 109 _ASM_EXTABLE_UA(14b, .Lcopy_user_tail) 110 _ASM_EXTABLE_UA(15b, .Lcopy_user_tail) 111 _ASM_EXTABLE_UA(16b, .Lcopy_user_tail) 112 _ASM_EXTABLE_UA(17b, .Lcopy_user_tail) 113 _ASM_EXTABLE_UA(20b, .Lcopy_user_tail) 114 _ASM_EXTABLE_UA(21b, .Lcopy_user_tail) 115 _ASM_EXTABLE_UA(22b, .Lcopy_user_tail) 116 _ASM_EXTABLE_UA(23b, .Lcopy_user_tail) 117 _ASM_EXTABLE_UA(24b, .Lcopy_user_tail) 118 _ASM_EXTABLE_UA(25b, .Lcopy_user_tail) 119 _ASM_EXTABLE_UA(26b, .Lcopy_user_tail) 120 _ASM_EXTABLE_UA(27b, .Lcopy_user_tail) 121SYM_FUNC_END(rep_movs_alternative) 122EXPORT_SYMBOL(rep_movs_alternative) 123