13fc21751SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 2ad2fc2cdSVitaly Mayatskikh/* 3ad2fc2cdSVitaly Mayatskikh * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> 4ad2fc2cdSVitaly Mayatskikh * Copyright 2002 Andi Kleen, SuSE Labs. 5185f3d38SThomas Gleixner * 6185f3d38SThomas Gleixner * Functions to copy from and to user space. 7185f3d38SThomas Gleixner */ 8185f3d38SThomas Gleixner 9185f3d38SThomas Gleixner#include <linux/linkage.h> 1047ee3f1dSLinus Torvalds#include <asm/cpufeatures.h> 1147ee3f1dSLinus Torvalds#include <asm/alternative.h> 129732da8cSH. Peter Anvin#include <asm/asm.h> 13784d5699SAl Viro#include <asm/export.h> 14185f3d38SThomas Gleixner 15427fda2cSLinus Torvalds/* 16427fda2cSLinus Torvalds * rep_movs_alternative - memory copy with exception handling. 17427fda2cSLinus Torvalds * This version is for CPUs that don't have FSRM (Fast Short Rep Movs) 18427fda2cSLinus Torvalds * 19427fda2cSLinus Torvalds * Input: 20427fda2cSLinus Torvalds * rdi destination 21427fda2cSLinus Torvalds * rsi source 22427fda2cSLinus Torvalds * rcx count 23427fda2cSLinus Torvalds * 24427fda2cSLinus Torvalds * Output: 25427fda2cSLinus Torvalds * rcx uncopied bytes or 0 if successful. 26427fda2cSLinus Torvalds * 27427fda2cSLinus Torvalds * NOTE! The calling convention is very intentionally the same as 28427fda2cSLinus Torvalds * for 'rep movs', so that we can rewrite the function call with 29427fda2cSLinus Torvalds * just a plain 'rep movs' on machines that have FSRM. But to make 30*ca96b162SMateusz Guzik * it simpler for us, we can clobber rsi/rdi and rax freely. 31427fda2cSLinus Torvalds */ 32427fda2cSLinus TorvaldsSYM_FUNC_START(rep_movs_alternative) 33427fda2cSLinus Torvalds cmpq $64,%rcx 3447ee3f1dSLinus Torvalds jae .Llarge 35427fda2cSLinus Torvalds 36427fda2cSLinus Torvalds cmp $8,%ecx 37427fda2cSLinus Torvalds jae .Lword 38427fda2cSLinus Torvalds 39427fda2cSLinus Torvalds testl %ecx,%ecx 40427fda2cSLinus Torvalds je .Lexit 41427fda2cSLinus Torvalds 42427fda2cSLinus Torvalds.Lcopy_user_tail: 43427fda2cSLinus Torvalds0: movb (%rsi),%al 44427fda2cSLinus Torvalds1: movb %al,(%rdi) 45427fda2cSLinus Torvalds inc %rdi 46427fda2cSLinus Torvalds inc %rsi 47427fda2cSLinus Torvalds dec %rcx 48427fda2cSLinus Torvalds jne .Lcopy_user_tail 49427fda2cSLinus Torvalds.Lexit: 50427fda2cSLinus Torvalds RET 51427fda2cSLinus Torvalds 52427fda2cSLinus Torvalds _ASM_EXTABLE_UA( 0b, .Lexit) 53427fda2cSLinus Torvalds _ASM_EXTABLE_UA( 1b, .Lexit) 54427fda2cSLinus Torvalds 55427fda2cSLinus Torvalds .p2align 4 56427fda2cSLinus Torvalds.Lword: 57427fda2cSLinus Torvalds2: movq (%rsi),%rax 58427fda2cSLinus Torvalds3: movq %rax,(%rdi) 59427fda2cSLinus Torvalds addq $8,%rsi 60427fda2cSLinus Torvalds addq $8,%rdi 61427fda2cSLinus Torvalds sub $8,%ecx 62427fda2cSLinus Torvalds je .Lexit 63427fda2cSLinus Torvalds cmp $8,%ecx 64427fda2cSLinus Torvalds jae .Lword 65427fda2cSLinus Torvalds jmp .Lcopy_user_tail 66427fda2cSLinus Torvalds 67427fda2cSLinus Torvalds _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail) 68427fda2cSLinus Torvalds _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail) 69427fda2cSLinus Torvalds 7047ee3f1dSLinus Torvalds.Llarge: 71*ca96b162SMateusz Guzik0: ALTERNATIVE "jmp .Llarge_movsq", "rep movsb", X86_FEATURE_ERMS 7247ee3f1dSLinus Torvalds1: RET 7347ee3f1dSLinus Torvalds 7447ee3f1dSLinus Torvalds _ASM_EXTABLE_UA( 0b, 1b) 7547ee3f1dSLinus Torvalds 76*ca96b162SMateusz Guzik.Llarge_movsq: 77*ca96b162SMateusz Guzik movq %rcx,%rax 78*ca96b162SMateusz Guzik shrq $3,%rcx 79*ca96b162SMateusz Guzik andl $7,%eax 80*ca96b162SMateusz Guzik0: rep movsq 81*ca96b162SMateusz Guzik movl %eax,%ecx 82427fda2cSLinus Torvalds testl %ecx,%ecx 83427fda2cSLinus Torvalds jne .Lcopy_user_tail 84427fda2cSLinus Torvalds RET 85427fda2cSLinus Torvalds 86*ca96b162SMateusz Guzik1: leaq (%rax,%rcx,8),%rcx 87*ca96b162SMateusz Guzik jmp .Lcopy_user_tail 88*ca96b162SMateusz Guzik 89*ca96b162SMateusz Guzik _ASM_EXTABLE_UA( 0b, 1b) 90427fda2cSLinus TorvaldsSYM_FUNC_END(rep_movs_alternative) 91427fda2cSLinus TorvaldsEXPORT_SYMBOL(rep_movs_alternative) 92