1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ 3 4#include <linux/linkage.h> 5#include <asm/asm.h> 6 7#ifndef CONFIG_UML 8 9#ifdef CONFIG_X86_MCE 10 11/* 12 * copy_mc_fragile - copy memory with indication if an exception / fault happened 13 * 14 * The 'fragile' version is opted into by platform quirks and takes 15 * pains to avoid unrecoverable corner cases like 'fast-string' 16 * instruction sequences, and consuming poison across a cacheline 17 * boundary. The non-fragile version is equivalent to memcpy() 18 * regardless of CPU machine-check-recovery capability. 19 */ 20SYM_FUNC_START(copy_mc_fragile) 21 cmpl $8, %edx 22 /* Less than 8 bytes? Go to byte copy loop */ 23 jb .L_no_whole_words 24 25 /* Check for bad alignment of source */ 26 testl $7, %esi 27 /* Already aligned */ 28 jz .L_8byte_aligned 29 30 /* Copy one byte at a time until source is 8-byte aligned */ 31 movl %esi, %ecx 32 andl $7, %ecx 33 subl $8, %ecx 34 negl %ecx 35 subl %ecx, %edx 36.L_read_leading_bytes: 37 movb (%rsi), %al 38.L_write_leading_bytes: 39 movb %al, (%rdi) 40 incq %rsi 41 incq %rdi 42 decl %ecx 43 jnz .L_read_leading_bytes 44 45.L_8byte_aligned: 46 movl %edx, %ecx 47 andl $7, %edx 48 shrl $3, %ecx 49 jz .L_no_whole_words 50 51.L_read_words: 52 movq (%rsi), %r8 53.L_write_words: 54 movq %r8, (%rdi) 55 addq $8, %rsi 56 addq $8, %rdi 57 decl %ecx 58 jnz .L_read_words 59 60 /* Any trailing bytes? */ 61.L_no_whole_words: 62 andl %edx, %edx 63 jz .L_done_memcpy_trap 64 65 /* Copy trailing bytes */ 66 movl %edx, %ecx 67.L_read_trailing_bytes: 68 movb (%rsi), %al 69.L_write_trailing_bytes: 70 movb %al, (%rdi) 71 incq %rsi 72 incq %rdi 73 decl %ecx 74 jnz .L_read_trailing_bytes 75 76 /* Copy successful. Return zero */ 77.L_done_memcpy_trap: 78 xorl %eax, %eax 79.L_done: 80 RET 81 82 /* 83 * Return number of bytes not copied for any failure. Note that 84 * there is no "tail" handling since the source buffer is 8-byte 85 * aligned and poison is cacheline aligned. 86 */ 87.E_read_words: 88 shll $3, %ecx 89.E_leading_bytes: 90 addl %edx, %ecx 91.E_trailing_bytes: 92 mov %ecx, %eax 93 jmp .L_done 94 95 /* 96 * For write fault handling, given the destination is unaligned, 97 * we handle faults on multi-byte writes with a byte-by-byte 98 * copy up to the write-protected page. 99 */ 100.E_write_words: 101 shll $3, %ecx 102 addl %edx, %ecx 103 movl %ecx, %edx 104 jmp copy_mc_fragile_handle_tail 105 106 _ASM_EXTABLE_TYPE(.L_read_leading_bytes, .E_leading_bytes, EX_TYPE_DEFAULT_MCE_SAFE) 107 _ASM_EXTABLE_TYPE(.L_read_words, .E_read_words, EX_TYPE_DEFAULT_MCE_SAFE) 108 _ASM_EXTABLE_TYPE(.L_read_trailing_bytes, .E_trailing_bytes, EX_TYPE_DEFAULT_MCE_SAFE) 109 _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) 110 _ASM_EXTABLE(.L_write_words, .E_write_words) 111 _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) 112 113SYM_FUNC_END(copy_mc_fragile) 114#endif /* CONFIG_X86_MCE */ 115 116/* 117 * copy_mc_enhanced_fast_string - memory copy with exception handling 118 * 119 * Fast string copy + fault / exception handling. If the CPU does 120 * support machine check exception recovery, but does not support 121 * recovering from fast-string exceptions then this CPU needs to be 122 * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any 123 * machine check recovery support this version should be no slower than 124 * standard memcpy. 125 */ 126SYM_FUNC_START(copy_mc_enhanced_fast_string) 127 movq %rdi, %rax 128 movq %rdx, %rcx 129.L_copy: 130 rep movsb 131 /* Copy successful. Return zero */ 132 xorl %eax, %eax 133 RET 134 135.E_copy: 136 /* 137 * On fault %rcx is updated such that the copy instruction could 138 * optionally be restarted at the fault position, i.e. it 139 * contains 'bytes remaining'. A non-zero return indicates error 140 * to copy_mc_generic() users, or indicate short transfers to 141 * user-copy routines. 142 */ 143 movq %rcx, %rax 144 RET 145 146 _ASM_EXTABLE_TYPE(.L_copy, .E_copy, EX_TYPE_DEFAULT_MCE_SAFE) 147 148SYM_FUNC_END(copy_mc_enhanced_fast_string) 149#endif /* !CONFIG_UML */ 150