1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ 3 4#include <linux/linkage.h> 5#include <asm/copy_mc_test.h> 6#include <asm/export.h> 7#include <asm/asm.h> 8 9#ifndef CONFIG_UML 10 11#ifdef CONFIG_X86_MCE 12COPY_MC_TEST_CTL 13 14/* 15 * copy_mc_fragile - copy memory with indication if an exception / fault happened 16 * 17 * The 'fragile' version is opted into by platform quirks and takes 18 * pains to avoid unrecoverable corner cases like 'fast-string' 19 * instruction sequences, and consuming poison across a cacheline 20 * boundary. The non-fragile version is equivalent to memcpy() 21 * regardless of CPU machine-check-recovery capability. 22 */ 23SYM_FUNC_START(copy_mc_fragile) 24 cmpl $8, %edx 25 /* Less than 8 bytes? Go to byte copy loop */ 26 jb .L_no_whole_words 27 28 /* Check for bad alignment of source */ 29 testl $7, %esi 30 /* Already aligned */ 31 jz .L_8byte_aligned 32 33 /* Copy one byte at a time until source is 8-byte aligned */ 34 movl %esi, %ecx 35 andl $7, %ecx 36 subl $8, %ecx 37 negl %ecx 38 subl %ecx, %edx 39.L_read_leading_bytes: 40 movb (%rsi), %al 41 COPY_MC_TEST_SRC %rsi 1 .E_leading_bytes 42 COPY_MC_TEST_DST %rdi 1 .E_leading_bytes 43.L_write_leading_bytes: 44 movb %al, (%rdi) 45 incq %rsi 46 incq %rdi 47 decl %ecx 48 jnz .L_read_leading_bytes 49 50.L_8byte_aligned: 51 movl %edx, %ecx 52 andl $7, %edx 53 shrl $3, %ecx 54 jz .L_no_whole_words 55 56.L_read_words: 57 movq (%rsi), %r8 58 COPY_MC_TEST_SRC %rsi 8 .E_read_words 59 COPY_MC_TEST_DST %rdi 8 .E_write_words 60.L_write_words: 61 movq %r8, (%rdi) 62 addq $8, %rsi 63 addq $8, %rdi 64 decl %ecx 65 jnz .L_read_words 66 67 /* Any trailing bytes? */ 68.L_no_whole_words: 69 andl %edx, %edx 70 jz .L_done_memcpy_trap 71 72 /* Copy trailing bytes */ 73 movl %edx, %ecx 74.L_read_trailing_bytes: 75 movb (%rsi), %al 76 COPY_MC_TEST_SRC %rsi 1 .E_trailing_bytes 77 COPY_MC_TEST_DST %rdi 1 .E_trailing_bytes 78.L_write_trailing_bytes: 79 movb %al, (%rdi) 80 incq %rsi 81 incq %rdi 82 decl %ecx 83 jnz .L_read_trailing_bytes 84 85 /* Copy successful. Return zero */ 86.L_done_memcpy_trap: 87 xorl %eax, %eax 88.L_done: 89 ret 90SYM_FUNC_END(copy_mc_fragile) 91EXPORT_SYMBOL_GPL(copy_mc_fragile) 92 93 .section .fixup, "ax" 94 /* 95 * Return number of bytes not copied for any failure. Note that 96 * there is no "tail" handling since the source buffer is 8-byte 97 * aligned and poison is cacheline aligned. 98 */ 99.E_read_words: 100 shll $3, %ecx 101.E_leading_bytes: 102 addl %edx, %ecx 103.E_trailing_bytes: 104 mov %ecx, %eax 105 jmp .L_done 106 107 /* 108 * For write fault handling, given the destination is unaligned, 109 * we handle faults on multi-byte writes with a byte-by-byte 110 * copy up to the write-protected page. 111 */ 112.E_write_words: 113 shll $3, %ecx 114 addl %edx, %ecx 115 movl %ecx, %edx 116 jmp copy_mc_fragile_handle_tail 117 118 .previous 119 120 _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes) 121 _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words) 122 _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes) 123 _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) 124 _ASM_EXTABLE(.L_write_words, .E_write_words) 125 _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) 126#endif /* CONFIG_X86_MCE */ 127 128/* 129 * copy_mc_enhanced_fast_string - memory copy with exception handling 130 * 131 * Fast string copy + fault / exception handling. If the CPU does 132 * support machine check exception recovery, but does not support 133 * recovering from fast-string exceptions then this CPU needs to be 134 * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any 135 * machine check recovery support this version should be no slower than 136 * standard memcpy. 137 */ 138SYM_FUNC_START(copy_mc_enhanced_fast_string) 139 movq %rdi, %rax 140 movq %rdx, %rcx 141.L_copy: 142 rep movsb 143 /* Copy successful. Return zero */ 144 xorl %eax, %eax 145 ret 146SYM_FUNC_END(copy_mc_enhanced_fast_string) 147 148 .section .fixup, "ax" 149.E_copy: 150 /* 151 * On fault %rcx is updated such that the copy instruction could 152 * optionally be restarted at the fault position, i.e. it 153 * contains 'bytes remaining'. A non-zero return indicates error 154 * to copy_mc_generic() users, or indicate short transfers to 155 * user-copy routines. 156 */ 157 movq %rcx, %rax 158 ret 159 160 .previous 161 162 _ASM_EXTABLE_FAULT(.L_copy, .E_copy) 163#endif /* !CONFIG_UML */ 164