1/* 2 * linux/arch/arm/lib/memcpy.S 3 * 4 * Author: Nicolas Pitre 5 * Created: Sep 28, 2005 6 * Copyright: MontaVista Software, Inc. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13#include <linux/linkage.h> 14#include <asm/assembler.h> 15 16#define LDR1W_SHIFT 0 17#define STR1W_SHIFT 0 18 19 .macro ldr1w ptr reg abort 20 W(ldr) \reg, [\ptr], #4 21 .endm 22 23 .macro ldr4w ptr reg1 reg2 reg3 reg4 abort 24 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4} 25 .endm 26 27 .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 28 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 29 .endm 30 31 .macro ldr1b ptr reg cond=al abort 32 ldrb\cond\() \reg, [\ptr], #1 33 .endm 34 35 .macro str1w ptr reg abort 36 W(str) \reg, [\ptr], #4 37 .endm 38 39 .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 40 stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 41 .endm 42 43 .macro str1b ptr reg cond=al abort 44 strb\cond\() \reg, [\ptr], #1 45 .endm 46 47 .macro enter reg1 reg2 48 stmdb sp!, {r0, \reg1, \reg2} 49 .endm 50 51 .macro exit reg1 reg2 52 ldmfd sp!, {r0, \reg1, \reg2} 53 .endm 54 55 .text 56 57/* Prototype: void *memcpy(void *dest, const void *src, size_t n); */ 58 .syntax unified 59#if CONFIG_IS_ENABLED(SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD) 60 .thumb 61 .thumb_func 62#endif 63ENTRY(memcpy) 64 cmp r0, r1 65 moveq pc, lr 66 67 enter r4, lr 68 69 subs r2, r2, #4 70 blt 8f 71 ands ip, r0, #3 72 PLD( pld [r1, #0] ) 73 bne 9f 74 ands ip, r1, #3 75 bne 10f 76 771: subs r2, r2, #(28) 78 stmfd sp!, {r5 - r8} 79 blt 5f 80 81 CALGN( ands ip, r0, #31 ) 82 CALGN( rsb r3, ip, #32 ) 83 CALGN( sbcsne r4, r3, r2 ) @ C is always set here 84 CALGN( bcs 2f ) 85 CALGN( adr r4, 6f ) 86 CALGN( subs r2, r2, r3 ) @ C gets set 87 CALGN( add pc, r4, ip ) 88 89 PLD( pld [r1, #0] ) 902: PLD( subs r2, r2, #96 ) 91 PLD( pld [r1, #28] ) 92 PLD( blt 4f ) 93 PLD( pld [r1, #60] ) 94 PLD( pld [r1, #92] ) 95 963: PLD( pld [r1, #124] ) 974: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 98 subs r2, r2, #32 99 str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 100 bge 3b 101 PLD( cmn r2, #96 ) 102 PLD( bge 4b ) 103 1045: ands ip, r2, #28 105 rsb ip, ip, #32 106#if LDR1W_SHIFT > 0 107 lsl ip, ip, #LDR1W_SHIFT 108#endif 109 addne pc, pc, ip @ C is always clear here 110 b 7f 1116: 112 .rept (1 << LDR1W_SHIFT) 113 W(nop) 114 .endr 115 ldr1w r1, r3, abort=20f 116 ldr1w r1, r4, abort=20f 117 ldr1w r1, r5, abort=20f 118 ldr1w r1, r6, abort=20f 119 ldr1w r1, r7, abort=20f 120 ldr1w r1, r8, abort=20f 121 ldr1w r1, lr, abort=20f 122 123#if LDR1W_SHIFT < STR1W_SHIFT 124 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT 125#elif LDR1W_SHIFT > STR1W_SHIFT 126 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT 127#endif 128 add pc, pc, ip 129 nop 130 .rept (1 << STR1W_SHIFT) 131 W(nop) 132 .endr 133 str1w r0, r3, abort=20f 134 str1w r0, r4, abort=20f 135 str1w r0, r5, abort=20f 136 str1w r0, r6, abort=20f 137 str1w r0, r7, abort=20f 138 str1w r0, r8, abort=20f 139 str1w r0, lr, abort=20f 140 141 CALGN( bcs 2b ) 142 1437: ldmfd sp!, {r5 - r8} 144 1458: movs r2, r2, lsl #31 146 ldr1b r1, r3, ne, abort=21f 147 ldr1b r1, r4, cs, abort=21f 148 ldr1b r1, ip, cs, abort=21f 149 str1b r0, r3, ne, abort=21f 150 str1b r0, r4, cs, abort=21f 151 str1b r0, ip, cs, abort=21f 152 153 exit r4, pc 154 1559: rsb ip, ip, #4 156 cmp ip, #2 157 ldr1b r1, r3, gt, abort=21f 158 ldr1b r1, r4, ge, abort=21f 159 ldr1b r1, lr, abort=21f 160 str1b r0, r3, gt, abort=21f 161 str1b r0, r4, ge, abort=21f 162 subs r2, r2, ip 163 str1b r0, lr, abort=21f 164 blt 8b 165 ands ip, r1, #3 166 beq 1b 167 16810: bic r1, r1, #3 169 cmp ip, #2 170 ldr1w r1, lr, abort=21f 171 beq 17f 172 bgt 18f 173 174 175 .macro forward_copy_shift pull push 176 177 subs r2, r2, #28 178 blt 14f 179 180 CALGN( ands ip, r0, #31 ) 181 CALGN( rsb ip, ip, #32 ) 182 CALGN( sbcsne r4, ip, r2 ) @ C is always set here 183 CALGN( subcc r2, r2, ip ) 184 CALGN( bcc 15f ) 185 18611: stmfd sp!, {r5 - r9} 187 188 PLD( pld [r1, #0] ) 189 PLD( subs r2, r2, #96 ) 190 PLD( pld [r1, #28] ) 191 PLD( blt 13f ) 192 PLD( pld [r1, #60] ) 193 PLD( pld [r1, #92] ) 194 19512: PLD( pld [r1, #124] ) 19613: ldr4w r1, r4, r5, r6, r7, abort=19f 197 mov r3, lr, lspull #\pull 198 subs r2, r2, #32 199 ldr4w r1, r8, r9, ip, lr, abort=19f 200 orr r3, r3, r4, lspush #\push 201 mov r4, r4, lspull #\pull 202 orr r4, r4, r5, lspush #\push 203 mov r5, r5, lspull #\pull 204 orr r5, r5, r6, lspush #\push 205 mov r6, r6, lspull #\pull 206 orr r6, r6, r7, lspush #\push 207 mov r7, r7, lspull #\pull 208 orr r7, r7, r8, lspush #\push 209 mov r8, r8, lspull #\pull 210 orr r8, r8, r9, lspush #\push 211 mov r9, r9, lspull #\pull 212 orr r9, r9, ip, lspush #\push 213 mov ip, ip, lspull #\pull 214 orr ip, ip, lr, lspush #\push 215 str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f 216 bge 12b 217 PLD( cmn r2, #96 ) 218 PLD( bge 13b ) 219 220 ldmfd sp!, {r5 - r9} 221 22214: ands ip, r2, #28 223 beq 16f 224 22515: mov r3, lr, lspull #\pull 226 ldr1w r1, lr, abort=21f 227 subs ip, ip, #4 228 orr r3, r3, lr, lspush #\push 229 str1w r0, r3, abort=21f 230 bgt 15b 231 CALGN( cmp r2, #0 ) 232 CALGN( bge 11b ) 233 23416: sub r1, r1, #(\push / 8) 235 b 8b 236 237 .endm 238 239 240 forward_copy_shift pull=8 push=24 241 24217: forward_copy_shift pull=16 push=16 243 24418: forward_copy_shift pull=24 push=8 245 246 247/* 248 * Abort preamble and completion macros. 249 * If a fixup handler is required then those macros must surround it. 250 * It is assumed that the fixup code will handle the private part of 251 * the exit macro. 252 */ 253 254 .macro copy_abort_preamble 25519: ldmfd sp!, {r5 - r9} 256 b 21f 25720: ldmfd sp!, {r5 - r8} 25821: 259 .endm 260 261 .macro copy_abort_end 262 ldmfd sp!, {r4, pc} 263 .endm 264 265ENDPROC(memcpy) 266