1/* 2 * linux/arch/arm/lib/memcpy.S 3 * 4 * Author: Nicolas Pitre 5 * Created: Sep 28, 2005 6 * Copyright: MontaVista Software, Inc. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13#include <linux/linkage.h> 14#include <asm/assembler.h> 15 16#if defined(CONFIG_SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD) 17#define W(instr) instr.w 18#else 19#define W(instr) instr 20#endif 21 22#define LDR1W_SHIFT 0 23#define STR1W_SHIFT 0 24 25 .macro ldr1w ptr reg abort 26 W(ldr) \reg, [\ptr], #4 27 .endm 28 29 .macro ldr4w ptr reg1 reg2 reg3 reg4 abort 30 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4} 31 .endm 32 33 .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 34 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 35 .endm 36 37 .macro ldr1b ptr reg cond=al abort 38 ldrb\cond\() \reg, [\ptr], #1 39 .endm 40 41 .macro str1w ptr reg abort 42 W(str) \reg, [\ptr], #4 43 .endm 44 45 .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 46 stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 47 .endm 48 49 .macro str1b ptr reg cond=al abort 50 strb\cond\() \reg, [\ptr], #1 51 .endm 52 53 .macro enter reg1 reg2 54 stmdb sp!, {r0, \reg1, \reg2} 55 .endm 56 57 .macro exit reg1 reg2 58 ldmfd sp!, {r0, \reg1, \reg2} 59 .endm 60 61 .text 62 63/* Prototype: void *memcpy(void *dest, const void *src, size_t n); */ 64 .syntax unified 65#if defined(CONFIG_SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD) 66 .thumb 67 .thumb_func 68#endif 69ENTRY(memcpy) 70 cmp r0, r1 71 moveq pc, lr 72 73 enter r4, lr 74 75 subs r2, r2, #4 76 blt 8f 77 ands ip, r0, #3 78 PLD( pld [r1, #0] ) 79 bne 9f 80 ands ip, r1, #3 81 bne 10f 82 831: subs r2, r2, #(28) 84 stmfd sp!, {r5 - r8} 85 blt 5f 86 87 CALGN( ands ip, r0, #31 ) 88 CALGN( rsb r3, ip, #32 ) 89 CALGN( sbcsne r4, r3, r2 ) @ C is always set here 90 CALGN( bcs 2f ) 91 CALGN( adr r4, 6f ) 92 CALGN( subs r2, r2, r3 ) @ C gets set 93 CALGN( add pc, r4, ip ) 94 95 PLD( pld [r1, #0] ) 962: PLD( subs r2, r2, #96 ) 97 PLD( pld [r1, #28] ) 98 PLD( blt 4f ) 99 PLD( pld [r1, #60] ) 100 PLD( pld [r1, #92] ) 101 1023: PLD( pld [r1, #124] ) 1034: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 104 subs r2, r2, #32 105 str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 106 bge 3b 107 PLD( cmn r2, #96 ) 108 PLD( bge 4b ) 109 1105: ands ip, r2, #28 111 rsb ip, ip, #32 112#if LDR1W_SHIFT > 0 113 lsl ip, ip, #LDR1W_SHIFT 114#endif 115 addne pc, pc, ip @ C is always clear here 116 b 7f 1176: 118 .rept (1 << LDR1W_SHIFT) 119 W(nop) 120 .endr 121 ldr1w r1, r3, abort=20f 122 ldr1w r1, r4, abort=20f 123 ldr1w r1, r5, abort=20f 124 ldr1w r1, r6, abort=20f 125 ldr1w r1, r7, abort=20f 126 ldr1w r1, r8, abort=20f 127 ldr1w r1, lr, abort=20f 128 129#if LDR1W_SHIFT < STR1W_SHIFT 130 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT 131#elif LDR1W_SHIFT > STR1W_SHIFT 132 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT 133#endif 134 add pc, pc, ip 135 nop 136 .rept (1 << STR1W_SHIFT) 137 W(nop) 138 .endr 139 str1w r0, r3, abort=20f 140 str1w r0, r4, abort=20f 141 str1w r0, r5, abort=20f 142 str1w r0, r6, abort=20f 143 str1w r0, r7, abort=20f 144 str1w r0, r8, abort=20f 145 str1w r0, lr, abort=20f 146 147 CALGN( bcs 2b ) 148 1497: ldmfd sp!, {r5 - r8} 150 1518: movs r2, r2, lsl #31 152 ldr1b r1, r3, ne, abort=21f 153 ldr1b r1, r4, cs, abort=21f 154 ldr1b r1, ip, cs, abort=21f 155 str1b r0, r3, ne, abort=21f 156 str1b r0, r4, cs, abort=21f 157 str1b r0, ip, cs, abort=21f 158 159 exit r4, pc 160 1619: rsb ip, ip, #4 162 cmp ip, #2 163 ldr1b r1, r3, gt, abort=21f 164 ldr1b r1, r4, ge, abort=21f 165 ldr1b r1, lr, abort=21f 166 str1b r0, r3, gt, abort=21f 167 str1b r0, r4, ge, abort=21f 168 subs r2, r2, ip 169 str1b r0, lr, abort=21f 170 blt 8b 171 ands ip, r1, #3 172 beq 1b 173 17410: bic r1, r1, #3 175 cmp ip, #2 176 ldr1w r1, lr, abort=21f 177 beq 17f 178 bgt 18f 179 180 181 .macro forward_copy_shift pull push 182 183 subs r2, r2, #28 184 blt 14f 185 186 CALGN( ands ip, r0, #31 ) 187 CALGN( rsb ip, ip, #32 ) 188 CALGN( sbcsne r4, ip, r2 ) @ C is always set here 189 CALGN( subcc r2, r2, ip ) 190 CALGN( bcc 15f ) 191 19211: stmfd sp!, {r5 - r9} 193 194 PLD( pld [r1, #0] ) 195 PLD( subs r2, r2, #96 ) 196 PLD( pld [r1, #28] ) 197 PLD( blt 13f ) 198 PLD( pld [r1, #60] ) 199 PLD( pld [r1, #92] ) 200 20112: PLD( pld [r1, #124] ) 20213: ldr4w r1, r4, r5, r6, r7, abort=19f 203 mov r3, lr, lspull #\pull 204 subs r2, r2, #32 205 ldr4w r1, r8, r9, ip, lr, abort=19f 206 orr r3, r3, r4, lspush #\push 207 mov r4, r4, lspull #\pull 208 orr r4, r4, r5, lspush #\push 209 mov r5, r5, lspull #\pull 210 orr r5, r5, r6, lspush #\push 211 mov r6, r6, lspull #\pull 212 orr r6, r6, r7, lspush #\push 213 mov r7, r7, lspull #\pull 214 orr r7, r7, r8, lspush #\push 215 mov r8, r8, lspull #\pull 216 orr r8, r8, r9, lspush #\push 217 mov r9, r9, lspull #\pull 218 orr r9, r9, ip, lspush #\push 219 mov ip, ip, lspull #\pull 220 orr ip, ip, lr, lspush #\push 221 str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f 222 bge 12b 223 PLD( cmn r2, #96 ) 224 PLD( bge 13b ) 225 226 ldmfd sp!, {r5 - r9} 227 22814: ands ip, r2, #28 229 beq 16f 230 23115: mov r3, lr, lspull #\pull 232 ldr1w r1, lr, abort=21f 233 subs ip, ip, #4 234 orr r3, r3, lr, lspush #\push 235 str1w r0, r3, abort=21f 236 bgt 15b 237 CALGN( cmp r2, #0 ) 238 CALGN( bge 11b ) 239 24016: sub r1, r1, #(\push / 8) 241 b 8b 242 243 .endm 244 245 246 forward_copy_shift pull=8 push=24 247 24817: forward_copy_shift pull=16 push=16 249 25018: forward_copy_shift pull=24 push=8 251 252 253/* 254 * Abort preamble and completion macros. 255 * If a fixup handler is required then those macros must surround it. 256 * It is assumed that the fixup code will handle the private part of 257 * the exit macro. 258 */ 259 260 .macro copy_abort_preamble 26119: ldmfd sp!, {r5 - r9} 262 b 21f 26320: ldmfd sp!, {r5 - r8} 26421: 265 .endm 266 267 .macro copy_abort_end 268 ldmfd sp!, {r4, pc} 269 .endm 270 271ENDPROC(memcpy) 272