1/* 2 * linux/arch/arm/lib/memcpy.S 3 * 4 * Author: Nicolas Pitre 5 * Created: Sep 28, 2005 6 * Copyright: MontaVista Software, Inc. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13#include <asm/assembler.h> 14 15#define W(instr) instr 16 17#define LDR1W_SHIFT 0 18#define STR1W_SHIFT 0 19 20 .macro ldr1w ptr reg abort 21 W(ldr) \reg, [\ptr], #4 22 .endm 23 24 .macro ldr4w ptr reg1 reg2 reg3 reg4 abort 25 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4} 26 .endm 27 28 .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 29 ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 30 .endm 31 32 .macro ldr1b ptr reg cond=al abort 33 ldr\cond\()b \reg, [\ptr], #1 34 .endm 35 36 .macro str1w ptr reg abort 37 W(str) \reg, [\ptr], #4 38 .endm 39 40 .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort 41 stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8} 42 .endm 43 44 .macro str1b ptr reg cond=al abort 45 str\cond\()b \reg, [\ptr], #1 46 .endm 47 48 .macro enter reg1 reg2 49 stmdb sp!, {r0, \reg1, \reg2} 50 .endm 51 52 .macro exit reg1 reg2 53 ldmfd sp!, {r0, \reg1, \reg2} 54 .endm 55 56 .text 57 58/* Prototype: void *memcpy(void *dest, const void *src, size_t n); */ 59 60.globl memcpy 61memcpy: 62 63 enter r4, lr 64 65 subs r2, r2, #4 66 blt 8f 67 ands ip, r0, #3 68 PLD( pld [r1, #0] ) 69 bne 9f 70 ands ip, r1, #3 71 bne 10f 72 731: subs r2, r2, #(28) 74 stmfd sp!, {r5 - r8} 75 blt 5f 76 77 CALGN( ands ip, r0, #31 ) 78 CALGN( rsb r3, ip, #32 ) 79 CALGN( sbcnes r4, r3, r2 ) @ C is always set here 80 CALGN( bcs 2f ) 81 CALGN( adr r4, 6f ) 82 CALGN( subs r2, r2, r3 ) @ C gets set 83 CALGN( add pc, r4, ip ) 84 85 PLD( pld [r1, #0] ) 862: PLD( subs r2, r2, #96 ) 87 PLD( pld [r1, #28] ) 88 PLD( blt 4f ) 89 PLD( pld [r1, #60] ) 90 PLD( pld [r1, #92] ) 91 923: PLD( pld [r1, #124] ) 934: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 94 subs r2, r2, #32 95 str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f 96 bge 3b 97 PLD( cmn r2, #96 ) 98 PLD( bge 4b ) 99 1005: ands ip, r2, #28 101 rsb ip, ip, #32 102#if LDR1W_SHIFT > 0 103 lsl ip, ip, #LDR1W_SHIFT 104#endif 105 addne pc, pc, ip @ C is always clear here 106 b 7f 1076: 108 .rept (1 << LDR1W_SHIFT) 109 W(nop) 110 .endr 111 ldr1w r1, r3, abort=20f 112 ldr1w r1, r4, abort=20f 113 ldr1w r1, r5, abort=20f 114 ldr1w r1, r6, abort=20f 115 ldr1w r1, r7, abort=20f 116 ldr1w r1, r8, abort=20f 117 ldr1w r1, lr, abort=20f 118 119#if LDR1W_SHIFT < STR1W_SHIFT 120 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT 121#elif LDR1W_SHIFT > STR1W_SHIFT 122 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT 123#endif 124 add pc, pc, ip 125 nop 126 .rept (1 << STR1W_SHIFT) 127 W(nop) 128 .endr 129 str1w r0, r3, abort=20f 130 str1w r0, r4, abort=20f 131 str1w r0, r5, abort=20f 132 str1w r0, r6, abort=20f 133 str1w r0, r7, abort=20f 134 str1w r0, r8, abort=20f 135 str1w r0, lr, abort=20f 136 137 CALGN( bcs 2b ) 138 1397: ldmfd sp!, {r5 - r8} 140 1418: movs r2, r2, lsl #31 142 ldr1b r1, r3, ne, abort=21f 143 ldr1b r1, r4, cs, abort=21f 144 ldr1b r1, ip, cs, abort=21f 145 str1b r0, r3, ne, abort=21f 146 str1b r0, r4, cs, abort=21f 147 str1b r0, ip, cs, abort=21f 148 149 exit r4, pc 150 1519: rsb ip, ip, #4 152 cmp ip, #2 153 ldr1b r1, r3, gt, abort=21f 154 ldr1b r1, r4, ge, abort=21f 155 ldr1b r1, lr, abort=21f 156 str1b r0, r3, gt, abort=21f 157 str1b r0, r4, ge, abort=21f 158 subs r2, r2, ip 159 str1b r0, lr, abort=21f 160 blt 8b 161 ands ip, r1, #3 162 beq 1b 163 16410: bic r1, r1, #3 165 cmp ip, #2 166 ldr1w r1, lr, abort=21f 167 beq 17f 168 bgt 18f 169 170 171 .macro forward_copy_shift pull push 172 173 subs r2, r2, #28 174 blt 14f 175 176 CALGN( ands ip, r0, #31 ) 177 CALGN( rsb ip, ip, #32 ) 178 CALGN( sbcnes r4, ip, r2 ) @ C is always set here 179 CALGN( subcc r2, r2, ip ) 180 CALGN( bcc 15f ) 181 18211: stmfd sp!, {r5 - r9} 183 184 PLD( pld [r1, #0] ) 185 PLD( subs r2, r2, #96 ) 186 PLD( pld [r1, #28] ) 187 PLD( blt 13f ) 188 PLD( pld [r1, #60] ) 189 PLD( pld [r1, #92] ) 190 19112: PLD( pld [r1, #124] ) 19213: ldr4w r1, r4, r5, r6, r7, abort=19f 193 mov r3, lr, pull #\pull 194 subs r2, r2, #32 195 ldr4w r1, r8, r9, ip, lr, abort=19f 196 orr r3, r3, r4, push #\push 197 mov r4, r4, pull #\pull 198 orr r4, r4, r5, push #\push 199 mov r5, r5, pull #\pull 200 orr r5, r5, r6, push #\push 201 mov r6, r6, pull #\pull 202 orr r6, r6, r7, push #\push 203 mov r7, r7, pull #\pull 204 orr r7, r7, r8, push #\push 205 mov r8, r8, pull #\pull 206 orr r8, r8, r9, push #\push 207 mov r9, r9, pull #\pull 208 orr r9, r9, ip, push #\push 209 mov ip, ip, pull #\pull 210 orr ip, ip, lr, push #\push 211 str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f 212 bge 12b 213 PLD( cmn r2, #96 ) 214 PLD( bge 13b ) 215 216 ldmfd sp!, {r5 - r9} 217 21814: ands ip, r2, #28 219 beq 16f 220 22115: mov r3, lr, pull #\pull 222 ldr1w r1, lr, abort=21f 223 subs ip, ip, #4 224 orr r3, r3, lr, push #\push 225 str1w r0, r3, abort=21f 226 bgt 15b 227 CALGN( cmp r2, #0 ) 228 CALGN( bge 11b ) 229 23016: sub r1, r1, #(\push / 8) 231 b 8b 232 233 .endm 234 235 236 forward_copy_shift pull=8 push=24 237 23817: forward_copy_shift pull=16 push=16 239 24018: forward_copy_shift pull=24 push=8 241 242