xref: /openbmc/u-boot/arch/arm/lib/memcpy.S (revision 63e22517) 
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  linux/arch/arm/lib/memcpy.S
4 *
5 *  Author:	Nicolas Pitre
6 *  Created:	Sep 28, 2005
7 *  Copyright:	MontaVista Software, Inc.
8 */
9
10#include <linux/linkage.h>
11#include <asm/assembler.h>
12
13#define LDR1W_SHIFT	0
14#define STR1W_SHIFT	0
15
16	.macro ldr1w ptr reg abort
17	W(ldr) \reg, [\ptr], #4
18	.endm
19
20	.macro ldr4w ptr reg1 reg2 reg3 reg4 abort
21	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
22	.endm
23
24	.macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
25	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
26	.endm
27
28	.macro ldr1b ptr reg cond=al abort
29	ldrb\cond\() \reg, [\ptr], #1
30	.endm
31
32	.macro str1w ptr reg abort
33	W(str) \reg, [\ptr], #4
34	.endm
35
36	.macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
37	stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
38	.endm
39
40	.macro str1b ptr reg cond=al abort
41	strb\cond\() \reg, [\ptr], #1
42	.endm
43
44	.macro enter reg1 reg2
45	stmdb sp!, {r0, \reg1, \reg2}
46	.endm
47
48	.macro exit reg1 reg2
49	ldmfd sp!, {r0, \reg1, \reg2}
50	.endm
51
52	.text
53
54/* Prototype: void *memcpy(void *dest, const void *src, size_t n); */
55	.syntax unified
56#if CONFIG_IS_ENABLED(SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD)
57	.thumb
58	.thumb_func
59#endif
60ENTRY(memcpy)
61		cmp	r0, r1
62		moveq	pc, lr
63
64		enter	r4, lr
65
66		subs	r2, r2, #4
67		blt	8f
68		ands	ip, r0, #3
69	PLD(	pld	[r1, #0]		)
70		bne	9f
71		ands	ip, r1, #3
72		bne	10f
73
741:		subs	r2, r2, #(28)
75		stmfd	sp!, {r5 - r8}
76		blt	5f
77
78	CALGN(	ands	ip, r0, #31		)
79	CALGN(	rsb	r3, ip, #32		)
80	CALGN(	sbcsne	r4, r3, r2		)  @ C is always set here
81	CALGN(	bcs	2f			)
82	CALGN(	adr	r4, 6f			)
83	CALGN(	subs	r2, r2, r3		)  @ C gets set
84	CALGN(	add	pc, r4, ip		)
85
86	PLD(	pld	[r1, #0]		)
872:	PLD(	subs	r2, r2, #96		)
88	PLD(	pld	[r1, #28]		)
89	PLD(	blt	4f			)
90	PLD(	pld	[r1, #60]		)
91	PLD(	pld	[r1, #92]		)
92
933:	PLD(	pld	[r1, #124]		)
944:		ldr8w	r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
95		subs	r2, r2, #32
96		str8w	r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
97		bge	3b
98	PLD(	cmn	r2, #96			)
99	PLD(	bge	4b			)
100
1015:		ands	ip, r2, #28
102		rsb	ip, ip, #32
103#if LDR1W_SHIFT > 0
104		lsl	ip, ip, #LDR1W_SHIFT
105#endif
106		addne	pc, pc, ip		@ C is always clear here
107		b	7f
1086:
109		.rept	(1 << LDR1W_SHIFT)
110		W(nop)
111		.endr
112		ldr1w	r1, r3, abort=20f
113		ldr1w	r1, r4, abort=20f
114		ldr1w	r1, r5, abort=20f
115		ldr1w	r1, r6, abort=20f
116		ldr1w	r1, r7, abort=20f
117		ldr1w	r1, r8, abort=20f
118		ldr1w	r1, lr, abort=20f
119
120#if LDR1W_SHIFT < STR1W_SHIFT
121		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
122#elif LDR1W_SHIFT > STR1W_SHIFT
123		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
124#endif
125		add	pc, pc, ip
126		nop
127		.rept	(1 << STR1W_SHIFT)
128		W(nop)
129		.endr
130		str1w	r0, r3, abort=20f
131		str1w	r0, r4, abort=20f
132		str1w	r0, r5, abort=20f
133		str1w	r0, r6, abort=20f
134		str1w	r0, r7, abort=20f
135		str1w	r0, r8, abort=20f
136		str1w	r0, lr, abort=20f
137
138	CALGN(	bcs	2b			)
139
1407:		ldmfd	sp!, {r5 - r8}
141
1428:		movs	r2, r2, lsl #31
143		ldr1b	r1, r3, ne, abort=21f
144		ldr1b	r1, r4, cs, abort=21f
145		ldr1b	r1, ip, cs, abort=21f
146		str1b	r0, r3, ne, abort=21f
147		str1b	r0, r4, cs, abort=21f
148		str1b	r0, ip, cs, abort=21f
149
150		exit	r4, pc
151
1529:		rsb	ip, ip, #4
153		cmp	ip, #2
154		ldr1b	r1, r3, gt, abort=21f
155		ldr1b	r1, r4, ge, abort=21f
156		ldr1b	r1, lr, abort=21f
157		str1b	r0, r3, gt, abort=21f
158		str1b	r0, r4, ge, abort=21f
159		subs	r2, r2, ip
160		str1b	r0, lr, abort=21f
161		blt	8b
162		ands	ip, r1, #3
163		beq	1b
164
16510:		bic	r1, r1, #3
166		cmp	ip, #2
167		ldr1w	r1, lr, abort=21f
168		beq	17f
169		bgt	18f
170
171
172		.macro	forward_copy_shift pull push
173
174		subs	r2, r2, #28
175		blt	14f
176
177	CALGN(	ands	ip, r0, #31		)
178	CALGN(	rsb	ip, ip, #32		)
179	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
180	CALGN(	subcc	r2, r2, ip		)
181	CALGN(	bcc	15f			)
182
18311:		stmfd	sp!, {r5 - r9}
184
185	PLD(	pld	[r1, #0]		)
186	PLD(	subs	r2, r2, #96		)
187	PLD(	pld	[r1, #28]		)
188	PLD(	blt	13f			)
189	PLD(	pld	[r1, #60]		)
190	PLD(	pld	[r1, #92]		)
191
19212:	PLD(	pld	[r1, #124]		)
19313:		ldr4w	r1, r4, r5, r6, r7, abort=19f
194		mov	r3, lr, lspull #\pull
195		subs	r2, r2, #32
196		ldr4w	r1, r8, r9, ip, lr, abort=19f
197		orr	r3, r3, r4, lspush #\push
198		mov	r4, r4, lspull #\pull
199		orr	r4, r4, r5, lspush #\push
200		mov	r5, r5, lspull #\pull
201		orr	r5, r5, r6, lspush #\push
202		mov	r6, r6, lspull #\pull
203		orr	r6, r6, r7, lspush #\push
204		mov	r7, r7, lspull #\pull
205		orr	r7, r7, r8, lspush #\push
206		mov	r8, r8, lspull #\pull
207		orr	r8, r8, r9, lspush #\push
208		mov	r9, r9, lspull #\pull
209		orr	r9, r9, ip, lspush #\push
210		mov	ip, ip, lspull #\pull
211		orr	ip, ip, lr, lspush #\push
212		str8w	r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
213		bge	12b
214	PLD(	cmn	r2, #96			)
215	PLD(	bge	13b			)
216
217		ldmfd	sp!, {r5 - r9}
218
21914:		ands	ip, r2, #28
220		beq	16f
221
22215:		mov	r3, lr, lspull #\pull
223		ldr1w	r1, lr, abort=21f
224		subs	ip, ip, #4
225		orr	r3, r3, lr, lspush #\push
226		str1w	r0, r3, abort=21f
227		bgt	15b
228	CALGN(	cmp	r2, #0			)
229	CALGN(	bge	11b			)
230
23116:		sub	r1, r1, #(\push / 8)
232		b	8b
233
234		.endm
235
236
237		forward_copy_shift	pull=8	push=24
238
23917:		forward_copy_shift	pull=16	push=16
240
24118:		forward_copy_shift	pull=24	push=8
242
243
244/*
245 * Abort preamble and completion macros.
246 * If a fixup handler is required then those macros must surround it.
247 * It is assumed that the fixup code will handle the private part of
248 * the exit macro.
249 */
250
251	.macro	copy_abort_preamble
25219:	ldmfd	sp!, {r5 - r9}
253	b	21f
25420:	ldmfd	sp!, {r5 - r8}
25521:
256	.endm
257
258	.macro	copy_abort_end
259	ldmfd	sp!, {r4, pc}
260	.endm
261
262ENDPROC(memcpy)
263