xref: /openbmc/u-boot/arch/arm/lib/memcpy.S (revision 06bda125) 
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  linux/arch/arm/lib/memcpy.S
4 *
5 *  Author:	Nicolas Pitre
6 *  Created:	Sep 28, 2005
7 *  Copyright:	MontaVista Software, Inc.
8 */
9
10#include <linux/linkage.h>
11#include <asm/assembler.h>
12
13#define LDR1W_SHIFT	0
14#define STR1W_SHIFT	0
15
16	.macro ldr1w ptr reg abort
17	W(ldr) \reg, [\ptr], #4
18	.endm
19
20	.macro ldr4w ptr reg1 reg2 reg3 reg4 abort
21	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
22	.endm
23
24	.macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
25	ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
26	.endm
27
28	.macro ldr1b ptr reg cond=al abort
29	ldrb\cond\() \reg, [\ptr], #1
30	.endm
31
32	.macro str1w ptr reg abort
33	W(str) \reg, [\ptr], #4
34	.endm
35
36	.macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
37	stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
38	.endm
39
40	.macro str1b ptr reg cond=al abort
41	strb\cond\() \reg, [\ptr], #1
42	.endm
43
44	.macro enter reg1 reg2
45	stmdb sp!, {r0, \reg1, \reg2}
46	.endm
47
48	.macro exit reg1 reg2
49	ldmfd sp!, {r0, \reg1, \reg2}
50	.endm
51
52	.text
53
54/* Prototype: void *memcpy(void *dest, const void *src, size_t n); */
55	.syntax unified
56#if CONFIG_IS_ENABLED(SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD)
57	.thumb
58	.thumb_func
59#endif
60ENTRY(memcpy)
61		cmp	r0, r1
62		bxeq	lr
63
64		enter	r4, lr
65
66		subs	r2, r2, #4
67		blt	8f
68		ands	ip, r0, #3
69	PLD(	pld	[r1, #0]		)
70		bne	9f
71		ands	ip, r1, #3
72		bne	10f
73
741:		subs	r2, r2, #(28)
75		stmfd	sp!, {r5 - r8}
76		blt	5f
77
78	CALGN(	ands	ip, r0, #31		)
79	CALGN(	rsb	r3, ip, #32		)
80	CALGN(	sbcsne	r4, r3, r2		)  @ C is always set here
81	CALGN(	bcs	2f			)
82	CALGN(	adr	r4, 6f			)
83	CALGN(	subs	r2, r2, r3		)  @ C gets set
84	CALGN(	add	pc, r4, ip		)
85
86	PLD(	pld	[r1, #0]		)
872:	PLD(	subs	r2, r2, #96		)
88	PLD(	pld	[r1, #28]		)
89	PLD(	blt	4f			)
90	PLD(	pld	[r1, #60]		)
91	PLD(	pld	[r1, #92]		)
92
933:	PLD(	pld	[r1, #124]		)
944:		ldr8w	r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
95		subs	r2, r2, #32
96		str8w	r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
97		bge	3b
98	PLD(	cmn	r2, #96			)
99	PLD(	bge	4b			)
100
1015:		ands	ip, r2, #28
102		rsb	ip, ip, #32
103#if LDR1W_SHIFT > 0
104		lsl	ip, ip, #LDR1W_SHIFT
105#endif
106		addne	pc, pc, ip		@ C is always clear here
107		b	7f
1086:
109		.rept	(1 << LDR1W_SHIFT)
110		W(nop)
111		.endr
112		ldr1w	r1, r3, abort=20f
113		ldr1w	r1, r4, abort=20f
114		ldr1w	r1, r5, abort=20f
115		ldr1w	r1, r6, abort=20f
116		ldr1w	r1, r7, abort=20f
117		ldr1w	r1, r8, abort=20f
118		ldr1w	r1, lr, abort=20f
119
120#if LDR1W_SHIFT < STR1W_SHIFT
121		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
122#elif LDR1W_SHIFT > STR1W_SHIFT
123		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
124#endif
125		add	pc, pc, ip
126		nop
127		.rept	(1 << STR1W_SHIFT)
128		W(nop)
129		.endr
130		str1w	r0, r3, abort=20f
131		str1w	r0, r4, abort=20f
132		str1w	r0, r5, abort=20f
133		str1w	r0, r6, abort=20f
134		str1w	r0, r7, abort=20f
135		str1w	r0, r8, abort=20f
136		str1w	r0, lr, abort=20f
137
138	CALGN(	bcs	2b			)
139
1407:		ldmfd	sp!, {r5 - r8}
141
1428:		movs	r2, r2, lsl #31
143		ldr1b	r1, r3, ne, abort=21f
144		ldr1b	r1, r4, cs, abort=21f
145		ldr1b	r1, ip, cs, abort=21f
146		str1b	r0, r3, ne, abort=21f
147		str1b	r0, r4, cs, abort=21f
148		str1b	r0, ip, cs, abort=21f
149
150		exit	r4, lr
151		bx	lr
152
1539:		rsb	ip, ip, #4
154		cmp	ip, #2
155		ldr1b	r1, r3, gt, abort=21f
156		ldr1b	r1, r4, ge, abort=21f
157		ldr1b	r1, lr, abort=21f
158		str1b	r0, r3, gt, abort=21f
159		str1b	r0, r4, ge, abort=21f
160		subs	r2, r2, ip
161		str1b	r0, lr, abort=21f
162		blt	8b
163		ands	ip, r1, #3
164		beq	1b
165
16610:		bic	r1, r1, #3
167		cmp	ip, #2
168		ldr1w	r1, lr, abort=21f
169		beq	17f
170		bgt	18f
171
172
173		.macro	forward_copy_shift pull push
174
175		subs	r2, r2, #28
176		blt	14f
177
178	CALGN(	ands	ip, r0, #31		)
179	CALGN(	rsb	ip, ip, #32		)
180	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
181	CALGN(	subcc	r2, r2, ip		)
182	CALGN(	bcc	15f			)
183
18411:		stmfd	sp!, {r5 - r9}
185
186	PLD(	pld	[r1, #0]		)
187	PLD(	subs	r2, r2, #96		)
188	PLD(	pld	[r1, #28]		)
189	PLD(	blt	13f			)
190	PLD(	pld	[r1, #60]		)
191	PLD(	pld	[r1, #92]		)
192
19312:	PLD(	pld	[r1, #124]		)
19413:		ldr4w	r1, r4, r5, r6, r7, abort=19f
195		mov	r3, lr, lspull #\pull
196		subs	r2, r2, #32
197		ldr4w	r1, r8, r9, ip, lr, abort=19f
198		orr	r3, r3, r4, lspush #\push
199		mov	r4, r4, lspull #\pull
200		orr	r4, r4, r5, lspush #\push
201		mov	r5, r5, lspull #\pull
202		orr	r5, r5, r6, lspush #\push
203		mov	r6, r6, lspull #\pull
204		orr	r6, r6, r7, lspush #\push
205		mov	r7, r7, lspull #\pull
206		orr	r7, r7, r8, lspush #\push
207		mov	r8, r8, lspull #\pull
208		orr	r8, r8, r9, lspush #\push
209		mov	r9, r9, lspull #\pull
210		orr	r9, r9, ip, lspush #\push
211		mov	ip, ip, lspull #\pull
212		orr	ip, ip, lr, lspush #\push
213		str8w	r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
214		bge	12b
215	PLD(	cmn	r2, #96			)
216	PLD(	bge	13b			)
217
218		ldmfd	sp!, {r5 - r9}
219
22014:		ands	ip, r2, #28
221		beq	16f
222
22315:		mov	r3, lr, lspull #\pull
224		ldr1w	r1, lr, abort=21f
225		subs	ip, ip, #4
226		orr	r3, r3, lr, lspush #\push
227		str1w	r0, r3, abort=21f
228		bgt	15b
229	CALGN(	cmp	r2, #0			)
230	CALGN(	bge	11b			)
231
23216:		sub	r1, r1, #(\push / 8)
233		b	8b
234
235		.endm
236
237
238		forward_copy_shift	pull=8	push=24
239
24017:		forward_copy_shift	pull=16	push=16
241
24218:		forward_copy_shift	pull=24	push=8
243
244
245/*
246 * Abort preamble and completion macros.
247 * If a fixup handler is required then those macros must surround it.
248 * It is assumed that the fixup code will handle the private part of
249 * the exit macro.
250 */
251
252	.macro	copy_abort_preamble
25319:	ldmfd	sp!, {r5 - r9}
254	b	21f
25520:	ldmfd	sp!, {r5 - r8}
25621:
257	.endm
258
259	.macro	copy_abort_end
260	ldmfd	sp!, {r4, lr}
261	bx	lr
262	.endm
263
264ENDPROC(memcpy)
265