xref: /openbmc/linux/arch/arm/lib/copy_template.S (revision 81de3bf3) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 *  linux/arch/arm/lib/copy_template.s
4 *
5 *  Code template for optimized memory copy functions
6 *
7 *  Author:	Nicolas Pitre
8 *  Created:	Sep 28, 2005
9 *  Copyright:	MontaVista Software, Inc.
10 */
11
12/*
13 * Theory of operation
14 * -------------------
15 *
16 * This file provides the core code for a forward memory copy used in
17 * the implementation of memcopy(), copy_to_user() and copy_from_user().
18 *
19 * The including file must define the following accessor macros
20 * according to the need of the given function:
21 *
22 * ldr1w ptr reg abort
23 *
24 *	This loads one word from 'ptr', stores it in 'reg' and increments
25 *	'ptr' to the next word. The 'abort' argument is used for fixup tables.
26 *
27 * ldr4w ptr reg1 reg2 reg3 reg4 abort
28 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
29 *
30 *	This loads four or eight words starting from 'ptr', stores them
31 *	in provided registers and increments 'ptr' past those words.
32 *	The'abort' argument is used for fixup tables.
33 *
34 * ldr1b ptr reg cond abort
35 *
36 *	Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
37 *	It also must apply the condition code if provided, otherwise the
38 *	"al" condition is assumed by default.
39 *
40 * str1w ptr reg abort
41 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
42 * str1b ptr reg cond abort
43 *
44 *	Same as their ldr* counterparts, but data is stored to 'ptr' location
45 *	rather than being loaded.
46 *
47 * enter reg1 reg2
48 *
49 *	Preserve the provided registers on the stack plus any additional
50 *	data as needed by the implementation including this code. Called
51 *	upon code entry.
52 *
53 * usave reg1 reg2
54 *
55 *	Unwind annotation macro is corresponding for 'enter' macro.
56 *	It tell unwinder that preserved some provided registers on the stack
57 *	and additional data by a prior 'enter' macro.
58 *
59 * exit reg1 reg2
60 *
61 *	Restore registers with the values previously saved with the
62 *	'preserv' macro. Called upon code termination.
63 *
64 * LDR1W_SHIFT
65 * STR1W_SHIFT
66 *
67 *	Correction to be applied to the "ip" register when branching into
68 *	the ldr1w or str1w instructions (some of these macros may expand to
69 *	than one 32bit instruction in Thumb-2)
70 */
71
72
73	UNWIND(	.fnstart			)
74		enter	r4, lr
75	UNWIND(	.fnend				)
76
77	UNWIND(	.fnstart			)
78		usave	r4, lr			  @ in first stmdb block
79
80		subs	r2, r2, #4
81		blt	8f
82		ands	ip, r0, #3
83	PLD(	pld	[r1, #0]		)
84		bne	9f
85		ands	ip, r1, #3
86		bne	10f
87
881:		subs	r2, r2, #(28)
89		stmfd	sp!, {r5 - r8}
90	UNWIND(	.fnend				)
91
92	UNWIND(	.fnstart			)
93		usave	r4, lr
94	UNWIND(	.save	{r5 - r8}		) @ in second stmfd block
95		blt	5f
96
97	CALGN(	ands	ip, r0, #31		)
98	CALGN(	rsb	r3, ip, #32		)
99	CALGN(	sbcsne	r4, r3, r2		)  @ C is always set here
100	CALGN(	bcs	2f			)
101	CALGN(	adr	r4, 6f			)
102	CALGN(	subs	r2, r2, r3		)  @ C gets set
103	CALGN(	add	pc, r4, ip		)
104
105	PLD(	pld	[r1, #0]		)
1062:	PLD(	subs	r2, r2, #96		)
107	PLD(	pld	[r1, #28]		)
108	PLD(	blt	4f			)
109	PLD(	pld	[r1, #60]		)
110	PLD(	pld	[r1, #92]		)
111
1123:	PLD(	pld	[r1, #124]		)
1134:		ldr8w	r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
114		subs	r2, r2, #32
115		str8w	r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
116		bge	3b
117	PLD(	cmn	r2, #96			)
118	PLD(	bge	4b			)
119
1205:		ands	ip, r2, #28
121		rsb	ip, ip, #32
122#if LDR1W_SHIFT > 0
123		lsl	ip, ip, #LDR1W_SHIFT
124#endif
125		addne	pc, pc, ip		@ C is always clear here
126		b	7f
1276:
128		.rept	(1 << LDR1W_SHIFT)
129		W(nop)
130		.endr
131		ldr1w	r1, r3, abort=20f
132		ldr1w	r1, r4, abort=20f
133		ldr1w	r1, r5, abort=20f
134		ldr1w	r1, r6, abort=20f
135		ldr1w	r1, r7, abort=20f
136		ldr1w	r1, r8, abort=20f
137		ldr1w	r1, lr, abort=20f
138
139#if LDR1W_SHIFT < STR1W_SHIFT
140		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
141#elif LDR1W_SHIFT > STR1W_SHIFT
142		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
143#endif
144		add	pc, pc, ip
145		nop
146		.rept	(1 << STR1W_SHIFT)
147		W(nop)
148		.endr
149		str1w	r0, r3, abort=20f
150		str1w	r0, r4, abort=20f
151		str1w	r0, r5, abort=20f
152		str1w	r0, r6, abort=20f
153		str1w	r0, r7, abort=20f
154		str1w	r0, r8, abort=20f
155		str1w	r0, lr, abort=20f
156
157	CALGN(	bcs	2b			)
158
1597:		ldmfd	sp!, {r5 - r8}
160	UNWIND(	.fnend				) @ end of second stmfd block
161
162	UNWIND(	.fnstart			)
163		usave	r4, lr			  @ still in first stmdb block
1648:		movs	r2, r2, lsl #31
165		ldr1b	r1, r3, ne, abort=21f
166		ldr1b	r1, r4, cs, abort=21f
167		ldr1b	r1, ip, cs, abort=21f
168		str1b	r0, r3, ne, abort=21f
169		str1b	r0, r4, cs, abort=21f
170		str1b	r0, ip, cs, abort=21f
171
172		exit	r4, pc
173
1749:		rsb	ip, ip, #4
175		cmp	ip, #2
176		ldr1b	r1, r3, gt, abort=21f
177		ldr1b	r1, r4, ge, abort=21f
178		ldr1b	r1, lr, abort=21f
179		str1b	r0, r3, gt, abort=21f
180		str1b	r0, r4, ge, abort=21f
181		subs	r2, r2, ip
182		str1b	r0, lr, abort=21f
183		blt	8b
184		ands	ip, r1, #3
185		beq	1b
186
18710:		bic	r1, r1, #3
188		cmp	ip, #2
189		ldr1w	r1, lr, abort=21f
190		beq	17f
191		bgt	18f
192	UNWIND(	.fnend				)
193
194
195		.macro	forward_copy_shift pull push
196
197	UNWIND(	.fnstart			)
198		usave	r4, lr			  @ still in first stmdb block
199		subs	r2, r2, #28
200		blt	14f
201
202	CALGN(	ands	ip, r0, #31		)
203	CALGN(	rsb	ip, ip, #32		)
204	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
205	CALGN(	subcc	r2, r2, ip		)
206	CALGN(	bcc	15f			)
207
20811:		stmfd	sp!, {r5 - r9}
209	UNWIND(	.fnend				)
210
211	UNWIND(	.fnstart			)
212		usave	r4, lr
213	UNWIND(	.save	{r5 - r9}		) @ in new second stmfd block
214	PLD(	pld	[r1, #0]		)
215	PLD(	subs	r2, r2, #96		)
216	PLD(	pld	[r1, #28]		)
217	PLD(	blt	13f			)
218	PLD(	pld	[r1, #60]		)
219	PLD(	pld	[r1, #92]		)
220
22112:	PLD(	pld	[r1, #124]		)
22213:		ldr4w	r1, r4, r5, r6, r7, abort=19f
223		mov	r3, lr, lspull #\pull
224		subs	r2, r2, #32
225		ldr4w	r1, r8, r9, ip, lr, abort=19f
226		orr	r3, r3, r4, lspush #\push
227		mov	r4, r4, lspull #\pull
228		orr	r4, r4, r5, lspush #\push
229		mov	r5, r5, lspull #\pull
230		orr	r5, r5, r6, lspush #\push
231		mov	r6, r6, lspull #\pull
232		orr	r6, r6, r7, lspush #\push
233		mov	r7, r7, lspull #\pull
234		orr	r7, r7, r8, lspush #\push
235		mov	r8, r8, lspull #\pull
236		orr	r8, r8, r9, lspush #\push
237		mov	r9, r9, lspull #\pull
238		orr	r9, r9, ip, lspush #\push
239		mov	ip, ip, lspull #\pull
240		orr	ip, ip, lr, lspush #\push
241		str8w	r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f
242		bge	12b
243	PLD(	cmn	r2, #96			)
244	PLD(	bge	13b			)
245
246		ldmfd	sp!, {r5 - r9}
247	UNWIND(	.fnend				) @ end of the second stmfd block
248
249	UNWIND(	.fnstart			)
250		usave	r4, lr			  @ still in first stmdb block
25114:		ands	ip, r2, #28
252		beq	16f
253
25415:		mov	r3, lr, lspull #\pull
255		ldr1w	r1, lr, abort=21f
256		subs	ip, ip, #4
257		orr	r3, r3, lr, lspush #\push
258		str1w	r0, r3, abort=21f
259		bgt	15b
260	CALGN(	cmp	r2, #0			)
261	CALGN(	bge	11b			)
262
26316:		sub	r1, r1, #(\push / 8)
264		b	8b
265	UNWIND(	.fnend				)
266
267		.endm
268
269
270		forward_copy_shift	pull=8	push=24
271
27217:		forward_copy_shift	pull=16	push=16
273
27418:		forward_copy_shift	pull=24	push=8
275
276
277/*
278 * Abort preamble and completion macros.
279 * If a fixup handler is required then those macros must surround it.
280 * It is assumed that the fixup code will handle the private part of
281 * the exit macro.
282 */
283
284	.macro	copy_abort_preamble
28519:	ldmfd	sp!, {r5 - r9}
286	b	21f
28720:	ldmfd	sp!, {r5 - r8}
28821:
289	.endm
290
291	.macro	copy_abort_end
292	ldmfd	sp!, {r4, pc}
293	.endm
294
295