xref: /openbmc/linux/arch/arm/lib/copy_template.S (revision 293d5b43) 
1/*
2 *  linux/arch/arm/lib/copy_template.s
3 *
4 *  Code template for optimized memory copy functions
5 *
6 *  Author:	Nicolas Pitre
7 *  Created:	Sep 28, 2005
8 *  Copyright:	MontaVista Software, Inc.
9 *
10 *  This program is free software; you can redistribute it and/or modify
11 *  it under the terms of the GNU General Public License version 2 as
12 *  published by the Free Software Foundation.
13 */
14
15/*
16 * Theory of operation
17 * -------------------
18 *
19 * This file provides the core code for a forward memory copy used in
20 * the implementation of memcopy(), copy_to_user() and copy_from_user().
21 *
22 * The including file must define the following accessor macros
23 * according to the need of the given function:
24 *
25 * ldr1w ptr reg abort
26 *
27 *	This loads one word from 'ptr', stores it in 'reg' and increments
28 *	'ptr' to the next word. The 'abort' argument is used for fixup tables.
29 *
30 * ldr4w ptr reg1 reg2 reg3 reg4 abort
31 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
32 *
33 *	This loads four or eight words starting from 'ptr', stores them
34 *	in provided registers and increments 'ptr' past those words.
35 *	The'abort' argument is used for fixup tables.
36 *
37 * ldr1b ptr reg cond abort
38 *
39 *	Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
40 *	It also must apply the condition code if provided, otherwise the
41 *	"al" condition is assumed by default.
42 *
43 * str1w ptr reg abort
44 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
45 * str1b ptr reg cond abort
46 *
47 *	Same as their ldr* counterparts, but data is stored to 'ptr' location
48 *	rather than being loaded.
49 *
50 * enter reg1 reg2
51 *
52 *	Preserve the provided registers on the stack plus any additional
53 *	data as needed by the implementation including this code. Called
54 *	upon code entry.
55 *
56 * usave reg1 reg2
57 *
58 *	Unwind annotation macro is corresponding for 'enter' macro.
59 *	It tell unwinder that preserved some provided registers on the stack
60 *	and additional data by a prior 'enter' macro.
61 *
62 * exit reg1 reg2
63 *
64 *	Restore registers with the values previously saved with the
65 *	'preserv' macro. Called upon code termination.
66 *
67 * LDR1W_SHIFT
68 * STR1W_SHIFT
69 *
70 *	Correction to be applied to the "ip" register when branching into
71 *	the ldr1w or str1w instructions (some of these macros may expand to
72 *	than one 32bit instruction in Thumb-2)
73 */
74
75
76	UNWIND(	.fnstart			)
77		enter	r4, lr
78	UNWIND(	.fnend				)
79
80	UNWIND(	.fnstart			)
81		usave	r4, lr			  @ in first stmdb block
82
83		subs	r2, r2, #4
84		blt	8f
85		ands	ip, r0, #3
86	PLD(	pld	[r1, #0]		)
87		bne	9f
88		ands	ip, r1, #3
89		bne	10f
90
911:		subs	r2, r2, #(28)
92		stmfd	sp!, {r5 - r8}
93	UNWIND(	.fnend				)
94
95	UNWIND(	.fnstart			)
96		usave	r4, lr
97	UNWIND(	.save	{r5 - r8}		) @ in second stmfd block
98		blt	5f
99
100	CALGN(	ands	ip, r0, #31		)
101	CALGN(	rsb	r3, ip, #32		)
102	CALGN(	sbcnes	r4, r3, r2		)  @ C is always set here
103	CALGN(	bcs	2f			)
104	CALGN(	adr	r4, 6f			)
105	CALGN(	subs	r2, r2, r3		)  @ C gets set
106	CALGN(	add	pc, r4, ip		)
107
108	PLD(	pld	[r1, #0]		)
1092:	PLD(	subs	r2, r2, #96		)
110	PLD(	pld	[r1, #28]		)
111	PLD(	blt	4f			)
112	PLD(	pld	[r1, #60]		)
113	PLD(	pld	[r1, #92]		)
114
1153:	PLD(	pld	[r1, #124]		)
1164:		ldr8w	r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
117		subs	r2, r2, #32
118		str8w	r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
119		bge	3b
120	PLD(	cmn	r2, #96			)
121	PLD(	bge	4b			)
122
1235:		ands	ip, r2, #28
124		rsb	ip, ip, #32
125#if LDR1W_SHIFT > 0
126		lsl	ip, ip, #LDR1W_SHIFT
127#endif
128		addne	pc, pc, ip		@ C is always clear here
129		b	7f
1306:
131		.rept	(1 << LDR1W_SHIFT)
132		W(nop)
133		.endr
134		ldr1w	r1, r3, abort=20f
135		ldr1w	r1, r4, abort=20f
136		ldr1w	r1, r5, abort=20f
137		ldr1w	r1, r6, abort=20f
138		ldr1w	r1, r7, abort=20f
139		ldr1w	r1, r8, abort=20f
140		ldr1w	r1, lr, abort=20f
141
142#if LDR1W_SHIFT < STR1W_SHIFT
143		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
144#elif LDR1W_SHIFT > STR1W_SHIFT
145		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
146#endif
147		add	pc, pc, ip
148		nop
149		.rept	(1 << STR1W_SHIFT)
150		W(nop)
151		.endr
152		str1w	r0, r3, abort=20f
153		str1w	r0, r4, abort=20f
154		str1w	r0, r5, abort=20f
155		str1w	r0, r6, abort=20f
156		str1w	r0, r7, abort=20f
157		str1w	r0, r8, abort=20f
158		str1w	r0, lr, abort=20f
159
160	CALGN(	bcs	2b			)
161
1627:		ldmfd	sp!, {r5 - r8}
163	UNWIND(	.fnend				) @ end of second stmfd block
164
165	UNWIND(	.fnstart			)
166		usave	r4, lr			  @ still in first stmdb block
1678:		movs	r2, r2, lsl #31
168		ldr1b	r1, r3, ne, abort=21f
169		ldr1b	r1, r4, cs, abort=21f
170		ldr1b	r1, ip, cs, abort=21f
171		str1b	r0, r3, ne, abort=21f
172		str1b	r0, r4, cs, abort=21f
173		str1b	r0, ip, cs, abort=21f
174
175		exit	r4, pc
176
1779:		rsb	ip, ip, #4
178		cmp	ip, #2
179		ldr1b	r1, r3, gt, abort=21f
180		ldr1b	r1, r4, ge, abort=21f
181		ldr1b	r1, lr, abort=21f
182		str1b	r0, r3, gt, abort=21f
183		str1b	r0, r4, ge, abort=21f
184		subs	r2, r2, ip
185		str1b	r0, lr, abort=21f
186		blt	8b
187		ands	ip, r1, #3
188		beq	1b
189
19010:		bic	r1, r1, #3
191		cmp	ip, #2
192		ldr1w	r1, lr, abort=21f
193		beq	17f
194		bgt	18f
195	UNWIND(	.fnend				)
196
197
198		.macro	forward_copy_shift pull push
199
200	UNWIND(	.fnstart			)
201		usave	r4, lr			  @ still in first stmdb block
202		subs	r2, r2, #28
203		blt	14f
204
205	CALGN(	ands	ip, r0, #31		)
206	CALGN(	rsb	ip, ip, #32		)
207	CALGN(	sbcnes	r4, ip, r2		)  @ C is always set here
208	CALGN(	subcc	r2, r2, ip		)
209	CALGN(	bcc	15f			)
210
21111:		stmfd	sp!, {r5 - r9}
212	UNWIND(	.fnend				)
213
214	UNWIND(	.fnstart			)
215		usave	r4, lr
216	UNWIND(	.save	{r5 - r9}		) @ in new second stmfd block
217	PLD(	pld	[r1, #0]		)
218	PLD(	subs	r2, r2, #96		)
219	PLD(	pld	[r1, #28]		)
220	PLD(	blt	13f			)
221	PLD(	pld	[r1, #60]		)
222	PLD(	pld	[r1, #92]		)
223
22412:	PLD(	pld	[r1, #124]		)
22513:		ldr4w	r1, r4, r5, r6, r7, abort=19f
226		mov	r3, lr, lspull #\pull
227		subs	r2, r2, #32
228		ldr4w	r1, r8, r9, ip, lr, abort=19f
229		orr	r3, r3, r4, lspush #\push
230		mov	r4, r4, lspull #\pull
231		orr	r4, r4, r5, lspush #\push
232		mov	r5, r5, lspull #\pull
233		orr	r5, r5, r6, lspush #\push
234		mov	r6, r6, lspull #\pull
235		orr	r6, r6, r7, lspush #\push
236		mov	r7, r7, lspull #\pull
237		orr	r7, r7, r8, lspush #\push
238		mov	r8, r8, lspull #\pull
239		orr	r8, r8, r9, lspush #\push
240		mov	r9, r9, lspull #\pull
241		orr	r9, r9, ip, lspush #\push
242		mov	ip, ip, lspull #\pull
243		orr	ip, ip, lr, lspush #\push
244		str8w	r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
245		bge	12b
246	PLD(	cmn	r2, #96			)
247	PLD(	bge	13b			)
248
249		ldmfd	sp!, {r5 - r9}
250	UNWIND(	.fnend				) @ end of the second stmfd block
251
252	UNWIND(	.fnstart			)
253		usave	r4, lr			  @ still in first stmdb block
25414:		ands	ip, r2, #28
255		beq	16f
256
25715:		mov	r3, lr, lspull #\pull
258		ldr1w	r1, lr, abort=21f
259		subs	ip, ip, #4
260		orr	r3, r3, lr, lspush #\push
261		str1w	r0, r3, abort=21f
262		bgt	15b
263	CALGN(	cmp	r2, #0			)
264	CALGN(	bge	11b			)
265
26616:		sub	r1, r1, #(\push / 8)
267		b	8b
268	UNWIND(	.fnend				)
269
270		.endm
271
272
273		forward_copy_shift	pull=8	push=24
274
27517:		forward_copy_shift	pull=16	push=16
276
27718:		forward_copy_shift	pull=24	push=8
278
279
280/*
281 * Abort preamble and completion macros.
282 * If a fixup handler is required then those macros must surround it.
283 * It is assumed that the fixup code will handle the private part of
284 * the exit macro.
285 */
286
287	.macro	copy_abort_preamble
28819:	ldmfd	sp!, {r5 - r9}
289	b	21f
29020:	ldmfd	sp!, {r5 - r8}
29121:
292	.endm
293
294	.macro	copy_abort_end
295	ldmfd	sp!, {r4, pc}
296	.endm
297
298