xref: /openbmc/linux/arch/arm/lib/copy_template.S (revision afb46f79) 
1/*
2 *  linux/arch/arm/lib/copy_template.s
3 *
4 *  Code template for optimized memory copy functions
5 *
6 *  Author:	Nicolas Pitre
7 *  Created:	Sep 28, 2005
8 *  Copyright:	MontaVista Software, Inc.
9 *
10 *  This program is free software; you can redistribute it and/or modify
11 *  it under the terms of the GNU General Public License version 2 as
12 *  published by the Free Software Foundation.
13 */
14
15/*
16 * Theory of operation
17 * -------------------
18 *
19 * This file provides the core code for a forward memory copy used in
20 * the implementation of memcopy(), copy_to_user() and copy_from_user().
21 *
22 * The including file must define the following accessor macros
23 * according to the need of the given function:
24 *
25 * ldr1w ptr reg abort
26 *
27 *	This loads one word from 'ptr', stores it in 'reg' and increments
28 *	'ptr' to the next word. The 'abort' argument is used for fixup tables.
29 *
30 * ldr4w ptr reg1 reg2 reg3 reg4 abort
31 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
32 *
33 *	This loads four or eight words starting from 'ptr', stores them
34 *	in provided registers and increments 'ptr' past those words.
35 *	The'abort' argument is used for fixup tables.
36 *
37 * ldr1b ptr reg cond abort
38 *
39 *	Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
40 *	It also must apply the condition code if provided, otherwise the
41 *	"al" condition is assumed by default.
42 *
43 * str1w ptr reg abort
44 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
45 * str1b ptr reg cond abort
46 *
47 *	Same as their ldr* counterparts, but data is stored to 'ptr' location
48 *	rather than being loaded.
49 *
50 * enter reg1 reg2
51 *
52 *	Preserve the provided registers on the stack plus any additional
53 *	data as needed by the implementation including this code. Called
54 *	upon code entry.
55 *
56 * exit reg1 reg2
57 *
58 *	Restore registers with the values previously saved with the
59 *	'preserv' macro. Called upon code termination.
60 *
61 * LDR1W_SHIFT
62 * STR1W_SHIFT
63 *
64 *	Correction to be applied to the "ip" register when branching into
65 *	the ldr1w or str1w instructions (some of these macros may expand to
66 *	than one 32bit instruction in Thumb-2)
67 */
68
69
70		enter	r4, lr
71
72		subs	r2, r2, #4
73		blt	8f
74		ands	ip, r0, #3
75	PLD(	pld	[r1, #0]		)
76		bne	9f
77		ands	ip, r1, #3
78		bne	10f
79
801:		subs	r2, r2, #(28)
81		stmfd	sp!, {r5 - r8}
82		blt	5f
83
84	CALGN(	ands	ip, r0, #31		)
85	CALGN(	rsb	r3, ip, #32		)
86	CALGN(	sbcnes	r4, r3, r2		)  @ C is always set here
87	CALGN(	bcs	2f			)
88	CALGN(	adr	r4, 6f			)
89	CALGN(	subs	r2, r2, r3		)  @ C gets set
90	CALGN(	add	pc, r4, ip		)
91
92	PLD(	pld	[r1, #0]		)
932:	PLD(	subs	r2, r2, #96		)
94	PLD(	pld	[r1, #28]		)
95	PLD(	blt	4f			)
96	PLD(	pld	[r1, #60]		)
97	PLD(	pld	[r1, #92]		)
98
993:	PLD(	pld	[r1, #124]		)
1004:		ldr8w	r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
101		subs	r2, r2, #32
102		str8w	r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
103		bge	3b
104	PLD(	cmn	r2, #96			)
105	PLD(	bge	4b			)
106
1075:		ands	ip, r2, #28
108		rsb	ip, ip, #32
109#if LDR1W_SHIFT > 0
110		lsl	ip, ip, #LDR1W_SHIFT
111#endif
112		addne	pc, pc, ip		@ C is always clear here
113		b	7f
1146:
115		.rept	(1 << LDR1W_SHIFT)
116		W(nop)
117		.endr
118		ldr1w	r1, r3, abort=20f
119		ldr1w	r1, r4, abort=20f
120		ldr1w	r1, r5, abort=20f
121		ldr1w	r1, r6, abort=20f
122		ldr1w	r1, r7, abort=20f
123		ldr1w	r1, r8, abort=20f
124		ldr1w	r1, lr, abort=20f
125
126#if LDR1W_SHIFT < STR1W_SHIFT
127		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
128#elif LDR1W_SHIFT > STR1W_SHIFT
129		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
130#endif
131		add	pc, pc, ip
132		nop
133		.rept	(1 << STR1W_SHIFT)
134		W(nop)
135		.endr
136		str1w	r0, r3, abort=20f
137		str1w	r0, r4, abort=20f
138		str1w	r0, r5, abort=20f
139		str1w	r0, r6, abort=20f
140		str1w	r0, r7, abort=20f
141		str1w	r0, r8, abort=20f
142		str1w	r0, lr, abort=20f
143
144	CALGN(	bcs	2b			)
145
1467:		ldmfd	sp!, {r5 - r8}
147
1488:		movs	r2, r2, lsl #31
149		ldr1b	r1, r3, ne, abort=21f
150		ldr1b	r1, r4, cs, abort=21f
151		ldr1b	r1, ip, cs, abort=21f
152		str1b	r0, r3, ne, abort=21f
153		str1b	r0, r4, cs, abort=21f
154		str1b	r0, ip, cs, abort=21f
155
156		exit	r4, pc
157
1589:		rsb	ip, ip, #4
159		cmp	ip, #2
160		ldr1b	r1, r3, gt, abort=21f
161		ldr1b	r1, r4, ge, abort=21f
162		ldr1b	r1, lr, abort=21f
163		str1b	r0, r3, gt, abort=21f
164		str1b	r0, r4, ge, abort=21f
165		subs	r2, r2, ip
166		str1b	r0, lr, abort=21f
167		blt	8b
168		ands	ip, r1, #3
169		beq	1b
170
17110:		bic	r1, r1, #3
172		cmp	ip, #2
173		ldr1w	r1, lr, abort=21f
174		beq	17f
175		bgt	18f
176
177
178		.macro	forward_copy_shift pull push
179
180		subs	r2, r2, #28
181		blt	14f
182
183	CALGN(	ands	ip, r0, #31		)
184	CALGN(	rsb	ip, ip, #32		)
185	CALGN(	sbcnes	r4, ip, r2		)  @ C is always set here
186	CALGN(	subcc	r2, r2, ip		)
187	CALGN(	bcc	15f			)
188
18911:		stmfd	sp!, {r5 - r9}
190
191	PLD(	pld	[r1, #0]		)
192	PLD(	subs	r2, r2, #96		)
193	PLD(	pld	[r1, #28]		)
194	PLD(	blt	13f			)
195	PLD(	pld	[r1, #60]		)
196	PLD(	pld	[r1, #92]		)
197
19812:	PLD(	pld	[r1, #124]		)
19913:		ldr4w	r1, r4, r5, r6, r7, abort=19f
200		mov	r3, lr, lspull #\pull
201		subs	r2, r2, #32
202		ldr4w	r1, r8, r9, ip, lr, abort=19f
203		orr	r3, r3, r4, lspush #\push
204		mov	r4, r4, lspull #\pull
205		orr	r4, r4, r5, lspush #\push
206		mov	r5, r5, lspull #\pull
207		orr	r5, r5, r6, lspush #\push
208		mov	r6, r6, lspull #\pull
209		orr	r6, r6, r7, lspush #\push
210		mov	r7, r7, lspull #\pull
211		orr	r7, r7, r8, lspush #\push
212		mov	r8, r8, lspull #\pull
213		orr	r8, r8, r9, lspush #\push
214		mov	r9, r9, lspull #\pull
215		orr	r9, r9, ip, lspush #\push
216		mov	ip, ip, lspull #\pull
217		orr	ip, ip, lr, lspush #\push
218		str8w	r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
219		bge	12b
220	PLD(	cmn	r2, #96			)
221	PLD(	bge	13b			)
222
223		ldmfd	sp!, {r5 - r9}
224
22514:		ands	ip, r2, #28
226		beq	16f
227
22815:		mov	r3, lr, lspull #\pull
229		ldr1w	r1, lr, abort=21f
230		subs	ip, ip, #4
231		orr	r3, r3, lr, lspush #\push
232		str1w	r0, r3, abort=21f
233		bgt	15b
234	CALGN(	cmp	r2, #0			)
235	CALGN(	bge	11b			)
236
23716:		sub	r1, r1, #(\push / 8)
238		b	8b
239
240		.endm
241
242
243		forward_copy_shift	pull=8	push=24
244
24517:		forward_copy_shift	pull=16	push=16
246
24718:		forward_copy_shift	pull=24	push=8
248
249
250/*
251 * Abort preamble and completion macros.
252 * If a fixup handler is required then those macros must surround it.
253 * It is assumed that the fixup code will handle the private part of
254 * the exit macro.
255 */
256
257	.macro	copy_abort_preamble
25819:	ldmfd	sp!, {r5 - r9}
259	b	21f
26020:	ldmfd	sp!, {r5 - r8}
26121:
262	.endm
263
264	.macro	copy_abort_end
265	ldmfd	sp!, {r4, pc}
266	.endm
267
268