xref: /openbmc/linux/arch/arm64/lib/memcpy.S (revision 0f61f6be)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (c) 2012-2021, Arm Limited.
4 *
5 * Adapted from the original at:
6 * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/memcpy.S
7 */
8
9#include <linux/linkage.h>
10#include <asm/assembler.h>
11
12/* Assumptions:
13 *
14 * ARMv8-a, AArch64, unaligned accesses.
15 *
16 */
17
18#define L(label) .L ## label
19
20#define dstin	x0
21#define src	x1
22#define count	x2
23#define dst	x3
24#define srcend	x4
25#define dstend	x5
26#define A_l	x6
27#define A_lw	w6
28#define A_h	x7
29#define B_l	x8
30#define B_lw	w8
31#define B_h	x9
32#define C_l	x10
33#define C_lw	w10
34#define C_h	x11
35#define D_l	x12
36#define D_h	x13
37#define E_l	x14
38#define E_h	x15
39#define F_l	x16
40#define F_h	x17
41#define G_l	count
42#define G_h	dst
43#define H_l	src
44#define H_h	srcend
45#define tmp1	x14
46
47/* This implementation handles overlaps and supports both memcpy and memmove
48   from a single entry point.  It uses unaligned accesses and branchless
49   sequences to keep the code small, simple and improve performance.
50
51   Copies are split into 3 main cases: small copies of up to 32 bytes, medium
52   copies of up to 128 bytes, and large copies.  The overhead of the overlap
53   check is negligible since it is only required for large copies.
54
55   Large copies use a software pipelined loop processing 64 bytes per iteration.
56   The destination pointer is 16-byte aligned to minimize unaligned accesses.
57   The loop tail is handled by always copying 64 bytes from the end.
58*/
59
60SYM_FUNC_START(__pi_memcpy)
61	add	srcend, src, count
62	add	dstend, dstin, count
63	cmp	count, 128
64	b.hi	L(copy_long)
65	cmp	count, 32
66	b.hi	L(copy32_128)
67
68	/* Small copies: 0..32 bytes.  */
69	cmp	count, 16
70	b.lo	L(copy16)
71	ldp	A_l, A_h, [src]
72	ldp	D_l, D_h, [srcend, -16]
73	stp	A_l, A_h, [dstin]
74	stp	D_l, D_h, [dstend, -16]
75	ret
76
77	/* Copy 8-15 bytes.  */
78L(copy16):
79	tbz	count, 3, L(copy8)
80	ldr	A_l, [src]
81	ldr	A_h, [srcend, -8]
82	str	A_l, [dstin]
83	str	A_h, [dstend, -8]
84	ret
85
86	.p2align 3
87	/* Copy 4-7 bytes.  */
88L(copy8):
89	tbz	count, 2, L(copy4)
90	ldr	A_lw, [src]
91	ldr	B_lw, [srcend, -4]
92	str	A_lw, [dstin]
93	str	B_lw, [dstend, -4]
94	ret
95
96	/* Copy 0..3 bytes using a branchless sequence.  */
97L(copy4):
98	cbz	count, L(copy0)
99	lsr	tmp1, count, 1
100	ldrb	A_lw, [src]
101	ldrb	C_lw, [srcend, -1]
102	ldrb	B_lw, [src, tmp1]
103	strb	A_lw, [dstin]
104	strb	B_lw, [dstin, tmp1]
105	strb	C_lw, [dstend, -1]
106L(copy0):
107	ret
108
109	.p2align 4
110	/* Medium copies: 33..128 bytes.  */
111L(copy32_128):
112	ldp	A_l, A_h, [src]
113	ldp	B_l, B_h, [src, 16]
114	ldp	C_l, C_h, [srcend, -32]
115	ldp	D_l, D_h, [srcend, -16]
116	cmp	count, 64
117	b.hi	L(copy128)
118	stp	A_l, A_h, [dstin]
119	stp	B_l, B_h, [dstin, 16]
120	stp	C_l, C_h, [dstend, -32]
121	stp	D_l, D_h, [dstend, -16]
122	ret
123
124	.p2align 4
125	/* Copy 65..128 bytes.  */
126L(copy128):
127	ldp	E_l, E_h, [src, 32]
128	ldp	F_l, F_h, [src, 48]
129	cmp	count, 96
130	b.ls	L(copy96)
131	ldp	G_l, G_h, [srcend, -64]
132	ldp	H_l, H_h, [srcend, -48]
133	stp	G_l, G_h, [dstend, -64]
134	stp	H_l, H_h, [dstend, -48]
135L(copy96):
136	stp	A_l, A_h, [dstin]
137	stp	B_l, B_h, [dstin, 16]
138	stp	E_l, E_h, [dstin, 32]
139	stp	F_l, F_h, [dstin, 48]
140	stp	C_l, C_h, [dstend, -32]
141	stp	D_l, D_h, [dstend, -16]
142	ret
143
144	.p2align 4
145	/* Copy more than 128 bytes.  */
146L(copy_long):
147	/* Use backwards copy if there is an overlap.  */
148	sub	tmp1, dstin, src
149	cbz	tmp1, L(copy0)
150	cmp	tmp1, count
151	b.lo	L(copy_long_backwards)
152
153	/* Copy 16 bytes and then align dst to 16-byte alignment.  */
154
155	ldp	D_l, D_h, [src]
156	and	tmp1, dstin, 15
157	bic	dst, dstin, 15
158	sub	src, src, tmp1
159	add	count, count, tmp1	/* Count is now 16 too large.  */
160	ldp	A_l, A_h, [src, 16]
161	stp	D_l, D_h, [dstin]
162	ldp	B_l, B_h, [src, 32]
163	ldp	C_l, C_h, [src, 48]
164	ldp	D_l, D_h, [src, 64]!
165	subs	count, count, 128 + 16	/* Test and readjust count.  */
166	b.ls	L(copy64_from_end)
167
168L(loop64):
169	stp	A_l, A_h, [dst, 16]
170	ldp	A_l, A_h, [src, 16]
171	stp	B_l, B_h, [dst, 32]
172	ldp	B_l, B_h, [src, 32]
173	stp	C_l, C_h, [dst, 48]
174	ldp	C_l, C_h, [src, 48]
175	stp	D_l, D_h, [dst, 64]!
176	ldp	D_l, D_h, [src, 64]!
177	subs	count, count, 64
178	b.hi	L(loop64)
179
180	/* Write the last iteration and copy 64 bytes from the end.  */
181L(copy64_from_end):
182	ldp	E_l, E_h, [srcend, -64]
183	stp	A_l, A_h, [dst, 16]
184	ldp	A_l, A_h, [srcend, -48]
185	stp	B_l, B_h, [dst, 32]
186	ldp	B_l, B_h, [srcend, -32]
187	stp	C_l, C_h, [dst, 48]
188	ldp	C_l, C_h, [srcend, -16]
189	stp	D_l, D_h, [dst, 64]
190	stp	E_l, E_h, [dstend, -64]
191	stp	A_l, A_h, [dstend, -48]
192	stp	B_l, B_h, [dstend, -32]
193	stp	C_l, C_h, [dstend, -16]
194	ret
195
196	.p2align 4
197
198	/* Large backwards copy for overlapping copies.
199	   Copy 16 bytes and then align dst to 16-byte alignment.  */
200L(copy_long_backwards):
201	ldp	D_l, D_h, [srcend, -16]
202	and	tmp1, dstend, 15
203	sub	srcend, srcend, tmp1
204	sub	count, count, tmp1
205	ldp	A_l, A_h, [srcend, -16]
206	stp	D_l, D_h, [dstend, -16]
207	ldp	B_l, B_h, [srcend, -32]
208	ldp	C_l, C_h, [srcend, -48]
209	ldp	D_l, D_h, [srcend, -64]!
210	sub	dstend, dstend, tmp1
211	subs	count, count, 128
212	b.ls	L(copy64_from_start)
213
214L(loop64_backwards):
215	stp	A_l, A_h, [dstend, -16]
216	ldp	A_l, A_h, [srcend, -16]
217	stp	B_l, B_h, [dstend, -32]
218	ldp	B_l, B_h, [srcend, -32]
219	stp	C_l, C_h, [dstend, -48]
220	ldp	C_l, C_h, [srcend, -48]
221	stp	D_l, D_h, [dstend, -64]!
222	ldp	D_l, D_h, [srcend, -64]!
223	subs	count, count, 64
224	b.hi	L(loop64_backwards)
225
226	/* Write the last iteration and copy 64 bytes from the start.  */
227L(copy64_from_start):
228	ldp	G_l, G_h, [src, 48]
229	stp	A_l, A_h, [dstend, -16]
230	ldp	A_l, A_h, [src, 32]
231	stp	B_l, B_h, [dstend, -32]
232	ldp	B_l, B_h, [src, 16]
233	stp	C_l, C_h, [dstend, -48]
234	ldp	C_l, C_h, [src]
235	stp	D_l, D_h, [dstend, -64]
236	stp	G_l, G_h, [dstin, 48]
237	stp	A_l, A_h, [dstin, 32]
238	stp	B_l, B_h, [dstin, 16]
239	stp	C_l, C_h, [dstin]
240	ret
241SYM_FUNC_END(__pi_memcpy)
242
243SYM_FUNC_ALIAS(__memcpy, __pi_memcpy)
244EXPORT_SYMBOL(__memcpy)
245SYM_FUNC_ALIAS_WEAK(memcpy, __memcpy)
246EXPORT_SYMBOL(memcpy)
247
248SYM_FUNC_ALIAS(__pi_memmove, __pi_memcpy)
249
250SYM_FUNC_ALIAS(__memmove, __pi_memmove)
251EXPORT_SYMBOL(__memmove)
252SYM_FUNC_ALIAS_WEAK(memmove, __memmove)
253EXPORT_SYMBOL(memmove)
254