xref: /openbmc/u-boot/arch/x86/lib/string.c (revision 95390360)
1 /*
2  * Copyright (C) 1991,1992,1993,1997,1998,2003, 2005 Free Software Foundation, Inc.
3  * This file is part of the GNU C Library.
4  * Copyright (c) 2011 The Chromium OS Authors.
5  *
6  * SPDX-License-Identifier:	GPL-2.0+
7  */
8 
9 /* From glibc-2.14, sysdeps/i386/memset.c */
10 
11 #include <linux/types.h>
12 #include <linux/compiler.h>
13 #include <asm/string.h>
14 
15 typedef uint32_t op_t;
16 
17 void *memset(void *dstpp, int c, size_t len)
18 {
19 	int d0;
20 	unsigned long int dstp = (unsigned long int) dstpp;
21 
22 	/* This explicit register allocation improves code very much indeed. */
23 	register op_t x asm("ax");
24 
25 	x = (unsigned char) c;
26 
27 	/* Clear the direction flag, so filling will move forward.  */
28 	asm volatile("cld");
29 
30 	/* This threshold value is optimal.  */
31 	if (len >= 12) {
32 		/* Fill X with four copies of the char we want to fill with. */
33 		x |= (x << 8);
34 		x |= (x << 16);
35 
36 		/* Adjust LEN for the bytes handled in the first loop.  */
37 		len -= (-dstp) % sizeof(op_t);
38 
39 		/*
40 		 * There are at least some bytes to set. No need to test for
41 		 * LEN == 0 in this alignment loop.
42 		 */
43 
44 		/* Fill bytes until DSTP is aligned on a longword boundary. */
45 		asm volatile(
46 			"rep\n"
47 			"stosb" /* %0, %2, %3 */ :
48 			"=D" (dstp), "=c" (d0) :
49 			"0" (dstp), "1" ((-dstp) % sizeof(op_t)), "a" (x) :
50 			"memory");
51 
52 		/* Fill longwords.  */
53 		asm volatile(
54 			"rep\n"
55 			"stosl" /* %0, %2, %3 */ :
56 			"=D" (dstp), "=c" (d0) :
57 			"0" (dstp), "1" (len / sizeof(op_t)), "a" (x) :
58 			"memory");
59 		len %= sizeof(op_t);
60 	}
61 
62 	/* Write the last few bytes. */
63 	asm volatile(
64 		"rep\n"
65 		"stosb" /* %0, %2, %3 */ :
66 		"=D" (dstp), "=c" (d0) :
67 		"0" (dstp), "1" (len), "a" (x) :
68 		"memory");
69 
70 	return dstpp;
71 }
72 
73 #define	OP_T_THRES	8
74 #define OPSIZ	(sizeof(op_t))
75 
76 #define BYTE_COPY_FWD(dst_bp, src_bp, nbytes)				  \
77 do {									  \
78 	int __d0;							  \
79 	asm volatile(							  \
80 		/* Clear the direction flag, so copying goes forward.  */ \
81 		"cld\n"							  \
82 		/* Copy bytes.  */					  \
83 		"rep\n"							  \
84 		"movsb" :						  \
85 		"=D" (dst_bp), "=S" (src_bp), "=c" (__d0) :		  \
86 		"0" (dst_bp), "1" (src_bp), "2" (nbytes) :		  \
87 		"memory");						  \
88 } while (0)
89 
90 #define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes)		  \
91 do {									  \
92 	int __d0;							  \
93 	asm volatile(							  \
94 		/* Clear the direction flag, so copying goes forward.  */ \
95 		"cld\n"							  \
96 		/* Copy longwords.  */					  \
97 		"rep\n"							  \
98 		"movsl" :						  \
99 		"=D" (dst_bp), "=S" (src_bp), "=c" (__d0) :		  \
100 		"0" (dst_bp), "1" (src_bp), "2" ((nbytes) / 4) :	  \
101 		"memory");						  \
102 	(nbytes_left) = (nbytes) % 4;					  \
103 } while (0)
104 
105 void *memcpy(void *dstpp, const void *srcpp, size_t len)
106 {
107 	unsigned long int dstp = (long int)dstpp;
108 	unsigned long int srcp = (long int)srcpp;
109 
110 	/* Copy from the beginning to the end.  */
111 
112 	/* If there not too few bytes to copy, use word copy.  */
113 	if (len >= OP_T_THRES) {
114 		/* Copy just a few bytes to make DSTP aligned.  */
115 		len -= (-dstp) % OPSIZ;
116 		BYTE_COPY_FWD(dstp, srcp, (-dstp) % OPSIZ);
117 
118 		/* Copy from SRCP to DSTP taking advantage of the known
119 		 * alignment of DSTP.  Number of bytes remaining is put
120 		 * in the third argument, i.e. in LEN.  This number may
121 		 * vary from machine to machine.
122 		 */
123 		WORD_COPY_FWD(dstp, srcp, len, len);
124 
125 		/* Fall out and copy the tail.  */
126 	}
127 
128 	/* There are just a few bytes to copy. Use byte memory operations. */
129 	BYTE_COPY_FWD(dstp, srcp, len);
130 
131 	return dstpp;
132 }
133 
134 void *memmove(void *dest, const void *src, size_t n)
135 {
136 	int d0, d1, d2, d3, d4, d5;
137 	char *ret = dest;
138 
139 	__asm__ __volatile__(
140 		/* Handle more 16 bytes in loop */
141 		"cmp $0x10, %0\n\t"
142 		"jb	1f\n\t"
143 
144 		/* Decide forward/backward copy mode */
145 		"cmp %2, %1\n\t"
146 		"jb	2f\n\t"
147 
148 		/*
149 		 * movs instruction have many startup latency
150 		 * so we handle small size by general register.
151 		 */
152 		"cmp  $680, %0\n\t"
153 		"jb 3f\n\t"
154 		/* movs instruction is only good for aligned case */
155 		"mov %1, %3\n\t"
156 		"xor %2, %3\n\t"
157 		"and $0xff, %3\n\t"
158 		"jz 4f\n\t"
159 		"3:\n\t"
160 		"sub $0x10, %0\n\t"
161 
162 		/* We gobble 16 bytes forward in each loop */
163 		"3:\n\t"
164 		"sub $0x10, %0\n\t"
165 		"mov 0*4(%1), %3\n\t"
166 		"mov 1*4(%1), %4\n\t"
167 		"mov  %3, 0*4(%2)\n\t"
168 		"mov  %4, 1*4(%2)\n\t"
169 		"mov 2*4(%1), %3\n\t"
170 		"mov 3*4(%1), %4\n\t"
171 		"mov  %3, 2*4(%2)\n\t"
172 		"mov  %4, 3*4(%2)\n\t"
173 		"lea  0x10(%1), %1\n\t"
174 		"lea  0x10(%2), %2\n\t"
175 		"jae 3b\n\t"
176 		"add $0x10, %0\n\t"
177 		"jmp 1f\n\t"
178 
179 		/* Handle data forward by movs */
180 		".p2align 4\n\t"
181 		"4:\n\t"
182 		"mov -4(%1, %0), %3\n\t"
183 		"lea -4(%2, %0), %4\n\t"
184 		"shr $2, %0\n\t"
185 		"rep movsl\n\t"
186 		"mov %3, (%4)\n\t"
187 		"jmp 11f\n\t"
188 		/* Handle data backward by movs */
189 		".p2align 4\n\t"
190 		"6:\n\t"
191 		"mov (%1), %3\n\t"
192 		"mov %2, %4\n\t"
193 		"lea -4(%1, %0), %1\n\t"
194 		"lea -4(%2, %0), %2\n\t"
195 		"shr $2, %0\n\t"
196 		"std\n\t"
197 		"rep movsl\n\t"
198 		"mov %3,(%4)\n\t"
199 		"cld\n\t"
200 		"jmp 11f\n\t"
201 
202 		/* Start to prepare for backward copy */
203 		".p2align 4\n\t"
204 		"2:\n\t"
205 		"cmp  $680, %0\n\t"
206 		"jb 5f\n\t"
207 		"mov %1, %3\n\t"
208 		"xor %2, %3\n\t"
209 		"and $0xff, %3\n\t"
210 		"jz 6b\n\t"
211 
212 		/* Calculate copy position to tail */
213 		"5:\n\t"
214 		"add %0, %1\n\t"
215 		"add %0, %2\n\t"
216 		"sub $0x10, %0\n\t"
217 
218 		/* We gobble 16 bytes backward in each loop */
219 		"7:\n\t"
220 		"sub $0x10, %0\n\t"
221 
222 		"mov -1*4(%1), %3\n\t"
223 		"mov -2*4(%1), %4\n\t"
224 		"mov  %3, -1*4(%2)\n\t"
225 		"mov  %4, -2*4(%2)\n\t"
226 		"mov -3*4(%1), %3\n\t"
227 		"mov -4*4(%1), %4\n\t"
228 		"mov  %3, -3*4(%2)\n\t"
229 		"mov  %4, -4*4(%2)\n\t"
230 		"lea  -0x10(%1), %1\n\t"
231 		"lea  -0x10(%2), %2\n\t"
232 		"jae 7b\n\t"
233 		/* Calculate copy position to head */
234 		"add $0x10, %0\n\t"
235 		"sub %0, %1\n\t"
236 		"sub %0, %2\n\t"
237 
238 		/* Move data from 8 bytes to 15 bytes */
239 		".p2align 4\n\t"
240 		"1:\n\t"
241 		"cmp $8, %0\n\t"
242 		"jb 8f\n\t"
243 		"mov 0*4(%1), %3\n\t"
244 		"mov 1*4(%1), %4\n\t"
245 		"mov -2*4(%1, %0), %5\n\t"
246 		"mov -1*4(%1, %0), %1\n\t"
247 
248 		"mov  %3, 0*4(%2)\n\t"
249 		"mov  %4, 1*4(%2)\n\t"
250 		"mov  %5, -2*4(%2, %0)\n\t"
251 		"mov  %1, -1*4(%2, %0)\n\t"
252 		"jmp 11f\n\t"
253 
254 		/* Move data from 4 bytes to 7 bytes */
255 		".p2align 4\n\t"
256 		"8:\n\t"
257 		"cmp $4, %0\n\t"
258 		"jb 9f\n\t"
259 		"mov 0*4(%1), %3\n\t"
260 		"mov -1*4(%1, %0), %4\n\t"
261 		"mov  %3, 0*4(%2)\n\t"
262 		"mov  %4, -1*4(%2, %0)\n\t"
263 		"jmp 11f\n\t"
264 
265 		/* Move data from 2 bytes to 3 bytes */
266 		".p2align 4\n\t"
267 		"9:\n\t"
268 		"cmp $2, %0\n\t"
269 		"jb 10f\n\t"
270 		"movw 0*2(%1), %%dx\n\t"
271 		"movw -1*2(%1, %0), %%bx\n\t"
272 		"movw %%dx, 0*2(%2)\n\t"
273 		"movw %%bx, -1*2(%2, %0)\n\t"
274 		"jmp 11f\n\t"
275 
276 		/* Move data for 1 byte */
277 		".p2align 4\n\t"
278 		"10:\n\t"
279 		"cmp $1, %0\n\t"
280 		"jb 11f\n\t"
281 		"movb (%1), %%cl\n\t"
282 		"movb %%cl, (%2)\n\t"
283 		".p2align 4\n\t"
284 		"11:"
285 		: "=&c" (d0), "=&S" (d1), "=&D" (d2),
286 		  "=r" (d3), "=r" (d4), "=r"(d5)
287 		: "0" (n),
288 		 "1" (src),
289 		 "2" (dest)
290 		: "memory");
291 
292 	return ret;
293 }
294