1/*
2 * Copyright (C) 2002 Paul Mackerras, IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9#include <asm/processor.h>
10#include <asm/ppc_asm.h>
11
12	.align	7
13_GLOBAL_TOC(memcpy)
14BEGIN_FTR_SECTION
15#ifdef __LITTLE_ENDIAN__
16	cmpdi	cr7,r5,0
17#else
18	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* save destination pointer for return value */
19#endif
20FTR_SECTION_ELSE
21#ifndef SELFTEST
22	b	memcpy_power7
23#endif
24ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
25#ifdef __LITTLE_ENDIAN__
26	/* dumb little-endian memcpy that will get replaced at runtime */
27	addi r9,r3,-1
28	addi r4,r4,-1
29	beqlr cr7
30	mtctr r5
311:	lbzu r10,1(r4)
32	stbu r10,1(r9)
33	bdnz 1b
34	blr
35#else
36	PPC_MTOCRF(0x01,r5)
37	cmpldi	cr1,r5,16
38	neg	r6,r3		# LS 3 bits = # bytes to 8-byte dest bdry
39	andi.	r6,r6,7
40	dcbt	0,r4
41	blt	cr1,.Lshort_copy
42/* Below we want to nop out the bne if we're on a CPU that has the
43   CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
44   cleared.
45   At the time of writing the only CPU that has this combination of bits
46   set is Power6. */
47BEGIN_FTR_SECTION
48	nop
49FTR_SECTION_ELSE
50	bne	.Ldst_unaligned
51ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
52                    CPU_FTR_UNALIGNED_LD_STD)
53.Ldst_aligned:
54	addi	r3,r3,-16
55BEGIN_FTR_SECTION
56	andi.	r0,r4,7
57	bne	.Lsrc_unaligned
58END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
59	srdi	r7,r5,4
60	ld	r9,0(r4)
61	addi	r4,r4,-8
62	mtctr	r7
63	andi.	r5,r5,7
64	bf	cr7*4+0,2f
65	addi	r3,r3,8
66	addi	r4,r4,8
67	mr	r8,r9
68	blt	cr1,3f
691:	ld	r9,8(r4)
70	std	r8,8(r3)
712:	ldu	r8,16(r4)
72	stdu	r9,16(r3)
73	bdnz	1b
743:	std	r8,8(r3)
75	beq	3f
76	addi	r3,r3,16
77.Ldo_tail:
78	bf	cr7*4+1,1f
79	lwz	r9,8(r4)
80	addi	r4,r4,4
81	stw	r9,0(r3)
82	addi	r3,r3,4
831:	bf	cr7*4+2,2f
84	lhz	r9,8(r4)
85	addi	r4,r4,2
86	sth	r9,0(r3)
87	addi	r3,r3,2
882:	bf	cr7*4+3,3f
89	lbz	r9,8(r4)
90	stb	r9,0(r3)
913:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
92	blr
93
94.Lsrc_unaligned:
95	srdi	r6,r5,3
96	addi	r5,r5,-16
97	subf	r4,r0,r4
98	srdi	r7,r5,4
99	sldi	r10,r0,3
100	cmpdi	cr6,r6,3
101	andi.	r5,r5,7
102	mtctr	r7
103	subfic	r11,r10,64
104	add	r5,r5,r0
105
106	bt	cr7*4+0,0f
107
108	ld	r9,0(r4)	# 3+2n loads, 2+2n stores
109	ld	r0,8(r4)
110	sld	r6,r9,r10
111	ldu	r9,16(r4)
112	srd	r7,r0,r11
113	sld	r8,r0,r10
114	or	r7,r7,r6
115	blt	cr6,4f
116	ld	r0,8(r4)
117	# s1<< in r8, d0=(s0<<|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12
118	b	2f
119
1200:	ld	r0,0(r4)	# 4+2n loads, 3+2n stores
121	ldu	r9,8(r4)
122	sld	r8,r0,r10
123	addi	r3,r3,-8
124	blt	cr6,5f
125	ld	r0,8(r4)
126	srd	r12,r9,r11
127	sld	r6,r9,r10
128	ldu	r9,16(r4)
129	or	r12,r8,r12
130	srd	r7,r0,r11
131	sld	r8,r0,r10
132	addi	r3,r3,16
133	beq	cr6,3f
134
135	# d0=(s0<<|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9
1361:	or	r7,r7,r6
137	ld	r0,8(r4)
138	std	r12,8(r3)
1392:	srd	r12,r9,r11
140	sld	r6,r9,r10
141	ldu	r9,16(r4)
142	or	r12,r8,r12
143	stdu	r7,16(r3)
144	srd	r7,r0,r11
145	sld	r8,r0,r10
146	bdnz	1b
147
1483:	std	r12,8(r3)
149	or	r7,r7,r6
1504:	std	r7,16(r3)
1515:	srd	r12,r9,r11
152	or	r12,r8,r12
153	std	r12,24(r3)
154	beq	4f
155	cmpwi	cr1,r5,8
156	addi	r3,r3,32
157	sld	r9,r9,r10
158	ble	cr1,6f
159	ld	r0,8(r4)
160	srd	r7,r0,r11
161	or	r9,r7,r9
1626:
163	bf	cr7*4+1,1f
164	rotldi	r9,r9,32
165	stw	r9,0(r3)
166	addi	r3,r3,4
1671:	bf	cr7*4+2,2f
168	rotldi	r9,r9,16
169	sth	r9,0(r3)
170	addi	r3,r3,2
1712:	bf	cr7*4+3,3f
172	rotldi	r9,r9,8
173	stb	r9,0(r3)
1743:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
175	blr
176
177.Ldst_unaligned:
178	PPC_MTOCRF(0x01,r6)		# put #bytes to 8B bdry into cr7
179	subf	r5,r6,r5
180	li	r7,0
181	cmpldi	cr1,r5,16
182	bf	cr7*4+3,1f
183	lbz	r0,0(r4)
184	stb	r0,0(r3)
185	addi	r7,r7,1
1861:	bf	cr7*4+2,2f
187	lhzx	r0,r7,r4
188	sthx	r0,r7,r3
189	addi	r7,r7,2
1902:	bf	cr7*4+1,3f
191	lwzx	r0,r7,r4
192	stwx	r0,r7,r3
1933:	PPC_MTOCRF(0x01,r5)
194	add	r4,r6,r4
195	add	r3,r6,r3
196	b	.Ldst_aligned
197
198.Lshort_copy:
199	bf	cr7*4+0,1f
200	lwz	r0,0(r4)
201	lwz	r9,4(r4)
202	addi	r4,r4,8
203	stw	r0,0(r3)
204	stw	r9,4(r3)
205	addi	r3,r3,8
2061:	bf	cr7*4+1,2f
207	lwz	r0,0(r4)
208	addi	r4,r4,4
209	stw	r0,0(r3)
210	addi	r3,r3,4
2112:	bf	cr7*4+2,3f
212	lhz	r0,0(r4)
213	addi	r4,r4,2
214	sth	r0,0(r3)
215	addi	r3,r3,2
2163:	bf	cr7*4+3,4f
217	lbz	r0,0(r4)
218	stb	r0,0(r3)
2194:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
220	blr
221#endif
222