xref: /openbmc/linux/arch/alpha/lib/ev6-copy_user.S (revision 9cfc5c90) 
1/*
2 * arch/alpha/lib/ev6-copy_user.S
3 *
4 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
5 *
6 * Copy to/from user space, handling exceptions as we go..  This
7 * isn't exactly pretty.
8 *
9 * This is essentially the same as "memcpy()", but with a few twists.
10 * Notably, we have to make sure that $0 is always up-to-date and
11 * contains the right "bytes left to copy" value (and that it is updated
12 * only _after_ a successful copy). There is also some rather minor
13 * exception setup stuff..
14 *
15 * NOTE! This is not directly C-callable, because the calling semantics are
16 * different:
17 *
18 * Inputs:
19 *	length in $0
20 *	destination address in $6
21 *	source address in $7
22 *	return address in $28
23 *
24 * Outputs:
25 *	bytes left to copy in $0
26 *
27 * Clobbers:
28 *	$1,$2,$3,$4,$5,$6,$7
29 *
30 * Much of the information about 21264 scheduling/coding comes from:
31 *	Compiler Writer's Guide for the Alpha 21264
32 *	abbreviated as 'CWG' in other comments here
33 *	ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
34 * Scheduling notation:
35 *	E	- either cluster
36 *	U	- upper subcluster; U0 - subcluster U0; U1 - subcluster U1
37 *	L	- lower subcluster; L0 - subcluster L0; L1 - subcluster L1
38 */
39
40/* Allow an exception for an insn; exit if we get one.  */
41#define EXI(x,y...)			\
42	99: x,##y;			\
43	.section __ex_table,"a";	\
44	.long 99b - .;			\
45	lda $31, $exitin-99b($31);	\
46	.previous
47
48#define EXO(x,y...)			\
49	99: x,##y;			\
50	.section __ex_table,"a";	\
51	.long 99b - .;			\
52	lda $31, $exitout-99b($31);	\
53	.previous
54
55	.set noat
56	.align 4
57	.globl __copy_user
58	.ent __copy_user
59				# Pipeline info: Slotting & Comments
60__copy_user:
61	.prologue 0
62	subq $0, 32, $1		# .. E  .. ..	: Is this going to be a small copy?
63	beq $0, $zerolength	# U  .. .. ..	: U L U L
64
65	and $6,7,$3		# .. .. .. E	: is leading dest misalignment
66	ble $1, $onebyteloop	# .. .. U  ..	: 1st branch : small amount of data
67	beq $3, $destaligned	# .. U  .. ..	: 2nd (one cycle fetcher stall)
68	subq $3, 8, $3		# E  .. .. ..	: L U U L : trip counter
69/*
70 * The fetcher stall also hides the 1 cycle cross-cluster stall for $3 (L --> U)
71 * This loop aligns the destination a byte at a time
72 * We know we have at least one trip through this loop
73 */
74$aligndest:
75	EXI( ldbu $1,0($7) )	# .. .. .. L	: Keep loads separate from stores
76	addq $6,1,$6		# .. .. E  ..	: Section 3.8 in the CWG
77	addq $3,1,$3		# .. E  .. ..	:
78	nop			# E  .. .. ..	: U L U L
79
80/*
81 * the -1 is to compensate for the inc($6) done in a previous quadpack
82 * which allows us zero dependencies within either quadpack in the loop
83 */
84	EXO( stb $1,-1($6) )	# .. .. .. L	:
85	addq $7,1,$7		# .. .. E  ..	: Section 3.8 in the CWG
86	subq $0,1,$0		# .. E  .. ..	:
87	bne $3, $aligndest	# U  .. .. ..	: U L U L
88
89/*
90 * If we fell through into here, we have a minimum of 33 - 7 bytes
91 * If we arrived via branch, we have a minimum of 32 bytes
92 */
93$destaligned:
94	and $7,7,$1		# .. .. .. E	: Check _current_ source alignment
95	bic $0,7,$4		# .. .. E  ..	: number bytes as a quadword loop
96	EXI( ldq_u $3,0($7) )	# .. L  .. ..	: Forward fetch for fallthrough code
97	beq $1,$quadaligned	# U  .. .. ..	: U L U L
98
99/*
100 * In the worst case, we've just executed an ldq_u here from 0($7)
101 * and we'll repeat it once if we take the branch
102 */
103
104/* Misaligned quadword loop - not unrolled.  Leave it that way. */
105$misquad:
106	EXI( ldq_u $2,8($7) )	# .. .. .. L	:
107	subq $4,8,$4		# .. .. E  ..	:
108	extql $3,$7,$3		# .. U  .. ..	:
109	extqh $2,$7,$1		# U  .. .. ..	: U U L L
110
111	bis $3,$1,$1		# .. .. .. E	:
112	EXO( stq $1,0($6) )	# .. .. L  ..	:
113	addq $7,8,$7		# .. E  .. ..	:
114	subq $0,8,$0		# E  .. .. ..	: U L L U
115
116	addq $6,8,$6		# .. .. .. E	:
117	bis $2,$2,$3		# .. .. E  ..	:
118	nop			# .. E  .. ..	:
119	bne $4,$misquad		# U  .. .. ..	: U L U L
120
121	nop			# .. .. .. E
122	nop			# .. .. E  ..
123	nop			# .. E  .. ..
124	beq $0,$zerolength	# U  .. .. ..	: U L U L
125
126/* We know we have at least one trip through the byte loop */
127	EXI ( ldbu $2,0($7) )	# .. .. .. L	: No loads in the same quad
128	addq $6,1,$6		# .. .. E  ..	: as the store (Section 3.8 in CWG)
129	nop			# .. E  .. ..	:
130	br $31, $dirtyentry	# L0 .. .. ..	: L U U L
131/* Do the trailing byte loop load, then hop into the store part of the loop */
132
133/*
134 * A minimum of (33 - 7) bytes to do a quad at a time.
135 * Based upon the usage context, it's worth the effort to unroll this loop
136 * $0 - number of bytes to be moved
137 * $4 - number of bytes to move as quadwords
138 * $6 is current destination address
139 * $7 is current source address
140 */
141$quadaligned:
142	subq	$4, 32, $2	# .. .. .. E	: do not unroll for small stuff
143	nop			# .. .. E  ..
144	nop			# .. E  .. ..
145	blt	$2, $onequad	# U  .. .. ..	: U L U L
146
147/*
148 * There is a significant assumption here that the source and destination
149 * addresses differ by more than 32 bytes.  In this particular case, a
150 * sparsity of registers further bounds this to be a minimum of 8 bytes.
151 * But if this isn't met, then the output result will be incorrect.
152 * Furthermore, due to a lack of available registers, we really can't
153 * unroll this to be an 8x loop (which would enable us to use the wh64
154 * instruction memory hint instruction).
155 */
156$unroll4:
157	EXI( ldq $1,0($7) )	# .. .. .. L
158	EXI( ldq $2,8($7) )	# .. .. L  ..
159	subq	$4,32,$4	# .. E  .. ..
160	nop			# E  .. .. ..	: U U L L
161
162	addq	$7,16,$7	# .. .. .. E
163	EXO( stq $1,0($6) )	# .. .. L  ..
164	EXO( stq $2,8($6) )	# .. L  .. ..
165	subq	$0,16,$0	# E  .. .. ..	: U L L U
166
167	addq	$6,16,$6	# .. .. .. E
168	EXI( ldq $1,0($7) )	# .. .. L  ..
169	EXI( ldq $2,8($7) )	# .. L  .. ..
170	subq	$4, 32, $3	# E  .. .. ..	: U U L L : is there enough for another trip?
171
172	EXO( stq $1,0($6) )	# .. .. .. L
173	EXO( stq $2,8($6) )	# .. .. L  ..
174	subq	$0,16,$0	# .. E  .. ..
175	addq	$7,16,$7	# E  .. .. ..	: U L L U
176
177	nop			# .. .. .. E
178	nop			# .. .. E  ..
179	addq	$6,16,$6	# .. E  .. ..
180	bgt	$3,$unroll4	# U  .. .. ..	: U L U L
181
182	nop
183	nop
184	nop
185	beq	$4, $noquads
186
187$onequad:
188	EXI( ldq $1,0($7) )
189	subq	$4,8,$4
190	addq	$7,8,$7
191	nop
192
193	EXO( stq $1,0($6) )
194	subq	$0,8,$0
195	addq	$6,8,$6
196	bne	$4,$onequad
197
198$noquads:
199	nop
200	nop
201	nop
202	beq $0,$zerolength
203
204/*
205 * For small copies (or the tail of a larger copy), do a very simple byte loop.
206 * There's no point in doing a lot of complex alignment calculations to try to
207 * to quadword stuff for a small amount of data.
208 *	$0 - remaining number of bytes left to copy
209 *	$6 - current dest addr
210 *	$7 - current source addr
211 */
212
213$onebyteloop:
214	EXI ( ldbu $2,0($7) )	# .. .. .. L	: No loads in the same quad
215	addq $6,1,$6		# .. .. E  ..	: as the store (Section 3.8 in CWG)
216	nop			# .. E  .. ..	:
217	nop			# E  .. .. ..	: U L U L
218
219$dirtyentry:
220/*
221 * the -1 is to compensate for the inc($6) done in a previous quadpack
222 * which allows us zero dependencies within either quadpack in the loop
223 */
224	EXO ( stb $2,-1($6) )	# .. .. .. L	:
225	addq $7,1,$7		# .. .. E  ..	: quadpack as the load
226	subq $0,1,$0		# .. E  .. ..	: change count _after_ copy
227	bgt $0,$onebyteloop	# U  .. .. ..	: U L U L
228
229$zerolength:
230$exitout:			# Destination for exception recovery(?)
231	nop			# .. .. .. E
232	nop			# .. .. E  ..
233	nop			# .. E  .. ..
234	ret $31,($28),1		# L0 .. .. ..	: L U L U
235
236$exitin:
237
238	/* A stupid byte-by-byte zeroing of the rest of the output
239	   buffer.  This cures security holes by never leaving
240	   random kernel data around to be copied elsewhere.  */
241
242	nop
243	nop
244	nop
245	mov	$0,$1
246
247$101:
248	EXO ( stb $31,0($6) )	# L
249	subq $1,1,$1		# E
250	addq $6,1,$6		# E
251	bgt $1,$101		# U
252
253	nop
254	nop
255	nop
256	ret $31,($28),1		# L0
257
258	.end __copy_user
259
260