mips/cavium-octeon/octeon-memcpy.S

18 #include <asm/asm-offsets.h>
30  *   - src and dst don't overlap
31  *   - src is readable
32  *   - dst is writable
40  *     - src is readable  (no exceptions when reading src)
42  *     - dst is writable  (no exceptions when writing dst)
43  * __copy_user uses a non-standard calling convention; see
57  *  1- AT contain the address of the byte just past the end of the source
59  *  2- src_entry <= src < AT, and
60  *  3- (dst - src) == (dst_entry - src_entry),
81  * Only on the 64-bit kernel we can made use of 64-bit registers.
132 #define FIRST(unit) ((unit)*NBYTES)  argument
133 #define REST(unit)  (FIRST(unit)+NBYTES-1)  argument
134 #define UNIT(unit)  FIRST(unit)  argument
136 #define ADDRMASK (NBYTES-1)
180 	 pref	0, 128(src)		# We must not prefetch invalid addresses
183 2:	pref	0, 256(src)		# We must not prefetch invalid addresses
187 EXC(	LOAD	t0, UNIT(0)(src),	l_exc)
188 EXC(	LOAD	t1, UNIT(1)(src),	l_exc_copy)
189 EXC(	LOAD	t2, UNIT(2)(src),	l_exc_copy)
190 EXC(	LOAD	t3, UNIT(3)(src),	l_exc_copy)
192 EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p16u)
193 EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p15u)
194 EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p14u)
195 EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p13u)
196 EXC(	LOAD	t0, UNIT(4)(src),	l_exc_copy)
197 EXC(	LOAD	t1, UNIT(5)(src),	l_exc_copy)
198 EXC(	LOAD	t2, UNIT(6)(src),	l_exc_copy)
199 EXC(	LOAD	t3, UNIT(7)(src),	l_exc_copy)
200 EXC(	STORE	t0, UNIT(4)(dst),	s_exc_p12u)
201 EXC(	STORE	t1, UNIT(5)(dst),	s_exc_p11u)
202 EXC(	STORE	t2, UNIT(6)(dst),	s_exc_p10u)
204 EXC(	STORE	t3, UNIT(7)(dst),	s_exc_p9u)
206 EXC(	LOAD	t0, UNIT(-8)(src),	l_exc_copy_rewind16)
207 EXC(	LOAD	t1, UNIT(-7)(src),	l_exc_copy_rewind16)
208 EXC(	LOAD	t2, UNIT(-6)(src),	l_exc_copy_rewind16)
209 EXC(	LOAD	t3, UNIT(-5)(src),	l_exc_copy_rewind16)
210 EXC(	STORE	t0, UNIT(-8)(dst),	s_exc_p8u)
211 EXC(	STORE	t1, UNIT(-7)(dst),	s_exc_p7u)
212 EXC(	STORE	t2, UNIT(-6)(dst),	s_exc_p6u)
213 EXC(	STORE	t3, UNIT(-5)(dst),	s_exc_p5u)
214 EXC(	LOAD	t0, UNIT(-4)(src),	l_exc_copy_rewind16)
215 EXC(	LOAD	t1, UNIT(-3)(src),	l_exc_copy_rewind16)
216 EXC(	LOAD	t2, UNIT(-2)(src),	l_exc_copy_rewind16)
217 EXC(	LOAD	t3, UNIT(-1)(src),	l_exc_copy_rewind16)
218 EXC(	STORE	t0, UNIT(-4)(dst),	s_exc_p4u)
219 EXC(	STORE	t1, UNIT(-3)(dst),	s_exc_p3u)
220 EXC(	STORE	t2, UNIT(-2)(dst),	s_exc_p2u)
221 EXC(	STORE	t3, UNIT(-1)(dst),	s_exc_p1u)
235 EXC(	LOAD	t0, UNIT(0)(src),	l_exc)
236 EXC(	LOAD	t1, UNIT(1)(src),	l_exc_copy)
237 EXC(	LOAD	t2, UNIT(2)(src),	l_exc_copy)
238 EXC(	LOAD	t3, UNIT(3)(src),	l_exc_copy)
240 EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p8u)
241 EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p7u)
242 EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p6u)
243 EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p5u)
244 EXC(	LOAD	t0, UNIT(4)(src),	l_exc_copy)
245 EXC(	LOAD	t1, UNIT(5)(src),	l_exc_copy)
246 EXC(	LOAD	t2, UNIT(6)(src),	l_exc_copy)
247 EXC(	LOAD	t3, UNIT(7)(src),	l_exc_copy)
248 EXC(	STORE	t0, UNIT(4)(dst),	s_exc_p4u)
249 EXC(	STORE	t1, UNIT(5)(dst),	s_exc_p3u)
250 EXC(	STORE	t2, UNIT(6)(dst),	s_exc_p2u)
251 EXC(	STORE	t3, UNIT(7)(dst),	s_exc_p1u)
262 EXC(	LOAD	t0, UNIT(0)(src),	l_exc)
263 EXC(	LOAD	t1, UNIT(1)(src),	l_exc_copy)
264 EXC(	LOAD	t2, UNIT(2)(src),	l_exc_copy)
265 EXC(	LOAD	t3, UNIT(3)(src),	l_exc_copy)
267 EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p4u)
268 EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p3u)
269 EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p2u)
270 EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p1u)
310 EXC(	 STORE	t0, -8(dst),		s_exc_p1u)
316 	 and	rem, len, (4*NBYTES-1)	 # rem = len % 4*NBYTES
321  * It's OK to load FIRST(N+1) before REST(N) because the two addresses
322  * are to the same unit (unless src is aligned, but it's not).
334 EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p4u)
335 EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p3u)
336 EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p2u)
337 EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p1u)
343 	 and	rem, len, NBYTES-1  # rem = len % NBYTES
372 EXC(	lb	t0, NBYTES-2(src), l_exc)
375 EXC(	 sb	t0, NBYTES-2(dst), s_exc_p1)
401 	sb	t1, 0(dst)	# can't fault -- we're copy_from_user
446 	sltu	t0, a1, t0			# dst + len <= src -> memcpy
447 	sltu	t1, a0, t1			# dst >= src + len -> memcpy
463 	lb	t0, -1(a1)
465 	sb	t0, -1(a0)