1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License.  See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Unified implementation of memcpy, memmove and the __copy_user backend.
7 *
8 * Copyright (C) 1998, 99, 2000, 01, 2002 Ralf Baechle (ralf@gnu.org)
9 * Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc.
10 * Copyright (C) 2002 Broadcom, Inc.
11 *   memcpy/copy_user author: Mark Vandevoorde
12 *
13 * Mnemonic names for arguments to memcpy/__copy_user
14 */
15
16#include <linux/export.h>
17#include <asm/asm.h>
18#include <asm/asm-offsets.h>
19#include <asm/regdef.h>
20
21#define dst a0
22#define src a1
23#define len a2
24
25/*
26 * Spec
27 *
28 * memcpy copies len bytes from src to dst and sets v0 to dst.
29 * It assumes that
30 *   - src and dst don't overlap
31 *   - src is readable
32 *   - dst is writable
33 * memcpy uses the standard calling convention
34 *
35 * __copy_user copies up to len bytes from src to dst and sets a2 (len) to
36 * the number of uncopied bytes due to an exception caused by a read or write.
37 * __copy_user assumes that src and dst don't overlap, and that the call is
38 * implementing one of the following:
39 *   copy_to_user
40 *     - src is readable  (no exceptions when reading src)
41 *   copy_from_user
42 *     - dst is writable  (no exceptions when writing dst)
43 * __copy_user uses a non-standard calling convention; see
44 * arch/mips/include/asm/uaccess.h
45 *
46 * When an exception happens on a load, the handler must
47 # ensure that all of the destination buffer is overwritten to prevent
48 * leaking information to user mode programs.
49 */
50
51/*
52 * Implementation
53 */
54
55/*
56 * The exception handler for loads requires that:
57 *  1- AT contain the address of the byte just past the end of the source
58 *     of the copy,
59 *  2- src_entry <= src < AT, and
60 *  3- (dst - src) == (dst_entry - src_entry),
61 * The _entry suffix denotes values when __copy_user was called.
62 *
63 * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user
64 * (2) is met by incrementing src by the number of bytes copied
65 * (3) is met by not doing loads between a pair of increments of dst and src
66 *
67 * The exception handlers for stores adjust len (if necessary) and return.
68 * These handlers do not need to overwrite any data.
69 *
70 * For __rmemcpy and memmove an exception is always a kernel bug, therefore
71 * they're not protected.
72 */
73
74#define EXC(inst_reg,addr,handler)		\
759:	inst_reg, addr;				\
76	.section __ex_table,"a";		\
77	PTR_WD	9b, handler;			\
78	.previous
79
80/*
81 * Only on the 64-bit kernel we can made use of 64-bit registers.
82 */
83
84#define LOAD   ld
85#define LOADL  ldl
86#define LOADR  ldr
87#define STOREL sdl
88#define STORER sdr
89#define STORE  sd
90#define ADD    daddu
91#define SUB    dsubu
92#define SRL    dsrl
93#define SRA    dsra
94#define SLL    dsll
95#define SLLV   dsllv
96#define SRLV   dsrlv
97#define NBYTES 8
98#define LOG_NBYTES 3
99
100/*
101 * As we are sharing code base with the mips32 tree (which use the o32 ABI
102 * register definitions). We need to redefine the register definitions from
103 * the n64 ABI register naming to the o32 ABI register naming.
104 */
105#undef t0
106#undef t1
107#undef t2
108#undef t3
109#define t0	$8
110#define t1	$9
111#define t2	$10
112#define t3	$11
113#define t4	$12
114#define t5	$13
115#define t6	$14
116#define t7	$15
117
118#ifdef CONFIG_CPU_LITTLE_ENDIAN
119#define LDFIRST LOADR
120#define LDREST	LOADL
121#define STFIRST STORER
122#define STREST	STOREL
123#define SHIFT_DISCARD SLLV
124#else
125#define LDFIRST LOADL
126#define LDREST	LOADR
127#define STFIRST STOREL
128#define STREST	STORER
129#define SHIFT_DISCARD SRLV
130#endif
131
132#define FIRST(unit) ((unit)*NBYTES)
133#define REST(unit)  (FIRST(unit)+NBYTES-1)
134#define UNIT(unit)  FIRST(unit)
135
136#define ADDRMASK (NBYTES-1)
137
138	.text
139	.set	noreorder
140	.set	noat
141
142/*
143 * A combined memcpy/__copy_user
144 * __copy_user sets len to 0 for success; else to an upper bound of
145 * the number of uncopied bytes.
146 * memcpy sets v0 to dst.
147 */
148	.align	5
149LEAF(memcpy)					/* a0=dst a1=src a2=len */
150EXPORT_SYMBOL(memcpy)
151	move	v0, dst				/* return value */
152__memcpy:
153FEXPORT(__raw_copy_from_user)
154EXPORT_SYMBOL(__raw_copy_from_user)
155FEXPORT(__raw_copy_to_user)
156EXPORT_SYMBOL(__raw_copy_to_user)
157	/*
158	 * Note: dst & src may be unaligned, len may be 0
159	 * Temps
160	 */
161	#
162	# Octeon doesn't care if the destination is unaligned. The hardware
163	# can fix it faster than we can special case the assembly.
164	#
165	pref	0, 0(src)
166	sltu	t0, len, NBYTES		# Check if < 1 word
167	bnez	t0, copy_bytes_checklen
168	 and	t0, src, ADDRMASK	# Check if src unaligned
169	bnez	t0, src_unaligned
170	 sltu	t0, len, 4*NBYTES	# Check if < 4 words
171	bnez	t0, less_than_4units
172	 sltu	t0, len, 8*NBYTES	# Check if < 8 words
173	bnez	t0, less_than_8units
174	 sltu	t0, len, 16*NBYTES	# Check if < 16 words
175	bnez	t0, cleanup_both_aligned
176	 sltu	t0, len, 128+1		# Check if len < 129
177	bnez	t0, 1f			# Skip prefetch if len is too short
178	 sltu	t0, len, 256+1		# Check if len < 257
179	bnez	t0, 1f			# Skip prefetch if len is too short
180	 pref	0, 128(src)		# We must not prefetch invalid addresses
181	#
182	# This is where we loop if there is more than 128 bytes left
1832:	pref	0, 256(src)		# We must not prefetch invalid addresses
184	#
185	# This is where we loop if we can't prefetch anymore
1861:
187EXC(	LOAD	t0, UNIT(0)(src),	l_exc)
188EXC(	LOAD	t1, UNIT(1)(src),	l_exc_copy)
189EXC(	LOAD	t2, UNIT(2)(src),	l_exc_copy)
190EXC(	LOAD	t3, UNIT(3)(src),	l_exc_copy)
191	SUB	len, len, 16*NBYTES
192EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p16u)
193EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p15u)
194EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p14u)
195EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p13u)
196EXC(	LOAD	t0, UNIT(4)(src),	l_exc_copy)
197EXC(	LOAD	t1, UNIT(5)(src),	l_exc_copy)
198EXC(	LOAD	t2, UNIT(6)(src),	l_exc_copy)
199EXC(	LOAD	t3, UNIT(7)(src),	l_exc_copy)
200EXC(	STORE	t0, UNIT(4)(dst),	s_exc_p12u)
201EXC(	STORE	t1, UNIT(5)(dst),	s_exc_p11u)
202EXC(	STORE	t2, UNIT(6)(dst),	s_exc_p10u)
203	ADD	src, src, 16*NBYTES
204EXC(	STORE	t3, UNIT(7)(dst),	s_exc_p9u)
205	ADD	dst, dst, 16*NBYTES
206EXC(	LOAD	t0, UNIT(-8)(src),	l_exc_copy_rewind16)
207EXC(	LOAD	t1, UNIT(-7)(src),	l_exc_copy_rewind16)
208EXC(	LOAD	t2, UNIT(-6)(src),	l_exc_copy_rewind16)
209EXC(	LOAD	t3, UNIT(-5)(src),	l_exc_copy_rewind16)
210EXC(	STORE	t0, UNIT(-8)(dst),	s_exc_p8u)
211EXC(	STORE	t1, UNIT(-7)(dst),	s_exc_p7u)
212EXC(	STORE	t2, UNIT(-6)(dst),	s_exc_p6u)
213EXC(	STORE	t3, UNIT(-5)(dst),	s_exc_p5u)
214EXC(	LOAD	t0, UNIT(-4)(src),	l_exc_copy_rewind16)
215EXC(	LOAD	t1, UNIT(-3)(src),	l_exc_copy_rewind16)
216EXC(	LOAD	t2, UNIT(-2)(src),	l_exc_copy_rewind16)
217EXC(	LOAD	t3, UNIT(-1)(src),	l_exc_copy_rewind16)
218EXC(	STORE	t0, UNIT(-4)(dst),	s_exc_p4u)
219EXC(	STORE	t1, UNIT(-3)(dst),	s_exc_p3u)
220EXC(	STORE	t2, UNIT(-2)(dst),	s_exc_p2u)
221EXC(	STORE	t3, UNIT(-1)(dst),	s_exc_p1u)
222	sltu	t0, len, 256+1		# See if we can prefetch more
223	beqz	t0, 2b
224	 sltu	t0, len, 128		# See if we can loop more time
225	beqz	t0, 1b
226	 nop
227	#
228	# Jump here if there are less than 16*NBYTES left.
229	#
230cleanup_both_aligned:
231	beqz	len, done
232	 sltu	t0, len, 8*NBYTES
233	bnez	t0, less_than_8units
234	 nop
235EXC(	LOAD	t0, UNIT(0)(src),	l_exc)
236EXC(	LOAD	t1, UNIT(1)(src),	l_exc_copy)
237EXC(	LOAD	t2, UNIT(2)(src),	l_exc_copy)
238EXC(	LOAD	t3, UNIT(3)(src),	l_exc_copy)
239	SUB	len, len, 8*NBYTES
240EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p8u)
241EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p7u)
242EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p6u)
243EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p5u)
244EXC(	LOAD	t0, UNIT(4)(src),	l_exc_copy)
245EXC(	LOAD	t1, UNIT(5)(src),	l_exc_copy)
246EXC(	LOAD	t2, UNIT(6)(src),	l_exc_copy)
247EXC(	LOAD	t3, UNIT(7)(src),	l_exc_copy)
248EXC(	STORE	t0, UNIT(4)(dst),	s_exc_p4u)
249EXC(	STORE	t1, UNIT(5)(dst),	s_exc_p3u)
250EXC(	STORE	t2, UNIT(6)(dst),	s_exc_p2u)
251EXC(	STORE	t3, UNIT(7)(dst),	s_exc_p1u)
252	ADD	src, src, 8*NBYTES
253	beqz	len, done
254	 ADD	dst, dst, 8*NBYTES
255	#
256	# Jump here if there are less than 8*NBYTES left.
257	#
258less_than_8units:
259	sltu	t0, len, 4*NBYTES
260	bnez	t0, less_than_4units
261	 nop
262EXC(	LOAD	t0, UNIT(0)(src),	l_exc)
263EXC(	LOAD	t1, UNIT(1)(src),	l_exc_copy)
264EXC(	LOAD	t2, UNIT(2)(src),	l_exc_copy)
265EXC(	LOAD	t3, UNIT(3)(src),	l_exc_copy)
266	SUB	len, len, 4*NBYTES
267EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p4u)
268EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p3u)
269EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p2u)
270EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p1u)
271	ADD	src, src, 4*NBYTES
272	beqz	len, done
273	 ADD	dst, dst, 4*NBYTES
274	#
275	# Jump here if there are less than 4*NBYTES left. This means
276	# we may need to copy up to 3 NBYTES words.
277	#
278less_than_4units:
279	sltu	t0, len, 1*NBYTES
280	bnez	t0, copy_bytes_checklen
281	 nop
282	#
283	# 1) Copy NBYTES, then check length again
284	#
285EXC(	LOAD	t0, 0(src),		l_exc)
286	SUB	len, len, NBYTES
287	sltu	t1, len, 8
288EXC(	STORE	t0, 0(dst),		s_exc_p1u)
289	ADD	src, src, NBYTES
290	bnez	t1, copy_bytes_checklen
291	 ADD	dst, dst, NBYTES
292	#
293	# 2) Copy NBYTES, then check length again
294	#
295EXC(	LOAD	t0, 0(src),		l_exc)
296	SUB	len, len, NBYTES
297	sltu	t1, len, 8
298EXC(	STORE	t0, 0(dst),		s_exc_p1u)
299	ADD	src, src, NBYTES
300	bnez	t1, copy_bytes_checklen
301	 ADD	dst, dst, NBYTES
302	#
303	# 3) Copy NBYTES, then check length again
304	#
305EXC(	LOAD	t0, 0(src),		l_exc)
306	SUB	len, len, NBYTES
307	ADD	src, src, NBYTES
308	ADD	dst, dst, NBYTES
309	b copy_bytes_checklen
310EXC(	 STORE	t0, -8(dst),		s_exc_p1u)
311
312src_unaligned:
313#define rem t8
314	SRL	t0, len, LOG_NBYTES+2	 # +2 for 4 units/iter
315	beqz	t0, cleanup_src_unaligned
316	 and	rem, len, (4*NBYTES-1)	 # rem = len % 4*NBYTES
3171:
318/*
319 * Avoid consecutive LD*'s to the same register since some mips
320 * implementations can't issue them in the same cycle.
321 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
322 * are to the same unit (unless src is aligned, but it's not).
323 */
324EXC(	LDFIRST t0, FIRST(0)(src),	l_exc)
325EXC(	LDFIRST t1, FIRST(1)(src),	l_exc_copy)
326	SUB	len, len, 4*NBYTES
327EXC(	LDREST	t0, REST(0)(src),	l_exc_copy)
328EXC(	LDREST	t1, REST(1)(src),	l_exc_copy)
329EXC(	LDFIRST t2, FIRST(2)(src),	l_exc_copy)
330EXC(	LDFIRST t3, FIRST(3)(src),	l_exc_copy)
331EXC(	LDREST	t2, REST(2)(src),	l_exc_copy)
332EXC(	LDREST	t3, REST(3)(src),	l_exc_copy)
333	ADD	src, src, 4*NBYTES
334EXC(	STORE	t0, UNIT(0)(dst),	s_exc_p4u)
335EXC(	STORE	t1, UNIT(1)(dst),	s_exc_p3u)
336EXC(	STORE	t2, UNIT(2)(dst),	s_exc_p2u)
337EXC(	STORE	t3, UNIT(3)(dst),	s_exc_p1u)
338	bne	len, rem, 1b
339	 ADD	dst, dst, 4*NBYTES
340
341cleanup_src_unaligned:
342	beqz	len, done
343	 and	rem, len, NBYTES-1  # rem = len % NBYTES
344	beq	rem, len, copy_bytes
345	 nop
3461:
347EXC(	LDFIRST t0, FIRST(0)(src),	l_exc)
348EXC(	LDREST	t0, REST(0)(src),	l_exc_copy)
349	SUB	len, len, NBYTES
350EXC(	STORE	t0, 0(dst),		s_exc_p1u)
351	ADD	src, src, NBYTES
352	bne	len, rem, 1b
353	 ADD	dst, dst, NBYTES
354
355copy_bytes_checklen:
356	beqz	len, done
357	 nop
358copy_bytes:
359	/* 0 < len < NBYTES  */
360#define COPY_BYTE(N)			\
361EXC(	lb	t0, N(src), l_exc);	\
362	SUB	len, len, 1;		\
363	beqz	len, done;		\
364EXC(	 sb	t0, N(dst), s_exc_p1)
365
366	COPY_BYTE(0)
367	COPY_BYTE(1)
368	COPY_BYTE(2)
369	COPY_BYTE(3)
370	COPY_BYTE(4)
371	COPY_BYTE(5)
372EXC(	lb	t0, NBYTES-2(src), l_exc)
373	SUB	len, len, 1
374	jr	ra
375EXC(	 sb	t0, NBYTES-2(dst), s_exc_p1)
376done:
377	jr	ra
378	 nop
379	END(memcpy)
380
381l_exc_copy_rewind16:
382	/* Rewind src and dst by 16*NBYTES for l_exc_copy */
383	SUB	src, src, 16*NBYTES
384	SUB	dst, dst, 16*NBYTES
385l_exc_copy:
386	/*
387	 * Copy bytes from src until faulting load address (or until a
388	 * lb faults)
389	 *
390	 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
391	 * may be more than a byte beyond the last address.
392	 * Hence, the lb below may get an exception.
393	 *
394	 * Assumes src < THREAD_BUADDR($28)
395	 */
396	LOAD	t0, TI_TASK($28)
397	LOAD	t0, THREAD_BUADDR(t0)
3981:
399EXC(	lb	t1, 0(src),	l_exc)
400	ADD	src, src, 1
401	sb	t1, 0(dst)	# can't fault -- we're copy_from_user
402	bne	src, t0, 1b
403	 ADD	dst, dst, 1
404l_exc:
405	LOAD	t0, TI_TASK($28)
406	LOAD	t0, THREAD_BUADDR(t0)	# t0 is just past last good address
407	SUB	len, AT, t0		# len number of uncopied bytes
408	jr	ra
409	 nop
410
411
412#define SEXC(n)				\
413s_exc_p ## n ## u:			\
414	jr	ra;			\
415	 ADD	len, len, n*NBYTES
416
417SEXC(16)
418SEXC(15)
419SEXC(14)
420SEXC(13)
421SEXC(12)
422SEXC(11)
423SEXC(10)
424SEXC(9)
425SEXC(8)
426SEXC(7)
427SEXC(6)
428SEXC(5)
429SEXC(4)
430SEXC(3)
431SEXC(2)
432SEXC(1)
433
434s_exc_p1:
435	jr	ra
436	 ADD	len, len, 1
437s_exc:
438	jr	ra
439	 nop
440
441	.align	5
442LEAF(memmove)
443EXPORT_SYMBOL(memmove)
444	ADD	t0, a0, a2
445	ADD	t1, a1, a2
446	sltu	t0, a1, t0			# dst + len <= src -> memcpy
447	sltu	t1, a0, t1			# dst >= src + len -> memcpy
448	and	t0, t1
449	beqz	t0, __memcpy
450	 move	v0, a0				/* return value */
451	beqz	a2, r_out
452	END(memmove)
453
454	/* fall through to __rmemcpy */
455LEAF(__rmemcpy)					/* a0=dst a1=src a2=len */
456	 sltu	t0, a1, a0
457	beqz	t0, r_end_bytes_up		# src >= dst
458	 nop
459	ADD	a0, a2				# dst = dst + len
460	ADD	a1, a2				# src = src + len
461
462r_end_bytes:
463	lb	t0, -1(a1)
464	SUB	a2, a2, 0x1
465	sb	t0, -1(a0)
466	SUB	a1, a1, 0x1
467	bnez	a2, r_end_bytes
468	 SUB	a0, a0, 0x1
469
470r_out:
471	jr	ra
472	 move	a2, zero
473
474r_end_bytes_up:
475	lb	t0, (a1)
476	SUB	a2, a2, 0x1
477	sb	t0, (a0)
478	ADD	a1, a1, 0x1
479	bnez	a2, r_end_bytes_up
480	 ADD	a0, a0, 0x1
481
482	jr	ra
483	 move	a2, zero
484	END(__rmemcpy)
485