xref: /openbmc/linux/arch/riscv/lib/uaccess.S (revision 4b05b993)
1#include <linux/linkage.h>
2#include <asm-generic/export.h>
3#include <asm/asm.h>
4#include <asm/asm-extable.h>
5#include <asm/csr.h>
6
7	.macro fixup op reg addr lbl
8100:
9	\op \reg, \addr
10	_asm_extable	100b, \lbl
11	.endm
12
13ENTRY(__asm_copy_to_user)
14ENTRY(__asm_copy_from_user)
15
16	/* Enable access to user memory */
17	li t6, SR_SUM
18	csrs CSR_STATUS, t6
19
20	/*
21	 * Save the terminal address which will be used to compute the number
22	 * of bytes copied in case of a fixup exception.
23	 */
24	add	t5, a0, a2
25
26	/*
27	 * Register allocation for code below:
28	 * a0 - start of uncopied dst
29	 * a1 - start of uncopied src
30	 * a2 - size
31	 * t0 - end of uncopied dst
32	 */
33	add	t0, a0, a2
34
35	/*
36	 * Use byte copy only if too small.
37	 * SZREG holds 4 for RV32 and 8 for RV64
38	 */
39	li	a3, 9*SZREG /* size must be larger than size in word_copy */
40	bltu	a2, a3, .Lbyte_copy_tail
41
42	/*
43	 * Copy first bytes until dst is aligned to word boundary.
44	 * a0 - start of dst
45	 * t1 - start of aligned dst
46	 */
47	addi	t1, a0, SZREG-1
48	andi	t1, t1, ~(SZREG-1)
49	/* dst is already aligned, skip */
50	beq	a0, t1, .Lskip_align_dst
511:
52	/* a5 - one byte for copying data */
53	fixup lb      a5, 0(a1), 10f
54	addi	a1, a1, 1	/* src */
55	fixup sb      a5, 0(a0), 10f
56	addi	a0, a0, 1	/* dst */
57	bltu	a0, t1, 1b	/* t1 - start of aligned dst */
58
59.Lskip_align_dst:
60	/*
61	 * Now dst is aligned.
62	 * Use shift-copy if src is misaligned.
63	 * Use word-copy if both src and dst are aligned because
64	 * can not use shift-copy which do not require shifting
65	 */
66	/* a1 - start of src */
67	andi	a3, a1, SZREG-1
68	bnez	a3, .Lshift_copy
69
70.Lword_copy:
71        /*
72	 * Both src and dst are aligned, unrolled word copy
73	 *
74	 * a0 - start of aligned dst
75	 * a1 - start of aligned src
76	 * t0 - end of aligned dst
77	 */
78	addi	t0, t0, -(8*SZREG) /* not to over run */
792:
80	fixup REG_L   a4,        0(a1), 10f
81	fixup REG_L   a5,    SZREG(a1), 10f
82	fixup REG_L   a6,  2*SZREG(a1), 10f
83	fixup REG_L   a7,  3*SZREG(a1), 10f
84	fixup REG_L   t1,  4*SZREG(a1), 10f
85	fixup REG_L   t2,  5*SZREG(a1), 10f
86	fixup REG_L   t3,  6*SZREG(a1), 10f
87	fixup REG_L   t4,  7*SZREG(a1), 10f
88	fixup REG_S   a4,        0(a0), 10f
89	fixup REG_S   a5,    SZREG(a0), 10f
90	fixup REG_S   a6,  2*SZREG(a0), 10f
91	fixup REG_S   a7,  3*SZREG(a0), 10f
92	fixup REG_S   t1,  4*SZREG(a0), 10f
93	fixup REG_S   t2,  5*SZREG(a0), 10f
94	fixup REG_S   t3,  6*SZREG(a0), 10f
95	fixup REG_S   t4,  7*SZREG(a0), 10f
96	addi	a0, a0, 8*SZREG
97	addi	a1, a1, 8*SZREG
98	bltu	a0, t0, 2b
99
100	addi	t0, t0, 8*SZREG /* revert to original value */
101	j	.Lbyte_copy_tail
102
103.Lshift_copy:
104
105	/*
106	 * Word copy with shifting.
107	 * For misaligned copy we still perform aligned word copy, but
108	 * we need to use the value fetched from the previous iteration and
109	 * do some shifts.
110	 * This is safe because reading is less than a word size.
111	 *
112	 * a0 - start of aligned dst
113	 * a1 - start of src
114	 * a3 - a1 & mask:(SZREG-1)
115	 * t0 - end of uncopied dst
116	 * t1 - end of aligned dst
117	 */
118	/* calculating aligned word boundary for dst */
119	andi	t1, t0, ~(SZREG-1)
120	/* Converting unaligned src to aligned src */
121	andi	a1, a1, ~(SZREG-1)
122
123	/*
124	 * Calculate shifts
125	 * t3 - prev shift
126	 * t4 - current shift
127	 */
128	slli	t3, a3, 3 /* converting bytes in a3 to bits */
129	li	a5, SZREG*8
130	sub	t4, a5, t3
131
132	/* Load the first word to combine with second word */
133	fixup REG_L   a5, 0(a1), 10f
134
1353:
136	/* Main shifting copy
137	 *
138	 * a0 - start of aligned dst
139	 * a1 - start of aligned src
140	 * t1 - end of aligned dst
141	 */
142
143	/* At least one iteration will be executed */
144	srl	a4, a5, t3
145	fixup REG_L   a5, SZREG(a1), 10f
146	addi	a1, a1, SZREG
147	sll	a2, a5, t4
148	or	a2, a2, a4
149	fixup REG_S   a2, 0(a0), 10f
150	addi	a0, a0, SZREG
151	bltu	a0, t1, 3b
152
153	/* Revert src to original unaligned value  */
154	add	a1, a1, a3
155
156.Lbyte_copy_tail:
157	/*
158	 * Byte copy anything left.
159	 *
160	 * a0 - start of remaining dst
161	 * a1 - start of remaining src
162	 * t0 - end of remaining dst
163	 */
164	bgeu	a0, t0, .Lout_copy_user  /* check if end of copy */
1654:
166	fixup lb      a5, 0(a1), 10f
167	addi	a1, a1, 1	/* src */
168	fixup sb      a5, 0(a0), 10f
169	addi	a0, a0, 1	/* dst */
170	bltu	a0, t0, 4b	/* t0 - end of dst */
171
172.Lout_copy_user:
173	/* Disable access to user memory */
174	csrc CSR_STATUS, t6
175	li	a0, 0
176	ret
177
178	/* Exception fixup code */
17910:
180	/* Disable access to user memory */
181	csrc CSR_STATUS, t6
182	sub a0, t5, a0
183	ret
184ENDPROC(__asm_copy_to_user)
185ENDPROC(__asm_copy_from_user)
186EXPORT_SYMBOL(__asm_copy_to_user)
187EXPORT_SYMBOL(__asm_copy_from_user)
188
189
190ENTRY(__clear_user)
191
192	/* Enable access to user memory */
193	li t6, SR_SUM
194	csrs CSR_STATUS, t6
195
196	add a3, a0, a1
197	addi t0, a0, SZREG-1
198	andi t1, a3, ~(SZREG-1)
199	andi t0, t0, ~(SZREG-1)
200	/*
201	 * a3: terminal address of target region
202	 * t0: lowest doubleword-aligned address in target region
203	 * t1: highest doubleword-aligned address in target region
204	 */
205	bgeu t0, t1, 2f
206	bltu a0, t0, 4f
2071:
208	fixup REG_S, zero, (a0), 11f
209	addi a0, a0, SZREG
210	bltu a0, t1, 1b
2112:
212	bltu a0, a3, 5f
213
2143:
215	/* Disable access to user memory */
216	csrc CSR_STATUS, t6
217	li a0, 0
218	ret
2194: /* Edge case: unalignment */
220	fixup sb, zero, (a0), 11f
221	addi a0, a0, 1
222	bltu a0, t0, 4b
223	j 1b
2245: /* Edge case: remainder */
225	fixup sb, zero, (a0), 11f
226	addi a0, a0, 1
227	bltu a0, a3, 5b
228	j 3b
229
230	/* Exception fixup code */
23111:
232	/* Disable access to user memory */
233	csrc CSR_STATUS, t6
234	sub a0, a3, a0
235	ret
236ENDPROC(__clear_user)
237EXPORT_SYMBOL(__clear_user)
238