1#include <linux/linkage.h> 2#include <asm-generic/export.h> 3#include <asm/asm.h> 4#include <asm/asm-extable.h> 5#include <asm/csr.h> 6 7 .macro fixup op reg addr lbl 8100: 9 \op \reg, \addr 10 _asm_extable 100b, \lbl 11 .endm 12 13ENTRY(__asm_copy_to_user) 14ENTRY(__asm_copy_from_user) 15 16 /* Enable access to user memory */ 17 li t6, SR_SUM 18 csrs CSR_STATUS, t6 19 20 /* 21 * Save the terminal address which will be used to compute the number 22 * of bytes copied in case of a fixup exception. 23 */ 24 add t5, a0, a2 25 26 /* 27 * Register allocation for code below: 28 * a0 - start of uncopied dst 29 * a1 - start of uncopied src 30 * a2 - size 31 * t0 - end of uncopied dst 32 */ 33 add t0, a0, a2 34 35 /* 36 * Use byte copy only if too small. 37 * SZREG holds 4 for RV32 and 8 for RV64 38 */ 39 li a3, 9*SZREG /* size must be larger than size in word_copy */ 40 bltu a2, a3, .Lbyte_copy_tail 41 42 /* 43 * Copy first bytes until dst is aligned to word boundary. 44 * a0 - start of dst 45 * t1 - start of aligned dst 46 */ 47 addi t1, a0, SZREG-1 48 andi t1, t1, ~(SZREG-1) 49 /* dst is already aligned, skip */ 50 beq a0, t1, .Lskip_align_dst 511: 52 /* a5 - one byte for copying data */ 53 fixup lb a5, 0(a1), 10f 54 addi a1, a1, 1 /* src */ 55 fixup sb a5, 0(a0), 10f 56 addi a0, a0, 1 /* dst */ 57 bltu a0, t1, 1b /* t1 - start of aligned dst */ 58 59.Lskip_align_dst: 60 /* 61 * Now dst is aligned. 62 * Use shift-copy if src is misaligned. 63 * Use word-copy if both src and dst are aligned because 64 * can not use shift-copy which do not require shifting 65 */ 66 /* a1 - start of src */ 67 andi a3, a1, SZREG-1 68 bnez a3, .Lshift_copy 69 70.Lword_copy: 71 /* 72 * Both src and dst are aligned, unrolled word copy 73 * 74 * a0 - start of aligned dst 75 * a1 - start of aligned src 76 * t0 - end of aligned dst 77 */ 78 addi t0, t0, -(8*SZREG) /* not to over run */ 792: 80 fixup REG_L a4, 0(a1), 10f 81 fixup REG_L a5, SZREG(a1), 10f 82 fixup REG_L a6, 2*SZREG(a1), 10f 83 fixup REG_L a7, 3*SZREG(a1), 10f 84 fixup REG_L t1, 4*SZREG(a1), 10f 85 fixup REG_L t2, 5*SZREG(a1), 10f 86 fixup REG_L t3, 6*SZREG(a1), 10f 87 fixup REG_L t4, 7*SZREG(a1), 10f 88 fixup REG_S a4, 0(a0), 10f 89 fixup REG_S a5, SZREG(a0), 10f 90 fixup REG_S a6, 2*SZREG(a0), 10f 91 fixup REG_S a7, 3*SZREG(a0), 10f 92 fixup REG_S t1, 4*SZREG(a0), 10f 93 fixup REG_S t2, 5*SZREG(a0), 10f 94 fixup REG_S t3, 6*SZREG(a0), 10f 95 fixup REG_S t4, 7*SZREG(a0), 10f 96 addi a0, a0, 8*SZREG 97 addi a1, a1, 8*SZREG 98 bltu a0, t0, 2b 99 100 addi t0, t0, 8*SZREG /* revert to original value */ 101 j .Lbyte_copy_tail 102 103.Lshift_copy: 104 105 /* 106 * Word copy with shifting. 107 * For misaligned copy we still perform aligned word copy, but 108 * we need to use the value fetched from the previous iteration and 109 * do some shifts. 110 * This is safe because reading is less than a word size. 111 * 112 * a0 - start of aligned dst 113 * a1 - start of src 114 * a3 - a1 & mask:(SZREG-1) 115 * t0 - end of uncopied dst 116 * t1 - end of aligned dst 117 */ 118 /* calculating aligned word boundary for dst */ 119 andi t1, t0, ~(SZREG-1) 120 /* Converting unaligned src to aligned src */ 121 andi a1, a1, ~(SZREG-1) 122 123 /* 124 * Calculate shifts 125 * t3 - prev shift 126 * t4 - current shift 127 */ 128 slli t3, a3, 3 /* converting bytes in a3 to bits */ 129 li a5, SZREG*8 130 sub t4, a5, t3 131 132 /* Load the first word to combine with second word */ 133 fixup REG_L a5, 0(a1), 10f 134 1353: 136 /* Main shifting copy 137 * 138 * a0 - start of aligned dst 139 * a1 - start of aligned src 140 * t1 - end of aligned dst 141 */ 142 143 /* At least one iteration will be executed */ 144 srl a4, a5, t3 145 fixup REG_L a5, SZREG(a1), 10f 146 addi a1, a1, SZREG 147 sll a2, a5, t4 148 or a2, a2, a4 149 fixup REG_S a2, 0(a0), 10f 150 addi a0, a0, SZREG 151 bltu a0, t1, 3b 152 153 /* Revert src to original unaligned value */ 154 add a1, a1, a3 155 156.Lbyte_copy_tail: 157 /* 158 * Byte copy anything left. 159 * 160 * a0 - start of remaining dst 161 * a1 - start of remaining src 162 * t0 - end of remaining dst 163 */ 164 bgeu a0, t0, .Lout_copy_user /* check if end of copy */ 1654: 166 fixup lb a5, 0(a1), 10f 167 addi a1, a1, 1 /* src */ 168 fixup sb a5, 0(a0), 10f 169 addi a0, a0, 1 /* dst */ 170 bltu a0, t0, 4b /* t0 - end of dst */ 171 172.Lout_copy_user: 173 /* Disable access to user memory */ 174 csrc CSR_STATUS, t6 175 li a0, 0 176 ret 177 178 /* Exception fixup code */ 17910: 180 /* Disable access to user memory */ 181 csrc CSR_STATUS, t6 182 sub a0, t5, a0 183 ret 184ENDPROC(__asm_copy_to_user) 185ENDPROC(__asm_copy_from_user) 186EXPORT_SYMBOL(__asm_copy_to_user) 187EXPORT_SYMBOL(__asm_copy_from_user) 188 189 190ENTRY(__clear_user) 191 192 /* Enable access to user memory */ 193 li t6, SR_SUM 194 csrs CSR_STATUS, t6 195 196 add a3, a0, a1 197 addi t0, a0, SZREG-1 198 andi t1, a3, ~(SZREG-1) 199 andi t0, t0, ~(SZREG-1) 200 /* 201 * a3: terminal address of target region 202 * t0: lowest doubleword-aligned address in target region 203 * t1: highest doubleword-aligned address in target region 204 */ 205 bgeu t0, t1, 2f 206 bltu a0, t0, 4f 2071: 208 fixup REG_S, zero, (a0), 11f 209 addi a0, a0, SZREG 210 bltu a0, t1, 1b 2112: 212 bltu a0, a3, 5f 213 2143: 215 /* Disable access to user memory */ 216 csrc CSR_STATUS, t6 217 li a0, 0 218 ret 2194: /* Edge case: unalignment */ 220 fixup sb, zero, (a0), 11f 221 addi a0, a0, 1 222 bltu a0, t0, 4b 223 j 1b 2245: /* Edge case: remainder */ 225 fixup sb, zero, (a0), 11f 226 addi a0, a0, 1 227 bltu a0, a3, 5b 228 j 3b 229 230 /* Exception fixup code */ 23111: 232 /* Disable access to user memory */ 233 csrc CSR_STATUS, t6 234 sub a0, a3, a0 235 ret 236ENDPROC(__clear_user) 237EXPORT_SYMBOL(__clear_user) 238