1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 6#include <asm/alternative-asm.h> 7#include <asm/asm.h> 8#include <asm/asmmacro.h> 9#include <asm/asm-extable.h> 10#include <asm/cpu.h> 11#include <asm/export.h> 12#include <asm/regdef.h> 13 14.irp to, 0, 1, 2, 3, 4, 5, 6, 7 15.L_fixup_handle_\to\(): 16 sub.d a0, a2, a0 17 addi.d a0, a0, (\to) * (-8) 18 jr ra 19.endr 20 21.irp to, 0, 2, 4 22.L_fixup_handle_s\to\(): 23 addi.d a0, a2, -\to 24 jr ra 25.endr 26 27SYM_FUNC_START(__copy_user) 28 /* 29 * Some CPUs support hardware unaligned access 30 */ 31 ALTERNATIVE "b __copy_user_generic", \ 32 "b __copy_user_fast", CPU_FEATURE_UAL 33SYM_FUNC_END(__copy_user) 34 35EXPORT_SYMBOL(__copy_user) 36 37/* 38 * unsigned long __copy_user_generic(void *to, const void *from, size_t n) 39 * 40 * a0: to 41 * a1: from 42 * a2: n 43 */ 44SYM_FUNC_START(__copy_user_generic) 45 beqz a2, 3f 46 471: ld.b t0, a1, 0 482: st.b t0, a0, 0 49 addi.d a0, a0, 1 50 addi.d a1, a1, 1 51 addi.d a2, a2, -1 52 bgtz a2, 1b 53 543: move a0, a2 55 jr ra 56 57 _asm_extable 1b, .L_fixup_handle_s0 58 _asm_extable 2b, .L_fixup_handle_s0 59SYM_FUNC_END(__copy_user_generic) 60 61/* 62 * unsigned long __copy_user_fast(void *to, const void *from, unsigned long n) 63 * 64 * a0: to 65 * a1: from 66 * a2: n 67 */ 68SYM_FUNC_START(__copy_user_fast) 69 sltui t0, a2, 9 70 bnez t0, .Lsmall 71 72 add.d a3, a1, a2 73 add.d a2, a0, a2 740: ld.d t0, a1, 0 751: st.d t0, a0, 0 76 77 /* align up destination address */ 78 andi t1, a0, 7 79 sub.d t0, zero, t1 80 addi.d t0, t0, 8 81 add.d a1, a1, t0 82 add.d a0, a0, t0 83 84 addi.d a4, a3, -64 85 bgeu a1, a4, .Llt64 86 87 /* copy 64 bytes at a time */ 88.Lloop64: 892: ld.d t0, a1, 0 903: ld.d t1, a1, 8 914: ld.d t2, a1, 16 925: ld.d t3, a1, 24 936: ld.d t4, a1, 32 947: ld.d t5, a1, 40 958: ld.d t6, a1, 48 969: ld.d t7, a1, 56 97 addi.d a1, a1, 64 9810: st.d t0, a0, 0 9911: st.d t1, a0, 8 10012: st.d t2, a0, 16 10113: st.d t3, a0, 24 10214: st.d t4, a0, 32 10315: st.d t5, a0, 40 10416: st.d t6, a0, 48 10517: st.d t7, a0, 56 106 addi.d a0, a0, 64 107 bltu a1, a4, .Lloop64 108 109 /* copy the remaining bytes */ 110.Llt64: 111 addi.d a4, a3, -32 112 bgeu a1, a4, .Llt32 11318: ld.d t0, a1, 0 11419: ld.d t1, a1, 8 11520: ld.d t2, a1, 16 11621: ld.d t3, a1, 24 117 addi.d a1, a1, 32 11822: st.d t0, a0, 0 11923: st.d t1, a0, 8 12024: st.d t2, a0, 16 12125: st.d t3, a0, 24 122 addi.d a0, a0, 32 123 124.Llt32: 125 addi.d a4, a3, -16 126 bgeu a1, a4, .Llt16 12726: ld.d t0, a1, 0 12827: ld.d t1, a1, 8 129 addi.d a1, a1, 16 13028: st.d t0, a0, 0 13129: st.d t1, a0, 8 132 addi.d a0, a0, 16 133 134.Llt16: 135 addi.d a4, a3, -8 136 bgeu a1, a4, .Llt8 13730: ld.d t0, a1, 0 13831: st.d t0, a0, 0 139 addi.d a0, a0, 8 140 141.Llt8: 14232: ld.d t0, a3, -8 14333: st.d t0, a2, -8 144 145 /* return */ 146 move a0, zero 147 jr ra 148 149 .align 5 150.Lsmall: 151 pcaddi t0, 8 152 slli.d a3, a2, 5 153 add.d t0, t0, a3 154 jr t0 155 156 .align 5 157 move a0, zero 158 jr ra 159 160 .align 5 16134: ld.b t0, a1, 0 16235: st.b t0, a0, 0 163 move a0, zero 164 jr ra 165 166 .align 5 16736: ld.h t0, a1, 0 16837: st.h t0, a0, 0 169 move a0, zero 170 jr ra 171 172 .align 5 17338: ld.h t0, a1, 0 17439: ld.b t1, a1, 2 17540: st.h t0, a0, 0 17641: st.b t1, a0, 2 177 move a0, zero 178 jr ra 179 180 .align 5 18142: ld.w t0, a1, 0 18243: st.w t0, a0, 0 183 move a0, zero 184 jr ra 185 186 .align 5 18744: ld.w t0, a1, 0 18845: ld.b t1, a1, 4 18946: st.w t0, a0, 0 19047: st.b t1, a0, 4 191 move a0, zero 192 jr ra 193 194 .align 5 19548: ld.w t0, a1, 0 19649: ld.h t1, a1, 4 19750: st.w t0, a0, 0 19851: st.h t1, a0, 4 199 move a0, zero 200 jr ra 201 202 .align 5 20352: ld.w t0, a1, 0 20453: ld.w t1, a1, 3 20554: st.w t0, a0, 0 20655: st.w t1, a0, 3 207 move a0, zero 208 jr ra 209 210 .align 5 21156: ld.d t0, a1, 0 21257: st.d t0, a0, 0 213 move a0, zero 214 jr ra 215 216 /* fixup and ex_table */ 217 _asm_extable 0b, .L_fixup_handle_0 218 _asm_extable 1b, .L_fixup_handle_0 219 _asm_extable 2b, .L_fixup_handle_0 220 _asm_extable 3b, .L_fixup_handle_0 221 _asm_extable 4b, .L_fixup_handle_0 222 _asm_extable 5b, .L_fixup_handle_0 223 _asm_extable 6b, .L_fixup_handle_0 224 _asm_extable 7b, .L_fixup_handle_0 225 _asm_extable 8b, .L_fixup_handle_0 226 _asm_extable 9b, .L_fixup_handle_0 227 _asm_extable 10b, .L_fixup_handle_0 228 _asm_extable 11b, .L_fixup_handle_1 229 _asm_extable 12b, .L_fixup_handle_2 230 _asm_extable 13b, .L_fixup_handle_3 231 _asm_extable 14b, .L_fixup_handle_4 232 _asm_extable 15b, .L_fixup_handle_5 233 _asm_extable 16b, .L_fixup_handle_6 234 _asm_extable 17b, .L_fixup_handle_7 235 _asm_extable 18b, .L_fixup_handle_0 236 _asm_extable 19b, .L_fixup_handle_0 237 _asm_extable 20b, .L_fixup_handle_0 238 _asm_extable 21b, .L_fixup_handle_0 239 _asm_extable 22b, .L_fixup_handle_0 240 _asm_extable 23b, .L_fixup_handle_1 241 _asm_extable 24b, .L_fixup_handle_2 242 _asm_extable 25b, .L_fixup_handle_3 243 _asm_extable 26b, .L_fixup_handle_0 244 _asm_extable 27b, .L_fixup_handle_0 245 _asm_extable 28b, .L_fixup_handle_0 246 _asm_extable 29b, .L_fixup_handle_1 247 _asm_extable 30b, .L_fixup_handle_0 248 _asm_extable 31b, .L_fixup_handle_0 249 _asm_extable 32b, .L_fixup_handle_0 250 _asm_extable 33b, .L_fixup_handle_0 251 _asm_extable 34b, .L_fixup_handle_s0 252 _asm_extable 35b, .L_fixup_handle_s0 253 _asm_extable 36b, .L_fixup_handle_s0 254 _asm_extable 37b, .L_fixup_handle_s0 255 _asm_extable 38b, .L_fixup_handle_s0 256 _asm_extable 39b, .L_fixup_handle_s0 257 _asm_extable 40b, .L_fixup_handle_s0 258 _asm_extable 41b, .L_fixup_handle_s2 259 _asm_extable 42b, .L_fixup_handle_s0 260 _asm_extable 43b, .L_fixup_handle_s0 261 _asm_extable 44b, .L_fixup_handle_s0 262 _asm_extable 45b, .L_fixup_handle_s0 263 _asm_extable 46b, .L_fixup_handle_s0 264 _asm_extable 47b, .L_fixup_handle_s4 265 _asm_extable 48b, .L_fixup_handle_s0 266 _asm_extable 49b, .L_fixup_handle_s0 267 _asm_extable 50b, .L_fixup_handle_s0 268 _asm_extable 51b, .L_fixup_handle_s4 269 _asm_extable 52b, .L_fixup_handle_s0 270 _asm_extable 53b, .L_fixup_handle_s0 271 _asm_extable 54b, .L_fixup_handle_s0 272 _asm_extable 55b, .L_fixup_handle_s4 273 _asm_extable 56b, .L_fixup_handle_s0 274 _asm_extable 57b, .L_fixup_handle_s0 275SYM_FUNC_END(__copy_user_fast) 276