1559671e0SHuacai Chen/* SPDX-License-Identifier: GPL-2.0 */ 2559671e0SHuacai Chen/* 3559671e0SHuacai Chen * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4559671e0SHuacai Chen */ 5559671e0SHuacai Chen 655b46ff9SMasahiro Yamada#include <linux/export.h> 7a275a82dSHuacai Chen#include <asm/alternative-asm.h> 8559671e0SHuacai Chen#include <asm/asm.h> 9559671e0SHuacai Chen#include <asm/asmmacro.h> 10508f28c6SYouling Tang#include <asm/asm-extable.h> 11a275a82dSHuacai Chen#include <asm/cpu.h> 12559671e0SHuacai Chen#include <asm/regdef.h> 13559671e0SHuacai Chen 14a275a82dSHuacai ChenSYM_FUNC_START(__clear_user) 15559671e0SHuacai Chen /* 16a275a82dSHuacai Chen * Some CPUs support hardware unaligned access 17a275a82dSHuacai Chen */ 18a275a82dSHuacai Chen ALTERNATIVE "b __clear_user_generic", \ 19a275a82dSHuacai Chen "b __clear_user_fast", CPU_FEATURE_UAL 20a275a82dSHuacai ChenSYM_FUNC_END(__clear_user) 21a275a82dSHuacai Chen 22a275a82dSHuacai ChenEXPORT_SYMBOL(__clear_user) 23a275a82dSHuacai Chen 24a275a82dSHuacai Chen/* 25a275a82dSHuacai Chen * unsigned long __clear_user_generic(void *addr, size_t size) 26559671e0SHuacai Chen * 27559671e0SHuacai Chen * a0: addr 28559671e0SHuacai Chen * a1: size 29559671e0SHuacai Chen */ 30a275a82dSHuacai ChenSYM_FUNC_START(__clear_user_generic) 31559671e0SHuacai Chen beqz a1, 2f 32559671e0SHuacai Chen 33559671e0SHuacai Chen1: st.b zero, a0, 0 34559671e0SHuacai Chen addi.d a0, a0, 1 35559671e0SHuacai Chen addi.d a1, a1, -1 361fdb9a92SWANG Xuerui bgtz a1, 1b 37559671e0SHuacai Chen 38559671e0SHuacai Chen2: move a0, a1 39559671e0SHuacai Chen jr ra 40559671e0SHuacai Chen 41*937f6593SWeihao Li _asm_extable 1b, 2b 42a275a82dSHuacai ChenSYM_FUNC_END(__clear_user_generic) 43559671e0SHuacai Chen 44a275a82dSHuacai Chen/* 45a275a82dSHuacai Chen * unsigned long __clear_user_fast(void *addr, unsigned long size) 46a275a82dSHuacai Chen * 47a275a82dSHuacai Chen * a0: addr 48a275a82dSHuacai Chen * a1: size 49a275a82dSHuacai Chen */ 50a275a82dSHuacai ChenSYM_FUNC_START(__clear_user_fast) 518941e93cSWANG Rui sltui t0, a1, 9 528941e93cSWANG Rui bnez t0, .Lsmall 53a275a82dSHuacai Chen 548941e93cSWANG Rui add.d a2, a0, a1 558941e93cSWANG Rui0: st.d zero, a0, 0 568941e93cSWANG Rui 578941e93cSWANG Rui /* align up address */ 588941e93cSWANG Rui addi.d a0, a0, 8 598941e93cSWANG Rui bstrins.d a0, zero, 2, 0 608941e93cSWANG Rui 618941e93cSWANG Rui addi.d a3, a2, -64 628941e93cSWANG Rui bgeu a0, a3, .Llt64 63a275a82dSHuacai Chen 64a275a82dSHuacai Chen /* set 64 bytes at a time */ 658941e93cSWANG Rui.Lloop64: 66a275a82dSHuacai Chen1: st.d zero, a0, 0 67a275a82dSHuacai Chen2: st.d zero, a0, 8 68a275a82dSHuacai Chen3: st.d zero, a0, 16 69a275a82dSHuacai Chen4: st.d zero, a0, 24 70a275a82dSHuacai Chen5: st.d zero, a0, 32 71a275a82dSHuacai Chen6: st.d zero, a0, 40 72a275a82dSHuacai Chen7: st.d zero, a0, 48 73a275a82dSHuacai Chen8: st.d zero, a0, 56 74a275a82dSHuacai Chen addi.d a0, a0, 64 758941e93cSWANG Rui bltu a0, a3, .Lloop64 76a275a82dSHuacai Chen 77a275a82dSHuacai Chen /* set the remaining bytes */ 788941e93cSWANG Rui.Llt64: 798941e93cSWANG Rui addi.d a3, a2, -32 808941e93cSWANG Rui bgeu a0, a3, .Llt32 818941e93cSWANG Rui9: st.d zero, a0, 0 828941e93cSWANG Rui10: st.d zero, a0, 8 838941e93cSWANG Rui11: st.d zero, a0, 16 848941e93cSWANG Rui12: st.d zero, a0, 24 858941e93cSWANG Rui addi.d a0, a0, 32 868941e93cSWANG Rui 878941e93cSWANG Rui.Llt32: 888941e93cSWANG Rui addi.d a3, a2, -16 898941e93cSWANG Rui bgeu a0, a3, .Llt16 908941e93cSWANG Rui13: st.d zero, a0, 0 918941e93cSWANG Rui14: st.d zero, a0, 8 928941e93cSWANG Rui addi.d a0, a0, 16 938941e93cSWANG Rui 948941e93cSWANG Rui.Llt16: 958941e93cSWANG Rui addi.d a3, a2, -8 968941e93cSWANG Rui bgeu a0, a3, .Llt8 978941e93cSWANG Rui15: st.d zero, a0, 0 98e66d511fSWANG Rui addi.d a0, a0, 8 998941e93cSWANG Rui 1008941e93cSWANG Rui.Llt8: 1018941e93cSWANG Rui16: st.d zero, a2, -8 102a275a82dSHuacai Chen 103a275a82dSHuacai Chen /* return */ 1048941e93cSWANG Rui move a0, zero 1058941e93cSWANG Rui jr ra 1068941e93cSWANG Rui 1078941e93cSWANG Rui .align 4 1088941e93cSWANG Rui.Lsmall: 1098941e93cSWANG Rui pcaddi t0, 4 1108941e93cSWANG Rui slli.d a2, a1, 4 1118941e93cSWANG Rui add.d t0, t0, a2 1128941e93cSWANG Rui jr t0 1138941e93cSWANG Rui 1148941e93cSWANG Rui .align 4 1158941e93cSWANG Rui move a0, zero 1168941e93cSWANG Rui jr ra 1178941e93cSWANG Rui 1188941e93cSWANG Rui .align 4 1198941e93cSWANG Rui17: st.b zero, a0, 0 1208941e93cSWANG Rui move a0, zero 1218941e93cSWANG Rui jr ra 1228941e93cSWANG Rui 1238941e93cSWANG Rui .align 4 1248941e93cSWANG Rui18: st.h zero, a0, 0 1258941e93cSWANG Rui move a0, zero 1268941e93cSWANG Rui jr ra 1278941e93cSWANG Rui 1288941e93cSWANG Rui .align 4 1298941e93cSWANG Rui19: st.h zero, a0, 0 1308941e93cSWANG Rui20: st.b zero, a0, 2 1318941e93cSWANG Rui move a0, zero 1328941e93cSWANG Rui jr ra 1338941e93cSWANG Rui 1348941e93cSWANG Rui .align 4 1358941e93cSWANG Rui21: st.w zero, a0, 0 1368941e93cSWANG Rui move a0, zero 1378941e93cSWANG Rui jr ra 1388941e93cSWANG Rui 1398941e93cSWANG Rui .align 4 1408941e93cSWANG Rui22: st.w zero, a0, 0 1418941e93cSWANG Rui23: st.b zero, a0, 4 1428941e93cSWANG Rui move a0, zero 1438941e93cSWANG Rui jr ra 1448941e93cSWANG Rui 1458941e93cSWANG Rui .align 4 1468941e93cSWANG Rui24: st.w zero, a0, 0 1478941e93cSWANG Rui25: st.h zero, a0, 4 1488941e93cSWANG Rui move a0, zero 1498941e93cSWANG Rui jr ra 1508941e93cSWANG Rui 1518941e93cSWANG Rui .align 4 1528941e93cSWANG Rui26: st.w zero, a0, 0 1538941e93cSWANG Rui27: st.w zero, a0, 3 1548941e93cSWANG Rui move a0, zero 1558941e93cSWANG Rui jr ra 1568941e93cSWANG Rui 1578941e93cSWANG Rui .align 4 1588941e93cSWANG Rui28: st.d zero, a0, 0 1598941e93cSWANG Rui move a0, zero 160a275a82dSHuacai Chen jr ra 161a275a82dSHuacai Chen 162a275a82dSHuacai Chen /* fixup and ex_table */ 163*937f6593SWeihao Li.Llarge_fixup: 164*937f6593SWeihao Li sub.d a1, a2, a0 165*937f6593SWeihao Li 166*937f6593SWeihao Li.Lsmall_fixup: 167*937f6593SWeihao Li29: st.b zero, a0, 0 168*937f6593SWeihao Li addi.d a0, a0, 1 169*937f6593SWeihao Li addi.d a1, a1, -1 170*937f6593SWeihao Li bgt a1, zero, 29b 171*937f6593SWeihao Li 172*937f6593SWeihao Li.Lexit: 173*937f6593SWeihao Li move a0, a1 174*937f6593SWeihao Li jr ra 175*937f6593SWeihao Li 176*937f6593SWeihao Li _asm_extable 0b, .Lsmall_fixup 177*937f6593SWeihao Li _asm_extable 1b, .Llarge_fixup 178*937f6593SWeihao Li _asm_extable 2b, .Llarge_fixup 179*937f6593SWeihao Li _asm_extable 3b, .Llarge_fixup 180*937f6593SWeihao Li _asm_extable 4b, .Llarge_fixup 181*937f6593SWeihao Li _asm_extable 5b, .Llarge_fixup 182*937f6593SWeihao Li _asm_extable 6b, .Llarge_fixup 183*937f6593SWeihao Li _asm_extable 7b, .Llarge_fixup 184*937f6593SWeihao Li _asm_extable 8b, .Llarge_fixup 185*937f6593SWeihao Li _asm_extable 9b, .Llarge_fixup 186*937f6593SWeihao Li _asm_extable 10b, .Llarge_fixup 187*937f6593SWeihao Li _asm_extable 11b, .Llarge_fixup 188*937f6593SWeihao Li _asm_extable 12b, .Llarge_fixup 189*937f6593SWeihao Li _asm_extable 13b, .Llarge_fixup 190*937f6593SWeihao Li _asm_extable 14b, .Llarge_fixup 191*937f6593SWeihao Li _asm_extable 15b, .Llarge_fixup 192*937f6593SWeihao Li _asm_extable 16b, .Llarge_fixup 193*937f6593SWeihao Li _asm_extable 17b, .Lexit 194*937f6593SWeihao Li _asm_extable 18b, .Lsmall_fixup 195*937f6593SWeihao Li _asm_extable 19b, .Lsmall_fixup 196*937f6593SWeihao Li _asm_extable 20b, .Lsmall_fixup 197*937f6593SWeihao Li _asm_extable 21b, .Lsmall_fixup 198*937f6593SWeihao Li _asm_extable 22b, .Lsmall_fixup 199*937f6593SWeihao Li _asm_extable 23b, .Lsmall_fixup 200*937f6593SWeihao Li _asm_extable 24b, .Lsmall_fixup 201*937f6593SWeihao Li _asm_extable 25b, .Lsmall_fixup 202*937f6593SWeihao Li _asm_extable 26b, .Lsmall_fixup 203*937f6593SWeihao Li _asm_extable 27b, .Lsmall_fixup 204*937f6593SWeihao Li _asm_extable 28b, .Lsmall_fixup 205*937f6593SWeihao Li _asm_extable 29b, .Lexit 206a275a82dSHuacai ChenSYM_FUNC_END(__clear_user_fast) 207