1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 6#include <linux/export.h> 7#include <asm/alternative-asm.h> 8#include <asm/asm.h> 9#include <asm/asmmacro.h> 10#include <asm/asm-extable.h> 11#include <asm/cpu.h> 12#include <asm/regdef.h> 13 14SYM_FUNC_START(__clear_user) 15 /* 16 * Some CPUs support hardware unaligned access 17 */ 18 ALTERNATIVE "b __clear_user_generic", \ 19 "b __clear_user_fast", CPU_FEATURE_UAL 20SYM_FUNC_END(__clear_user) 21 22EXPORT_SYMBOL(__clear_user) 23 24/* 25 * unsigned long __clear_user_generic(void *addr, size_t size) 26 * 27 * a0: addr 28 * a1: size 29 */ 30SYM_FUNC_START(__clear_user_generic) 31 beqz a1, 2f 32 331: st.b zero, a0, 0 34 addi.d a0, a0, 1 35 addi.d a1, a1, -1 36 bgtz a1, 1b 37 382: move a0, a1 39 jr ra 40 41 _asm_extable 1b, 2b 42SYM_FUNC_END(__clear_user_generic) 43 44/* 45 * unsigned long __clear_user_fast(void *addr, unsigned long size) 46 * 47 * a0: addr 48 * a1: size 49 */ 50SYM_FUNC_START(__clear_user_fast) 51 sltui t0, a1, 9 52 bnez t0, .Lsmall 53 54 add.d a2, a0, a1 550: st.d zero, a0, 0 56 57 /* align up address */ 58 addi.d a0, a0, 8 59 bstrins.d a0, zero, 2, 0 60 61 addi.d a3, a2, -64 62 bgeu a0, a3, .Llt64 63 64 /* set 64 bytes at a time */ 65.Lloop64: 661: st.d zero, a0, 0 672: st.d zero, a0, 8 683: st.d zero, a0, 16 694: st.d zero, a0, 24 705: st.d zero, a0, 32 716: st.d zero, a0, 40 727: st.d zero, a0, 48 738: st.d zero, a0, 56 74 addi.d a0, a0, 64 75 bltu a0, a3, .Lloop64 76 77 /* set the remaining bytes */ 78.Llt64: 79 addi.d a3, a2, -32 80 bgeu a0, a3, .Llt32 819: st.d zero, a0, 0 8210: st.d zero, a0, 8 8311: st.d zero, a0, 16 8412: st.d zero, a0, 24 85 addi.d a0, a0, 32 86 87.Llt32: 88 addi.d a3, a2, -16 89 bgeu a0, a3, .Llt16 9013: st.d zero, a0, 0 9114: st.d zero, a0, 8 92 addi.d a0, a0, 16 93 94.Llt16: 95 addi.d a3, a2, -8 96 bgeu a0, a3, .Llt8 9715: st.d zero, a0, 0 98 addi.d a0, a0, 8 99 100.Llt8: 10116: st.d zero, a2, -8 102 103 /* return */ 104 move a0, zero 105 jr ra 106 107 .align 4 108.Lsmall: 109 pcaddi t0, 4 110 slli.d a2, a1, 4 111 add.d t0, t0, a2 112 jr t0 113 114 .align 4 115 move a0, zero 116 jr ra 117 118 .align 4 11917: st.b zero, a0, 0 120 move a0, zero 121 jr ra 122 123 .align 4 12418: st.h zero, a0, 0 125 move a0, zero 126 jr ra 127 128 .align 4 12919: st.h zero, a0, 0 13020: st.b zero, a0, 2 131 move a0, zero 132 jr ra 133 134 .align 4 13521: st.w zero, a0, 0 136 move a0, zero 137 jr ra 138 139 .align 4 14022: st.w zero, a0, 0 14123: st.b zero, a0, 4 142 move a0, zero 143 jr ra 144 145 .align 4 14624: st.w zero, a0, 0 14725: st.h zero, a0, 4 148 move a0, zero 149 jr ra 150 151 .align 4 15226: st.w zero, a0, 0 15327: st.w zero, a0, 3 154 move a0, zero 155 jr ra 156 157 .align 4 15828: st.d zero, a0, 0 159 move a0, zero 160 jr ra 161 162 /* fixup and ex_table */ 163.Llarge_fixup: 164 sub.d a1, a2, a0 165 166.Lsmall_fixup: 16729: st.b zero, a0, 0 168 addi.d a0, a0, 1 169 addi.d a1, a1, -1 170 bgt a1, zero, 29b 171 172.Lexit: 173 move a0, a1 174 jr ra 175 176 _asm_extable 0b, .Lsmall_fixup 177 _asm_extable 1b, .Llarge_fixup 178 _asm_extable 2b, .Llarge_fixup 179 _asm_extable 3b, .Llarge_fixup 180 _asm_extable 4b, .Llarge_fixup 181 _asm_extable 5b, .Llarge_fixup 182 _asm_extable 6b, .Llarge_fixup 183 _asm_extable 7b, .Llarge_fixup 184 _asm_extable 8b, .Llarge_fixup 185 _asm_extable 9b, .Llarge_fixup 186 _asm_extable 10b, .Llarge_fixup 187 _asm_extable 11b, .Llarge_fixup 188 _asm_extable 12b, .Llarge_fixup 189 _asm_extable 13b, .Llarge_fixup 190 _asm_extable 14b, .Llarge_fixup 191 _asm_extable 15b, .Llarge_fixup 192 _asm_extable 16b, .Llarge_fixup 193 _asm_extable 17b, .Lexit 194 _asm_extable 18b, .Lsmall_fixup 195 _asm_extable 19b, .Lsmall_fixup 196 _asm_extable 20b, .Lsmall_fixup 197 _asm_extable 21b, .Lsmall_fixup 198 _asm_extable 22b, .Lsmall_fixup 199 _asm_extable 23b, .Lsmall_fixup 200 _asm_extable 24b, .Lsmall_fixup 201 _asm_extable 25b, .Lsmall_fixup 202 _asm_extable 26b, .Lsmall_fixup 203 _asm_extable 27b, .Lsmall_fixup 204 _asm_extable 28b, .Lsmall_fixup 205 _asm_extable 29b, .Lexit 206SYM_FUNC_END(__clear_user_fast) 207