1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * arch/arm64/lib/xor-neon.c 4 * 5 * Authors: Jackie Liu <liuyun01@kylinos.cn> 6 * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. 7 */ 8 9 #include <linux/raid/xor.h> 10 #include <linux/module.h> 11 #include <asm/neon-intrinsics.h> 12 13 static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1, 14 const unsigned long * __restrict p2) 15 { 16 uint64_t *dp1 = (uint64_t *)p1; 17 uint64_t *dp2 = (uint64_t *)p2; 18 19 register uint64x2_t v0, v1, v2, v3; 20 long lines = bytes / (sizeof(uint64x2_t) * 4); 21 22 do { 23 /* p1 ^= p2 */ 24 v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); 25 v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); 26 v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); 27 v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); 28 29 /* store */ 30 vst1q_u64(dp1 + 0, v0); 31 vst1q_u64(dp1 + 2, v1); 32 vst1q_u64(dp1 + 4, v2); 33 vst1q_u64(dp1 + 6, v3); 34 35 dp1 += 8; 36 dp2 += 8; 37 } while (--lines > 0); 38 } 39 40 static void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1, 41 const unsigned long * __restrict p2, 42 const unsigned long * __restrict p3) 43 { 44 uint64_t *dp1 = (uint64_t *)p1; 45 uint64_t *dp2 = (uint64_t *)p2; 46 uint64_t *dp3 = (uint64_t *)p3; 47 48 register uint64x2_t v0, v1, v2, v3; 49 long lines = bytes / (sizeof(uint64x2_t) * 4); 50 51 do { 52 /* p1 ^= p2 */ 53 v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); 54 v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); 55 v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); 56 v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); 57 58 /* p1 ^= p3 */ 59 v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); 60 v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); 61 v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); 62 v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); 63 64 /* store */ 65 vst1q_u64(dp1 + 0, v0); 66 vst1q_u64(dp1 + 2, v1); 67 vst1q_u64(dp1 + 4, v2); 68 vst1q_u64(dp1 + 6, v3); 69 70 dp1 += 8; 71 dp2 += 8; 72 dp3 += 8; 73 } while (--lines > 0); 74 } 75 76 static void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1, 77 const unsigned long * __restrict p2, 78 const unsigned long * __restrict p3, 79 const unsigned long * __restrict p4) 80 { 81 uint64_t *dp1 = (uint64_t *)p1; 82 uint64_t *dp2 = (uint64_t *)p2; 83 uint64_t *dp3 = (uint64_t *)p3; 84 uint64_t *dp4 = (uint64_t *)p4; 85 86 register uint64x2_t v0, v1, v2, v3; 87 long lines = bytes / (sizeof(uint64x2_t) * 4); 88 89 do { 90 /* p1 ^= p2 */ 91 v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); 92 v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); 93 v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); 94 v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); 95 96 /* p1 ^= p3 */ 97 v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); 98 v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); 99 v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); 100 v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); 101 102 /* p1 ^= p4 */ 103 v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); 104 v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); 105 v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); 106 v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); 107 108 /* store */ 109 vst1q_u64(dp1 + 0, v0); 110 vst1q_u64(dp1 + 2, v1); 111 vst1q_u64(dp1 + 4, v2); 112 vst1q_u64(dp1 + 6, v3); 113 114 dp1 += 8; 115 dp2 += 8; 116 dp3 += 8; 117 dp4 += 8; 118 } while (--lines > 0); 119 } 120 121 static void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1, 122 const unsigned long * __restrict p2, 123 const unsigned long * __restrict p3, 124 const unsigned long * __restrict p4, 125 const unsigned long * __restrict p5) 126 { 127 uint64_t *dp1 = (uint64_t *)p1; 128 uint64_t *dp2 = (uint64_t *)p2; 129 uint64_t *dp3 = (uint64_t *)p3; 130 uint64_t *dp4 = (uint64_t *)p4; 131 uint64_t *dp5 = (uint64_t *)p5; 132 133 register uint64x2_t v0, v1, v2, v3; 134 long lines = bytes / (sizeof(uint64x2_t) * 4); 135 136 do { 137 /* p1 ^= p2 */ 138 v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0)); 139 v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2)); 140 v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4)); 141 v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6)); 142 143 /* p1 ^= p3 */ 144 v0 = veorq_u64(v0, vld1q_u64(dp3 + 0)); 145 v1 = veorq_u64(v1, vld1q_u64(dp3 + 2)); 146 v2 = veorq_u64(v2, vld1q_u64(dp3 + 4)); 147 v3 = veorq_u64(v3, vld1q_u64(dp3 + 6)); 148 149 /* p1 ^= p4 */ 150 v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); 151 v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); 152 v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); 153 v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); 154 155 /* p1 ^= p5 */ 156 v0 = veorq_u64(v0, vld1q_u64(dp5 + 0)); 157 v1 = veorq_u64(v1, vld1q_u64(dp5 + 2)); 158 v2 = veorq_u64(v2, vld1q_u64(dp5 + 4)); 159 v3 = veorq_u64(v3, vld1q_u64(dp5 + 6)); 160 161 /* store */ 162 vst1q_u64(dp1 + 0, v0); 163 vst1q_u64(dp1 + 2, v1); 164 vst1q_u64(dp1 + 4, v2); 165 vst1q_u64(dp1 + 6, v3); 166 167 dp1 += 8; 168 dp2 += 8; 169 dp3 += 8; 170 dp4 += 8; 171 dp5 += 8; 172 } while (--lines > 0); 173 } 174 175 struct xor_block_template xor_block_inner_neon __ro_after_init = { 176 .name = "__inner_neon__", 177 .do_2 = xor_arm64_neon_2, 178 .do_3 = xor_arm64_neon_3, 179 .do_4 = xor_arm64_neon_4, 180 .do_5 = xor_arm64_neon_5, 181 }; 182 EXPORT_SYMBOL(xor_block_inner_neon); 183 184 static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) 185 { 186 uint64x2_t res; 187 188 asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" 189 "eor3 %0.16b, %1.16b, %2.16b, %3.16b" 190 : "=w"(res) : "w"(p), "w"(q), "w"(r)); 191 return res; 192 } 193 194 static void xor_arm64_eor3_3(unsigned long bytes, 195 unsigned long * __restrict p1, 196 const unsigned long * __restrict p2, 197 const unsigned long * __restrict p3) 198 { 199 uint64_t *dp1 = (uint64_t *)p1; 200 uint64_t *dp2 = (uint64_t *)p2; 201 uint64_t *dp3 = (uint64_t *)p3; 202 203 register uint64x2_t v0, v1, v2, v3; 204 long lines = bytes / (sizeof(uint64x2_t) * 4); 205 206 do { 207 /* p1 ^= p2 ^ p3 */ 208 v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), 209 vld1q_u64(dp3 + 0)); 210 v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), 211 vld1q_u64(dp3 + 2)); 212 v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), 213 vld1q_u64(dp3 + 4)); 214 v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), 215 vld1q_u64(dp3 + 6)); 216 217 /* store */ 218 vst1q_u64(dp1 + 0, v0); 219 vst1q_u64(dp1 + 2, v1); 220 vst1q_u64(dp1 + 4, v2); 221 vst1q_u64(dp1 + 6, v3); 222 223 dp1 += 8; 224 dp2 += 8; 225 dp3 += 8; 226 } while (--lines > 0); 227 } 228 229 static void xor_arm64_eor3_4(unsigned long bytes, 230 unsigned long * __restrict p1, 231 const unsigned long * __restrict p2, 232 const unsigned long * __restrict p3, 233 const unsigned long * __restrict p4) 234 { 235 uint64_t *dp1 = (uint64_t *)p1; 236 uint64_t *dp2 = (uint64_t *)p2; 237 uint64_t *dp3 = (uint64_t *)p3; 238 uint64_t *dp4 = (uint64_t *)p4; 239 240 register uint64x2_t v0, v1, v2, v3; 241 long lines = bytes / (sizeof(uint64x2_t) * 4); 242 243 do { 244 /* p1 ^= p2 ^ p3 */ 245 v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), 246 vld1q_u64(dp3 + 0)); 247 v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), 248 vld1q_u64(dp3 + 2)); 249 v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), 250 vld1q_u64(dp3 + 4)); 251 v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), 252 vld1q_u64(dp3 + 6)); 253 254 /* p1 ^= p4 */ 255 v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); 256 v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); 257 v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); 258 v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); 259 260 /* store */ 261 vst1q_u64(dp1 + 0, v0); 262 vst1q_u64(dp1 + 2, v1); 263 vst1q_u64(dp1 + 4, v2); 264 vst1q_u64(dp1 + 6, v3); 265 266 dp1 += 8; 267 dp2 += 8; 268 dp3 += 8; 269 dp4 += 8; 270 } while (--lines > 0); 271 } 272 273 static void xor_arm64_eor3_5(unsigned long bytes, 274 unsigned long * __restrict p1, 275 const unsigned long * __restrict p2, 276 const unsigned long * __restrict p3, 277 const unsigned long * __restrict p4, 278 const unsigned long * __restrict p5) 279 { 280 uint64_t *dp1 = (uint64_t *)p1; 281 uint64_t *dp2 = (uint64_t *)p2; 282 uint64_t *dp3 = (uint64_t *)p3; 283 uint64_t *dp4 = (uint64_t *)p4; 284 uint64_t *dp5 = (uint64_t *)p5; 285 286 register uint64x2_t v0, v1, v2, v3; 287 long lines = bytes / (sizeof(uint64x2_t) * 4); 288 289 do { 290 /* p1 ^= p2 ^ p3 */ 291 v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), 292 vld1q_u64(dp3 + 0)); 293 v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), 294 vld1q_u64(dp3 + 2)); 295 v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), 296 vld1q_u64(dp3 + 4)); 297 v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), 298 vld1q_u64(dp3 + 6)); 299 300 /* p1 ^= p4 ^ p5 */ 301 v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0)); 302 v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2)); 303 v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4)); 304 v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6)); 305 306 /* store */ 307 vst1q_u64(dp1 + 0, v0); 308 vst1q_u64(dp1 + 2, v1); 309 vst1q_u64(dp1 + 4, v2); 310 vst1q_u64(dp1 + 6, v3); 311 312 dp1 += 8; 313 dp2 += 8; 314 dp3 += 8; 315 dp4 += 8; 316 dp5 += 8; 317 } while (--lines > 0); 318 } 319 320 static int __init xor_neon_init(void) 321 { 322 if (IS_ENABLED(CONFIG_AS_HAS_SHA3) && cpu_have_named_feature(SHA3)) { 323 xor_block_inner_neon.do_3 = xor_arm64_eor3_3; 324 xor_block_inner_neon.do_4 = xor_arm64_eor3_4; 325 xor_block_inner_neon.do_5 = xor_arm64_eor3_5; 326 } 327 return 0; 328 } 329 module_init(xor_neon_init); 330 331 static void __exit xor_neon_exit(void) 332 { 333 } 334 module_exit(xor_neon_exit); 335 336 MODULE_AUTHOR("Jackie Liu <liuyun01@kylinos.cn>"); 337 MODULE_DESCRIPTION("ARMv8 XOR Extensions"); 338 MODULE_LICENSE("GPL"); 339