1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions. 4 * 5 * Copyright (C) 2015 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 */ 7 8#include <linux/linkage.h> 9#include <asm/assembler.h> 10 11 .arch armv8-a 12 .fpu crypto-neon-fp-armv8 13 14 SHASH .req q0 15 T1 .req q1 16 XL .req q2 17 XM .req q3 18 XH .req q4 19 IN1 .req q4 20 21 SHASH_L .req d0 22 SHASH_H .req d1 23 T1_L .req d2 24 T1_H .req d3 25 XL_L .req d4 26 XL_H .req d5 27 XM_L .req d6 28 XM_H .req d7 29 XH_L .req d8 30 31 t0l .req d10 32 t0h .req d11 33 t1l .req d12 34 t1h .req d13 35 t2l .req d14 36 t2h .req d15 37 t3l .req d16 38 t3h .req d17 39 t4l .req d18 40 t4h .req d19 41 42 t0q .req q5 43 t1q .req q6 44 t2q .req q7 45 t3q .req q8 46 t4q .req q9 47 T2 .req q9 48 49 s1l .req d20 50 s1h .req d21 51 s2l .req d22 52 s2h .req d23 53 s3l .req d24 54 s3h .req d25 55 s4l .req d26 56 s4h .req d27 57 58 MASK .req d28 59 SHASH2_p8 .req d28 60 61 k16 .req d29 62 k32 .req d30 63 k48 .req d31 64 SHASH2_p64 .req d31 65 66 HH .req q10 67 HH3 .req q11 68 HH4 .req q12 69 HH34 .req q13 70 71 HH_L .req d20 72 HH_H .req d21 73 HH3_L .req d22 74 HH3_H .req d23 75 HH4_L .req d24 76 HH4_H .req d25 77 HH34_L .req d26 78 HH34_H .req d27 79 SHASH2_H .req d29 80 81 XL2 .req q5 82 XM2 .req q6 83 XH2 .req q7 84 T3 .req q8 85 86 XL2_L .req d10 87 XL2_H .req d11 88 XM2_L .req d12 89 XM2_H .req d13 90 T3_L .req d16 91 T3_H .req d17 92 93 .text 94 95 .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4 96 vmull.p64 \rd, \rn, \rm 97 .endm 98 99 /* 100 * This implementation of 64x64 -> 128 bit polynomial multiplication 101 * using vmull.p8 instructions (8x8 -> 16) is taken from the paper 102 * "Fast Software Polynomial Multiplication on ARM Processors Using 103 * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and 104 * Ricardo Dahab (https://hal.inria.fr/hal-01506572) 105 * 106 * It has been slightly tweaked for in-order performance, and to allow 107 * 'rq' to overlap with 'ad' or 'bd'. 108 */ 109 .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l 110 vext.8 t0l, \ad, \ad, #1 @ A1 111 .ifc \b1, t4l 112 vext.8 t4l, \bd, \bd, #1 @ B1 113 .endif 114 vmull.p8 t0q, t0l, \bd @ F = A1*B 115 vext.8 t1l, \ad, \ad, #2 @ A2 116 vmull.p8 t4q, \ad, \b1 @ E = A*B1 117 .ifc \b2, t3l 118 vext.8 t3l, \bd, \bd, #2 @ B2 119 .endif 120 vmull.p8 t1q, t1l, \bd @ H = A2*B 121 vext.8 t2l, \ad, \ad, #3 @ A3 122 vmull.p8 t3q, \ad, \b2 @ G = A*B2 123 veor t0q, t0q, t4q @ L = E + F 124 .ifc \b3, t4l 125 vext.8 t4l, \bd, \bd, #3 @ B3 126 .endif 127 vmull.p8 t2q, t2l, \bd @ J = A3*B 128 veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 129 veor t1q, t1q, t3q @ M = G + H 130 .ifc \b4, t3l 131 vext.8 t3l, \bd, \bd, #4 @ B4 132 .endif 133 vmull.p8 t4q, \ad, \b3 @ I = A*B3 134 veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 135 vmull.p8 t3q, \ad, \b4 @ K = A*B4 136 vand t0h, t0h, k48 137 vand t1h, t1h, k32 138 veor t2q, t2q, t4q @ N = I + J 139 veor t0l, t0l, t0h 140 veor t1l, t1l, t1h 141 veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 142 vand t2h, t2h, k16 143 veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 144 vmov.i64 t3h, #0 145 vext.8 t0q, t0q, t0q, #15 146 veor t2l, t2l, t2h 147 vext.8 t1q, t1q, t1q, #14 148 vmull.p8 \rq, \ad, \bd @ D = A*B 149 vext.8 t2q, t2q, t2q, #13 150 vext.8 t3q, t3q, t3q, #12 151 veor t0q, t0q, t1q 152 veor t2q, t2q, t3q 153 veor \rq, \rq, t0q 154 veor \rq, \rq, t2q 155 .endm 156 157 // 158 // PMULL (64x64->128) based reduction for CPUs that can do 159 // it in a single instruction. 160 // 161 .macro __pmull_reduce_p64 162 vmull.p64 T1, XL_L, MASK 163 164 veor XH_L, XH_L, XM_H 165 vext.8 T1, T1, T1, #8 166 veor XL_H, XL_H, XM_L 167 veor T1, T1, XL 168 169 vmull.p64 XL, T1_H, MASK 170 .endm 171 172 // 173 // Alternative reduction for CPUs that lack support for the 174 // 64x64->128 PMULL instruction 175 // 176 .macro __pmull_reduce_p8 177 veor XL_H, XL_H, XM_L 178 veor XH_L, XH_L, XM_H 179 180 vshl.i64 T1, XL, #57 181 vshl.i64 T2, XL, #62 182 veor T1, T1, T2 183 vshl.i64 T2, XL, #63 184 veor T1, T1, T2 185 veor XL_H, XL_H, T1_L 186 veor XH_L, XH_L, T1_H 187 188 vshr.u64 T1, XL, #1 189 veor XH, XH, XL 190 veor XL, XL, T1 191 vshr.u64 T1, T1, #6 192 vshr.u64 XL, XL, #1 193 .endm 194 195 .macro ghash_update, pn 196 vld1.64 {XL}, [r1] 197 198 /* do the head block first, if supplied */ 199 ldr ip, [sp] 200 teq ip, #0 201 beq 0f 202 vld1.64 {T1}, [ip] 203 teq r0, #0 204 b 3f 205 2060: .ifc \pn, p64 207 tst r0, #3 // skip until #blocks is a 208 bne 2f // round multiple of 4 209 210 vld1.8 {XL2-XM2}, [r2]! 2111: vld1.8 {T3-T2}, [r2]! 212 vrev64.8 XL2, XL2 213 vrev64.8 XM2, XM2 214 215 subs r0, r0, #4 216 217 vext.8 T1, XL2, XL2, #8 218 veor XL2_H, XL2_H, XL_L 219 veor XL, XL, T1 220 221 vrev64.8 T3, T3 222 vrev64.8 T1, T2 223 224 vmull.p64 XH, HH4_H, XL_H // a1 * b1 225 veor XL2_H, XL2_H, XL_H 226 vmull.p64 XL, HH4_L, XL_L // a0 * b0 227 vmull.p64 XM, HH34_H, XL2_H // (a1 + a0)(b1 + b0) 228 229 vmull.p64 XH2, HH3_H, XM2_L // a1 * b1 230 veor XM2_L, XM2_L, XM2_H 231 vmull.p64 XL2, HH3_L, XM2_H // a0 * b0 232 vmull.p64 XM2, HH34_L, XM2_L // (a1 + a0)(b1 + b0) 233 234 veor XH, XH, XH2 235 veor XL, XL, XL2 236 veor XM, XM, XM2 237 238 vmull.p64 XH2, HH_H, T3_L // a1 * b1 239 veor T3_L, T3_L, T3_H 240 vmull.p64 XL2, HH_L, T3_H // a0 * b0 241 vmull.p64 XM2, SHASH2_H, T3_L // (a1 + a0)(b1 + b0) 242 243 veor XH, XH, XH2 244 veor XL, XL, XL2 245 veor XM, XM, XM2 246 247 vmull.p64 XH2, SHASH_H, T1_L // a1 * b1 248 veor T1_L, T1_L, T1_H 249 vmull.p64 XL2, SHASH_L, T1_H // a0 * b0 250 vmull.p64 XM2, SHASH2_p64, T1_L // (a1 + a0)(b1 + b0) 251 252 veor XH, XH, XH2 253 veor XL, XL, XL2 254 veor XM, XM, XM2 255 256 beq 4f 257 258 vld1.8 {XL2-XM2}, [r2]! 259 260 veor T1, XL, XH 261 veor XM, XM, T1 262 263 __pmull_reduce_p64 264 265 veor T1, T1, XH 266 veor XL, XL, T1 267 268 b 1b 269 .endif 270 2712: vld1.64 {T1}, [r2]! 272 subs r0, r0, #1 273 2743: /* multiply XL by SHASH in GF(2^128) */ 275#ifndef CONFIG_CPU_BIG_ENDIAN 276 vrev64.8 T1, T1 277#endif 278 vext.8 IN1, T1, T1, #8 279 veor T1_L, T1_L, XL_H 280 veor XL, XL, IN1 281 282 __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 283 veor T1, T1, XL 284 __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 285 __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0) 286 2874: veor T1, XL, XH 288 veor XM, XM, T1 289 290 __pmull_reduce_\pn 291 292 veor T1, T1, XH 293 veor XL, XL, T1 294 295 bne 0b 296 297 vst1.64 {XL}, [r1] 298 bx lr 299 .endm 300 301 /* 302 * void pmull_ghash_update(int blocks, u64 dg[], const char *src, 303 * struct ghash_key const *k, const char *head) 304 */ 305ENTRY(pmull_ghash_update_p64) 306 vld1.64 {SHASH}, [r3]! 307 vld1.64 {HH}, [r3]! 308 vld1.64 {HH3-HH4}, [r3] 309 310 veor SHASH2_p64, SHASH_L, SHASH_H 311 veor SHASH2_H, HH_L, HH_H 312 veor HH34_L, HH3_L, HH3_H 313 veor HH34_H, HH4_L, HH4_H 314 315 vmov.i8 MASK, #0xe1 316 vshl.u64 MASK, MASK, #57 317 318 ghash_update p64 319ENDPROC(pmull_ghash_update_p64) 320 321ENTRY(pmull_ghash_update_p8) 322 vld1.64 {SHASH}, [r3] 323 veor SHASH2_p8, SHASH_L, SHASH_H 324 325 vext.8 s1l, SHASH_L, SHASH_L, #1 326 vext.8 s2l, SHASH_L, SHASH_L, #2 327 vext.8 s3l, SHASH_L, SHASH_L, #3 328 vext.8 s4l, SHASH_L, SHASH_L, #4 329 vext.8 s1h, SHASH_H, SHASH_H, #1 330 vext.8 s2h, SHASH_H, SHASH_H, #2 331 vext.8 s3h, SHASH_H, SHASH_H, #3 332 vext.8 s4h, SHASH_H, SHASH_H, #4 333 334 vmov.i64 k16, #0xffff 335 vmov.i64 k32, #0xffffffff 336 vmov.i64 k48, #0xffffffffffff 337 338 ghash_update p8 339ENDPROC(pmull_ghash_update_p8) 340