1 /*
2 * crypto_helper.c - emulate v8 Crypto Extensions instructions
3 *
4 * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 */
11
12 #include "qemu/osdep.h"
13
14 #include "cpu.h"
15 #include "exec/helper-proto.h"
16 #include "tcg/tcg-gvec-desc.h"
17 #include "crypto/aes-round.h"
18 #include "crypto/sm4.h"
19 #include "vec_internal.h"
20
21 union CRYPTO_STATE {
22 uint8_t bytes[16];
23 uint32_t words[4];
24 uint64_t l[2];
25 };
26
27 #if HOST_BIG_ENDIAN
28 #define CR_ST_BYTE(state, i) ((state).bytes[(15 - (i)) ^ 8])
29 #define CR_ST_WORD(state, i) ((state).words[(3 - (i)) ^ 2])
30 #else
31 #define CR_ST_BYTE(state, i) ((state).bytes[i])
32 #define CR_ST_WORD(state, i) ((state).words[i])
33 #endif
34
35 /*
36 * The caller has not been converted to full gvec, and so only
37 * modifies the low 16 bytes of the vector register.
38 */
clear_tail_16(void * vd,uint32_t desc)39 static void clear_tail_16(void *vd, uint32_t desc)
40 {
41 int opr_sz = simd_oprsz(desc);
42 int max_sz = simd_maxsz(desc);
43
44 assert(opr_sz == 16);
45 clear_tail(vd, opr_sz, max_sz);
46 }
47
48 static const AESState aes_zero = { };
49
HELPER(crypto_aese)50 void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc)
51 {
52 intptr_t i, opr_sz = simd_oprsz(desc);
53
54 for (i = 0; i < opr_sz; i += 16) {
55 AESState *ad = (AESState *)(vd + i);
56 AESState *st = (AESState *)(vn + i);
57 AESState *rk = (AESState *)(vm + i);
58 AESState t;
59
60 /*
61 * Our uint64_t are in the wrong order for big-endian.
62 * The Arm AddRoundKey comes first, while the API AddRoundKey
63 * comes last: perform the xor here, and provide zero to API.
64 */
65 if (HOST_BIG_ENDIAN) {
66 t.d[0] = st->d[1] ^ rk->d[1];
67 t.d[1] = st->d[0] ^ rk->d[0];
68 aesenc_SB_SR_AK(&t, &t, &aes_zero, false);
69 ad->d[0] = t.d[1];
70 ad->d[1] = t.d[0];
71 } else {
72 t.v = st->v ^ rk->v;
73 aesenc_SB_SR_AK(ad, &t, &aes_zero, false);
74 }
75 }
76 clear_tail(vd, opr_sz, simd_maxsz(desc));
77 }
78
HELPER(crypto_aesd)79 void HELPER(crypto_aesd)(void *vd, void *vn, void *vm, uint32_t desc)
80 {
81 intptr_t i, opr_sz = simd_oprsz(desc);
82
83 for (i = 0; i < opr_sz; i += 16) {
84 AESState *ad = (AESState *)(vd + i);
85 AESState *st = (AESState *)(vn + i);
86 AESState *rk = (AESState *)(vm + i);
87 AESState t;
88
89 /* Our uint64_t are in the wrong order for big-endian. */
90 if (HOST_BIG_ENDIAN) {
91 t.d[0] = st->d[1] ^ rk->d[1];
92 t.d[1] = st->d[0] ^ rk->d[0];
93 aesdec_ISB_ISR_AK(&t, &t, &aes_zero, false);
94 ad->d[0] = t.d[1];
95 ad->d[1] = t.d[0];
96 } else {
97 t.v = st->v ^ rk->v;
98 aesdec_ISB_ISR_AK(ad, &t, &aes_zero, false);
99 }
100 }
101 clear_tail(vd, opr_sz, simd_maxsz(desc));
102 }
103
HELPER(crypto_aesmc)104 void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc)
105 {
106 intptr_t i, opr_sz = simd_oprsz(desc);
107
108 for (i = 0; i < opr_sz; i += 16) {
109 AESState *ad = (AESState *)(vd + i);
110 AESState *st = (AESState *)(vm + i);
111 AESState t;
112
113 /* Our uint64_t are in the wrong order for big-endian. */
114 if (HOST_BIG_ENDIAN) {
115 t.d[0] = st->d[1];
116 t.d[1] = st->d[0];
117 aesenc_MC(&t, &t, false);
118 ad->d[0] = t.d[1];
119 ad->d[1] = t.d[0];
120 } else {
121 aesenc_MC(ad, st, false);
122 }
123 }
124 clear_tail(vd, opr_sz, simd_maxsz(desc));
125 }
126
HELPER(crypto_aesimc)127 void HELPER(crypto_aesimc)(void *vd, void *vm, uint32_t desc)
128 {
129 intptr_t i, opr_sz = simd_oprsz(desc);
130
131 for (i = 0; i < opr_sz; i += 16) {
132 AESState *ad = (AESState *)(vd + i);
133 AESState *st = (AESState *)(vm + i);
134 AESState t;
135
136 /* Our uint64_t are in the wrong order for big-endian. */
137 if (HOST_BIG_ENDIAN) {
138 t.d[0] = st->d[1];
139 t.d[1] = st->d[0];
140 aesdec_IMC(&t, &t, false);
141 ad->d[0] = t.d[1];
142 ad->d[1] = t.d[0];
143 } else {
144 aesdec_IMC(ad, st, false);
145 }
146 }
147 clear_tail(vd, opr_sz, simd_maxsz(desc));
148 }
149
150 /*
151 * SHA-1 logical functions
152 */
153
cho(uint32_t x,uint32_t y,uint32_t z)154 static uint32_t cho(uint32_t x, uint32_t y, uint32_t z)
155 {
156 return (x & (y ^ z)) ^ z;
157 }
158
par(uint32_t x,uint32_t y,uint32_t z)159 static uint32_t par(uint32_t x, uint32_t y, uint32_t z)
160 {
161 return x ^ y ^ z;
162 }
163
maj(uint32_t x,uint32_t y,uint32_t z)164 static uint32_t maj(uint32_t x, uint32_t y, uint32_t z)
165 {
166 return (x & y) | ((x | y) & z);
167 }
168
HELPER(crypto_sha1su0)169 void HELPER(crypto_sha1su0)(void *vd, void *vn, void *vm, uint32_t desc)
170 {
171 uint64_t *d = vd, *n = vn, *m = vm;
172 uint64_t d0, d1;
173
174 d0 = d[1] ^ d[0] ^ m[0];
175 d1 = n[0] ^ d[1] ^ m[1];
176 d[0] = d0;
177 d[1] = d1;
178
179 clear_tail_16(vd, desc);
180 }
181
crypto_sha1_3reg(uint64_t * rd,uint64_t * rn,uint64_t * rm,uint32_t desc,uint32_t (* fn)(union CRYPTO_STATE * d))182 static inline void crypto_sha1_3reg(uint64_t *rd, uint64_t *rn,
183 uint64_t *rm, uint32_t desc,
184 uint32_t (*fn)(union CRYPTO_STATE *d))
185 {
186 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
187 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
188 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
189 int i;
190
191 for (i = 0; i < 4; i++) {
192 uint32_t t = fn(&d);
193
194 t += rol32(CR_ST_WORD(d, 0), 5) + CR_ST_WORD(n, 0)
195 + CR_ST_WORD(m, i);
196
197 CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3);
198 CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
199 CR_ST_WORD(d, 2) = ror32(CR_ST_WORD(d, 1), 2);
200 CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
201 CR_ST_WORD(d, 0) = t;
202 }
203 rd[0] = d.l[0];
204 rd[1] = d.l[1];
205
206 clear_tail_16(rd, desc);
207 }
208
do_sha1c(union CRYPTO_STATE * d)209 static uint32_t do_sha1c(union CRYPTO_STATE *d)
210 {
211 return cho(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
212 }
213
HELPER(crypto_sha1c)214 void HELPER(crypto_sha1c)(void *vd, void *vn, void *vm, uint32_t desc)
215 {
216 crypto_sha1_3reg(vd, vn, vm, desc, do_sha1c);
217 }
218
do_sha1p(union CRYPTO_STATE * d)219 static uint32_t do_sha1p(union CRYPTO_STATE *d)
220 {
221 return par(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
222 }
223
HELPER(crypto_sha1p)224 void HELPER(crypto_sha1p)(void *vd, void *vn, void *vm, uint32_t desc)
225 {
226 crypto_sha1_3reg(vd, vn, vm, desc, do_sha1p);
227 }
228
do_sha1m(union CRYPTO_STATE * d)229 static uint32_t do_sha1m(union CRYPTO_STATE *d)
230 {
231 return maj(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
232 }
233
HELPER(crypto_sha1m)234 void HELPER(crypto_sha1m)(void *vd, void *vn, void *vm, uint32_t desc)
235 {
236 crypto_sha1_3reg(vd, vn, vm, desc, do_sha1m);
237 }
238
HELPER(crypto_sha1h)239 void HELPER(crypto_sha1h)(void *vd, void *vm, uint32_t desc)
240 {
241 uint64_t *rd = vd;
242 uint64_t *rm = vm;
243 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
244
245 CR_ST_WORD(m, 0) = ror32(CR_ST_WORD(m, 0), 2);
246 CR_ST_WORD(m, 1) = CR_ST_WORD(m, 2) = CR_ST_WORD(m, 3) = 0;
247
248 rd[0] = m.l[0];
249 rd[1] = m.l[1];
250
251 clear_tail_16(vd, desc);
252 }
253
HELPER(crypto_sha1su1)254 void HELPER(crypto_sha1su1)(void *vd, void *vm, uint32_t desc)
255 {
256 uint64_t *rd = vd;
257 uint64_t *rm = vm;
258 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
259 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
260
261 CR_ST_WORD(d, 0) = rol32(CR_ST_WORD(d, 0) ^ CR_ST_WORD(m, 1), 1);
262 CR_ST_WORD(d, 1) = rol32(CR_ST_WORD(d, 1) ^ CR_ST_WORD(m, 2), 1);
263 CR_ST_WORD(d, 2) = rol32(CR_ST_WORD(d, 2) ^ CR_ST_WORD(m, 3), 1);
264 CR_ST_WORD(d, 3) = rol32(CR_ST_WORD(d, 3) ^ CR_ST_WORD(d, 0), 1);
265
266 rd[0] = d.l[0];
267 rd[1] = d.l[1];
268
269 clear_tail_16(vd, desc);
270 }
271
272 /*
273 * The SHA-256 logical functions, according to
274 * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
275 */
276
S0(uint32_t x)277 static uint32_t S0(uint32_t x)
278 {
279 return ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22);
280 }
281
S1(uint32_t x)282 static uint32_t S1(uint32_t x)
283 {
284 return ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25);
285 }
286
s0(uint32_t x)287 static uint32_t s0(uint32_t x)
288 {
289 return ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3);
290 }
291
s1(uint32_t x)292 static uint32_t s1(uint32_t x)
293 {
294 return ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10);
295 }
296
HELPER(crypto_sha256h)297 void HELPER(crypto_sha256h)(void *vd, void *vn, void *vm, uint32_t desc)
298 {
299 uint64_t *rd = vd;
300 uint64_t *rn = vn;
301 uint64_t *rm = vm;
302 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
303 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
304 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
305 int i;
306
307 for (i = 0; i < 4; i++) {
308 uint32_t t = cho(CR_ST_WORD(n, 0), CR_ST_WORD(n, 1), CR_ST_WORD(n, 2))
309 + CR_ST_WORD(n, 3) + S1(CR_ST_WORD(n, 0))
310 + CR_ST_WORD(m, i);
311
312 CR_ST_WORD(n, 3) = CR_ST_WORD(n, 2);
313 CR_ST_WORD(n, 2) = CR_ST_WORD(n, 1);
314 CR_ST_WORD(n, 1) = CR_ST_WORD(n, 0);
315 CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3) + t;
316
317 t += maj(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
318 + S0(CR_ST_WORD(d, 0));
319
320 CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
321 CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
322 CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
323 CR_ST_WORD(d, 0) = t;
324 }
325
326 rd[0] = d.l[0];
327 rd[1] = d.l[1];
328
329 clear_tail_16(vd, desc);
330 }
331
HELPER(crypto_sha256h2)332 void HELPER(crypto_sha256h2)(void *vd, void *vn, void *vm, uint32_t desc)
333 {
334 uint64_t *rd = vd;
335 uint64_t *rn = vn;
336 uint64_t *rm = vm;
337 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
338 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
339 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
340 int i;
341
342 for (i = 0; i < 4; i++) {
343 uint32_t t = cho(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
344 + CR_ST_WORD(d, 3) + S1(CR_ST_WORD(d, 0))
345 + CR_ST_WORD(m, i);
346
347 CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
348 CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
349 CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
350 CR_ST_WORD(d, 0) = CR_ST_WORD(n, 3 - i) + t;
351 }
352
353 rd[0] = d.l[0];
354 rd[1] = d.l[1];
355
356 clear_tail_16(vd, desc);
357 }
358
HELPER(crypto_sha256su0)359 void HELPER(crypto_sha256su0)(void *vd, void *vm, uint32_t desc)
360 {
361 uint64_t *rd = vd;
362 uint64_t *rm = vm;
363 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
364 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
365
366 CR_ST_WORD(d, 0) += s0(CR_ST_WORD(d, 1));
367 CR_ST_WORD(d, 1) += s0(CR_ST_WORD(d, 2));
368 CR_ST_WORD(d, 2) += s0(CR_ST_WORD(d, 3));
369 CR_ST_WORD(d, 3) += s0(CR_ST_WORD(m, 0));
370
371 rd[0] = d.l[0];
372 rd[1] = d.l[1];
373
374 clear_tail_16(vd, desc);
375 }
376
HELPER(crypto_sha256su1)377 void HELPER(crypto_sha256su1)(void *vd, void *vn, void *vm, uint32_t desc)
378 {
379 uint64_t *rd = vd;
380 uint64_t *rn = vn;
381 uint64_t *rm = vm;
382 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
383 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
384 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
385
386 CR_ST_WORD(d, 0) += s1(CR_ST_WORD(m, 2)) + CR_ST_WORD(n, 1);
387 CR_ST_WORD(d, 1) += s1(CR_ST_WORD(m, 3)) + CR_ST_WORD(n, 2);
388 CR_ST_WORD(d, 2) += s1(CR_ST_WORD(d, 0)) + CR_ST_WORD(n, 3);
389 CR_ST_WORD(d, 3) += s1(CR_ST_WORD(d, 1)) + CR_ST_WORD(m, 0);
390
391 rd[0] = d.l[0];
392 rd[1] = d.l[1];
393
394 clear_tail_16(vd, desc);
395 }
396
397 /*
398 * The SHA-512 logical functions (same as above but using 64-bit operands)
399 */
400
cho512(uint64_t x,uint64_t y,uint64_t z)401 static uint64_t cho512(uint64_t x, uint64_t y, uint64_t z)
402 {
403 return (x & (y ^ z)) ^ z;
404 }
405
maj512(uint64_t x,uint64_t y,uint64_t z)406 static uint64_t maj512(uint64_t x, uint64_t y, uint64_t z)
407 {
408 return (x & y) | ((x | y) & z);
409 }
410
S0_512(uint64_t x)411 static uint64_t S0_512(uint64_t x)
412 {
413 return ror64(x, 28) ^ ror64(x, 34) ^ ror64(x, 39);
414 }
415
S1_512(uint64_t x)416 static uint64_t S1_512(uint64_t x)
417 {
418 return ror64(x, 14) ^ ror64(x, 18) ^ ror64(x, 41);
419 }
420
s0_512(uint64_t x)421 static uint64_t s0_512(uint64_t x)
422 {
423 return ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7);
424 }
425
s1_512(uint64_t x)426 static uint64_t s1_512(uint64_t x)
427 {
428 return ror64(x, 19) ^ ror64(x, 61) ^ (x >> 6);
429 }
430
HELPER(crypto_sha512h)431 void HELPER(crypto_sha512h)(void *vd, void *vn, void *vm, uint32_t desc)
432 {
433 uint64_t *rd = vd;
434 uint64_t *rn = vn;
435 uint64_t *rm = vm;
436 uint64_t d0 = rd[0];
437 uint64_t d1 = rd[1];
438
439 d1 += S1_512(rm[1]) + cho512(rm[1], rn[0], rn[1]);
440 d0 += S1_512(d1 + rm[0]) + cho512(d1 + rm[0], rm[1], rn[0]);
441
442 rd[0] = d0;
443 rd[1] = d1;
444
445 clear_tail_16(vd, desc);
446 }
447
HELPER(crypto_sha512h2)448 void HELPER(crypto_sha512h2)(void *vd, void *vn, void *vm, uint32_t desc)
449 {
450 uint64_t *rd = vd;
451 uint64_t *rn = vn;
452 uint64_t *rm = vm;
453 uint64_t d0 = rd[0];
454 uint64_t d1 = rd[1];
455
456 d1 += S0_512(rm[0]) + maj512(rn[0], rm[1], rm[0]);
457 d0 += S0_512(d1) + maj512(d1, rm[0], rm[1]);
458
459 rd[0] = d0;
460 rd[1] = d1;
461
462 clear_tail_16(vd, desc);
463 }
464
HELPER(crypto_sha512su0)465 void HELPER(crypto_sha512su0)(void *vd, void *vn, uint32_t desc)
466 {
467 uint64_t *rd = vd;
468 uint64_t *rn = vn;
469 uint64_t d0 = rd[0];
470 uint64_t d1 = rd[1];
471
472 d0 += s0_512(rd[1]);
473 d1 += s0_512(rn[0]);
474
475 rd[0] = d0;
476 rd[1] = d1;
477
478 clear_tail_16(vd, desc);
479 }
480
HELPER(crypto_sha512su1)481 void HELPER(crypto_sha512su1)(void *vd, void *vn, void *vm, uint32_t desc)
482 {
483 uint64_t *rd = vd;
484 uint64_t *rn = vn;
485 uint64_t *rm = vm;
486
487 rd[0] += s1_512(rn[0]) + rm[0];
488 rd[1] += s1_512(rn[1]) + rm[1];
489
490 clear_tail_16(vd, desc);
491 }
492
HELPER(crypto_sm3partw1)493 void HELPER(crypto_sm3partw1)(void *vd, void *vn, void *vm, uint32_t desc)
494 {
495 uint64_t *rd = vd;
496 uint64_t *rn = vn;
497 uint64_t *rm = vm;
498 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
499 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
500 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
501 uint32_t t;
502
503 t = CR_ST_WORD(d, 0) ^ CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 1), 17);
504 CR_ST_WORD(d, 0) = t ^ ror32(t, 17) ^ ror32(t, 9);
505
506 t = CR_ST_WORD(d, 1) ^ CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 2), 17);
507 CR_ST_WORD(d, 1) = t ^ ror32(t, 17) ^ ror32(t, 9);
508
509 t = CR_ST_WORD(d, 2) ^ CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 3), 17);
510 CR_ST_WORD(d, 2) = t ^ ror32(t, 17) ^ ror32(t, 9);
511
512 t = CR_ST_WORD(d, 3) ^ CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 0), 17);
513 CR_ST_WORD(d, 3) = t ^ ror32(t, 17) ^ ror32(t, 9);
514
515 rd[0] = d.l[0];
516 rd[1] = d.l[1];
517
518 clear_tail_16(vd, desc);
519 }
520
HELPER(crypto_sm3partw2)521 void HELPER(crypto_sm3partw2)(void *vd, void *vn, void *vm, uint32_t desc)
522 {
523 uint64_t *rd = vd;
524 uint64_t *rn = vn;
525 uint64_t *rm = vm;
526 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
527 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
528 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
529 uint32_t t = CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 0), 25);
530
531 CR_ST_WORD(d, 0) ^= t;
532 CR_ST_WORD(d, 1) ^= CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 1), 25);
533 CR_ST_WORD(d, 2) ^= CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 2), 25);
534 CR_ST_WORD(d, 3) ^= CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(m, 3), 25) ^
535 ror32(t, 17) ^ ror32(t, 2) ^ ror32(t, 26);
536
537 rd[0] = d.l[0];
538 rd[1] = d.l[1];
539
540 clear_tail_16(vd, desc);
541 }
542
543 static inline void QEMU_ALWAYS_INLINE
crypto_sm3tt(uint64_t * rd,uint64_t * rn,uint64_t * rm,uint32_t desc,uint32_t opcode)544 crypto_sm3tt(uint64_t *rd, uint64_t *rn, uint64_t *rm,
545 uint32_t desc, uint32_t opcode)
546 {
547 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
548 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
549 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
550 uint32_t imm2 = simd_data(desc);
551 uint32_t t;
552
553 assert(imm2 < 4);
554
555 if (opcode == 0 || opcode == 2) {
556 /* SM3TT1A, SM3TT2A */
557 t = par(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
558 } else if (opcode == 1) {
559 /* SM3TT1B */
560 t = maj(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
561 } else if (opcode == 3) {
562 /* SM3TT2B */
563 t = cho(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
564 } else {
565 qemu_build_not_reached();
566 }
567
568 t += CR_ST_WORD(d, 0) + CR_ST_WORD(m, imm2);
569
570 CR_ST_WORD(d, 0) = CR_ST_WORD(d, 1);
571
572 if (opcode < 2) {
573 /* SM3TT1A, SM3TT1B */
574 t += CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 3), 20);
575
576 CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 23);
577 } else {
578 /* SM3TT2A, SM3TT2B */
579 t += CR_ST_WORD(n, 3);
580 t ^= rol32(t, 9) ^ rol32(t, 17);
581
582 CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 13);
583 }
584
585 CR_ST_WORD(d, 2) = CR_ST_WORD(d, 3);
586 CR_ST_WORD(d, 3) = t;
587
588 rd[0] = d.l[0];
589 rd[1] = d.l[1];
590
591 clear_tail_16(rd, desc);
592 }
593
594 #define DO_SM3TT(NAME, OPCODE) \
595 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
596 { crypto_sm3tt(vd, vn, vm, desc, OPCODE); }
597
598 DO_SM3TT(crypto_sm3tt1a, 0)
599 DO_SM3TT(crypto_sm3tt1b, 1)
600 DO_SM3TT(crypto_sm3tt2a, 2)
601 DO_SM3TT(crypto_sm3tt2b, 3)
602
603 #undef DO_SM3TT
604
do_crypto_sm4e(uint64_t * rd,uint64_t * rn,uint64_t * rm)605 static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm)
606 {
607 union CRYPTO_STATE d = { .l = { rn[0], rn[1] } };
608 union CRYPTO_STATE n = { .l = { rm[0], rm[1] } };
609 uint32_t t, i;
610
611 for (i = 0; i < 4; i++) {
612 t = CR_ST_WORD(d, (i + 1) % 4) ^
613 CR_ST_WORD(d, (i + 2) % 4) ^
614 CR_ST_WORD(d, (i + 3) % 4) ^
615 CR_ST_WORD(n, i);
616
617 t = sm4_subword(t);
618
619 CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^
620 rol32(t, 24);
621 }
622
623 rd[0] = d.l[0];
624 rd[1] = d.l[1];
625 }
626
HELPER(crypto_sm4e)627 void HELPER(crypto_sm4e)(void *vd, void *vn, void *vm, uint32_t desc)
628 {
629 intptr_t i, opr_sz = simd_oprsz(desc);
630
631 for (i = 0; i < opr_sz; i += 16) {
632 do_crypto_sm4e(vd + i, vn + i, vm + i);
633 }
634 clear_tail(vd, opr_sz, simd_maxsz(desc));
635 }
636
do_crypto_sm4ekey(uint64_t * rd,uint64_t * rn,uint64_t * rm)637 static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm)
638 {
639 union CRYPTO_STATE d;
640 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
641 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
642 uint32_t t, i;
643
644 d = n;
645 for (i = 0; i < 4; i++) {
646 t = CR_ST_WORD(d, (i + 1) % 4) ^
647 CR_ST_WORD(d, (i + 2) % 4) ^
648 CR_ST_WORD(d, (i + 3) % 4) ^
649 CR_ST_WORD(m, i);
650
651 t = sm4_subword(t);
652
653 CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23);
654 }
655
656 rd[0] = d.l[0];
657 rd[1] = d.l[1];
658 }
659
HELPER(crypto_sm4ekey)660 void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm, uint32_t desc)
661 {
662 intptr_t i, opr_sz = simd_oprsz(desc);
663
664 for (i = 0; i < opr_sz; i += 16) {
665 do_crypto_sm4ekey(vd + i, vn + i, vm + i);
666 }
667 clear_tail(vd, opr_sz, simd_maxsz(desc));
668 }
669
HELPER(crypto_rax1)670 void HELPER(crypto_rax1)(void *vd, void *vn, void *vm, uint32_t desc)
671 {
672 intptr_t i, opr_sz = simd_oprsz(desc);
673 uint64_t *d = vd, *n = vn, *m = vm;
674
675 for (i = 0; i < opr_sz / 8; ++i) {
676 d[i] = n[i] ^ rol64(m[i], 1);
677 }
678 clear_tail(vd, opr_sz, simd_maxsz(desc));
679 }
680