powerpc/crypto/aes-gcm-p10.S

1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  # Accelerated AES-GCM stitched implementation for ppc64le.
5  # Copyright 2022- IBM Inc. All rights reserved
22  # Hash keys = v3 - v14
29  # v31 - counter 1
32  #     vs0 - vs14 for round keys
35  # This implementation uses stitched AES-GCM approach to improve overall performance.
48  # v15 - v18 - input states
49  # vs1 - vs9 - round keys
60 	vcipher	18, 18, 19
65 	vcipher	18, 18, 20
70 	vcipher	18, 18, 21
75 	vcipher	18, 18, 22
85 	vcipher	18, 18, 19
90 	vcipher	18, 18, 20
95 	vcipher	18, 18, 21
100 	vcipher	18, 18, 22
102 	xxlor	23+32, 9, 9
103 	vcipher	15, 15, 23
104 	vcipher	16, 16, 23
105 	vcipher	17, 17, 23
106 	vcipher	18, 18, 23
110  # v15 - v22 - input states
111  # vs1 - vs9 - round keys
114 	xxlor	23+32, 1, 1
119 	vcipher	15, 15, 23
120 	vcipher	16, 16, 23
121 	vcipher	17, 17, 23
122 	vcipher	18, 18, 23
123 	vcipher	19, 19, 23
124 	vcipher	20, 20, 23
125 	vcipher	21, 21, 23
126 	vcipher	22, 22, 23
131 	vcipher	18, 18, 24
140 	vcipher	18, 18, 25
149 	vcipher	18, 18, 26
155 	xxlor	23+32, 5, 5
160 	vcipher	15, 15, 23
161 	vcipher	16, 16, 23
162 	vcipher	17, 17, 23
163 	vcipher	18, 18, 23
164 	vcipher	19, 19, 23
165 	vcipher	20, 20, 23
166 	vcipher	21, 21, 23
167 	vcipher	22, 22, 23
172 	vcipher	18, 18, 24
181 	vcipher	18, 18, 25
190 	vcipher	18, 18, 26
196 	xxlor	23+32, 9, 9
197 	vcipher	15, 15, 23
198 	vcipher	16, 16, 23
199 	vcipher	17, 17, 23
200 	vcipher	18, 18, 23
201 	vcipher	19, 19, 23
202 	vcipher	20, 20, 23
203 	vcipher	21, 21, 23
204 	vcipher	22, 22, 23
238 	vpmsumd		23, 12, 15		# H4.L * X.L
241 	vpmsumd		26, 3, 18
243 	vxor		23, 23, 24
244 	vxor		23, 23, 25
245 	vxor		23, 23, 26		# L
250 	vpmsumd		27, 4, 18
257 	vpmsumd		28, 23, 2		# reduction
262 	vxor		23, 23, 26		# mL + L
264 	vsldoi		23, 23, 23, 8		# swap
265 	vxor		23, 23, 28
270 	vpmsumd		27, 5, 18
279 	vsldoi		27, 23, 23, 8		# swap
280 	vpmsumd		23, 23, 2
282 	vxor		23, 23, 27
284 	xxlor		32, 23+32, 23+32		# update hash
290  # v15 - v22 - input blocks
296 	vpmsumd		23, 12, 15		# H4.L * X.L
299 	vpmsumd		26, 3, 18
301 	vxor		23, 23, 24
302 	vxor		23, 23, 25
303 	vxor		23, 23, 26		# L
308 	vpmsumd		27, 4, 18
314 	vpmsumd		28, 23, 2		# reduction
321 	vxor		23, 23, 26		# mL + L
323 	vsldoi		23, 23, 23, 8		# swap
324 	vxor		23, 23, 28
329 	vpmsumd		27, 5, 18
338 	vsldoi		27, 23, 23, 8		# swap
339 	vpmsumd		23, 23, 2
341 	vxor		27, 23, 27		# 1st Xi
348 	vpmsumd		23, 12, 19		# H4.L * X.L
350 	vxor		23, 23, 24
351 	vxor		23, 23, 25
352 	vxor		23, 23, 26		# L
363 	vpmsumd		28, 23, 2		# reduction
370 	vxor		23, 23, 26		# mL + L
372 	vsldoi		23, 23, 23, 8		# swap
373 	vxor		23, 23, 28
387 	vsldoi		27, 23, 23, 8		# swap
388 	vpmsumd		23, 23, 2
390 	vxor		23, 23, 27
392 	xxlor		32, 23+32, 23+32		# update hash
405 	vpmsumd		23, 4, 28		# M
410 	vsldoi		25, 23, 19, 8		# mL
411 	vsldoi		26, 19, 23, 8		# mH
428 	stdu 1,-640(1)
435 	std	18,144(1)
446 	stvx	23, 9, 1
467 	stxv	18, 528(1)
480 	lxv	18, 528(1)
492 	lvx	23, 9, 1
515 	ld      18,144(1)
528 	# load Hash - h^4, h^3, h^2, h
564  #    r3 - inp
565  #    r4 - out
566  #    r5 - len
567  #    r6 - AES round keys
568  #    r7 - iv and other data
569  #    r8 - Xi, HPoli, hash keys
581 	# initialize ICB: GHASH( IV ), IV - r7
582 	lxvb16x	30+32, 0, 7	# load IV  - v30
605 	# load rounds - 10 (128), 12 (192), 14 (256)
611 	vxor	15, 30, 29	# IV + round key - add round key 0
650 	divdu	10, 12, 10	# n 128 bytes-blocks
659 	vxor	18, 30, 29
674 	li	18, 64
686 	lxvb16x		18, 17, 14	# load block
687 	lxvb16x		19, 18, 14	# load block
695 	xxlor	23+32, 10, 10
703 	vcipher	15, 15, 23
704 	vcipher	16, 16, 23
705 	vcipher	17, 17, 23
706 	vcipher	18, 18, 23
707 	vcipher	19, 19, 23
708 	vcipher	20, 20, 23
709 	vcipher	21, 21, 23
710 	vcipher	22, 22, 23
715 	vcipher	18, 18, 24
721 	xxlor	23+32, 12, 12
729 	vcipher	15, 15, 23
730 	vcipher	16, 16, 23
731 	vcipher	17, 17, 23
732 	vcipher	18, 18, 23
733 	vcipher	19, 19, 23
734 	vcipher	20, 20, 23
735 	vcipher	21, 21, 23
736 	vcipher	22, 22, 23
741 	vcipher	18, 18, 24
747 	xxlor	23+32, 14, 14
757 	vcipherlast     15, 15, 23
758 	vcipherlast     16, 16, 23
765 	vcipherlast     17, 17, 23
766 	vcipherlast     18, 18, 23
770 	xxlxor		50, 50, 18
773 	vcipherlast     19, 19, 23
774 	vcipherlast     20, 20, 23
777 	stxvb16x        51, 18, 9	# store output
781 	vcipherlast     21, 21, 23
782 	vcipherlast     22, 22, 23
803 	vxor    18, 30, 27
813 	addi    12, 12, -128
841 	xxlor	23+32, 10, 10
849 	vcipher	15, 15, 23
852 	xxlor	23+32, 12, 12
860 	vcipher	15, 15, 23
863 	xxlor	23+32, 14, 14
869 	vcipherlast     15, 15, 23
879 	addi		12, 12, -16
897 	xxlor	23+32, 10, 10
905 	vcipher	15, 15, 23
908 	xxlor	23+32, 12, 12
916 	vcipher	15, 15, 23
919 	xxlor	23+32, 14, 14
925 	vcipherlast     15, 15, 23
947 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
948 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
976 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
977 	vspltisb	17, 0		# second 16 bytes - 0x0000...00
997 	GEN_MASK 18, 15, 5
1002 	GEN_MASK 18, 15, 16
1010 	xxland	47, 47, 18
1017 	xxland	32+28, 32+17, 18
1037 	sub	17, 16, 15		# 16 - partial
1049 	addi		10, 9, -1
1054         lbzu		18, 1(16)
1055 	stbu		18, 1(10)
1071 	vxor	15, 30, 29		# IV + round key - add round key 0
1073 	std	15, 56(7)		# partial done - clear
1083  # r9 - output
1084  # r12 - remaining bytes
1085  # v15 - partial input data
1091 	addi		10, 9, -1
1122 	# initialize ICB: GHASH( IV ), IV - r7
1123 	lxvb16x	30+32, 0, 7	# load IV  - v30
1146 	# load rounds - 10 (128), 12 (192), 14 (256)
1152 	vxor	15, 30, 29	# IV + round key - add round key 0
1191 	divdu	10, 12, 10	# n 128 bytes-blocks
1200 	vxor	18, 30, 29
1215 	li	18, 64
1227 	lxvb16x		18, 17, 14	# load block
1228 	lxvb16x		19, 18, 14	# load block
1236 	xxlor	23+32, 10, 10
1244 	vcipher	15, 15, 23
1245 	vcipher	16, 16, 23
1246 	vcipher	17, 17, 23
1247 	vcipher	18, 18, 23
1248 	vcipher	19, 19, 23
1249 	vcipher	20, 20, 23
1250 	vcipher	21, 21, 23
1251 	vcipher	22, 22, 23
1256 	vcipher	18, 18, 24
1262 	xxlor	23+32, 12, 12
1270 	vcipher	15, 15, 23
1271 	vcipher	16, 16, 23
1272 	vcipher	17, 17, 23
1273 	vcipher	18, 18, 23
1274 	vcipher	19, 19, 23
1275 	vcipher	20, 20, 23
1276 	vcipher	21, 21, 23
1277 	vcipher	22, 22, 23
1282 	vcipher	18, 18, 24
1288 	xxlor	23+32, 14, 14
1298 	vcipherlast     15, 15, 23
1299 	vcipherlast     16, 16, 23
1306 	vcipherlast     17, 17, 23
1307 	vcipherlast     18, 18, 23
1311 	xxlxor		50, 50, 18
1314 	vcipherlast     19, 19, 23
1315 	vcipherlast     20, 20, 23
1318 	stxvb16x        51, 18, 9	# store output
1322 	vcipherlast     21, 21, 23
1323 	vcipherlast     22, 22, 23
1335 	xxlor           18+32, 18, 18
1353 	vxor    18, 30, 27
1363 	addi    12, 12, -128
1391 	xxlor	23+32, 10, 10
1399 	vcipher	15, 15, 23
1402 	xxlor	23+32, 12, 12
1410 	vcipher	15, 15, 23
1413 	xxlor	23+32, 14, 14
1419 	vcipherlast     15, 15, 23
1430 	addi		12, 12, -16
1448 	xxlor	23+32, 10, 10
1456 	vcipher	15, 15, 23
1459 	xxlor	23+32, 12, 12
1467 	vcipher	15, 15, 23
1470 	xxlor	23+32, 14, 14
1476 	vcipherlast     15, 15, 23
1497 	vspltisb	16, -1		# first 16 bytes - 0xffff...ff
1498 	vspltisb	17, 0		# second 16 bytes - 0x0000...00