1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * BLAKE2b digest algorithm, NEON accelerated
4 *
5 * Copyright 2020 Google LLC
6 *
7 * Author: Eric Biggers <ebiggers@google.com>
8 */
9
10#include <linux/linkage.h>
11
12	.text
13	.fpu		neon
14
15	// The arguments to blake2b_compress_neon()
16	STATE		.req	r0
17	BLOCK		.req	r1
18	NBLOCKS		.req	r2
19	INC		.req	r3
20
21	// Pointers to the rotation tables
22	ROR24_TABLE	.req	r4
23	ROR16_TABLE	.req	r5
24
25	// The original stack pointer
26	ORIG_SP		.req	r6
27
28	// NEON registers which contain the message words of the current block.
29	// M_0-M_3 are occasionally used for other purposes too.
30	M_0		.req	d16
31	M_1		.req	d17
32	M_2		.req	d18
33	M_3		.req	d19
34	M_4		.req	d20
35	M_5		.req	d21
36	M_6		.req	d22
37	M_7		.req	d23
38	M_8		.req	d24
39	M_9		.req	d25
40	M_10		.req	d26
41	M_11		.req	d27
42	M_12		.req	d28
43	M_13		.req	d29
44	M_14		.req	d30
45	M_15		.req	d31
46
47	.align		4
48	// Tables for computing ror64(x, 24) and ror64(x, 16) using the vtbl.8
49	// instruction.  This is the most efficient way to implement these
50	// rotation amounts with NEON.  (On Cortex-A53 it's the same speed as
51	// vshr.u64 + vsli.u64, while on Cortex-A7 it's faster.)
52.Lror24_table:
53	.byte		3, 4, 5, 6, 7, 0, 1, 2
54.Lror16_table:
55	.byte		2, 3, 4, 5, 6, 7, 0, 1
56	// The BLAKE2b initialization vector
57.Lblake2b_IV:
58	.quad		0x6a09e667f3bcc908, 0xbb67ae8584caa73b
59	.quad		0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1
60	.quad		0x510e527fade682d1, 0x9b05688c2b3e6c1f
61	.quad		0x1f83d9abfb41bd6b, 0x5be0cd19137e2179
62
63// Execute one round of BLAKE2b by updating the state matrix v[0..15] in the
64// NEON registers q0-q7.  The message block is in q8..q15 (M_0-M_15).  The stack
65// pointer points to a 32-byte aligned buffer containing a copy of q8 and q9
66// (M_0-M_3), so that they can be reloaded if they are used as temporary
67// registers.  The macro arguments s0-s15 give the order in which the message
68// words are used in this round.  'final' is 1 if this is the final round.
69.macro	_blake2b_round	s0, s1, s2, s3, s4, s5, s6, s7, \
70			s8, s9, s10, s11, s12, s13, s14, s15, final=0
71
72	// Mix the columns:
73	// (v[0], v[4], v[8], v[12]), (v[1], v[5], v[9], v[13]),
74	// (v[2], v[6], v[10], v[14]), and (v[3], v[7], v[11], v[15]).
75
76	// a += b + m[blake2b_sigma[r][2*i + 0]];
77	vadd.u64	q0, q0, q2
78	vadd.u64	q1, q1, q3
79	vadd.u64	d0, d0, M_\s0
80	vadd.u64	d1, d1, M_\s2
81	vadd.u64	d2, d2, M_\s4
82	vadd.u64	d3, d3, M_\s6
83
84	// d = ror64(d ^ a, 32);
85	veor		q6, q6, q0
86	veor		q7, q7, q1
87	vrev64.32	q6, q6
88	vrev64.32	q7, q7
89
90	// c += d;
91	vadd.u64	q4, q4, q6
92	vadd.u64	q5, q5, q7
93
94	// b = ror64(b ^ c, 24);
95	vld1.8		{M_0}, [ROR24_TABLE, :64]
96	veor		q2, q2, q4
97	veor		q3, q3, q5
98	vtbl.8		d4, {d4}, M_0
99	vtbl.8		d5, {d5}, M_0
100	vtbl.8		d6, {d6}, M_0
101	vtbl.8		d7, {d7}, M_0
102
103	// a += b + m[blake2b_sigma[r][2*i + 1]];
104	//
105	// M_0 got clobbered above, so we have to reload it if any of the four
106	// message words this step needs happens to be M_0.  Otherwise we don't
107	// need to reload it here, as it will just get clobbered again below.
108.if \s1 == 0 || \s3 == 0 || \s5 == 0 || \s7 == 0
109	vld1.8		{M_0}, [sp, :64]
110.endif
111	vadd.u64	q0, q0, q2
112	vadd.u64	q1, q1, q3
113	vadd.u64	d0, d0, M_\s1
114	vadd.u64	d1, d1, M_\s3
115	vadd.u64	d2, d2, M_\s5
116	vadd.u64	d3, d3, M_\s7
117
118	// d = ror64(d ^ a, 16);
119	vld1.8		{M_0}, [ROR16_TABLE, :64]
120	veor		q6, q6, q0
121	veor		q7, q7, q1
122	vtbl.8		d12, {d12}, M_0
123	vtbl.8		d13, {d13}, M_0
124	vtbl.8		d14, {d14}, M_0
125	vtbl.8		d15, {d15}, M_0
126
127	// c += d;
128	vadd.u64	q4, q4, q6
129	vadd.u64	q5, q5, q7
130
131	// b = ror64(b ^ c, 63);
132	//
133	// This rotation amount isn't a multiple of 8, so it has to be
134	// implemented using a pair of shifts, which requires temporary
135	// registers.  Use q8-q9 (M_0-M_3) for this, and reload them afterwards.
136	veor		q8, q2, q4
137	veor		q9, q3, q5
138	vshr.u64	q2, q8, #63
139	vshr.u64	q3, q9, #63
140	vsli.u64	q2, q8, #1
141	vsli.u64	q3, q9, #1
142	vld1.8		{q8-q9}, [sp, :256]
143
144	// Mix the diagonals:
145	// (v[0], v[5], v[10], v[15]), (v[1], v[6], v[11], v[12]),
146	// (v[2], v[7], v[8], v[13]), and (v[3], v[4], v[9], v[14]).
147	//
148	// There are two possible ways to do this: use 'vext' instructions to
149	// shift the rows of the matrix so that the diagonals become columns,
150	// and undo it afterwards; or just use 64-bit operations on 'd'
151	// registers instead of 128-bit operations on 'q' registers.  We use the
152	// latter approach, as it performs much better on Cortex-A7.
153
154	// a += b + m[blake2b_sigma[r][2*i + 0]];
155	vadd.u64	d0, d0, d5
156	vadd.u64	d1, d1, d6
157	vadd.u64	d2, d2, d7
158	vadd.u64	d3, d3, d4
159	vadd.u64	d0, d0, M_\s8
160	vadd.u64	d1, d1, M_\s10
161	vadd.u64	d2, d2, M_\s12
162	vadd.u64	d3, d3, M_\s14
163
164	// d = ror64(d ^ a, 32);
165	veor		d15, d15, d0
166	veor		d12, d12, d1
167	veor		d13, d13, d2
168	veor		d14, d14, d3
169	vrev64.32	d15, d15
170	vrev64.32	d12, d12
171	vrev64.32	d13, d13
172	vrev64.32	d14, d14
173
174	// c += d;
175	vadd.u64	d10, d10, d15
176	vadd.u64	d11, d11, d12
177	vadd.u64	d8, d8, d13
178	vadd.u64	d9, d9, d14
179
180	// b = ror64(b ^ c, 24);
181	vld1.8		{M_0}, [ROR24_TABLE, :64]
182	veor		d5, d5, d10
183	veor		d6, d6, d11
184	veor		d7, d7, d8
185	veor		d4, d4, d9
186	vtbl.8		d5, {d5}, M_0
187	vtbl.8		d6, {d6}, M_0
188	vtbl.8		d7, {d7}, M_0
189	vtbl.8		d4, {d4}, M_0
190
191	// a += b + m[blake2b_sigma[r][2*i + 1]];
192.if \s9 == 0 || \s11 == 0 || \s13 == 0 || \s15 == 0
193	vld1.8		{M_0}, [sp, :64]
194.endif
195	vadd.u64	d0, d0, d5
196	vadd.u64	d1, d1, d6
197	vadd.u64	d2, d2, d7
198	vadd.u64	d3, d3, d4
199	vadd.u64	d0, d0, M_\s9
200	vadd.u64	d1, d1, M_\s11
201	vadd.u64	d2, d2, M_\s13
202	vadd.u64	d3, d3, M_\s15
203
204	// d = ror64(d ^ a, 16);
205	vld1.8		{M_0}, [ROR16_TABLE, :64]
206	veor		d15, d15, d0
207	veor		d12, d12, d1
208	veor		d13, d13, d2
209	veor		d14, d14, d3
210	vtbl.8		d12, {d12}, M_0
211	vtbl.8		d13, {d13}, M_0
212	vtbl.8		d14, {d14}, M_0
213	vtbl.8		d15, {d15}, M_0
214
215	// c += d;
216	vadd.u64	d10, d10, d15
217	vadd.u64	d11, d11, d12
218	vadd.u64	d8, d8, d13
219	vadd.u64	d9, d9, d14
220
221	// b = ror64(b ^ c, 63);
222	veor		d16, d4, d9
223	veor		d17, d5, d10
224	veor		d18, d6, d11
225	veor		d19, d7, d8
226	vshr.u64	q2, q8, #63
227	vshr.u64	q3, q9, #63
228	vsli.u64	q2, q8, #1
229	vsli.u64	q3, q9, #1
230	// Reloading q8-q9 can be skipped on the final round.
231.if ! \final
232	vld1.8		{q8-q9}, [sp, :256]
233.endif
234.endm
235
236//
237// void blake2b_compress_neon(struct blake2b_state *state,
238//			      const u8 *block, size_t nblocks, u32 inc);
239//
240// Only the first three fields of struct blake2b_state are used:
241//	u64 h[8];	(inout)
242//	u64 t[2];	(inout)
243//	u64 f[2];	(in)
244//
245	.align		5
246ENTRY(blake2b_compress_neon)
247	push		{r4-r10}
248
249	// Allocate a 32-byte stack buffer that is 32-byte aligned.
250	mov		ORIG_SP, sp
251	sub		ip, sp, #32
252	bic		ip, ip, #31
253	mov		sp, ip
254
255	adr		ROR24_TABLE, .Lror24_table
256	adr		ROR16_TABLE, .Lror16_table
257
258	mov		ip, STATE
259	vld1.64		{q0-q1}, [ip]!		// Load h[0..3]
260	vld1.64		{q2-q3}, [ip]!		// Load h[4..7]
261.Lnext_block:
262	  adr		r10, .Lblake2b_IV
263	vld1.64		{q14-q15}, [ip]		// Load t[0..1] and f[0..1]
264	vld1.64		{q4-q5}, [r10]!		// Load IV[0..3]
265	  vmov		r7, r8, d28		// Copy t[0] to (r7, r8)
266	vld1.64		{q6-q7}, [r10]		// Load IV[4..7]
267	  adds		r7, r7, INC		// Increment counter
268	bcs		.Lslow_inc_ctr
269	vmov.i32	d28[0], r7
270	vst1.64		{d28}, [ip]		// Update t[0]
271.Linc_ctr_done:
272
273	// Load the next message block and finish initializing the state matrix
274	// 'v'.  Fortunately, there are exactly enough NEON registers to fit the
275	// entire state matrix in q0-q7 and the entire message block in q8-15.
276	//
277	// However, _blake2b_round also needs some extra registers for rotates,
278	// so we have to spill some registers.  It's better to spill the message
279	// registers than the state registers, as the message doesn't change.
280	// Therefore we store a copy of the first 32 bytes of the message block
281	// (q8-q9) in an aligned buffer on the stack so that they can be
282	// reloaded when needed.  (We could just reload directly from the
283	// message buffer, but it's faster to use aligned loads.)
284	vld1.8		{q8-q9}, [BLOCK]!
285	  veor		q6, q6, q14	// v[12..13] = IV[4..5] ^ t[0..1]
286	vld1.8		{q10-q11}, [BLOCK]!
287	  veor		q7, q7, q15	// v[14..15] = IV[6..7] ^ f[0..1]
288	vld1.8		{q12-q13}, [BLOCK]!
289	vst1.8		{q8-q9}, [sp, :256]
290	  mov		ip, STATE
291	vld1.8		{q14-q15}, [BLOCK]!
292
293	// Execute the rounds.  Each round is provided the order in which it
294	// needs to use the message words.
295	_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
296	_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3
297	_blake2b_round	11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4
298	_blake2b_round	7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8
299	_blake2b_round	9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13
300	_blake2b_round	2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9
301	_blake2b_round	12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11
302	_blake2b_round	13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10
303	_blake2b_round	6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5
304	_blake2b_round	10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0
305	_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
306	_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 \
307			final=1
308
309	// Fold the final state matrix into the hash chaining value:
310	//
311	//	for (i = 0; i < 8; i++)
312	//		h[i] ^= v[i] ^ v[i + 8];
313	//
314	  vld1.64	{q8-q9}, [ip]!		// Load old h[0..3]
315	veor		q0, q0, q4		// v[0..1] ^= v[8..9]
316	veor		q1, q1, q5		// v[2..3] ^= v[10..11]
317	  vld1.64	{q10-q11}, [ip]		// Load old h[4..7]
318	veor		q2, q2, q6		// v[4..5] ^= v[12..13]
319	veor		q3, q3, q7		// v[6..7] ^= v[14..15]
320	veor		q0, q0, q8		// v[0..1] ^= h[0..1]
321	veor		q1, q1, q9		// v[2..3] ^= h[2..3]
322	  mov		ip, STATE
323	  subs		NBLOCKS, NBLOCKS, #1	// nblocks--
324	  vst1.64	{q0-q1}, [ip]!		// Store new h[0..3]
325	veor		q2, q2, q10		// v[4..5] ^= h[4..5]
326	veor		q3, q3, q11		// v[6..7] ^= h[6..7]
327	  vst1.64	{q2-q3}, [ip]!		// Store new h[4..7]
328
329	// Advance to the next block, if there is one.
330	bne		.Lnext_block		// nblocks != 0?
331
332	mov		sp, ORIG_SP
333	pop		{r4-r10}
334	mov		pc, lr
335
336.Lslow_inc_ctr:
337	// Handle the case where the counter overflowed its low 32 bits, by
338	// carrying the overflow bit into the full 128-bit counter.
339	vmov		r9, r10, d29
340	adcs		r8, r8, #0
341	adcs		r9, r9, #0
342	adc		r10, r10, #0
343	vmov		d28, r7, r8
344	vmov		d29, r9, r10
345	vst1.64		{q14}, [ip]		// Update t[0] and t[1]
346	b		.Linc_ctr_done
347ENDPROC(blake2b_compress_neon)
348