1########################################################################
2# Implement fast SHA-256 with AVX2 instructions. (x86_64)
3#
4# Copyright (C) 2013 Intel Corporation.
5#
6# Authors:
7#     James Guilford <james.guilford@intel.com>
8#     Kirk Yap <kirk.s.yap@intel.com>
9#     Tim Chen <tim.c.chen@linux.intel.com>
10#
11# This software is available to you under a choice of one of two
12# licenses.  You may choose to be licensed under the terms of the GNU
13# General Public License (GPL) Version 2, available from the file
14# COPYING in the main directory of this source tree, or the
15# OpenIB.org BSD license below:
16#
17#     Redistribution and use in source and binary forms, with or
18#     without modification, are permitted provided that the following
19#     conditions are met:
20#
21#      - Redistributions of source code must retain the above
22#        copyright notice, this list of conditions and the following
23#        disclaimer.
24#
25#      - Redistributions in binary form must reproduce the above
26#        copyright notice, this list of conditions and the following
27#        disclaimer in the documentation and/or other materials
28#        provided with the distribution.
29#
30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37# SOFTWARE.
38#
39########################################################################
40#
41# This code is described in an Intel White-Paper:
42# "Fast SHA-256 Implementations on Intel Architecture Processors"
43#
44# To find it, surf to http://www.intel.com/p/en_US/embedded
45# and search for that title.
46#
47########################################################################
48# This code schedules 2 blocks at a time, with 4 lanes per block
49########################################################################
50
51#include <linux/linkage.h>
52
53## assume buffers not aligned
54#define	VMOVDQ vmovdqu
55
56################################ Define Macros
57
58# addm [mem], reg
59# Add reg to mem using reg-mem add and store
60.macro addm p1 p2
61	add	\p1, \p2
62	mov	\p2, \p1
63.endm
64
65################################
66
67X0 = %ymm4
68X1 = %ymm5
69X2 = %ymm6
70X3 = %ymm7
71
72# XMM versions of above
73XWORD0 = %xmm4
74XWORD1 = %xmm5
75XWORD2 = %xmm6
76XWORD3 = %xmm7
77
78XTMP0 = %ymm0
79XTMP1 = %ymm1
80XTMP2 = %ymm2
81XTMP3 = %ymm3
82XTMP4 = %ymm8
83XFER  = %ymm9
84XTMP5 = %ymm11
85
86SHUF_00BA =	%ymm10 # shuffle xBxA -> 00BA
87SHUF_DC00 =	%ymm12 # shuffle xDxC -> DC00
88BYTE_FLIP_MASK = %ymm13
89
90X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK
91
92NUM_BLKS = %rdx	# 3rd arg
93INP	= %rsi  # 2nd arg
94CTX	= %rdi	# 1st arg
95c	= %ecx
96d	= %r8d
97e       = %edx	# clobbers NUM_BLKS
98y3	= %esi	# clobbers INP
99
100SRND	= CTX	# SRND is same register as CTX
101
102a = %eax
103b = %ebx
104f = %r9d
105g = %r10d
106h = %r11d
107old_h = %r11d
108
109T1 = %r12d
110y0 = %r13d
111y1 = %r14d
112y2 = %r15d
113
114
115_XFER_SIZE	= 2*64*4	# 2 blocks, 64 rounds, 4 bytes/round
116_XMM_SAVE_SIZE	= 0
117_INP_END_SIZE	= 8
118_INP_SIZE	= 8
119_CTX_SIZE	= 8
120
121_XFER		= 0
122_XMM_SAVE	= _XFER     + _XFER_SIZE
123_INP_END	= _XMM_SAVE + _XMM_SAVE_SIZE
124_INP		= _INP_END  + _INP_END_SIZE
125_CTX		= _INP      + _INP_SIZE
126STACK_SIZE	= _CTX      + _CTX_SIZE
127
128# rotate_Xs
129# Rotate values of symbols X0...X3
130.macro rotate_Xs
131	X_ = X0
132	X0 = X1
133	X1 = X2
134	X2 = X3
135	X3 = X_
136.endm
137
138# ROTATE_ARGS
139# Rotate values of symbols a...h
140.macro ROTATE_ARGS
141	old_h = h
142	TMP_ = h
143	h = g
144	g = f
145	f = e
146	e = d
147	d = c
148	c = b
149	b = a
150	a = TMP_
151.endm
152
153.macro FOUR_ROUNDS_AND_SCHED disp
154################################### RND N + 0 ############################
155
156	mov	a, y3		# y3 = a                                # MAJA
157	rorx	$25, e, y0	# y0 = e >> 25				# S1A
158	rorx	$11, e, y1	# y1 = e >> 11				# S1B
159
160	addl	\disp(%rsp, SRND), h		# h = k + w + h         # --
161	or	c, y3		# y3 = a|c                              # MAJA
162	vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7]
163	mov	f, y2		# y2 = f                                # CH
164	rorx	$13, a, T1	# T1 = a >> 13				# S0B
165
166	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
167	xor	g, y2		# y2 = f^g                              # CH
168	vpaddd	X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1
169	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
170
171	and	e, y2		# y2 = (f^g)&e                          # CH
172	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
173	rorx	$22, a, y1	# y1 = a >> 22				# S0A
174	add	h, d		# d = k + w + h + d                     # --
175
176	and	b, y3		# y3 = (a|c)&b                          # MAJA
177	vpalignr $4, X0, X1, XTMP1	# XTMP1 = W[-15]
178	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
179	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
180
181	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
182	vpsrld	$7, XTMP1, XTMP2
183	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
184	mov	a, T1		# T1 = a                                # MAJB
185	and	c, T1		# T1 = a&c                              # MAJB
186
187	add	y0, y2		# y2 = S1 + CH                          # --
188	vpslld	$(32-7), XTMP1, XTMP3
189	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
190	add	y1, h		# h = k + w + h + S0                    # --
191
192	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
193	vpor	XTMP2, XTMP3, XTMP3	# XTMP3 = W[-15] ror 7
194
195	vpsrld	$18, XTMP1, XTMP2
196	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
197	add	y3, h		# h = t1 + S0 + MAJ                     # --
198
199
200	ROTATE_ARGS
201
202################################### RND N + 1 ############################
203
204	mov	a, y3		# y3 = a                                # MAJA
205	rorx	$25, e, y0	# y0 = e >> 25				# S1A
206	rorx	$11, e, y1	# y1 = e >> 11				# S1B
207	offset = \disp + 1*4
208	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
209	or	c, y3		# y3 = a|c                              # MAJA
210
211
212	vpsrld	$3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3
213	mov	f, y2		# y2 = f                                # CH
214	rorx	$13, a, T1	# T1 = a >> 13				# S0B
215	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
216	xor	g, y2		# y2 = f^g                              # CH
217
218
219	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
220	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
221	rorx	$22, a, y1	# y1 = a >> 22				# S0A
222	and	e, y2		# y2 = (f^g)&e                          # CH
223	add	h, d		# d = k + w + h + d                     # --
224
225	vpslld	$(32-18), XTMP1, XTMP1
226	and	b, y3		# y3 = (a|c)&b                          # MAJA
227	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
228
229	vpxor	XTMP1, XTMP3, XTMP3
230	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
231	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
232
233	vpxor	XTMP2, XTMP3, XTMP3	# XTMP3 = W[-15] ror 7 ^ W[-15] ror 18
234	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
235	mov	a, T1		# T1 = a                                # MAJB
236	and	c, T1		# T1 = a&c                              # MAJB
237	add	y0, y2		# y2 = S1 + CH                          # --
238
239	vpxor	XTMP4, XTMP3, XTMP1	# XTMP1 = s0
240	vpshufd	$0b11111010, X3, XTMP2	# XTMP2 = W[-2] {BBAA}
241	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
242	add	y1, h		# h = k + w + h + S0                    # --
243
244	vpaddd	XTMP1, XTMP0, XTMP0	# XTMP0 = W[-16] + W[-7] + s0
245	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
246	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
247	add	y3, h		# h = t1 + S0 + MAJ                     # --
248
249	vpsrld	$10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA}
250
251
252	ROTATE_ARGS
253
254################################### RND N + 2 ############################
255
256	mov	a, y3		# y3 = a                                # MAJA
257	rorx	$25, e, y0	# y0 = e >> 25				# S1A
258	offset = \disp + 2*4
259	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
260
261	vpsrlq	$19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA}
262	rorx	$11, e, y1	# y1 = e >> 11				# S1B
263	or	c, y3		# y3 = a|c                              # MAJA
264	mov	f, y2		# y2 = f                                # CH
265	xor	g, y2		# y2 = f^g                              # CH
266
267	rorx	$13, a, T1	# T1 = a >> 13				# S0B
268	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
269	vpsrlq	$17, XTMP2, XTMP2	# XTMP2 = W[-2] ror 17 {xBxA}
270	and	e, y2		# y2 = (f^g)&e                          # CH
271
272	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
273	vpxor	XTMP3, XTMP2, XTMP2
274	add	h, d		# d = k + w + h + d                     # --
275	and	b, y3		# y3 = (a|c)&b                          # MAJA
276
277	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
278	rorx	$22, a, y1	# y1 = a >> 22				# S0A
279	vpxor	XTMP2, XTMP4, XTMP4	# XTMP4 = s1 {xBxA}
280	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
281
282	vpshufb	SHUF_00BA, XTMP4, XTMP4	# XTMP4 = s1 {00BA}
283	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
284	rorx	$2, a ,T1	# T1 = (a >> 2)				# S0
285	vpaddd	XTMP4, XTMP0, XTMP0	# XTMP0 = {..., ..., W[1], W[0]}
286
287	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
288	mov	a, T1		# T1 = a                                # MAJB
289	and	c, T1		# T1 = a&c                              # MAJB
290	add	y0, y2		# y2 = S1 + CH                          # --
291	vpshufd	$0b01010000, XTMP0, XTMP2	# XTMP2 = W[-2] {DDCC}
292
293	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
294	add	y1,h		# h = k + w + h + S0                    # --
295	add	y2,d		# d = k + w + h + d + S1 + CH = d + t1  # --
296	add	y2,h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
297
298	add	y3,h		# h = t1 + S0 + MAJ                     # --
299
300
301	ROTATE_ARGS
302
303################################### RND N + 3 ############################
304
305	mov	a, y3		# y3 = a                                # MAJA
306	rorx	$25, e, y0	# y0 = e >> 25				# S1A
307	rorx	$11, e, y1	# y1 = e >> 11				# S1B
308	offset = \disp + 3*4
309	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
310	or	c, y3		# y3 = a|c                              # MAJA
311
312
313	vpsrld	$10, XTMP2, XTMP5	# XTMP5 = W[-2] >> 10 {DDCC}
314	mov	f, y2		# y2 = f                                # CH
315	rorx	$13, a, T1	# T1 = a >> 13				# S0B
316	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
317	xor	g, y2		# y2 = f^g                              # CH
318
319
320	vpsrlq	$19, XTMP2, XTMP3	# XTMP3 = W[-2] ror 19 {xDxC}
321	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
322	and	e, y2		# y2 = (f^g)&e                          # CH
323	add	h, d		# d = k + w + h + d                     # --
324	and	b, y3		# y3 = (a|c)&b                          # MAJA
325
326	vpsrlq	$17, XTMP2, XTMP2	# XTMP2 = W[-2] ror 17 {xDxC}
327	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
328	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
329
330	vpxor	XTMP3, XTMP2, XTMP2
331	rorx	$22, a, y1	# y1 = a >> 22				# S0A
332	add	y0, y2		# y2 = S1 + CH                          # --
333
334	vpxor	XTMP2, XTMP5, XTMP5	# XTMP5 = s1 {xDxC}
335	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
336	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
337
338	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
339	vpshufb	SHUF_DC00, XTMP5, XTMP5	# XTMP5 = s1 {DC00}
340
341	vpaddd	XTMP0, XTMP5, X0	# X0 = {W[3], W[2], W[1], W[0]}
342	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
343	mov	a, T1		# T1 = a                                # MAJB
344	and	c, T1		# T1 = a&c                              # MAJB
345	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
346
347	add	y1, h		# h = k + w + h + S0                    # --
348	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
349	add	y3, h		# h = t1 + S0 + MAJ                     # --
350
351	ROTATE_ARGS
352	rotate_Xs
353.endm
354
355.macro DO_4ROUNDS disp
356################################### RND N + 0 ###########################
357
358	mov	f, y2		# y2 = f                                # CH
359	rorx	$25, e, y0	# y0 = e >> 25				# S1A
360	rorx	$11, e, y1	# y1 = e >> 11				# S1B
361	xor	g, y2		# y2 = f^g                              # CH
362
363	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
364	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
365	and	e, y2		# y2 = (f^g)&e                          # CH
366
367	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
368	rorx	$13, a, T1	# T1 = a >> 13				# S0B
369	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
370	rorx	$22, a, y1	# y1 = a >> 22				# S0A
371	mov	a, y3		# y3 = a                                # MAJA
372
373	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
374	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
375	addl	\disp(%rsp, SRND), h		# h = k + w + h # --
376	or	c, y3		# y3 = a|c                              # MAJA
377
378	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
379	mov	a, T1		# T1 = a                                # MAJB
380	and	b, y3		# y3 = (a|c)&b                          # MAJA
381	and	c, T1		# T1 = a&c                              # MAJB
382	add	y0, y2		# y2 = S1 + CH                          # --
383
384
385	add	h, d		# d = k + w + h + d                     # --
386	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
387	add	y1, h		# h = k + w + h + S0                    # --
388	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
389
390	ROTATE_ARGS
391
392################################### RND N + 1 ###########################
393
394	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
395	mov	f, y2		# y2 = f                                # CH
396	rorx	$25, e, y0	# y0 = e >> 25				# S1A
397	rorx	$11, e, y1	# y1 = e >> 11				# S1B
398	xor	g, y2		# y2 = f^g                              # CH
399
400	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
401	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
402	and	e, y2		# y2 = (f^g)&e                          # CH
403	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
404
405	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
406	rorx	$13, a, T1	# T1 = a >> 13				# S0B
407	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
408	rorx	$22, a, y1	# y1 = a >> 22				# S0A
409	mov	a, y3		# y3 = a                                # MAJA
410
411	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
412	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
413	offset = 4*1 + \disp
414	addl	offset(%rsp, SRND), h		# h = k + w + h # --
415	or	c, y3		# y3 = a|c                              # MAJA
416
417	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
418	mov	a, T1		# T1 = a                                # MAJB
419	and	b, y3		# y3 = (a|c)&b                          # MAJA
420	and	c, T1		# T1 = a&c                              # MAJB
421	add	y0, y2		# y2 = S1 + CH                          # --
422
423
424	add	h, d		# d = k + w + h + d                     # --
425	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
426	add	y1, h		# h = k + w + h + S0                    # --
427
428	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
429
430	ROTATE_ARGS
431
432################################### RND N + 2 ##############################
433
434	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
435	mov	f, y2		# y2 = f                                # CH
436	rorx	$25, e, y0	# y0 = e >> 25				# S1A
437	rorx	$11, e, y1	# y1 = e >> 11				# S1B
438	xor	g, y2		# y2 = f^g                              # CH
439
440	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
441	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
442	and	e, y2		# y2 = (f^g)&e                          # CH
443	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
444
445	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
446	rorx	$13, a, T1	# T1 = a >> 13				# S0B
447	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
448	rorx	$22, a, y1	# y1 = a >> 22				# S0A
449	mov	a, y3		# y3 = a                                # MAJA
450
451	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
452	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
453	offset = 4*2 + \disp
454	addl	offset(%rsp, SRND), h		# h = k + w + h # --
455	or	c, y3		# y3 = a|c                              # MAJA
456
457	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
458	mov	a, T1		# T1 = a                                # MAJB
459	and	b, y3		# y3 = (a|c)&b                          # MAJA
460	and	c, T1		# T1 = a&c                              # MAJB
461	add	y0, y2		# y2 = S1 + CH                          # --
462
463
464	add	h, d		# d = k + w + h + d                     # --
465	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
466	add	y1, h		# h = k + w + h + S0                    # --
467
468	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
469
470	ROTATE_ARGS
471
472################################### RND N + 3 ###########################
473
474	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
475	mov	f, y2		# y2 = f                                # CH
476	rorx	$25, e, y0	# y0 = e >> 25				# S1A
477	rorx	$11, e, y1	# y1 = e >> 11				# S1B
478	xor	g, y2		# y2 = f^g                              # CH
479
480	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
481	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
482	and	e, y2		# y2 = (f^g)&e                          # CH
483	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
484
485	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
486	rorx	$13, a, T1	# T1 = a >> 13				# S0B
487	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
488	rorx	$22, a, y1	# y1 = a >> 22				# S0A
489	mov	a, y3		# y3 = a                                # MAJA
490
491	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
492	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
493	offset = 4*3 + \disp
494	addl	offset(%rsp, SRND), h		# h = k + w + h # --
495	or	c, y3		# y3 = a|c                              # MAJA
496
497	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
498	mov	a, T1		# T1 = a                                # MAJB
499	and	b, y3		# y3 = (a|c)&b                          # MAJA
500	and	c, T1		# T1 = a&c                              # MAJB
501	add	y0, y2		# y2 = S1 + CH                          # --
502
503
504	add	h, d		# d = k + w + h + d                     # --
505	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
506	add	y1, h		# h = k + w + h + S0                    # --
507
508	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
509
510
511	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
512
513	add	y3, h		# h = t1 + S0 + MAJ                     # --
514
515	ROTATE_ARGS
516
517.endm
518
519########################################################################
520## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks)
521## arg 1 : pointer to state
522## arg 2 : pointer to input data
523## arg 3 : Num blocks
524########################################################################
525.text
526SYM_FUNC_START(sha256_transform_rorx)
527.align 32
528	pushq	%rbx
529	pushq	%r12
530	pushq	%r13
531	pushq	%r14
532	pushq	%r15
533
534	push	%rbp
535	mov	%rsp, %rbp
536
537	subq	$STACK_SIZE, %rsp
538	and	$-32, %rsp	# align rsp to 32 byte boundary
539
540	shl	$6, NUM_BLKS	# convert to bytes
541	jz	done_hash
542	lea	-64(INP, NUM_BLKS), NUM_BLKS # pointer to last block
543	mov	NUM_BLKS, _INP_END(%rsp)
544
545	cmp	NUM_BLKS, INP
546	je	only_one_block
547
548	## load initial digest
549	mov	(CTX), a
550	mov	4*1(CTX), b
551	mov	4*2(CTX), c
552	mov	4*3(CTX), d
553	mov	4*4(CTX), e
554	mov	4*5(CTX), f
555	mov	4*6(CTX), g
556	mov	4*7(CTX), h
557
558	vmovdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
559	vmovdqa  _SHUF_00BA(%rip), SHUF_00BA
560	vmovdqa  _SHUF_DC00(%rip), SHUF_DC00
561
562	mov	CTX, _CTX(%rsp)
563
564loop0:
565	## Load first 16 dwords from two blocks
566	VMOVDQ	0*32(INP),XTMP0
567	VMOVDQ	1*32(INP),XTMP1
568	VMOVDQ	2*32(INP),XTMP2
569	VMOVDQ	3*32(INP),XTMP3
570
571	## byte swap data
572	vpshufb	BYTE_FLIP_MASK, XTMP0, XTMP0
573	vpshufb	BYTE_FLIP_MASK, XTMP1, XTMP1
574	vpshufb	BYTE_FLIP_MASK, XTMP2, XTMP2
575	vpshufb	BYTE_FLIP_MASK, XTMP3, XTMP3
576
577	## transpose data into high/low halves
578	vperm2i128	$0x20, XTMP2, XTMP0, X0
579	vperm2i128	$0x31, XTMP2, XTMP0, X1
580	vperm2i128	$0x20, XTMP3, XTMP1, X2
581	vperm2i128	$0x31, XTMP3, XTMP1, X3
582
583last_block_enter:
584	add	$64, INP
585	mov	INP, _INP(%rsp)
586
587	## schedule 48 input dwords, by doing 3 rounds of 12 each
588	xor	SRND, SRND
589
590.align 16
591loop1:
592	vpaddd	K256+0*32(SRND), X0, XFER
593	vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
594	FOUR_ROUNDS_AND_SCHED	_XFER + 0*32
595
596	vpaddd	K256+1*32(SRND), X0, XFER
597	vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
598	FOUR_ROUNDS_AND_SCHED	_XFER + 1*32
599
600	vpaddd	K256+2*32(SRND), X0, XFER
601	vmovdqa XFER, 2*32+_XFER(%rsp, SRND)
602	FOUR_ROUNDS_AND_SCHED	_XFER + 2*32
603
604	vpaddd	K256+3*32(SRND), X0, XFER
605	vmovdqa XFER, 3*32+_XFER(%rsp, SRND)
606	FOUR_ROUNDS_AND_SCHED	_XFER + 3*32
607
608	add	$4*32, SRND
609	cmp	$3*4*32, SRND
610	jb	loop1
611
612loop2:
613	## Do last 16 rounds with no scheduling
614	vpaddd	K256+0*32(SRND), X0, XFER
615	vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
616	DO_4ROUNDS	_XFER + 0*32
617
618	vpaddd	K256+1*32(SRND), X1, XFER
619	vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
620	DO_4ROUNDS	_XFER + 1*32
621	add	$2*32, SRND
622
623	vmovdqa	X2, X0
624	vmovdqa	X3, X1
625
626	cmp	$4*4*32, SRND
627	jb	loop2
628
629	mov	_CTX(%rsp), CTX
630	mov	_INP(%rsp), INP
631
632	addm    (4*0)(CTX),a
633	addm    (4*1)(CTX),b
634	addm    (4*2)(CTX),c
635	addm    (4*3)(CTX),d
636	addm    (4*4)(CTX),e
637	addm    (4*5)(CTX),f
638	addm    (4*6)(CTX),g
639	addm    (4*7)(CTX),h
640
641	cmp	_INP_END(%rsp), INP
642	ja	done_hash
643
644	#### Do second block using previously scheduled results
645	xor	SRND, SRND
646.align 16
647loop3:
648	DO_4ROUNDS	 _XFER + 0*32 + 16
649	DO_4ROUNDS	 _XFER + 1*32 + 16
650	add	$2*32, SRND
651	cmp	$4*4*32, SRND
652	jb	loop3
653
654	mov	_CTX(%rsp), CTX
655	mov	_INP(%rsp), INP
656	add	$64, INP
657
658	addm    (4*0)(CTX),a
659	addm    (4*1)(CTX),b
660	addm    (4*2)(CTX),c
661	addm    (4*3)(CTX),d
662	addm    (4*4)(CTX),e
663	addm    (4*5)(CTX),f
664	addm    (4*6)(CTX),g
665	addm    (4*7)(CTX),h
666
667	cmp	_INP_END(%rsp), INP
668	jb	loop0
669	ja	done_hash
670
671do_last_block:
672	VMOVDQ	0*16(INP),XWORD0
673	VMOVDQ	1*16(INP),XWORD1
674	VMOVDQ	2*16(INP),XWORD2
675	VMOVDQ	3*16(INP),XWORD3
676
677	vpshufb	X_BYTE_FLIP_MASK, XWORD0, XWORD0
678	vpshufb	X_BYTE_FLIP_MASK, XWORD1, XWORD1
679	vpshufb	X_BYTE_FLIP_MASK, XWORD2, XWORD2
680	vpshufb	X_BYTE_FLIP_MASK, XWORD3, XWORD3
681
682	jmp	last_block_enter
683
684only_one_block:
685
686	## load initial digest
687	mov	(4*0)(CTX),a
688	mov	(4*1)(CTX),b
689	mov	(4*2)(CTX),c
690	mov	(4*3)(CTX),d
691	mov	(4*4)(CTX),e
692	mov	(4*5)(CTX),f
693	mov	(4*6)(CTX),g
694	mov	(4*7)(CTX),h
695
696	vmovdqa	PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
697	vmovdqa	_SHUF_00BA(%rip), SHUF_00BA
698	vmovdqa	_SHUF_DC00(%rip), SHUF_DC00
699
700	mov	CTX, _CTX(%rsp)
701	jmp	do_last_block
702
703done_hash:
704
705	mov	%rbp, %rsp
706	pop	%rbp
707
708	popq	%r15
709	popq	%r14
710	popq	%r13
711	popq	%r12
712	popq	%rbx
713	ret
714SYM_FUNC_END(sha256_transform_rorx)
715
716.section	.rodata.cst512.K256, "aM", @progbits, 512
717.align 64
718K256:
719	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
720	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
721	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
722	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
723	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
724	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
725	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
726	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
727	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
728	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
729	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
730	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
731	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
732	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
733	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
734	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
735	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
736	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
737	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
738	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
739	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
740	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
741	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
742	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
743	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
744	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
745	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
746	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
747	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
748	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
749	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
750	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
751
752.section	.rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
753.align 32
754PSHUFFLE_BYTE_FLIP_MASK:
755	.octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203
756
757# shuffle xBxA -> 00BA
758.section	.rodata.cst32._SHUF_00BA, "aM", @progbits, 32
759.align 32
760_SHUF_00BA:
761	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100
762
763# shuffle xDxC -> DC00
764.section	.rodata.cst32._SHUF_DC00, "aM", @progbits, 32
765.align 32
766_SHUF_DC00:
767	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF
768