1########################################################################
2# Implement fast SHA-256 with AVX2 instructions. (x86_64)
3#
4# Copyright (C) 2013 Intel Corporation.
5#
6# Authors:
7#     James Guilford <james.guilford@intel.com>
8#     Kirk Yap <kirk.s.yap@intel.com>
9#     Tim Chen <tim.c.chen@linux.intel.com>
10#
11# This software is available to you under a choice of one of two
12# licenses.  You may choose to be licensed under the terms of the GNU
13# General Public License (GPL) Version 2, available from the file
14# COPYING in the main directory of this source tree, or the
15# OpenIB.org BSD license below:
16#
17#     Redistribution and use in source and binary forms, with or
18#     without modification, are permitted provided that the following
19#     conditions are met:
20#
21#      - Redistributions of source code must retain the above
22#        copyright notice, this list of conditions and the following
23#        disclaimer.
24#
25#      - Redistributions in binary form must reproduce the above
26#        copyright notice, this list of conditions and the following
27#        disclaimer in the documentation and/or other materials
28#        provided with the distribution.
29#
30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37# SOFTWARE.
38#
39########################################################################
40#
41# This code is described in an Intel White-Paper:
42# "Fast SHA-256 Implementations on Intel Architecture Processors"
43#
44# To find it, surf to http://www.intel.com/p/en_US/embedded
45# and search for that title.
46#
47########################################################################
48# This code schedules 2 blocks at a time, with 4 lanes per block
49########################################################################
50
51#ifdef CONFIG_AS_AVX2
52#include <linux/linkage.h>
53
54## assume buffers not aligned
55#define	VMOVDQ vmovdqu
56
57################################ Define Macros
58
59# addm [mem], reg
60# Add reg to mem using reg-mem add and store
61.macro addm p1 p2
62	add	\p1, \p2
63	mov	\p2, \p1
64.endm
65
66################################
67
68X0 = %ymm4
69X1 = %ymm5
70X2 = %ymm6
71X3 = %ymm7
72
73# XMM versions of above
74XWORD0 = %xmm4
75XWORD1 = %xmm5
76XWORD2 = %xmm6
77XWORD3 = %xmm7
78
79XTMP0 = %ymm0
80XTMP1 = %ymm1
81XTMP2 = %ymm2
82XTMP3 = %ymm3
83XTMP4 = %ymm8
84XFER  = %ymm9
85XTMP5 = %ymm11
86
87SHUF_00BA =	%ymm10 # shuffle xBxA -> 00BA
88SHUF_DC00 =	%ymm12 # shuffle xDxC -> DC00
89BYTE_FLIP_MASK = %ymm13
90
91X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK
92
93NUM_BLKS = %rdx	# 3rd arg
94INP	= %rsi  # 2nd arg
95CTX	= %rdi	# 1st arg
96c	= %ecx
97d	= %r8d
98e       = %edx	# clobbers NUM_BLKS
99y3	= %esi	# clobbers INP
100
101SRND	= CTX	# SRND is same register as CTX
102
103a = %eax
104b = %ebx
105f = %r9d
106g = %r10d
107h = %r11d
108old_h = %r11d
109
110T1 = %r12d
111y0 = %r13d
112y1 = %r14d
113y2 = %r15d
114
115
116_XFER_SIZE	= 2*64*4	# 2 blocks, 64 rounds, 4 bytes/round
117_XMM_SAVE_SIZE	= 0
118_INP_END_SIZE	= 8
119_INP_SIZE	= 8
120_CTX_SIZE	= 8
121_RSP_SIZE	= 8
122
123_XFER		= 0
124_XMM_SAVE	= _XFER     + _XFER_SIZE
125_INP_END	= _XMM_SAVE + _XMM_SAVE_SIZE
126_INP		= _INP_END  + _INP_END_SIZE
127_CTX		= _INP      + _INP_SIZE
128_RSP		= _CTX      + _CTX_SIZE
129STACK_SIZE	= _RSP      + _RSP_SIZE
130
131# rotate_Xs
132# Rotate values of symbols X0...X3
133.macro rotate_Xs
134	X_ = X0
135	X0 = X1
136	X1 = X2
137	X2 = X3
138	X3 = X_
139.endm
140
141# ROTATE_ARGS
142# Rotate values of symbols a...h
143.macro ROTATE_ARGS
144	old_h = h
145	TMP_ = h
146	h = g
147	g = f
148	f = e
149	e = d
150	d = c
151	c = b
152	b = a
153	a = TMP_
154.endm
155
156.macro FOUR_ROUNDS_AND_SCHED disp
157################################### RND N + 0 ############################
158
159	mov	a, y3		# y3 = a                                # MAJA
160	rorx	$25, e, y0	# y0 = e >> 25				# S1A
161	rorx	$11, e, y1	# y1 = e >> 11				# S1B
162
163	addl	\disp(%rsp, SRND), h		# h = k + w + h         # --
164	or	c, y3		# y3 = a|c                              # MAJA
165	vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7]
166	mov	f, y2		# y2 = f                                # CH
167	rorx	$13, a, T1	# T1 = a >> 13				# S0B
168
169	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
170	xor	g, y2		# y2 = f^g                              # CH
171	vpaddd	X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1
172	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
173
174	and	e, y2		# y2 = (f^g)&e                          # CH
175	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
176	rorx	$22, a, y1	# y1 = a >> 22				# S0A
177	add	h, d		# d = k + w + h + d                     # --
178
179	and	b, y3		# y3 = (a|c)&b                          # MAJA
180	vpalignr $4, X0, X1, XTMP1	# XTMP1 = W[-15]
181	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
182	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
183
184	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
185	vpsrld	$7, XTMP1, XTMP2
186	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
187	mov	a, T1		# T1 = a                                # MAJB
188	and	c, T1		# T1 = a&c                              # MAJB
189
190	add	y0, y2		# y2 = S1 + CH                          # --
191	vpslld	$(32-7), XTMP1, XTMP3
192	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
193	add	y1, h		# h = k + w + h + S0                    # --
194
195	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
196	vpor	XTMP2, XTMP3, XTMP3	# XTMP3 = W[-15] ror 7
197
198	vpsrld	$18, XTMP1, XTMP2
199	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
200	add	y3, h		# h = t1 + S0 + MAJ                     # --
201
202
203	ROTATE_ARGS
204
205################################### RND N + 1 ############################
206
207	mov	a, y3		# y3 = a                                # MAJA
208	rorx	$25, e, y0	# y0 = e >> 25				# S1A
209	rorx	$11, e, y1	# y1 = e >> 11				# S1B
210	offset = \disp + 1*4
211	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
212	or	c, y3		# y3 = a|c                              # MAJA
213
214
215	vpsrld	$3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3
216	mov	f, y2		# y2 = f                                # CH
217	rorx	$13, a, T1	# T1 = a >> 13				# S0B
218	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
219	xor	g, y2		# y2 = f^g                              # CH
220
221
222	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
223	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
224	rorx	$22, a, y1	# y1 = a >> 22				# S0A
225	and	e, y2		# y2 = (f^g)&e                          # CH
226	add	h, d		# d = k + w + h + d                     # --
227
228	vpslld	$(32-18), XTMP1, XTMP1
229	and	b, y3		# y3 = (a|c)&b                          # MAJA
230	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
231
232	vpxor	XTMP1, XTMP3, XTMP3
233	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
234	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
235
236	vpxor	XTMP2, XTMP3, XTMP3	# XTMP3 = W[-15] ror 7 ^ W[-15] ror 18
237	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
238	mov	a, T1		# T1 = a                                # MAJB
239	and	c, T1		# T1 = a&c                              # MAJB
240	add	y0, y2		# y2 = S1 + CH                          # --
241
242	vpxor	XTMP4, XTMP3, XTMP1	# XTMP1 = s0
243	vpshufd	$0b11111010, X3, XTMP2	# XTMP2 = W[-2] {BBAA}
244	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
245	add	y1, h		# h = k + w + h + S0                    # --
246
247	vpaddd	XTMP1, XTMP0, XTMP0	# XTMP0 = W[-16] + W[-7] + s0
248	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
249	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
250	add	y3, h		# h = t1 + S0 + MAJ                     # --
251
252	vpsrld	$10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA}
253
254
255	ROTATE_ARGS
256
257################################### RND N + 2 ############################
258
259	mov	a, y3		# y3 = a                                # MAJA
260	rorx	$25, e, y0	# y0 = e >> 25				# S1A
261	offset = \disp + 2*4
262	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
263
264	vpsrlq	$19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA}
265	rorx	$11, e, y1	# y1 = e >> 11				# S1B
266	or	c, y3		# y3 = a|c                              # MAJA
267	mov	f, y2		# y2 = f                                # CH
268	xor	g, y2		# y2 = f^g                              # CH
269
270	rorx	$13, a, T1	# T1 = a >> 13				# S0B
271	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
272	vpsrlq	$17, XTMP2, XTMP2	# XTMP2 = W[-2] ror 17 {xBxA}
273	and	e, y2		# y2 = (f^g)&e                          # CH
274
275	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
276	vpxor	XTMP3, XTMP2, XTMP2
277	add	h, d		# d = k + w + h + d                     # --
278	and	b, y3		# y3 = (a|c)&b                          # MAJA
279
280	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
281	rorx	$22, a, y1	# y1 = a >> 22				# S0A
282	vpxor	XTMP2, XTMP4, XTMP4	# XTMP4 = s1 {xBxA}
283	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
284
285	vpshufb	SHUF_00BA, XTMP4, XTMP4	# XTMP4 = s1 {00BA}
286	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
287	rorx	$2, a ,T1	# T1 = (a >> 2)				# S0
288	vpaddd	XTMP4, XTMP0, XTMP0	# XTMP0 = {..., ..., W[1], W[0]}
289
290	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
291	mov	a, T1		# T1 = a                                # MAJB
292	and	c, T1		# T1 = a&c                              # MAJB
293	add	y0, y2		# y2 = S1 + CH                          # --
294	vpshufd	$0b01010000, XTMP0, XTMP2	# XTMP2 = W[-2] {DDCC}
295
296	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
297	add	y1,h		# h = k + w + h + S0                    # --
298	add	y2,d		# d = k + w + h + d + S1 + CH = d + t1  # --
299	add	y2,h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
300
301	add	y3,h		# h = t1 + S0 + MAJ                     # --
302
303
304	ROTATE_ARGS
305
306################################### RND N + 3 ############################
307
308	mov	a, y3		# y3 = a                                # MAJA
309	rorx	$25, e, y0	# y0 = e >> 25				# S1A
310	rorx	$11, e, y1	# y1 = e >> 11				# S1B
311	offset = \disp + 3*4
312	addl	offset(%rsp, SRND), h	# h = k + w + h         # --
313	or	c, y3		# y3 = a|c                              # MAJA
314
315
316	vpsrld	$10, XTMP2, XTMP5	# XTMP5 = W[-2] >> 10 {DDCC}
317	mov	f, y2		# y2 = f                                # CH
318	rorx	$13, a, T1	# T1 = a >> 13				# S0B
319	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
320	xor	g, y2		# y2 = f^g                              # CH
321
322
323	vpsrlq	$19, XTMP2, XTMP3	# XTMP3 = W[-2] ror 19 {xDxC}
324	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
325	and	e, y2		# y2 = (f^g)&e                          # CH
326	add	h, d		# d = k + w + h + d                     # --
327	and	b, y3		# y3 = (a|c)&b                          # MAJA
328
329	vpsrlq	$17, XTMP2, XTMP2	# XTMP2 = W[-2] ror 17 {xDxC}
330	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
331	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
332
333	vpxor	XTMP3, XTMP2, XTMP2
334	rorx	$22, a, y1	# y1 = a >> 22				# S0A
335	add	y0, y2		# y2 = S1 + CH                          # --
336
337	vpxor	XTMP2, XTMP5, XTMP5	# XTMP5 = s1 {xDxC}
338	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
339	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
340
341	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
342	vpshufb	SHUF_DC00, XTMP5, XTMP5	# XTMP5 = s1 {DC00}
343
344	vpaddd	XTMP0, XTMP5, X0	# X0 = {W[3], W[2], W[1], W[0]}
345	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
346	mov	a, T1		# T1 = a                                # MAJB
347	and	c, T1		# T1 = a&c                              # MAJB
348	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
349
350	add	y1, h		# h = k + w + h + S0                    # --
351	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
352	add	y3, h		# h = t1 + S0 + MAJ                     # --
353
354	ROTATE_ARGS
355	rotate_Xs
356.endm
357
358.macro DO_4ROUNDS disp
359################################### RND N + 0 ###########################
360
361	mov	f, y2		# y2 = f                                # CH
362	rorx	$25, e, y0	# y0 = e >> 25				# S1A
363	rorx	$11, e, y1	# y1 = e >> 11				# S1B
364	xor	g, y2		# y2 = f^g                              # CH
365
366	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
367	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
368	and	e, y2		# y2 = (f^g)&e                          # CH
369
370	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
371	rorx	$13, a, T1	# T1 = a >> 13				# S0B
372	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
373	rorx	$22, a, y1	# y1 = a >> 22				# S0A
374	mov	a, y3		# y3 = a                                # MAJA
375
376	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
377	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
378	addl	\disp(%rsp, SRND), h		# h = k + w + h # --
379	or	c, y3		# y3 = a|c                              # MAJA
380
381	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
382	mov	a, T1		# T1 = a                                # MAJB
383	and	b, y3		# y3 = (a|c)&b                          # MAJA
384	and	c, T1		# T1 = a&c                              # MAJB
385	add	y0, y2		# y2 = S1 + CH                          # --
386
387
388	add	h, d		# d = k + w + h + d                     # --
389	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
390	add	y1, h		# h = k + w + h + S0                    # --
391	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
392
393	ROTATE_ARGS
394
395################################### RND N + 1 ###########################
396
397	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
398	mov	f, y2		# y2 = f                                # CH
399	rorx	$25, e, y0	# y0 = e >> 25				# S1A
400	rorx	$11, e, y1	# y1 = e >> 11				# S1B
401	xor	g, y2		# y2 = f^g                              # CH
402
403	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
404	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
405	and	e, y2		# y2 = (f^g)&e                          # CH
406	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
407
408	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
409	rorx	$13, a, T1	# T1 = a >> 13				# S0B
410	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
411	rorx	$22, a, y1	# y1 = a >> 22				# S0A
412	mov	a, y3		# y3 = a                                # MAJA
413
414	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
415	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
416	offset = 4*1 + \disp
417	addl	offset(%rsp, SRND), h		# h = k + w + h # --
418	or	c, y3		# y3 = a|c                              # MAJA
419
420	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
421	mov	a, T1		# T1 = a                                # MAJB
422	and	b, y3		# y3 = (a|c)&b                          # MAJA
423	and	c, T1		# T1 = a&c                              # MAJB
424	add	y0, y2		# y2 = S1 + CH                          # --
425
426
427	add	h, d		# d = k + w + h + d                     # --
428	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
429	add	y1, h		# h = k + w + h + S0                    # --
430
431	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
432
433	ROTATE_ARGS
434
435################################### RND N + 2 ##############################
436
437	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
438	mov	f, y2		# y2 = f                                # CH
439	rorx	$25, e, y0	# y0 = e >> 25				# S1A
440	rorx	$11, e, y1	# y1 = e >> 11				# S1B
441	xor	g, y2		# y2 = f^g                              # CH
442
443	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
444	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
445	and	e, y2		# y2 = (f^g)&e                          # CH
446	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
447
448	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
449	rorx	$13, a, T1	# T1 = a >> 13				# S0B
450	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
451	rorx	$22, a, y1	# y1 = a >> 22				# S0A
452	mov	a, y3		# y3 = a                                # MAJA
453
454	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
455	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
456	offset = 4*2 + \disp
457	addl	offset(%rsp, SRND), h		# h = k + w + h # --
458	or	c, y3		# y3 = a|c                              # MAJA
459
460	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
461	mov	a, T1		# T1 = a                                # MAJB
462	and	b, y3		# y3 = (a|c)&b                          # MAJA
463	and	c, T1		# T1 = a&c                              # MAJB
464	add	y0, y2		# y2 = S1 + CH                          # --
465
466
467	add	h, d		# d = k + w + h + d                     # --
468	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
469	add	y1, h		# h = k + w + h + S0                    # --
470
471	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
472
473	ROTATE_ARGS
474
475################################### RND N + 3 ###########################
476
477	add	y2, old_h	# h = k + w + h + S0 + S1 + CH = t1 + S0# --
478	mov	f, y2		# y2 = f                                # CH
479	rorx	$25, e, y0	# y0 = e >> 25				# S1A
480	rorx	$11, e, y1	# y1 = e >> 11				# S1B
481	xor	g, y2		# y2 = f^g                              # CH
482
483	xor	y1, y0		# y0 = (e>>25) ^ (e>>11)		# S1
484	rorx	$6, e, y1	# y1 = (e >> 6)				# S1
485	and	e, y2		# y2 = (f^g)&e                          # CH
486	add	y3, old_h	# h = t1 + S0 + MAJ                     # --
487
488	xor	y1, y0		# y0 = (e>>25) ^ (e>>11) ^ (e>>6)	# S1
489	rorx	$13, a, T1	# T1 = a >> 13				# S0B
490	xor	g, y2		# y2 = CH = ((f^g)&e)^g                 # CH
491	rorx	$22, a, y1	# y1 = a >> 22				# S0A
492	mov	a, y3		# y3 = a                                # MAJA
493
494	xor	T1, y1		# y1 = (a>>22) ^ (a>>13)		# S0
495	rorx	$2, a, T1	# T1 = (a >> 2)				# S0
496	offset = 4*3 + \disp
497	addl	offset(%rsp, SRND), h		# h = k + w + h # --
498	or	c, y3		# y3 = a|c                              # MAJA
499
500	xor	T1, y1		# y1 = (a>>22) ^ (a>>13) ^ (a>>2)	# S0
501	mov	a, T1		# T1 = a                                # MAJB
502	and	b, y3		# y3 = (a|c)&b                          # MAJA
503	and	c, T1		# T1 = a&c                              # MAJB
504	add	y0, y2		# y2 = S1 + CH                          # --
505
506
507	add	h, d		# d = k + w + h + d                     # --
508	or	T1, y3		# y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
509	add	y1, h		# h = k + w + h + S0                    # --
510
511	add	y2, d		# d = k + w + h + d + S1 + CH = d + t1  # --
512
513
514	add	y2, h		# h = k + w + h + S0 + S1 + CH = t1 + S0# --
515
516	add	y3, h		# h = t1 + S0 + MAJ                     # --
517
518	ROTATE_ARGS
519
520.endm
521
522########################################################################
523## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks)
524## arg 1 : pointer to state
525## arg 2 : pointer to input data
526## arg 3 : Num blocks
527########################################################################
528.text
529SYM_FUNC_START(sha256_transform_rorx)
530.align 32
531	pushq	%rbx
532	pushq	%r12
533	pushq	%r13
534	pushq	%r14
535	pushq	%r15
536
537	mov	%rsp, %rax
538	subq	$STACK_SIZE, %rsp
539	and	$-32, %rsp	# align rsp to 32 byte boundary
540	mov	%rax, _RSP(%rsp)
541
542
543	shl	$6, NUM_BLKS	# convert to bytes
544	jz	done_hash
545	lea	-64(INP, NUM_BLKS), NUM_BLKS # pointer to last block
546	mov	NUM_BLKS, _INP_END(%rsp)
547
548	cmp	NUM_BLKS, INP
549	je	only_one_block
550
551	## load initial digest
552	mov	(CTX), a
553	mov	4*1(CTX), b
554	mov	4*2(CTX), c
555	mov	4*3(CTX), d
556	mov	4*4(CTX), e
557	mov	4*5(CTX), f
558	mov	4*6(CTX), g
559	mov	4*7(CTX), h
560
561	vmovdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
562	vmovdqa  _SHUF_00BA(%rip), SHUF_00BA
563	vmovdqa  _SHUF_DC00(%rip), SHUF_DC00
564
565	mov	CTX, _CTX(%rsp)
566
567loop0:
568	## Load first 16 dwords from two blocks
569	VMOVDQ	0*32(INP),XTMP0
570	VMOVDQ	1*32(INP),XTMP1
571	VMOVDQ	2*32(INP),XTMP2
572	VMOVDQ	3*32(INP),XTMP3
573
574	## byte swap data
575	vpshufb	BYTE_FLIP_MASK, XTMP0, XTMP0
576	vpshufb	BYTE_FLIP_MASK, XTMP1, XTMP1
577	vpshufb	BYTE_FLIP_MASK, XTMP2, XTMP2
578	vpshufb	BYTE_FLIP_MASK, XTMP3, XTMP3
579
580	## transpose data into high/low halves
581	vperm2i128	$0x20, XTMP2, XTMP0, X0
582	vperm2i128	$0x31, XTMP2, XTMP0, X1
583	vperm2i128	$0x20, XTMP3, XTMP1, X2
584	vperm2i128	$0x31, XTMP3, XTMP1, X3
585
586last_block_enter:
587	add	$64, INP
588	mov	INP, _INP(%rsp)
589
590	## schedule 48 input dwords, by doing 3 rounds of 12 each
591	xor	SRND, SRND
592
593.align 16
594loop1:
595	vpaddd	K256+0*32(SRND), X0, XFER
596	vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
597	FOUR_ROUNDS_AND_SCHED	_XFER + 0*32
598
599	vpaddd	K256+1*32(SRND), X0, XFER
600	vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
601	FOUR_ROUNDS_AND_SCHED	_XFER + 1*32
602
603	vpaddd	K256+2*32(SRND), X0, XFER
604	vmovdqa XFER, 2*32+_XFER(%rsp, SRND)
605	FOUR_ROUNDS_AND_SCHED	_XFER + 2*32
606
607	vpaddd	K256+3*32(SRND), X0, XFER
608	vmovdqa XFER, 3*32+_XFER(%rsp, SRND)
609	FOUR_ROUNDS_AND_SCHED	_XFER + 3*32
610
611	add	$4*32, SRND
612	cmp	$3*4*32, SRND
613	jb	loop1
614
615loop2:
616	## Do last 16 rounds with no scheduling
617	vpaddd	K256+0*32(SRND), X0, XFER
618	vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
619	DO_4ROUNDS	_XFER + 0*32
620
621	vpaddd	K256+1*32(SRND), X1, XFER
622	vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
623	DO_4ROUNDS	_XFER + 1*32
624	add	$2*32, SRND
625
626	vmovdqa	X2, X0
627	vmovdqa	X3, X1
628
629	cmp	$4*4*32, SRND
630	jb	loop2
631
632	mov	_CTX(%rsp), CTX
633	mov	_INP(%rsp), INP
634
635	addm    (4*0)(CTX),a
636	addm    (4*1)(CTX),b
637	addm    (4*2)(CTX),c
638	addm    (4*3)(CTX),d
639	addm    (4*4)(CTX),e
640	addm    (4*5)(CTX),f
641	addm    (4*6)(CTX),g
642	addm    (4*7)(CTX),h
643
644	cmp	_INP_END(%rsp), INP
645	ja	done_hash
646
647	#### Do second block using previously scheduled results
648	xor	SRND, SRND
649.align 16
650loop3:
651	DO_4ROUNDS	 _XFER + 0*32 + 16
652	DO_4ROUNDS	 _XFER + 1*32 + 16
653	add	$2*32, SRND
654	cmp	$4*4*32, SRND
655	jb	loop3
656
657	mov	_CTX(%rsp), CTX
658	mov	_INP(%rsp), INP
659	add	$64, INP
660
661	addm    (4*0)(CTX),a
662	addm    (4*1)(CTX),b
663	addm    (4*2)(CTX),c
664	addm    (4*3)(CTX),d
665	addm    (4*4)(CTX),e
666	addm    (4*5)(CTX),f
667	addm    (4*6)(CTX),g
668	addm    (4*7)(CTX),h
669
670	cmp	_INP_END(%rsp), INP
671	jb	loop0
672	ja	done_hash
673
674do_last_block:
675	VMOVDQ	0*16(INP),XWORD0
676	VMOVDQ	1*16(INP),XWORD1
677	VMOVDQ	2*16(INP),XWORD2
678	VMOVDQ	3*16(INP),XWORD3
679
680	vpshufb	X_BYTE_FLIP_MASK, XWORD0, XWORD0
681	vpshufb	X_BYTE_FLIP_MASK, XWORD1, XWORD1
682	vpshufb	X_BYTE_FLIP_MASK, XWORD2, XWORD2
683	vpshufb	X_BYTE_FLIP_MASK, XWORD3, XWORD3
684
685	jmp	last_block_enter
686
687only_one_block:
688
689	## load initial digest
690	mov	(4*0)(CTX),a
691	mov	(4*1)(CTX),b
692	mov	(4*2)(CTX),c
693	mov	(4*3)(CTX),d
694	mov	(4*4)(CTX),e
695	mov	(4*5)(CTX),f
696	mov	(4*6)(CTX),g
697	mov	(4*7)(CTX),h
698
699	vmovdqa	PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
700	vmovdqa	_SHUF_00BA(%rip), SHUF_00BA
701	vmovdqa	_SHUF_DC00(%rip), SHUF_DC00
702
703	mov	CTX, _CTX(%rsp)
704	jmp	do_last_block
705
706done_hash:
707
708	mov	_RSP(%rsp), %rsp
709
710	popq	%r15
711	popq	%r14
712	popq	%r13
713	popq	%r12
714	popq	%rbx
715	ret
716SYM_FUNC_END(sha256_transform_rorx)
717
718.section	.rodata.cst512.K256, "aM", @progbits, 512
719.align 64
720K256:
721	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
722	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
723	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
724	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
725	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
726	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
727	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
728	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
729	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
730	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
731	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
732	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
733	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
734	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
735	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
736	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
737	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
738	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
739	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
740	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
741	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
742	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
743	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
744	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
745	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
746	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
747	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
748	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
749	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
750	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
751	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
752	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
753
754.section	.rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
755.align 32
756PSHUFFLE_BYTE_FLIP_MASK:
757	.octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203
758
759# shuffle xBxA -> 00BA
760.section	.rodata.cst32._SHUF_00BA, "aM", @progbits, 32
761.align 32
762_SHUF_00BA:
763	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100
764
765# shuffle xDxC -> DC00
766.section	.rodata.cst32._SHUF_DC00, "aM", @progbits, 32
767.align 32
768_SHUF_DC00:
769	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF
770
771#endif
772