1########################################################################
2# Implement fast SHA-256 with SSSE3 instructions. (x86_64)
3#
4# Copyright (C) 2013 Intel Corporation.
5#
6# Authors:
7#     James Guilford <james.guilford@intel.com>
8#     Kirk Yap <kirk.s.yap@intel.com>
9#     Tim Chen <tim.c.chen@linux.intel.com>
10#
11# This software is available to you under a choice of one of two
12# licenses.  You may choose to be licensed under the terms of the GNU
13# General Public License (GPL) Version 2, available from the file
14# COPYING in the main directory of this source tree, or the
15# OpenIB.org BSD license below:
16#
17#     Redistribution and use in source and binary forms, with or
18#     without modification, are permitted provided that the following
19#     conditions are met:
20#
21#      - Redistributions of source code must retain the above
22#        copyright notice, this list of conditions and the following
23#        disclaimer.
24#
25#      - Redistributions in binary form must reproduce the above
26#        copyright notice, this list of conditions and the following
27#        disclaimer in the documentation and/or other materials
28#        provided with the distribution.
29#
30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37# SOFTWARE.
38#
39########################################################################
40#
41# This code is described in an Intel White-Paper:
42# "Fast SHA-256 Implementations on Intel Architecture Processors"
43#
44# To find it, surf to http://www.intel.com/p/en_US/embedded
45# and search for that title.
46#
47########################################################################
48
49#include <linux/linkage.h>
50
51## assume buffers not aligned
52#define    MOVDQ movdqu
53
54################################ Define Macros
55
56# addm [mem], reg
57# Add reg to mem using reg-mem add and store
58.macro addm p1 p2
59        add     \p1, \p2
60        mov     \p2, \p1
61.endm
62
63################################
64
65# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
66# Load xmm with mem and byte swap each dword
67.macro COPY_XMM_AND_BSWAP p1 p2 p3
68        MOVDQ \p2, \p1
69        pshufb \p3, \p1
70.endm
71
72################################
73
74X0 = %xmm4
75X1 = %xmm5
76X2 = %xmm6
77X3 = %xmm7
78
79XTMP0 = %xmm0
80XTMP1 = %xmm1
81XTMP2 = %xmm2
82XTMP3 = %xmm3
83XTMP4 = %xmm8
84XFER = %xmm9
85
86SHUF_00BA = %xmm10      # shuffle xBxA -> 00BA
87SHUF_DC00 = %xmm11      # shuffle xDxC -> DC00
88BYTE_FLIP_MASK = %xmm12
89
90NUM_BLKS = %rdx   # 3rd arg
91INP = %rsi        # 2nd arg
92CTX = %rdi        # 1st arg
93
94SRND = %rsi       # clobbers INP
95c = %ecx
96d = %r8d
97e = %edx
98TBL = %r12
99a = %eax
100b = %ebx
101
102f = %r9d
103g = %r10d
104h = %r11d
105
106y0 = %r13d
107y1 = %r14d
108y2 = %r15d
109
110
111
112_INP_END_SIZE = 8
113_INP_SIZE = 8
114_XFER_SIZE = 16
115_XMM_SAVE_SIZE = 0
116
117_INP_END = 0
118_INP            = _INP_END  + _INP_END_SIZE
119_XFER           = _INP      + _INP_SIZE
120_XMM_SAVE       = _XFER     + _XFER_SIZE
121STACK_SIZE      = _XMM_SAVE + _XMM_SAVE_SIZE
122
123# rotate_Xs
124# Rotate values of symbols X0...X3
125.macro rotate_Xs
126X_ = X0
127X0 = X1
128X1 = X2
129X2 = X3
130X3 = X_
131.endm
132
133# ROTATE_ARGS
134# Rotate values of symbols a...h
135.macro ROTATE_ARGS
136TMP_ = h
137h = g
138g = f
139f = e
140e = d
141d = c
142c = b
143b = a
144a = TMP_
145.endm
146
147.macro FOUR_ROUNDS_AND_SCHED
148	## compute s0 four at a time and s1 two at a time
149	## compute W[-16] + W[-7] 4 at a time
150	movdqa  X3, XTMP0
151	mov     e, y0			# y0 = e
152	ror     $(25-11), y0            # y0 = e >> (25-11)
153	mov     a, y1                   # y1 = a
154	palignr $4, X2, XTMP0           # XTMP0 = W[-7]
155	ror     $(22-13), y1            # y1 = a >> (22-13)
156	xor     e, y0                   # y0 = e ^ (e >> (25-11))
157	mov     f, y2                   # y2 = f
158	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
159	movdqa  X1, XTMP1
160	xor     a, y1                   # y1 = a ^ (a >> (22-13)
161	xor     g, y2                   # y2 = f^g
162	paddd   X0, XTMP0               # XTMP0 = W[-7] + W[-16]
163	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
164	and     e, y2                   # y2 = (f^g)&e
165	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
166	## compute s0
167	palignr $4, X0, XTMP1           # XTMP1 = W[-15]
168	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
169	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
170	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
171	movdqa  XTMP1, XTMP2            # XTMP2 = W[-15]
172	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
173	add     y0, y2                  # y2 = S1 + CH
174	add     _XFER(%rsp) , y2        # y2 = k + w + S1 + CH
175	movdqa  XTMP1, XTMP3            # XTMP3 = W[-15]
176	mov     a, y0                   # y0 = a
177	add     y2, h                   # h = h + S1 + CH + k + w
178	mov     a, y2                   # y2 = a
179	pslld   $(32-7), XTMP1          #
180	or      c, y0                   # y0 = a|c
181	add     h, d                    # d = d + h + S1 + CH + k + w
182	and     c, y2                   # y2 = a&c
183	psrld   $7, XTMP2               #
184	and     b, y0                   # y0 = (a|c)&b
185	add     y1, h                   # h = h + S1 + CH + k + w + S0
186	por     XTMP2, XTMP1            # XTMP1 = W[-15] ror 7
187	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
188	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
189					#
190	ROTATE_ARGS                     #
191	movdqa  XTMP3, XTMP2            # XTMP2 = W[-15]
192	mov     e, y0                   # y0 = e
193	mov     a, y1                   # y1 = a
194	movdqa  XTMP3, XTMP4            # XTMP4 = W[-15]
195	ror     $(25-11), y0            # y0 = e >> (25-11)
196	xor     e, y0                   # y0 = e ^ (e >> (25-11))
197	mov     f, y2                   # y2 = f
198	ror     $(22-13), y1            # y1 = a >> (22-13)
199	pslld   $(32-18), XTMP3         #
200	xor     a, y1                   # y1 = a ^ (a >> (22-13)
201	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
202	xor     g, y2                   # y2 = f^g
203	psrld   $18, XTMP2              #
204	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
205	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
206	and     e, y2                   # y2 = (f^g)&e
207	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
208	pxor    XTMP3, XTMP1
209	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
210	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
211	psrld   $3, XTMP4               # XTMP4 = W[-15] >> 3
212	add     y0, y2                  # y2 = S1 + CH
213	add     (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
214	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
215	pxor    XTMP2, XTMP1            # XTMP1 = W[-15] ror 7 ^ W[-15] ror 18
216	mov     a, y0                   # y0 = a
217	add     y2, h                   # h = h + S1 + CH + k + w
218	mov     a, y2                   # y2 = a
219	pxor    XTMP4, XTMP1            # XTMP1 = s0
220	or      c, y0                   # y0 = a|c
221	add     h, d                    # d = d + h + S1 + CH + k + w
222	and     c, y2                   # y2 = a&c
223	## compute low s1
224	pshufd  $0b11111010, X3, XTMP2   # XTMP2 = W[-2] {BBAA}
225	and     b, y0			# y0 = (a|c)&b
226	add     y1, h                   # h = h + S1 + CH + k + w + S0
227	paddd   XTMP1, XTMP0            # XTMP0 = W[-16] + W[-7] + s0
228	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
229	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
230
231	ROTATE_ARGS
232	movdqa  XTMP2, XTMP3            # XTMP3 = W[-2] {BBAA}
233	mov     e, y0                   # y0 = e
234	mov     a, y1                   # y1 = a
235	ror     $(25-11), y0            # y0 = e >> (25-11)
236	movdqa  XTMP2, XTMP4            # XTMP4 = W[-2] {BBAA}
237	xor     e, y0                   # y0 = e ^ (e >> (25-11))
238	ror     $(22-13), y1            # y1 = a >> (22-13)
239	mov     f, y2                   # y2 = f
240	xor     a, y1                   # y1 = a ^ (a >> (22-13)
241	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
242	psrlq   $17, XTMP2              # XTMP2 = W[-2] ror 17 {xBxA}
243	xor     g, y2                   # y2 = f^g
244	psrlq   $19, XTMP3              # XTMP3 = W[-2] ror 19 {xBxA}
245	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
246	and     e, y2                   # y2 = (f^g)&e
247	psrld   $10, XTMP4              # XTMP4 = W[-2] >> 10 {BBAA}
248	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
249	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
250	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
251	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
252	pxor    XTMP3, XTMP2
253	add     y0, y2                  # y2 = S1 + CH
254	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
255	add     (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
256	pxor    XTMP2, XTMP4            # XTMP4 = s1 {xBxA}
257	mov     a, y0                   # y0 = a
258	add     y2, h                   # h = h + S1 + CH + k + w
259	mov     a, y2                   # y2 = a
260	pshufb  SHUF_00BA, XTMP4        # XTMP4 = s1 {00BA}
261	or      c, y0                   # y0 = a|c
262	add     h, d                    # d = d + h + S1 + CH + k + w
263	and     c, y2                   # y2 = a&c
264	paddd   XTMP4, XTMP0            # XTMP0 = {..., ..., W[1], W[0]}
265	and     b, y0                   # y0 = (a|c)&b
266	add     y1, h                   # h = h + S1 + CH + k + w + S0
267	## compute high s1
268	pshufd  $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {BBAA}
269	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
270	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
271					#
272	ROTATE_ARGS                     #
273	movdqa  XTMP2, XTMP3            # XTMP3 = W[-2] {DDCC}
274	mov     e, y0                   # y0 = e
275	ror     $(25-11), y0            # y0 = e >> (25-11)
276	mov     a, y1                   # y1 = a
277	movdqa  XTMP2, X0               # X0    = W[-2] {DDCC}
278	ror     $(22-13), y1            # y1 = a >> (22-13)
279	xor     e, y0                   # y0 = e ^ (e >> (25-11))
280	mov     f, y2                   # y2 = f
281	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
282	psrlq   $17, XTMP2              # XTMP2 = W[-2] ror 17 {xDxC}
283	xor     a, y1                   # y1 = a ^ (a >> (22-13)
284	xor     g, y2                   # y2 = f^g
285	psrlq   $19, XTMP3              # XTMP3 = W[-2] ror 19 {xDxC}
286	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25
287	and     e, y2                   # y2 = (f^g)&e
288	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
289	psrld   $10, X0                 # X0 = W[-2] >> 10 {DDCC}
290	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22
291	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>2
292	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
293	pxor    XTMP3, XTMP2            #
294	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>2
295	add     y0, y2                  # y2 = S1 + CH
296	add     (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
297	pxor    XTMP2, X0               # X0 = s1 {xDxC}
298	mov     a, y0                   # y0 = a
299	add     y2, h                   # h = h + S1 + CH + k + w
300	mov     a, y2                   # y2 = a
301	pshufb  SHUF_DC00, X0           # X0 = s1 {DC00}
302	or      c, y0                   # y0 = a|c
303	add     h, d                    # d = d + h + S1 + CH + k + w
304	and     c, y2                   # y2 = a&c
305	paddd   XTMP0, X0               # X0 = {W[3], W[2], W[1], W[0]}
306	and     b, y0                   # y0 = (a|c)&b
307	add     y1, h                   # h = h + S1 + CH + k + w + S0
308	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
309	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
310
311	ROTATE_ARGS
312	rotate_Xs
313.endm
314
315## input is [rsp + _XFER + %1 * 4]
316.macro DO_ROUND round
317	mov     e, y0                 # y0 = e
318	ror     $(25-11), y0          # y0 = e >> (25-11)
319	mov     a, y1                 # y1 = a
320	xor     e, y0                 # y0 = e ^ (e >> (25-11))
321	ror     $(22-13), y1          # y1 = a >> (22-13)
322	mov     f, y2                 # y2 = f
323	xor     a, y1                 # y1 = a ^ (a >> (22-13)
324	ror     $(11-6), y0           # y0 = (e >> (11-6)) ^ (e >> (25-6))
325	xor     g, y2                 # y2 = f^g
326	xor     e, y0                 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
327	ror     $(13-2), y1           # y1 = (a >> (13-2)) ^ (a >> (22-2))
328	and     e, y2                 # y2 = (f^g)&e
329	xor     a, y1                 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
330	ror     $6, y0                # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
331	xor     g, y2                 # y2 = CH = ((f^g)&e)^g
332	add     y0, y2                # y2 = S1 + CH
333	ror     $2, y1                # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
334	offset = \round * 4 + _XFER
335	add     offset(%rsp), y2      # y2 = k + w + S1 + CH
336	mov     a, y0                 # y0 = a
337	add     y2, h                 # h = h + S1 + CH + k + w
338	mov     a, y2                 # y2 = a
339	or      c, y0                 # y0 = a|c
340	add     h, d                  # d = d + h + S1 + CH + k + w
341	and     c, y2                 # y2 = a&c
342	and     b, y0                 # y0 = (a|c)&b
343	add     y1, h                 # h = h + S1 + CH + k + w + S0
344	or      y2, y0		      # y0 = MAJ = (a|c)&b)|(a&c)
345	add     y0, h		      # h = h + S1 + CH + k + w + S0 + MAJ
346	ROTATE_ARGS
347.endm
348
349########################################################################
350## void sha256_transform_ssse3(void *input_data, UINT32 digest[8], UINT64 num_blks)
351## arg 1 : pointer to digest
352## arg 2 : pointer to input data
353## arg 3 : Num blocks
354########################################################################
355.text
356ENTRY(sha256_transform_ssse3)
357.align 32
358	pushq   %rbx
359	pushq   %r12
360	pushq   %r13
361	pushq   %r14
362	pushq   %r15
363	pushq   %rbp
364	mov	%rsp, %rbp
365
366	subq    $STACK_SIZE, %rsp
367	and	$~15, %rsp
368
369	shl     $6, NUM_BLKS		 # convert to bytes
370	jz      done_hash
371	add     INP, NUM_BLKS
372	mov     NUM_BLKS, _INP_END(%rsp) # pointer to end of data
373
374	## load initial digest
375	mov     4*0(CTX), a
376	mov     4*1(CTX), b
377	mov     4*2(CTX), c
378	mov     4*3(CTX), d
379	mov     4*4(CTX), e
380	mov     4*5(CTX), f
381	mov     4*6(CTX), g
382	mov     4*7(CTX), h
383
384	movdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
385	movdqa  _SHUF_00BA(%rip), SHUF_00BA
386	movdqa  _SHUF_DC00(%rip), SHUF_DC00
387
388loop0:
389	lea     K256(%rip), TBL
390
391	## byte swap first 16 dwords
392	COPY_XMM_AND_BSWAP      X0, 0*16(INP), BYTE_FLIP_MASK
393	COPY_XMM_AND_BSWAP      X1, 1*16(INP), BYTE_FLIP_MASK
394	COPY_XMM_AND_BSWAP      X2, 2*16(INP), BYTE_FLIP_MASK
395	COPY_XMM_AND_BSWAP      X3, 3*16(INP), BYTE_FLIP_MASK
396
397	mov     INP, _INP(%rsp)
398
399	## schedule 48 input dwords, by doing 3 rounds of 16 each
400	mov     $3, SRND
401.align 16
402loop1:
403	movdqa  (TBL), XFER
404	paddd   X0, XFER
405	movdqa  XFER, _XFER(%rsp)
406	FOUR_ROUNDS_AND_SCHED
407
408	movdqa  1*16(TBL), XFER
409	paddd   X0, XFER
410	movdqa  XFER, _XFER(%rsp)
411	FOUR_ROUNDS_AND_SCHED
412
413	movdqa  2*16(TBL), XFER
414	paddd   X0, XFER
415	movdqa  XFER, _XFER(%rsp)
416	FOUR_ROUNDS_AND_SCHED
417
418	movdqa  3*16(TBL), XFER
419	paddd   X0, XFER
420	movdqa  XFER, _XFER(%rsp)
421	add     $4*16, TBL
422	FOUR_ROUNDS_AND_SCHED
423
424	sub     $1, SRND
425	jne     loop1
426
427	mov     $2, SRND
428loop2:
429	paddd   (TBL), X0
430	movdqa  X0, _XFER(%rsp)
431	DO_ROUND        0
432	DO_ROUND        1
433	DO_ROUND        2
434	DO_ROUND        3
435	paddd   1*16(TBL), X1
436	movdqa  X1, _XFER(%rsp)
437	add     $2*16, TBL
438	DO_ROUND        0
439	DO_ROUND        1
440	DO_ROUND        2
441	DO_ROUND        3
442
443	movdqa  X2, X0
444	movdqa  X3, X1
445
446	sub     $1, SRND
447	jne     loop2
448
449	addm    (4*0)(CTX),a
450	addm    (4*1)(CTX),b
451	addm    (4*2)(CTX),c
452	addm    (4*3)(CTX),d
453	addm    (4*4)(CTX),e
454	addm    (4*5)(CTX),f
455	addm    (4*6)(CTX),g
456	addm    (4*7)(CTX),h
457
458	mov     _INP(%rsp), INP
459	add     $64, INP
460	cmp     _INP_END(%rsp), INP
461	jne     loop0
462
463done_hash:
464
465	mov	%rbp, %rsp
466	popq	%rbp
467	popq    %r15
468	popq    %r14
469	popq    %r13
470	popq    %r12
471	popq    %rbx
472
473	ret
474ENDPROC(sha256_transform_ssse3)
475
476.section	.rodata.cst256.K256, "aM", @progbits, 256
477.align 64
478K256:
479        .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
480        .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
481        .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
482        .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
483        .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
484        .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
485        .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
486        .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
487        .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
488        .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
489        .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
490        .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
491        .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
492        .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
493        .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
494        .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
495
496.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
497.align 16
498PSHUFFLE_BYTE_FLIP_MASK:
499	.octa 0x0c0d0e0f08090a0b0405060700010203
500
501.section	.rodata.cst16._SHUF_00BA, "aM", @progbits, 16
502.align 16
503# shuffle xBxA -> 00BA
504_SHUF_00BA:
505	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
506
507.section	.rodata.cst16._SHUF_DC00, "aM", @progbits, 16
508.align 16
509# shuffle xDxC -> DC00
510_SHUF_DC00:
511	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
512