1######################################################################## 2# Implement fast SHA-256 with AVX1 instructions. (x86_64) 3# 4# Copyright (C) 2013 Intel Corporation. 5# 6# Authors: 7# James Guilford <james.guilford@intel.com> 8# Kirk Yap <kirk.s.yap@intel.com> 9# Tim Chen <tim.c.chen@linux.intel.com> 10# 11# This software is available to you under a choice of one of two 12# licenses. You may choose to be licensed under the terms of the GNU 13# General Public License (GPL) Version 2, available from the file 14# COPYING in the main directory of this source tree, or the 15# OpenIB.org BSD license below: 16# 17# Redistribution and use in source and binary forms, with or 18# without modification, are permitted provided that the following 19# conditions are met: 20# 21# - Redistributions of source code must retain the above 22# copyright notice, this list of conditions and the following 23# disclaimer. 24# 25# - Redistributions in binary form must reproduce the above 26# copyright notice, this list of conditions and the following 27# disclaimer in the documentation and/or other materials 28# provided with the distribution. 29# 30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 37# SOFTWARE. 38######################################################################## 39# 40# This code is described in an Intel White-Paper: 41# "Fast SHA-256 Implementations on Intel Architecture Processors" 42# 43# To find it, surf to http://www.intel.com/p/en_US/embedded 44# and search for that title. 45# 46######################################################################## 47# This code schedules 1 block at a time, with 4 lanes per block 48######################################################################## 49 50#include <linux/linkage.h> 51 52## assume buffers not aligned 53#define VMOVDQ vmovdqu 54 55################################ Define Macros 56 57# addm [mem], reg 58# Add reg to mem using reg-mem add and store 59.macro addm p1 p2 60 add \p1, \p2 61 mov \p2, \p1 62.endm 63 64 65.macro MY_ROR p1 p2 66 shld $(32-(\p1)), \p2, \p2 67.endm 68 69################################ 70 71# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask 72# Load xmm with mem and byte swap each dword 73.macro COPY_XMM_AND_BSWAP p1 p2 p3 74 VMOVDQ \p2, \p1 75 vpshufb \p3, \p1, \p1 76.endm 77 78################################ 79 80X0 = %xmm4 81X1 = %xmm5 82X2 = %xmm6 83X3 = %xmm7 84 85XTMP0 = %xmm0 86XTMP1 = %xmm1 87XTMP2 = %xmm2 88XTMP3 = %xmm3 89XTMP4 = %xmm8 90XFER = %xmm9 91XTMP5 = %xmm11 92 93SHUF_00BA = %xmm10 # shuffle xBxA -> 00BA 94SHUF_DC00 = %xmm12 # shuffle xDxC -> DC00 95BYTE_FLIP_MASK = %xmm13 96 97NUM_BLKS = %rdx # 3rd arg 98INP = %rsi # 2nd arg 99CTX = %rdi # 1st arg 100 101SRND = %rsi # clobbers INP 102c = %ecx 103d = %r8d 104e = %edx 105TBL = %r12 106a = %eax 107b = %ebx 108 109f = %r9d 110g = %r10d 111h = %r11d 112 113y0 = %r13d 114y1 = %r14d 115y2 = %r15d 116 117 118_INP_END_SIZE = 8 119_INP_SIZE = 8 120_XFER_SIZE = 16 121_XMM_SAVE_SIZE = 0 122 123_INP_END = 0 124_INP = _INP_END + _INP_END_SIZE 125_XFER = _INP + _INP_SIZE 126_XMM_SAVE = _XFER + _XFER_SIZE 127STACK_SIZE = _XMM_SAVE + _XMM_SAVE_SIZE 128 129# rotate_Xs 130# Rotate values of symbols X0...X3 131.macro rotate_Xs 132X_ = X0 133X0 = X1 134X1 = X2 135X2 = X3 136X3 = X_ 137.endm 138 139# ROTATE_ARGS 140# Rotate values of symbols a...h 141.macro ROTATE_ARGS 142TMP_ = h 143h = g 144g = f 145f = e 146e = d 147d = c 148c = b 149b = a 150a = TMP_ 151.endm 152 153.macro FOUR_ROUNDS_AND_SCHED 154 ## compute s0 four at a time and s1 two at a time 155 ## compute W[-16] + W[-7] 4 at a time 156 157 mov e, y0 # y0 = e 158 MY_ROR (25-11), y0 # y0 = e >> (25-11) 159 mov a, y1 # y1 = a 160 vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7] 161 MY_ROR (22-13), y1 # y1 = a >> (22-13) 162 xor e, y0 # y0 = e ^ (e >> (25-11)) 163 mov f, y2 # y2 = f 164 MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 165 xor a, y1 # y1 = a ^ (a >> (22-13) 166 xor g, y2 # y2 = f^g 167 vpaddd X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16] 168 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 169 and e, y2 # y2 = (f^g)&e 170 MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 171 ## compute s0 172 vpalignr $4, X0, X1, XTMP1 # XTMP1 = W[-15] 173 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 174 MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 175 xor g, y2 # y2 = CH = ((f^g)&e)^g 176 MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 177 add y0, y2 # y2 = S1 + CH 178 add _XFER(%rsp), y2 # y2 = k + w + S1 + CH 179 mov a, y0 # y0 = a 180 add y2, h # h = h + S1 + CH + k + w 181 mov a, y2 # y2 = a 182 vpsrld $7, XTMP1, XTMP2 183 or c, y0 # y0 = a|c 184 add h, d # d = d + h + S1 + CH + k + w 185 and c, y2 # y2 = a&c 186 vpslld $(32-7), XTMP1, XTMP3 187 and b, y0 # y0 = (a|c)&b 188 add y1, h # h = h + S1 + CH + k + w + S0 189 vpor XTMP2, XTMP3, XTMP3 # XTMP1 = W[-15] MY_ROR 7 190 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 191 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 192 ROTATE_ARGS 193 mov e, y0 # y0 = e 194 mov a, y1 # y1 = a 195 MY_ROR (25-11), y0 # y0 = e >> (25-11) 196 xor e, y0 # y0 = e ^ (e >> (25-11)) 197 mov f, y2 # y2 = f 198 MY_ROR (22-13), y1 # y1 = a >> (22-13) 199 vpsrld $18, XTMP1, XTMP2 # 200 xor a, y1 # y1 = a ^ (a >> (22-13) 201 MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 202 xor g, y2 # y2 = f^g 203 vpsrld $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3 204 MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 205 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 206 and e, y2 # y2 = (f^g)&e 207 MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 208 vpslld $(32-18), XTMP1, XTMP1 209 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 210 xor g, y2 # y2 = CH = ((f^g)&e)^g 211 vpxor XTMP1, XTMP3, XTMP3 # 212 add y0, y2 # y2 = S1 + CH 213 add (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH 214 MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 215 vpxor XTMP2, XTMP3, XTMP3 # XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR 216 mov a, y0 # y0 = a 217 add y2, h # h = h + S1 + CH + k + w 218 mov a, y2 # y2 = a 219 vpxor XTMP4, XTMP3, XTMP1 # XTMP1 = s0 220 or c, y0 # y0 = a|c 221 add h, d # d = d + h + S1 + CH + k + w 222 and c, y2 # y2 = a&c 223 ## compute low s1 224 vpshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} 225 and b, y0 # y0 = (a|c)&b 226 add y1, h # h = h + S1 + CH + k + w + S0 227 vpaddd XTMP1, XTMP0, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 228 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 229 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 230 ROTATE_ARGS 231 mov e, y0 # y0 = e 232 mov a, y1 # y1 = a 233 MY_ROR (25-11), y0 # y0 = e >> (25-11) 234 xor e, y0 # y0 = e ^ (e >> (25-11)) 235 MY_ROR (22-13), y1 # y1 = a >> (22-13) 236 mov f, y2 # y2 = f 237 xor a, y1 # y1 = a ^ (a >> (22-13) 238 MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 239 vpsrld $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} 240 xor g, y2 # y2 = f^g 241 vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] MY_ROR 19 {xBxA} 242 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 243 and e, y2 # y2 = (f^g)&e 244 vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] MY_ROR 17 {xBxA} 245 MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 246 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 247 xor g, y2 # y2 = CH = ((f^g)&e)^g 248 MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 249 vpxor XTMP3, XTMP2, XTMP2 # 250 add y0, y2 # y2 = S1 + CH 251 MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 252 add (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH 253 vpxor XTMP2, XTMP4, XTMP4 # XTMP4 = s1 {xBxA} 254 mov a, y0 # y0 = a 255 add y2, h # h = h + S1 + CH + k + w 256 mov a, y2 # y2 = a 257 vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA} 258 or c, y0 # y0 = a|c 259 add h, d # d = d + h + S1 + CH + k + w 260 and c, y2 # y2 = a&c 261 vpaddd XTMP4, XTMP0, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} 262 and b, y0 # y0 = (a|c)&b 263 add y1, h # h = h + S1 + CH + k + w + S0 264 ## compute high s1 265 vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC} 266 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 267 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 268 ROTATE_ARGS 269 mov e, y0 # y0 = e 270 MY_ROR (25-11), y0 # y0 = e >> (25-11) 271 mov a, y1 # y1 = a 272 MY_ROR (22-13), y1 # y1 = a >> (22-13) 273 xor e, y0 # y0 = e ^ (e >> (25-11)) 274 mov f, y2 # y2 = f 275 MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 276 vpsrld $10, XTMP2, XTMP5 # XTMP5 = W[-2] >> 10 {DDCC} 277 xor a, y1 # y1 = a ^ (a >> (22-13) 278 xor g, y2 # y2 = f^g 279 vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] MY_ROR 19 {xDxC} 280 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 281 and e, y2 # y2 = (f^g)&e 282 MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 283 vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] MY_ROR 17 {xDxC} 284 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 285 MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 286 xor g, y2 # y2 = CH = ((f^g)&e)^g 287 vpxor XTMP3, XTMP2, XTMP2 288 MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 289 add y0, y2 # y2 = S1 + CH 290 add (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH 291 vpxor XTMP2, XTMP5, XTMP5 # XTMP5 = s1 {xDxC} 292 mov a, y0 # y0 = a 293 add y2, h # h = h + S1 + CH + k + w 294 mov a, y2 # y2 = a 295 vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00} 296 or c, y0 # y0 = a|c 297 add h, d # d = d + h + S1 + CH + k + w 298 and c, y2 # y2 = a&c 299 vpaddd XTMP0, XTMP5, X0 # X0 = {W[3], W[2], W[1], W[0]} 300 and b, y0 # y0 = (a|c)&b 301 add y1, h # h = h + S1 + CH + k + w + S0 302 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 303 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 304 ROTATE_ARGS 305 rotate_Xs 306.endm 307 308## input is [rsp + _XFER + %1 * 4] 309.macro DO_ROUND round 310 mov e, y0 # y0 = e 311 MY_ROR (25-11), y0 # y0 = e >> (25-11) 312 mov a, y1 # y1 = a 313 xor e, y0 # y0 = e ^ (e >> (25-11)) 314 MY_ROR (22-13), y1 # y1 = a >> (22-13) 315 mov f, y2 # y2 = f 316 xor a, y1 # y1 = a ^ (a >> (22-13) 317 MY_ROR (11-6), y0 # y0 = (e >> (11-6)) ^ (e >> (25-6)) 318 xor g, y2 # y2 = f^g 319 xor e, y0 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) 320 MY_ROR (13-2), y1 # y1 = (a >> (13-2)) ^ (a >> (22-2)) 321 and e, y2 # y2 = (f^g)&e 322 xor a, y1 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) 323 MY_ROR 6, y0 # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) 324 xor g, y2 # y2 = CH = ((f^g)&e)^g 325 add y0, y2 # y2 = S1 + CH 326 MY_ROR 2, y1 # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) 327 offset = \round * 4 + _XFER # 328 add offset(%rsp), y2 # y2 = k + w + S1 + CH 329 mov a, y0 # y0 = a 330 add y2, h # h = h + S1 + CH + k + w 331 mov a, y2 # y2 = a 332 or c, y0 # y0 = a|c 333 add h, d # d = d + h + S1 + CH + k + w 334 and c, y2 # y2 = a&c 335 and b, y0 # y0 = (a|c)&b 336 add y1, h # h = h + S1 + CH + k + w + S0 337 or y2, y0 # y0 = MAJ = (a|c)&b)|(a&c) 338 add y0, h # h = h + S1 + CH + k + w + S0 + MAJ 339 ROTATE_ARGS 340.endm 341 342######################################################################## 343## void sha256_transform_avx(state sha256_state *state, const u8 *data, int blocks) 344## arg 1 : pointer to state 345## arg 2 : pointer to input data 346## arg 3 : Num blocks 347######################################################################## 348.text 349SYM_FUNC_START(sha256_transform_avx) 350.align 32 351 pushq %rbx 352 pushq %r12 353 pushq %r13 354 pushq %r14 355 pushq %r15 356 pushq %rbp 357 movq %rsp, %rbp 358 359 subq $STACK_SIZE, %rsp # allocate stack space 360 and $~15, %rsp # align stack pointer 361 362 shl $6, NUM_BLKS # convert to bytes 363 jz done_hash 364 add INP, NUM_BLKS # pointer to end of data 365 mov NUM_BLKS, _INP_END(%rsp) 366 367 ## load initial digest 368 mov 4*0(CTX), a 369 mov 4*1(CTX), b 370 mov 4*2(CTX), c 371 mov 4*3(CTX), d 372 mov 4*4(CTX), e 373 mov 4*5(CTX), f 374 mov 4*6(CTX), g 375 mov 4*7(CTX), h 376 377 vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK 378 vmovdqa _SHUF_00BA(%rip), SHUF_00BA 379 vmovdqa _SHUF_DC00(%rip), SHUF_DC00 380loop0: 381 lea K256(%rip), TBL 382 383 ## byte swap first 16 dwords 384 COPY_XMM_AND_BSWAP X0, 0*16(INP), BYTE_FLIP_MASK 385 COPY_XMM_AND_BSWAP X1, 1*16(INP), BYTE_FLIP_MASK 386 COPY_XMM_AND_BSWAP X2, 2*16(INP), BYTE_FLIP_MASK 387 COPY_XMM_AND_BSWAP X3, 3*16(INP), BYTE_FLIP_MASK 388 389 mov INP, _INP(%rsp) 390 391 ## schedule 48 input dwords, by doing 3 rounds of 16 each 392 mov $3, SRND 393.align 16 394loop1: 395 vpaddd (TBL), X0, XFER 396 vmovdqa XFER, _XFER(%rsp) 397 FOUR_ROUNDS_AND_SCHED 398 399 vpaddd 1*16(TBL), X0, XFER 400 vmovdqa XFER, _XFER(%rsp) 401 FOUR_ROUNDS_AND_SCHED 402 403 vpaddd 2*16(TBL), X0, XFER 404 vmovdqa XFER, _XFER(%rsp) 405 FOUR_ROUNDS_AND_SCHED 406 407 vpaddd 3*16(TBL), X0, XFER 408 vmovdqa XFER, _XFER(%rsp) 409 add $4*16, TBL 410 FOUR_ROUNDS_AND_SCHED 411 412 sub $1, SRND 413 jne loop1 414 415 mov $2, SRND 416loop2: 417 vpaddd (TBL), X0, XFER 418 vmovdqa XFER, _XFER(%rsp) 419 DO_ROUND 0 420 DO_ROUND 1 421 DO_ROUND 2 422 DO_ROUND 3 423 424 vpaddd 1*16(TBL), X1, XFER 425 vmovdqa XFER, _XFER(%rsp) 426 add $2*16, TBL 427 DO_ROUND 0 428 DO_ROUND 1 429 DO_ROUND 2 430 DO_ROUND 3 431 432 vmovdqa X2, X0 433 vmovdqa X3, X1 434 435 sub $1, SRND 436 jne loop2 437 438 addm (4*0)(CTX),a 439 addm (4*1)(CTX),b 440 addm (4*2)(CTX),c 441 addm (4*3)(CTX),d 442 addm (4*4)(CTX),e 443 addm (4*5)(CTX),f 444 addm (4*6)(CTX),g 445 addm (4*7)(CTX),h 446 447 mov _INP(%rsp), INP 448 add $64, INP 449 cmp _INP_END(%rsp), INP 450 jne loop0 451 452done_hash: 453 454 mov %rbp, %rsp 455 popq %rbp 456 popq %r15 457 popq %r14 458 popq %r13 459 popq %r12 460 popq %rbx 461 ret 462SYM_FUNC_END(sha256_transform_avx) 463 464.section .rodata.cst256.K256, "aM", @progbits, 256 465.align 64 466K256: 467 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 468 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 469 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 470 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 471 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc 472 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da 473 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 474 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 475 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 476 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 477 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 478 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 479 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 480 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 481 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 482 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 483 484.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16 485.align 16 486PSHUFFLE_BYTE_FLIP_MASK: 487 .octa 0x0c0d0e0f08090a0b0405060700010203 488 489.section .rodata.cst16._SHUF_00BA, "aM", @progbits, 16 490.align 16 491# shuffle xBxA -> 00BA 492_SHUF_00BA: 493 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100 494 495.section .rodata.cst16._SHUF_DC00, "aM", @progbits, 16 496.align 16 497# shuffle xDxC -> DC00 498_SHUF_DC00: 499 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF 500