1 /* 2 * Implement fast SHA-1 with AVX2 instructions. (x86_64) 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * Copyright(c) 2014 Intel Corporation. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * Contact Information: 21 * Ilya Albrekht <ilya.albrekht@intel.com> 22 * Maxim Locktyukhin <maxim.locktyukhin@intel.com> 23 * Ronen Zohar <ronen.zohar@intel.com> 24 * Chandramouli Narayanan <mouli@linux.intel.com> 25 * 26 * BSD LICENSE 27 * 28 * Copyright(c) 2014 Intel Corporation. 29 * 30 * Redistribution and use in source and binary forms, with or without 31 * modification, are permitted provided that the following conditions 32 * are met: 33 * 34 * Redistributions of source code must retain the above copyright 35 * notice, this list of conditions and the following disclaimer. 36 * Redistributions in binary form must reproduce the above copyright 37 * notice, this list of conditions and the following disclaimer in 38 * the documentation and/or other materials provided with the 39 * distribution. 40 * Neither the name of Intel Corporation nor the names of its 41 * contributors may be used to endorse or promote products derived 42 * from this software without specific prior written permission. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 45 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 46 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 47 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 48 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 49 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 50 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 51 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 52 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 53 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 54 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 55 * 56 */ 57 58 /* 59 * SHA-1 implementation with Intel(R) AVX2 instruction set extensions. 60 * 61 *This implementation is based on the previous SSSE3 release: 62 *Visit http://software.intel.com/en-us/articles/ 63 *and refer to improving-the-performance-of-the-secure-hash-algorithm-1/ 64 * 65 *Updates 20-byte SHA-1 record at start of 'state', from 'input', for 66 *even number of 'blocks' consecutive 64-byte blocks. 67 * 68 *extern "C" void sha1_transform_avx2( 69 * struct sha1_state *state, const u8* input, int blocks ); 70 */ 71 72 #include <linux/linkage.h> 73 74 #define CTX %rdi /* arg1 */ 75 #define BUF %rsi /* arg2 */ 76 #define CNT %rdx /* arg3 */ 77 78 #define REG_A %ecx 79 #define REG_B %esi 80 #define REG_C %edi 81 #define REG_D %eax 82 #define REG_E %edx 83 #define REG_TB %ebx 84 #define REG_TA %r12d 85 #define REG_RA %rcx 86 #define REG_RB %rsi 87 #define REG_RC %rdi 88 #define REG_RD %rax 89 #define REG_RE %rdx 90 #define REG_RTA %r12 91 #define REG_RTB %rbx 92 #define REG_T1 %r11d 93 #define xmm_mov vmovups 94 #define avx2_zeroupper vzeroupper 95 #define RND_F1 1 96 #define RND_F2 2 97 #define RND_F3 3 98 99 .macro REGALLOC 100 .set A, REG_A 101 .set B, REG_B 102 .set C, REG_C 103 .set D, REG_D 104 .set E, REG_E 105 .set TB, REG_TB 106 .set TA, REG_TA 107 108 .set RA, REG_RA 109 .set RB, REG_RB 110 .set RC, REG_RC 111 .set RD, REG_RD 112 .set RE, REG_RE 113 114 .set RTA, REG_RTA 115 .set RTB, REG_RTB 116 117 .set T1, REG_T1 118 .endm 119 120 #define HASH_PTR %r9 121 #define BLOCKS_CTR %r8 122 #define BUFFER_PTR %r10 123 #define BUFFER_PTR2 %r13 124 125 #define PRECALC_BUF %r14 126 #define WK_BUF %r15 127 128 #define W_TMP %xmm0 129 #define WY_TMP %ymm0 130 #define WY_TMP2 %ymm9 131 132 # AVX2 variables 133 #define WY0 %ymm3 134 #define WY4 %ymm5 135 #define WY08 %ymm7 136 #define WY12 %ymm8 137 #define WY16 %ymm12 138 #define WY20 %ymm13 139 #define WY24 %ymm14 140 #define WY28 %ymm15 141 142 #define YMM_SHUFB_BSWAP %ymm10 143 144 /* 145 * Keep 2 iterations precalculated at a time: 146 * - 80 DWORDs per iteration * 2 147 */ 148 #define W_SIZE (80*2*2 +16) 149 150 #define WK(t) ((((t) % 80) / 4)*32 + ( (t) % 4)*4 + ((t)/80)*16 )(WK_BUF) 151 #define PRECALC_WK(t) ((t)*2*2)(PRECALC_BUF) 152 153 154 .macro UPDATE_HASH hash, val 155 add \hash, \val 156 mov \val, \hash 157 .endm 158 159 .macro PRECALC_RESET_WY 160 .set WY_00, WY0 161 .set WY_04, WY4 162 .set WY_08, WY08 163 .set WY_12, WY12 164 .set WY_16, WY16 165 .set WY_20, WY20 166 .set WY_24, WY24 167 .set WY_28, WY28 168 .set WY_32, WY_00 169 .endm 170 171 .macro PRECALC_ROTATE_WY 172 /* Rotate macros */ 173 .set WY_32, WY_28 174 .set WY_28, WY_24 175 .set WY_24, WY_20 176 .set WY_20, WY_16 177 .set WY_16, WY_12 178 .set WY_12, WY_08 179 .set WY_08, WY_04 180 .set WY_04, WY_00 181 .set WY_00, WY_32 182 183 /* Define register aliases */ 184 .set WY, WY_00 185 .set WY_minus_04, WY_04 186 .set WY_minus_08, WY_08 187 .set WY_minus_12, WY_12 188 .set WY_minus_16, WY_16 189 .set WY_minus_20, WY_20 190 .set WY_minus_24, WY_24 191 .set WY_minus_28, WY_28 192 .set WY_minus_32, WY 193 .endm 194 195 .macro PRECALC_00_15 196 .if (i == 0) # Initialize and rotate registers 197 PRECALC_RESET_WY 198 PRECALC_ROTATE_WY 199 .endif 200 201 /* message scheduling pre-compute for rounds 0-15 */ 202 .if ((i & 7) == 0) 203 /* 204 * blended AVX2 and ALU instruction scheduling 205 * 1 vector iteration per 8 rounds 206 */ 207 vmovdqu (i * 2)(BUFFER_PTR), W_TMP 208 .elseif ((i & 7) == 1) 209 vinsertf128 $1, ((i-1) * 2)(BUFFER_PTR2),\ 210 WY_TMP, WY_TMP 211 .elseif ((i & 7) == 2) 212 vpshufb YMM_SHUFB_BSWAP, WY_TMP, WY 213 .elseif ((i & 7) == 4) 214 vpaddd K_XMM + K_XMM_AR(%rip), WY, WY_TMP 215 .elseif ((i & 7) == 7) 216 vmovdqu WY_TMP, PRECALC_WK(i&~7) 217 218 PRECALC_ROTATE_WY 219 .endif 220 .endm 221 222 .macro PRECALC_16_31 223 /* 224 * message scheduling pre-compute for rounds 16-31 225 * calculating last 32 w[i] values in 8 XMM registers 226 * pre-calculate K+w[i] values and store to mem 227 * for later load by ALU add instruction 228 * 229 * "brute force" vectorization for rounds 16-31 only 230 * due to w[i]->w[i-3] dependency 231 */ 232 .if ((i & 7) == 0) 233 /* 234 * blended AVX2 and ALU instruction scheduling 235 * 1 vector iteration per 8 rounds 236 */ 237 /* w[i-14] */ 238 vpalignr $8, WY_minus_16, WY_minus_12, WY 239 vpsrldq $4, WY_minus_04, WY_TMP /* w[i-3] */ 240 .elseif ((i & 7) == 1) 241 vpxor WY_minus_08, WY, WY 242 vpxor WY_minus_16, WY_TMP, WY_TMP 243 .elseif ((i & 7) == 2) 244 vpxor WY_TMP, WY, WY 245 vpslldq $12, WY, WY_TMP2 246 .elseif ((i & 7) == 3) 247 vpslld $1, WY, WY_TMP 248 vpsrld $31, WY, WY 249 .elseif ((i & 7) == 4) 250 vpor WY, WY_TMP, WY_TMP 251 vpslld $2, WY_TMP2, WY 252 .elseif ((i & 7) == 5) 253 vpsrld $30, WY_TMP2, WY_TMP2 254 vpxor WY, WY_TMP, WY_TMP 255 .elseif ((i & 7) == 7) 256 vpxor WY_TMP2, WY_TMP, WY 257 vpaddd K_XMM + K_XMM_AR(%rip), WY, WY_TMP 258 vmovdqu WY_TMP, PRECALC_WK(i&~7) 259 260 PRECALC_ROTATE_WY 261 .endif 262 .endm 263 264 .macro PRECALC_32_79 265 /* 266 * in SHA-1 specification: 267 * w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1 268 * instead we do equal: 269 * w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2 270 * allows more efficient vectorization 271 * since w[i]=>w[i-3] dependency is broken 272 */ 273 274 .if ((i & 7) == 0) 275 /* 276 * blended AVX2 and ALU instruction scheduling 277 * 1 vector iteration per 8 rounds 278 */ 279 vpalignr $8, WY_minus_08, WY_minus_04, WY_TMP 280 .elseif ((i & 7) == 1) 281 /* W is W_minus_32 before xor */ 282 vpxor WY_minus_28, WY, WY 283 .elseif ((i & 7) == 2) 284 vpxor WY_minus_16, WY_TMP, WY_TMP 285 .elseif ((i & 7) == 3) 286 vpxor WY_TMP, WY, WY 287 .elseif ((i & 7) == 4) 288 vpslld $2, WY, WY_TMP 289 .elseif ((i & 7) == 5) 290 vpsrld $30, WY, WY 291 vpor WY, WY_TMP, WY 292 .elseif ((i & 7) == 7) 293 vpaddd K_XMM + K_XMM_AR(%rip), WY, WY_TMP 294 vmovdqu WY_TMP, PRECALC_WK(i&~7) 295 296 PRECALC_ROTATE_WY 297 .endif 298 .endm 299 300 .macro PRECALC r, s 301 .set i, \r 302 303 .if (i < 40) 304 .set K_XMM, 32*0 305 .elseif (i < 80) 306 .set K_XMM, 32*1 307 .elseif (i < 120) 308 .set K_XMM, 32*2 309 .else 310 .set K_XMM, 32*3 311 .endif 312 313 .if (i<32) 314 PRECALC_00_15 \s 315 .elseif (i<64) 316 PRECALC_16_31 \s 317 .elseif (i < 160) 318 PRECALC_32_79 \s 319 .endif 320 .endm 321 322 .macro ROTATE_STATE 323 .set T_REG, E 324 .set E, D 325 .set D, C 326 .set C, B 327 .set B, TB 328 .set TB, A 329 .set A, T_REG 330 331 .set T_REG, RE 332 .set RE, RD 333 .set RD, RC 334 .set RC, RB 335 .set RB, RTB 336 .set RTB, RA 337 .set RA, T_REG 338 .endm 339 340 /* Macro relies on saved ROUND_Fx */ 341 342 .macro RND_FUN f, r 343 .if (\f == RND_F1) 344 ROUND_F1 \r 345 .elseif (\f == RND_F2) 346 ROUND_F2 \r 347 .elseif (\f == RND_F3) 348 ROUND_F3 \r 349 .endif 350 .endm 351 352 .macro RR r 353 .set round_id, (\r % 80) 354 355 .if (round_id == 0) /* Precalculate F for first round */ 356 .set ROUND_FUNC, RND_F1 357 mov B, TB 358 359 rorx $(32-30), B, B /* b>>>2 */ 360 andn D, TB, T1 361 and C, TB 362 xor T1, TB 363 .endif 364 365 RND_FUN ROUND_FUNC, \r 366 ROTATE_STATE 367 368 .if (round_id == 18) 369 .set ROUND_FUNC, RND_F2 370 .elseif (round_id == 38) 371 .set ROUND_FUNC, RND_F3 372 .elseif (round_id == 58) 373 .set ROUND_FUNC, RND_F2 374 .endif 375 376 .set round_id, ( (\r+1) % 80) 377 378 RND_FUN ROUND_FUNC, (\r+1) 379 ROTATE_STATE 380 .endm 381 382 .macro ROUND_F1 r 383 add WK(\r), E 384 385 andn C, A, T1 /* ~b&d */ 386 lea (RE,RTB), E /* Add F from the previous round */ 387 388 rorx $(32-5), A, TA /* T2 = A >>> 5 */ 389 rorx $(32-30),A, TB /* b>>>2 for next round */ 390 391 PRECALC (\r) /* msg scheduling for next 2 blocks */ 392 393 /* 394 * Calculate F for the next round 395 * (b & c) ^ andn[b, d] 396 */ 397 and B, A /* b&c */ 398 xor T1, A /* F1 = (b&c) ^ (~b&d) */ 399 400 lea (RE,RTA), E /* E += A >>> 5 */ 401 .endm 402 403 .macro ROUND_F2 r 404 add WK(\r), E 405 lea (RE,RTB), E /* Add F from the previous round */ 406 407 /* Calculate F for the next round */ 408 rorx $(32-5), A, TA /* T2 = A >>> 5 */ 409 .if ((round_id) < 79) 410 rorx $(32-30), A, TB /* b>>>2 for next round */ 411 .endif 412 PRECALC (\r) /* msg scheduling for next 2 blocks */ 413 414 .if ((round_id) < 79) 415 xor B, A 416 .endif 417 418 add TA, E /* E += A >>> 5 */ 419 420 .if ((round_id) < 79) 421 xor C, A 422 .endif 423 .endm 424 425 .macro ROUND_F3 r 426 add WK(\r), E 427 PRECALC (\r) /* msg scheduling for next 2 blocks */ 428 429 lea (RE,RTB), E /* Add F from the previous round */ 430 431 mov B, T1 432 or A, T1 433 434 rorx $(32-5), A, TA /* T2 = A >>> 5 */ 435 rorx $(32-30), A, TB /* b>>>2 for next round */ 436 437 /* Calculate F for the next round 438 * (b and c) or (d and (b or c)) 439 */ 440 and C, T1 441 and B, A 442 or T1, A 443 444 add TA, E /* E += A >>> 5 */ 445 446 .endm 447 448 /* Add constant only if (%2 > %3) condition met (uses RTA as temp) 449 * %1 + %2 >= %3 ? %4 : 0 450 */ 451 .macro ADD_IF_GE a, b, c, d 452 mov \a, RTA 453 add $\d, RTA 454 cmp $\c, \b 455 cmovge RTA, \a 456 .endm 457 458 /* 459 * macro implements 80 rounds of SHA-1, for multiple blocks with s/w pipelining 460 */ 461 .macro SHA1_PIPELINED_MAIN_BODY 462 463 REGALLOC 464 465 mov (HASH_PTR), A 466 mov 4(HASH_PTR), B 467 mov 8(HASH_PTR), C 468 mov 12(HASH_PTR), D 469 mov 16(HASH_PTR), E 470 471 mov %rsp, PRECALC_BUF 472 lea (2*4*80+32)(%rsp), WK_BUF 473 474 # Precalc WK for first 2 blocks 475 ADD_IF_GE BUFFER_PTR2, BLOCKS_CTR, 2, 64 476 .set i, 0 477 .rept 160 478 PRECALC i 479 .set i, i + 1 480 .endr 481 482 /* Go to next block if needed */ 483 ADD_IF_GE BUFFER_PTR, BLOCKS_CTR, 3, 128 484 ADD_IF_GE BUFFER_PTR2, BLOCKS_CTR, 4, 128 485 xchg WK_BUF, PRECALC_BUF 486 487 .align 32 488 .L_loop: 489 /* 490 * code loops through more than one block 491 * we use K_BASE value as a signal of a last block, 492 * it is set below by: cmovae BUFFER_PTR, K_BASE 493 */ 494 test BLOCKS_CTR, BLOCKS_CTR 495 jnz .L_begin 496 .align 32 497 jmp .L_end 498 .align 32 499 .L_begin: 500 501 /* 502 * Do first block 503 * rounds: 0,2,4,6,8 504 */ 505 .set j, 0 506 .rept 5 507 RR j 508 .set j, j+2 509 .endr 510 511 /* 512 * rounds: 513 * 10,12,14,16,18 514 * 20,22,24,26,28 515 * 30,32,34,36,38 516 * 40,42,44,46,48 517 * 50,52,54,56,58 518 */ 519 .rept 25 520 RR j 521 .set j, j+2 522 .endr 523 524 /* Update Counter */ 525 sub $1, BLOCKS_CTR 526 /* Move to the next block only if needed*/ 527 ADD_IF_GE BUFFER_PTR, BLOCKS_CTR, 4, 128 528 /* 529 * rounds 530 * 60,62,64,66,68 531 * 70,72,74,76,78 532 */ 533 .rept 10 534 RR j 535 .set j, j+2 536 .endr 537 538 UPDATE_HASH (HASH_PTR), A 539 UPDATE_HASH 4(HASH_PTR), TB 540 UPDATE_HASH 8(HASH_PTR), C 541 UPDATE_HASH 12(HASH_PTR), D 542 UPDATE_HASH 16(HASH_PTR), E 543 544 test BLOCKS_CTR, BLOCKS_CTR 545 jz .L_loop 546 547 mov TB, B 548 549 /* Process second block */ 550 /* 551 * rounds 552 * 0+80, 2+80, 4+80, 6+80, 8+80 553 * 10+80,12+80,14+80,16+80,18+80 554 */ 555 556 .set j, 0 557 .rept 10 558 RR j+80 559 .set j, j+2 560 .endr 561 562 /* 563 * rounds 564 * 20+80,22+80,24+80,26+80,28+80 565 * 30+80,32+80,34+80,36+80,38+80 566 */ 567 .rept 10 568 RR j+80 569 .set j, j+2 570 .endr 571 572 /* 573 * rounds 574 * 40+80,42+80,44+80,46+80,48+80 575 * 50+80,52+80,54+80,56+80,58+80 576 */ 577 .rept 10 578 RR j+80 579 .set j, j+2 580 .endr 581 582 /* update counter */ 583 sub $1, BLOCKS_CTR 584 /* Move to the next block only if needed*/ 585 ADD_IF_GE BUFFER_PTR2, BLOCKS_CTR, 4, 128 586 587 /* 588 * rounds 589 * 60+80,62+80,64+80,66+80,68+80 590 * 70+80,72+80,74+80,76+80,78+80 591 */ 592 .rept 10 593 RR j+80 594 .set j, j+2 595 .endr 596 597 UPDATE_HASH (HASH_PTR), A 598 UPDATE_HASH 4(HASH_PTR), TB 599 UPDATE_HASH 8(HASH_PTR), C 600 UPDATE_HASH 12(HASH_PTR), D 601 UPDATE_HASH 16(HASH_PTR), E 602 603 /* Reset state for AVX2 reg permutation */ 604 mov A, TA 605 mov TB, A 606 mov C, TB 607 mov E, C 608 mov D, B 609 mov TA, D 610 611 REGALLOC 612 613 xchg WK_BUF, PRECALC_BUF 614 615 jmp .L_loop 616 617 .align 32 618 .L_end: 619 620 .endm 621 /* 622 * macro implements SHA-1 function's body for several 64-byte blocks 623 * param: function's name 624 */ 625 .macro SHA1_VECTOR_ASM name 626 SYM_FUNC_START(\name) 627 628 push %rbx 629 push %r12 630 push %r13 631 push %r14 632 push %r15 633 634 RESERVE_STACK = (W_SIZE*4 + 8+24) 635 636 /* Align stack */ 637 push %rbp 638 mov %rsp, %rbp 639 and $~(0x20-1), %rsp 640 sub $RESERVE_STACK, %rsp 641 642 avx2_zeroupper 643 644 /* Setup initial values */ 645 mov CTX, HASH_PTR 646 mov BUF, BUFFER_PTR 647 648 mov BUF, BUFFER_PTR2 649 mov CNT, BLOCKS_CTR 650 651 xmm_mov BSWAP_SHUFB_CTL(%rip), YMM_SHUFB_BSWAP 652 653 SHA1_PIPELINED_MAIN_BODY 654 655 avx2_zeroupper 656 657 mov %rbp, %rsp 658 pop %rbp 659 660 pop %r15 661 pop %r14 662 pop %r13 663 pop %r12 664 pop %rbx 665 666 RET 667 668 SYM_FUNC_END(\name) 669 .endm 670 671 .section .rodata 672 673 #define K1 0x5a827999 674 #define K2 0x6ed9eba1 675 #define K3 0x8f1bbcdc 676 #define K4 0xca62c1d6 677 678 .align 128 679 K_XMM_AR: 680 .long K1, K1, K1, K1 681 .long K1, K1, K1, K1 682 .long K2, K2, K2, K2 683 .long K2, K2, K2, K2 684 .long K3, K3, K3, K3 685 .long K3, K3, K3, K3 686 .long K4, K4, K4, K4 687 .long K4, K4, K4, K4 688 689 BSWAP_SHUFB_CTL: 690 .long 0x00010203 691 .long 0x04050607 692 .long 0x08090a0b 693 .long 0x0c0d0e0f 694 .long 0x00010203 695 .long 0x04050607 696 .long 0x08090a0b 697 .long 0x0c0d0e0f 698 .text 699 700 SHA1_VECTOR_ASM sha1_transform_avx2 701