1bb6c8c46SArd Biesheuvel/* SPDX-License-Identifier: GPL-2.0 */ 2bb6c8c46SArd Biesheuvel/* 3bb6c8c46SArd Biesheuvel * sha512-ce-core.S - core SHA-384/SHA-512 transform using v8 Crypto Extensions 4bb6c8c46SArd Biesheuvel * 5bb6c8c46SArd Biesheuvel * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 6bb6c8c46SArd Biesheuvel * 7bb6c8c46SArd Biesheuvel * This program is free software; you can redistribute it and/or modify 8bb6c8c46SArd Biesheuvel * it under the terms of the GNU General Public License version 2 as 9bb6c8c46SArd Biesheuvel * published by the Free Software Foundation. 10bb6c8c46SArd Biesheuvel */ 11bb6c8c46SArd Biesheuvel 12bb6c8c46SArd Biesheuvel#include <linux/linkage.h> 13bb6c8c46SArd Biesheuvel#include <asm/assembler.h> 14bb6c8c46SArd Biesheuvel 15fb87127bSArd Biesheuvel .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19 16bb6c8c46SArd Biesheuvel .set .Lq\b, \b 17bb6c8c46SArd Biesheuvel .set .Lv\b\().2d, \b 18bb6c8c46SArd Biesheuvel .endr 19bb6c8c46SArd Biesheuvel 20bb6c8c46SArd Biesheuvel .macro sha512h, rd, rn, rm 21bb6c8c46SArd Biesheuvel .inst 0xce608000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) 22bb6c8c46SArd Biesheuvel .endm 23bb6c8c46SArd Biesheuvel 24bb6c8c46SArd Biesheuvel .macro sha512h2, rd, rn, rm 25bb6c8c46SArd Biesheuvel .inst 0xce608400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) 26bb6c8c46SArd Biesheuvel .endm 27bb6c8c46SArd Biesheuvel 28bb6c8c46SArd Biesheuvel .macro sha512su0, rd, rn 29bb6c8c46SArd Biesheuvel .inst 0xcec08000 | .L\rd | (.L\rn << 5) 30bb6c8c46SArd Biesheuvel .endm 31bb6c8c46SArd Biesheuvel 32bb6c8c46SArd Biesheuvel .macro sha512su1, rd, rn, rm 33bb6c8c46SArd Biesheuvel .inst 0xce608800 | .L\rd | (.L\rn << 5) | (.L\rm << 16) 34bb6c8c46SArd Biesheuvel .endm 35bb6c8c46SArd Biesheuvel 36bb6c8c46SArd Biesheuvel /* 37bb6c8c46SArd Biesheuvel * The SHA-512 round constants 38bb6c8c46SArd Biesheuvel */ 39fb87127bSArd Biesheuvel .section ".rodata", "a" 40bb6c8c46SArd Biesheuvel .align 4 41bb6c8c46SArd Biesheuvel.Lsha512_rcon: 42bb6c8c46SArd Biesheuvel .quad 0x428a2f98d728ae22, 0x7137449123ef65cd 43bb6c8c46SArd Biesheuvel .quad 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc 44bb6c8c46SArd Biesheuvel .quad 0x3956c25bf348b538, 0x59f111f1b605d019 45bb6c8c46SArd Biesheuvel .quad 0x923f82a4af194f9b, 0xab1c5ed5da6d8118 46bb6c8c46SArd Biesheuvel .quad 0xd807aa98a3030242, 0x12835b0145706fbe 47bb6c8c46SArd Biesheuvel .quad 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2 48bb6c8c46SArd Biesheuvel .quad 0x72be5d74f27b896f, 0x80deb1fe3b1696b1 49bb6c8c46SArd Biesheuvel .quad 0x9bdc06a725c71235, 0xc19bf174cf692694 50bb6c8c46SArd Biesheuvel .quad 0xe49b69c19ef14ad2, 0xefbe4786384f25e3 51bb6c8c46SArd Biesheuvel .quad 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65 52bb6c8c46SArd Biesheuvel .quad 0x2de92c6f592b0275, 0x4a7484aa6ea6e483 53bb6c8c46SArd Biesheuvel .quad 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5 54bb6c8c46SArd Biesheuvel .quad 0x983e5152ee66dfab, 0xa831c66d2db43210 55bb6c8c46SArd Biesheuvel .quad 0xb00327c898fb213f, 0xbf597fc7beef0ee4 56bb6c8c46SArd Biesheuvel .quad 0xc6e00bf33da88fc2, 0xd5a79147930aa725 57bb6c8c46SArd Biesheuvel .quad 0x06ca6351e003826f, 0x142929670a0e6e70 58bb6c8c46SArd Biesheuvel .quad 0x27b70a8546d22ffc, 0x2e1b21385c26c926 59bb6c8c46SArd Biesheuvel .quad 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df 60bb6c8c46SArd Biesheuvel .quad 0x650a73548baf63de, 0x766a0abb3c77b2a8 61bb6c8c46SArd Biesheuvel .quad 0x81c2c92e47edaee6, 0x92722c851482353b 62bb6c8c46SArd Biesheuvel .quad 0xa2bfe8a14cf10364, 0xa81a664bbc423001 63bb6c8c46SArd Biesheuvel .quad 0xc24b8b70d0f89791, 0xc76c51a30654be30 64bb6c8c46SArd Biesheuvel .quad 0xd192e819d6ef5218, 0xd69906245565a910 65bb6c8c46SArd Biesheuvel .quad 0xf40e35855771202a, 0x106aa07032bbd1b8 66bb6c8c46SArd Biesheuvel .quad 0x19a4c116b8d2d0c8, 0x1e376c085141ab53 67bb6c8c46SArd Biesheuvel .quad 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8 68bb6c8c46SArd Biesheuvel .quad 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb 69bb6c8c46SArd Biesheuvel .quad 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3 70bb6c8c46SArd Biesheuvel .quad 0x748f82ee5defb2fc, 0x78a5636f43172f60 71bb6c8c46SArd Biesheuvel .quad 0x84c87814a1f0ab72, 0x8cc702081a6439ec 72bb6c8c46SArd Biesheuvel .quad 0x90befffa23631e28, 0xa4506cebde82bde9 73bb6c8c46SArd Biesheuvel .quad 0xbef9a3f7b2c67915, 0xc67178f2e372532b 74bb6c8c46SArd Biesheuvel .quad 0xca273eceea26619c, 0xd186b8c721c0c207 75bb6c8c46SArd Biesheuvel .quad 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178 76bb6c8c46SArd Biesheuvel .quad 0x06f067aa72176fba, 0x0a637dc5a2c898a6 77bb6c8c46SArd Biesheuvel .quad 0x113f9804bef90dae, 0x1b710b35131c471b 78bb6c8c46SArd Biesheuvel .quad 0x28db77f523047d84, 0x32caab7b40c72493 79bb6c8c46SArd Biesheuvel .quad 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c 80bb6c8c46SArd Biesheuvel .quad 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a 81bb6c8c46SArd Biesheuvel .quad 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 82bb6c8c46SArd Biesheuvel 83bb6c8c46SArd Biesheuvel .macro dround, i0, i1, i2, i3, i4, rc0, rc1, in0, in1, in2, in3, in4 84bb6c8c46SArd Biesheuvel .ifnb \rc1 85fb87127bSArd Biesheuvel ld1 {v\rc1\().2d}, [x4], #16 86bb6c8c46SArd Biesheuvel .endif 87fb87127bSArd Biesheuvel add v5.2d, v\rc0\().2d, v\in0\().2d 88bb6c8c46SArd Biesheuvel ext v6.16b, v\i2\().16b, v\i3\().16b, #8 89fb87127bSArd Biesheuvel ext v5.16b, v5.16b, v5.16b, #8 90bb6c8c46SArd Biesheuvel ext v7.16b, v\i1\().16b, v\i2\().16b, #8 91fb87127bSArd Biesheuvel add v\i3\().2d, v\i3\().2d, v5.2d 92bb6c8c46SArd Biesheuvel .ifnb \in1 93fb87127bSArd Biesheuvel ext v5.16b, v\in3\().16b, v\in4\().16b, #8 94bb6c8c46SArd Biesheuvel sha512su0 v\in0\().2d, v\in1\().2d 95bb6c8c46SArd Biesheuvel .endif 96bb6c8c46SArd Biesheuvel sha512h q\i3, q6, v7.2d 97bb6c8c46SArd Biesheuvel .ifnb \in1 98fb87127bSArd Biesheuvel sha512su1 v\in0\().2d, v\in2\().2d, v5.2d 99bb6c8c46SArd Biesheuvel .endif 100bb6c8c46SArd Biesheuvel add v\i4\().2d, v\i1\().2d, v\i3\().2d 101bb6c8c46SArd Biesheuvel sha512h2 q\i3, q\i1, v\i0\().2d 102bb6c8c46SArd Biesheuvel .endm 103bb6c8c46SArd Biesheuvel 104bb6c8c46SArd Biesheuvel /* 105bb6c8c46SArd Biesheuvel * void sha512_ce_transform(struct sha512_state *sst, u8 const *src, 106bb6c8c46SArd Biesheuvel * int blocks) 107bb6c8c46SArd Biesheuvel */ 108fb87127bSArd Biesheuvel .text 1090e89640bSMark BrownSYM_FUNC_START(sha512_ce_transform) 110bb6c8c46SArd Biesheuvel /* load state */ 1115f6cb2e6SArd Biesheuvel ld1 {v8.2d-v11.2d}, [x0] 112fb87127bSArd Biesheuvel 113fb87127bSArd Biesheuvel /* load first 4 round constants */ 114fb87127bSArd Biesheuvel adr_l x3, .Lsha512_rcon 115fb87127bSArd Biesheuvel ld1 {v20.2d-v23.2d}, [x3], #64 116bb6c8c46SArd Biesheuvel 117bb6c8c46SArd Biesheuvel /* load input */ 1185f6cb2e6SArd Biesheuvel0: ld1 {v12.2d-v15.2d}, [x1], #64 1195f6cb2e6SArd Biesheuvel ld1 {v16.2d-v19.2d}, [x1], #64 1205f6cb2e6SArd Biesheuvel sub w2, w2, #1 121bb6c8c46SArd Biesheuvel 122bb6c8c46SArd BiesheuvelCPU_LE( rev64 v12.16b, v12.16b ) 123bb6c8c46SArd BiesheuvelCPU_LE( rev64 v13.16b, v13.16b ) 124bb6c8c46SArd BiesheuvelCPU_LE( rev64 v14.16b, v14.16b ) 125bb6c8c46SArd BiesheuvelCPU_LE( rev64 v15.16b, v15.16b ) 126bb6c8c46SArd BiesheuvelCPU_LE( rev64 v16.16b, v16.16b ) 127bb6c8c46SArd BiesheuvelCPU_LE( rev64 v17.16b, v17.16b ) 128bb6c8c46SArd BiesheuvelCPU_LE( rev64 v18.16b, v18.16b ) 129bb6c8c46SArd BiesheuvelCPU_LE( rev64 v19.16b, v19.16b ) 130bb6c8c46SArd Biesheuvel 131fb87127bSArd Biesheuvel mov x4, x3 // rc pointer 132bb6c8c46SArd Biesheuvel 133fb87127bSArd Biesheuvel mov v0.16b, v8.16b 134fb87127bSArd Biesheuvel mov v1.16b, v9.16b 135fb87127bSArd Biesheuvel mov v2.16b, v10.16b 136fb87127bSArd Biesheuvel mov v3.16b, v11.16b 137bb6c8c46SArd Biesheuvel 138bb6c8c46SArd Biesheuvel // v0 ab cd -- ef gh ab 139bb6c8c46SArd Biesheuvel // v1 cd -- ef gh ab cd 140bb6c8c46SArd Biesheuvel // v2 ef gh ab cd -- ef 141bb6c8c46SArd Biesheuvel // v3 gh ab cd -- ef gh 142bb6c8c46SArd Biesheuvel // v4 -- ef gh ab cd -- 143bb6c8c46SArd Biesheuvel 144fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 20, 24, 12, 13, 19, 16, 17 145fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 21, 25, 13, 14, 12, 17, 18 146fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 22, 26, 14, 15, 13, 18, 19 147fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 23, 27, 15, 16, 14, 19, 12 148fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 24, 28, 16, 17, 15, 12, 13 149bb6c8c46SArd Biesheuvel 150fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 25, 29, 17, 18, 16, 13, 14 151fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 26, 30, 18, 19, 17, 14, 15 152fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 27, 31, 19, 12, 18, 15, 16 153fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 28, 24, 12, 13, 19, 16, 17 154fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 29, 25, 13, 14, 12, 17, 18 155bb6c8c46SArd Biesheuvel 156fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 30, 26, 14, 15, 13, 18, 19 157fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 31, 27, 15, 16, 14, 19, 12 158fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 24, 28, 16, 17, 15, 12, 13 159fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 25, 29, 17, 18, 16, 13, 14 160fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 26, 30, 18, 19, 17, 14, 15 161bb6c8c46SArd Biesheuvel 162fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 27, 31, 19, 12, 18, 15, 16 163fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 28, 24, 12, 13, 19, 16, 17 164fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 29, 25, 13, 14, 12, 17, 18 165fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 30, 26, 14, 15, 13, 18, 19 166fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 31, 27, 15, 16, 14, 19, 12 167bb6c8c46SArd Biesheuvel 168fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 24, 28, 16, 17, 15, 12, 13 169fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 25, 29, 17, 18, 16, 13, 14 170fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 26, 30, 18, 19, 17, 14, 15 171fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 27, 31, 19, 12, 18, 15, 16 172fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 28, 24, 12, 13, 19, 16, 17 173bb6c8c46SArd Biesheuvel 174fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 29, 25, 13, 14, 12, 17, 18 175fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 30, 26, 14, 15, 13, 18, 19 176fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 31, 27, 15, 16, 14, 19, 12 177fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 24, 28, 16, 17, 15, 12, 13 178fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 25, 29, 17, 18, 16, 13, 14 179bb6c8c46SArd Biesheuvel 180fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 26, 30, 18, 19, 17, 14, 15 181fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 27, 31, 19, 12, 18, 15, 16 182fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 28, 24, 12 183fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 29, 25, 13 184fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 30, 26, 14 185bb6c8c46SArd Biesheuvel 186fb87127bSArd Biesheuvel dround 0, 1, 2, 3, 4, 31, 27, 15 187fb87127bSArd Biesheuvel dround 3, 0, 4, 2, 1, 24, , 16 188fb87127bSArd Biesheuvel dround 2, 3, 1, 4, 0, 25, , 17 189fb87127bSArd Biesheuvel dround 4, 2, 0, 1, 3, 26, , 18 190fb87127bSArd Biesheuvel dround 1, 4, 3, 0, 2, 27, , 19 191bb6c8c46SArd Biesheuvel 192bb6c8c46SArd Biesheuvel /* update state */ 193fb87127bSArd Biesheuvel add v8.2d, v8.2d, v0.2d 194fb87127bSArd Biesheuvel add v9.2d, v9.2d, v1.2d 195fb87127bSArd Biesheuvel add v10.2d, v10.2d, v2.2d 196fb87127bSArd Biesheuvel add v11.2d, v11.2d, v3.2d 197bb6c8c46SArd Biesheuvel 198*13150149SArd Biesheuvel cond_yield 3f, x4, x5 199bb6c8c46SArd Biesheuvel /* handled all input blocks? */ 2005f6cb2e6SArd Biesheuvel cbnz w2, 0b 201bb6c8c46SArd Biesheuvel 202bb6c8c46SArd Biesheuvel /* store new state */ 2035f6cb2e6SArd Biesheuvel3: st1 {v8.2d-v11.2d}, [x0] 2045f6cb2e6SArd Biesheuvel mov w0, w2 205bb6c8c46SArd Biesheuvel ret 2060e89640bSMark BrownSYM_FUNC_END(sha512_ce_transform) 207