1/* 2 * sha2-ce-core.S - core SHA-224/SHA-256 transform using v8 Crypto Extensions 3 * 4 * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11#include <linux/linkage.h> 12#include <asm/assembler.h> 13 14 .text 15 .arch armv8-a+crypto 16 17 dga .req q20 18 dgav .req v20 19 dgb .req q21 20 dgbv .req v21 21 22 t0 .req v22 23 t1 .req v23 24 25 dg0q .req q24 26 dg0v .req v24 27 dg1q .req q25 28 dg1v .req v25 29 dg2q .req q26 30 dg2v .req v26 31 32 .macro add_only, ev, rc, s0 33 mov dg2v.16b, dg0v.16b 34 .ifeq \ev 35 add t1.4s, v\s0\().4s, \rc\().4s 36 sha256h dg0q, dg1q, t0.4s 37 sha256h2 dg1q, dg2q, t0.4s 38 .else 39 .ifnb \s0 40 add t0.4s, v\s0\().4s, \rc\().4s 41 .endif 42 sha256h dg0q, dg1q, t1.4s 43 sha256h2 dg1q, dg2q, t1.4s 44 .endif 45 .endm 46 47 .macro add_update, ev, rc, s0, s1, s2, s3 48 sha256su0 v\s0\().4s, v\s1\().4s 49 add_only \ev, \rc, \s1 50 sha256su1 v\s0\().4s, v\s2\().4s, v\s3\().4s 51 .endm 52 53 /* 54 * The SHA-256 round constants 55 */ 56 .section ".rodata", "a" 57 .align 4 58.Lsha2_rcon: 59 .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 60 .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 61 .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 62 .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 63 .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc 64 .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da 65 .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 66 .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 67 .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 68 .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 69 .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 70 .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 71 .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 72 .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 73 .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 74 .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 75 76 /* 77 * void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, 78 * int blocks) 79 */ 80 .text 81ENTRY(sha2_ce_transform) 82 frame_push 3 83 84 mov x19, x0 85 mov x20, x1 86 mov x21, x2 87 88 /* load round constants */ 890: adr_l x8, .Lsha2_rcon 90 ld1 { v0.4s- v3.4s}, [x8], #64 91 ld1 { v4.4s- v7.4s}, [x8], #64 92 ld1 { v8.4s-v11.4s}, [x8], #64 93 ld1 {v12.4s-v15.4s}, [x8] 94 95 /* load state */ 96 ld1 {dgav.4s, dgbv.4s}, [x19] 97 98 /* load sha256_ce_state::finalize */ 99 ldr_l w4, sha256_ce_offsetof_finalize, x4 100 ldr w4, [x19, x4] 101 102 /* load input */ 1031: ld1 {v16.4s-v19.4s}, [x20], #64 104 sub w21, w21, #1 105 106CPU_LE( rev32 v16.16b, v16.16b ) 107CPU_LE( rev32 v17.16b, v17.16b ) 108CPU_LE( rev32 v18.16b, v18.16b ) 109CPU_LE( rev32 v19.16b, v19.16b ) 110 1112: add t0.4s, v16.4s, v0.4s 112 mov dg0v.16b, dgav.16b 113 mov dg1v.16b, dgbv.16b 114 115 add_update 0, v1, 16, 17, 18, 19 116 add_update 1, v2, 17, 18, 19, 16 117 add_update 0, v3, 18, 19, 16, 17 118 add_update 1, v4, 19, 16, 17, 18 119 120 add_update 0, v5, 16, 17, 18, 19 121 add_update 1, v6, 17, 18, 19, 16 122 add_update 0, v7, 18, 19, 16, 17 123 add_update 1, v8, 19, 16, 17, 18 124 125 add_update 0, v9, 16, 17, 18, 19 126 add_update 1, v10, 17, 18, 19, 16 127 add_update 0, v11, 18, 19, 16, 17 128 add_update 1, v12, 19, 16, 17, 18 129 130 add_only 0, v13, 17 131 add_only 1, v14, 18 132 add_only 0, v15, 19 133 add_only 1 134 135 /* update state */ 136 add dgav.4s, dgav.4s, dg0v.4s 137 add dgbv.4s, dgbv.4s, dg1v.4s 138 139 /* handled all input blocks? */ 140 cbz w21, 3f 141 142 if_will_cond_yield_neon 143 st1 {dgav.4s, dgbv.4s}, [x19] 144 do_cond_yield_neon 145 b 0b 146 endif_yield_neon 147 148 b 1b 149 150 /* 151 * Final block: add padding and total bit count. 152 * Skip if the input size was not a round multiple of the block size, 153 * the padding is handled by the C code in that case. 154 */ 1553: cbz x4, 4f 156 ldr_l w4, sha256_ce_offsetof_count, x4 157 ldr x4, [x19, x4] 158 movi v17.2d, #0 159 mov x8, #0x80000000 160 movi v18.2d, #0 161 ror x7, x4, #29 // ror(lsl(x4, 3), 32) 162 fmov d16, x8 163 mov x4, #0 164 mov v19.d[0], xzr 165 mov v19.d[1], x7 166 b 2b 167 168 /* store new state */ 1694: st1 {dgav.4s, dgbv.4s}, [x19] 170 frame_pop 171 ret 172ENDPROC(sha2_ce_transform) 173