1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions 4 * 5 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #include <asm/hwcap.h> 13 #include <asm/neon.h> 14 #include <asm/simd.h> 15 #include <asm/unaligned.h> 16 #include <crypto/internal/hash.h> 17 #include <crypto/internal/simd.h> 18 #include <crypto/sha3.h> 19 #include <linux/cpufeature.h> 20 #include <linux/crypto.h> 21 #include <linux/module.h> 22 23 MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions"); 24 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 25 MODULE_LICENSE("GPL v2"); 26 MODULE_ALIAS_CRYPTO("sha3-224"); 27 MODULE_ALIAS_CRYPTO("sha3-256"); 28 MODULE_ALIAS_CRYPTO("sha3-384"); 29 MODULE_ALIAS_CRYPTO("sha3-512"); 30 31 asmlinkage int sha3_ce_transform(u64 *st, const u8 *data, int blocks, 32 int md_len); 33 34 static int sha3_update(struct shash_desc *desc, const u8 *data, 35 unsigned int len) 36 { 37 struct sha3_state *sctx = shash_desc_ctx(desc); 38 unsigned int digest_size = crypto_shash_digestsize(desc->tfm); 39 40 if (!crypto_simd_usable()) 41 return crypto_sha3_update(desc, data, len); 42 43 if ((sctx->partial + len) >= sctx->rsiz) { 44 int blocks; 45 46 if (sctx->partial) { 47 int p = sctx->rsiz - sctx->partial; 48 49 memcpy(sctx->buf + sctx->partial, data, p); 50 kernel_neon_begin(); 51 sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); 52 kernel_neon_end(); 53 54 data += p; 55 len -= p; 56 sctx->partial = 0; 57 } 58 59 blocks = len / sctx->rsiz; 60 len %= sctx->rsiz; 61 62 while (blocks) { 63 int rem; 64 65 kernel_neon_begin(); 66 rem = sha3_ce_transform(sctx->st, data, blocks, 67 digest_size); 68 kernel_neon_end(); 69 data += (blocks - rem) * sctx->rsiz; 70 blocks = rem; 71 } 72 } 73 74 if (len) { 75 memcpy(sctx->buf + sctx->partial, data, len); 76 sctx->partial += len; 77 } 78 return 0; 79 } 80 81 static int sha3_final(struct shash_desc *desc, u8 *out) 82 { 83 struct sha3_state *sctx = shash_desc_ctx(desc); 84 unsigned int digest_size = crypto_shash_digestsize(desc->tfm); 85 __le64 *digest = (__le64 *)out; 86 int i; 87 88 if (!crypto_simd_usable()) 89 return crypto_sha3_final(desc, out); 90 91 sctx->buf[sctx->partial++] = 0x06; 92 memset(sctx->buf + sctx->partial, 0, sctx->rsiz - sctx->partial); 93 sctx->buf[sctx->rsiz - 1] |= 0x80; 94 95 kernel_neon_begin(); 96 sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size); 97 kernel_neon_end(); 98 99 for (i = 0; i < digest_size / 8; i++) 100 put_unaligned_le64(sctx->st[i], digest++); 101 102 if (digest_size & 4) 103 put_unaligned_le32(sctx->st[i], (__le32 *)digest); 104 105 memzero_explicit(sctx, sizeof(*sctx)); 106 return 0; 107 } 108 109 static struct shash_alg algs[] = { { 110 .digestsize = SHA3_224_DIGEST_SIZE, 111 .init = crypto_sha3_init, 112 .update = sha3_update, 113 .final = sha3_final, 114 .descsize = sizeof(struct sha3_state), 115 .base.cra_name = "sha3-224", 116 .base.cra_driver_name = "sha3-224-ce", 117 .base.cra_blocksize = SHA3_224_BLOCK_SIZE, 118 .base.cra_module = THIS_MODULE, 119 .base.cra_priority = 200, 120 }, { 121 .digestsize = SHA3_256_DIGEST_SIZE, 122 .init = crypto_sha3_init, 123 .update = sha3_update, 124 .final = sha3_final, 125 .descsize = sizeof(struct sha3_state), 126 .base.cra_name = "sha3-256", 127 .base.cra_driver_name = "sha3-256-ce", 128 .base.cra_blocksize = SHA3_256_BLOCK_SIZE, 129 .base.cra_module = THIS_MODULE, 130 .base.cra_priority = 200, 131 }, { 132 .digestsize = SHA3_384_DIGEST_SIZE, 133 .init = crypto_sha3_init, 134 .update = sha3_update, 135 .final = sha3_final, 136 .descsize = sizeof(struct sha3_state), 137 .base.cra_name = "sha3-384", 138 .base.cra_driver_name = "sha3-384-ce", 139 .base.cra_blocksize = SHA3_384_BLOCK_SIZE, 140 .base.cra_module = THIS_MODULE, 141 .base.cra_priority = 200, 142 }, { 143 .digestsize = SHA3_512_DIGEST_SIZE, 144 .init = crypto_sha3_init, 145 .update = sha3_update, 146 .final = sha3_final, 147 .descsize = sizeof(struct sha3_state), 148 .base.cra_name = "sha3-512", 149 .base.cra_driver_name = "sha3-512-ce", 150 .base.cra_blocksize = SHA3_512_BLOCK_SIZE, 151 .base.cra_module = THIS_MODULE, 152 .base.cra_priority = 200, 153 } }; 154 155 static int __init sha3_neon_mod_init(void) 156 { 157 return crypto_register_shashes(algs, ARRAY_SIZE(algs)); 158 } 159 160 static void __exit sha3_neon_mod_fini(void) 161 { 162 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 163 } 164 165 module_cpu_feature_match(SHA3, sha3_neon_mod_init); 166 module_exit(sha3_neon_mod_fini); 167