1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 22c98833aSArd Biesheuvel /* 32c98833aSArd Biesheuvel * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions 42c98833aSArd Biesheuvel * 50771f323SArd Biesheuvel * Copyright (C) 2014 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 62c98833aSArd Biesheuvel */ 72c98833aSArd Biesheuvel 82c98833aSArd Biesheuvel #include <asm/neon.h> 90771f323SArd Biesheuvel #include <asm/simd.h> 102c98833aSArd Biesheuvel #include <asm/unaligned.h> 112c98833aSArd Biesheuvel #include <crypto/internal/hash.h> 12e52b7023SEric Biggers #include <crypto/internal/simd.h> 13a24d22b2SEric Biggers #include <crypto/sha1.h> 1407eb54d3SArd Biesheuvel #include <crypto/sha1_base.h> 152c98833aSArd Biesheuvel #include <linux/cpufeature.h> 162c98833aSArd Biesheuvel #include <linux/crypto.h> 172c98833aSArd Biesheuvel #include <linux/module.h> 182c98833aSArd Biesheuvel 192c98833aSArd Biesheuvel MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); 202c98833aSArd Biesheuvel MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 212c98833aSArd Biesheuvel MODULE_LICENSE("GPL v2"); 22*0df07d81SArd Biesheuvel MODULE_ALIAS_CRYPTO("sha1"); 232c98833aSArd Biesheuvel 2407eb54d3SArd Biesheuvel struct sha1_ce_state { 2507eb54d3SArd Biesheuvel struct sha1_state sst; 2607eb54d3SArd Biesheuvel u32 finalize; 272c98833aSArd Biesheuvel }; 282c98833aSArd Biesheuvel 290f4128dcSHerbert Xu extern const u32 sha1_ce_offsetof_count; 300f4128dcSHerbert Xu extern const u32 sha1_ce_offsetof_finalize; 310f4128dcSHerbert Xu 3207eb54d3SArd Biesheuvel asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, 3307eb54d3SArd Biesheuvel int blocks); 3407eb54d3SArd Biesheuvel 356320a15eSSami Tolvanen static void __sha1_ce_transform(struct sha1_state *sst, u8 const *src, 366320a15eSSami Tolvanen int blocks) 376320a15eSSami Tolvanen { 386320a15eSSami Tolvanen sha1_ce_transform(container_of(sst, struct sha1_ce_state, sst), src, 396320a15eSSami Tolvanen blocks); 406320a15eSSami Tolvanen } 416320a15eSSami Tolvanen 42f4857f4cSArd Biesheuvel const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count); 43f4857f4cSArd Biesheuvel const u32 sha1_ce_offsetof_finalize = offsetof(struct sha1_ce_state, finalize); 44f4857f4cSArd Biesheuvel 4507eb54d3SArd Biesheuvel static int sha1_ce_update(struct shash_desc *desc, const u8 *data, 462c98833aSArd Biesheuvel unsigned int len) 472c98833aSArd Biesheuvel { 4807eb54d3SArd Biesheuvel struct sha1_ce_state *sctx = shash_desc_ctx(desc); 492c98833aSArd Biesheuvel 50e52b7023SEric Biggers if (!crypto_simd_usable()) 510771f323SArd Biesheuvel return crypto_sha1_update(desc, data, len); 520771f323SArd Biesheuvel 5307eb54d3SArd Biesheuvel sctx->finalize = 0; 540771f323SArd Biesheuvel kernel_neon_begin(); 556320a15eSSami Tolvanen sha1_base_do_update(desc, data, len, __sha1_ce_transform); 562c98833aSArd Biesheuvel kernel_neon_end(); 572c98833aSArd Biesheuvel 582c98833aSArd Biesheuvel return 0; 592c98833aSArd Biesheuvel } 602c98833aSArd Biesheuvel 6107eb54d3SArd Biesheuvel static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, 622c98833aSArd Biesheuvel unsigned int len, u8 *out) 632c98833aSArd Biesheuvel { 6407eb54d3SArd Biesheuvel struct sha1_ce_state *sctx = shash_desc_ctx(desc); 651d4aaf16SElena Petrova bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len; 662c98833aSArd Biesheuvel 67e52b7023SEric Biggers if (!crypto_simd_usable()) 680771f323SArd Biesheuvel return crypto_sha1_finup(desc, data, len, out); 690771f323SArd Biesheuvel 702c98833aSArd Biesheuvel /* 7107eb54d3SArd Biesheuvel * Allow the asm code to perform the finalization if there is no 7207eb54d3SArd Biesheuvel * partial data and the input is a round multiple of the block size. 732c98833aSArd Biesheuvel */ 7407eb54d3SArd Biesheuvel sctx->finalize = finalize; 752c98833aSArd Biesheuvel 760771f323SArd Biesheuvel kernel_neon_begin(); 776320a15eSSami Tolvanen sha1_base_do_update(desc, data, len, __sha1_ce_transform); 7807eb54d3SArd Biesheuvel if (!finalize) 796320a15eSSami Tolvanen sha1_base_do_finalize(desc, __sha1_ce_transform); 802c98833aSArd Biesheuvel kernel_neon_end(); 8107eb54d3SArd Biesheuvel return sha1_base_finish(desc, out); 822c98833aSArd Biesheuvel } 832c98833aSArd Biesheuvel 8407eb54d3SArd Biesheuvel static int sha1_ce_final(struct shash_desc *desc, u8 *out) 852c98833aSArd Biesheuvel { 86bf7883ebSArd Biesheuvel struct sha1_ce_state *sctx = shash_desc_ctx(desc); 87bf7883ebSArd Biesheuvel 88e52b7023SEric Biggers if (!crypto_simd_usable()) 890771f323SArd Biesheuvel return crypto_sha1_finup(desc, NULL, 0, out); 900771f323SArd Biesheuvel 91bf7883ebSArd Biesheuvel sctx->finalize = 0; 920771f323SArd Biesheuvel kernel_neon_begin(); 936320a15eSSami Tolvanen sha1_base_do_finalize(desc, __sha1_ce_transform); 9407eb54d3SArd Biesheuvel kernel_neon_end(); 9507eb54d3SArd Biesheuvel return sha1_base_finish(desc, out); 962c98833aSArd Biesheuvel } 972c98833aSArd Biesheuvel 98d282eeebSCorentin Labbe static int sha1_ce_export(struct shash_desc *desc, void *out) 99d282eeebSCorentin Labbe { 100d282eeebSCorentin Labbe struct sha1_ce_state *sctx = shash_desc_ctx(desc); 101d282eeebSCorentin Labbe 102d282eeebSCorentin Labbe memcpy(out, &sctx->sst, sizeof(struct sha1_state)); 103d282eeebSCorentin Labbe return 0; 104d282eeebSCorentin Labbe } 105d282eeebSCorentin Labbe 106d282eeebSCorentin Labbe static int sha1_ce_import(struct shash_desc *desc, const void *in) 107d282eeebSCorentin Labbe { 108d282eeebSCorentin Labbe struct sha1_ce_state *sctx = shash_desc_ctx(desc); 109d282eeebSCorentin Labbe 110d282eeebSCorentin Labbe memcpy(&sctx->sst, in, sizeof(struct sha1_state)); 111d282eeebSCorentin Labbe sctx->finalize = 0; 112d282eeebSCorentin Labbe return 0; 113d282eeebSCorentin Labbe } 114d282eeebSCorentin Labbe 1152c98833aSArd Biesheuvel static struct shash_alg alg = { 11607eb54d3SArd Biesheuvel .init = sha1_base_init, 11707eb54d3SArd Biesheuvel .update = sha1_ce_update, 11807eb54d3SArd Biesheuvel .final = sha1_ce_final, 11907eb54d3SArd Biesheuvel .finup = sha1_ce_finup, 120d282eeebSCorentin Labbe .import = sha1_ce_import, 121d282eeebSCorentin Labbe .export = sha1_ce_export, 12207eb54d3SArd Biesheuvel .descsize = sizeof(struct sha1_ce_state), 123d282eeebSCorentin Labbe .statesize = sizeof(struct sha1_state), 1242c98833aSArd Biesheuvel .digestsize = SHA1_DIGEST_SIZE, 1252c98833aSArd Biesheuvel .base = { 1262c98833aSArd Biesheuvel .cra_name = "sha1", 1272c98833aSArd Biesheuvel .cra_driver_name = "sha1-ce", 1282c98833aSArd Biesheuvel .cra_priority = 200, 1292c98833aSArd Biesheuvel .cra_blocksize = SHA1_BLOCK_SIZE, 1302c98833aSArd Biesheuvel .cra_module = THIS_MODULE, 1312c98833aSArd Biesheuvel } 1322c98833aSArd Biesheuvel }; 1332c98833aSArd Biesheuvel 1342c98833aSArd Biesheuvel static int __init sha1_ce_mod_init(void) 1352c98833aSArd Biesheuvel { 1362c98833aSArd Biesheuvel return crypto_register_shash(&alg); 1372c98833aSArd Biesheuvel } 1382c98833aSArd Biesheuvel 1392c98833aSArd Biesheuvel static void __exit sha1_ce_mod_fini(void) 1402c98833aSArd Biesheuvel { 1412c98833aSArd Biesheuvel crypto_unregister_shash(&alg); 1422c98833aSArd Biesheuvel } 1432c98833aSArd Biesheuvel 1442c98833aSArd Biesheuvel module_cpu_feature_match(SHA1, sha1_ce_mod_init); 1452c98833aSArd Biesheuvel module_exit(sha1_ce_mod_fini); 146