1 /* 2 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64 3 * 4 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the Free 8 * Software Foundation; either version 2 of the License, or (at your option) 9 * any later version. 10 * 11 */ 12 13 #include <asm/hwcap.h> 14 #include <asm/neon.h> 15 #include <asm/simd.h> 16 #include <crypto/internal/hash.h> 17 #include <crypto/sha.h> 18 #include <crypto/sha256_base.h> 19 #include <linux/cryptohash.h> 20 #include <linux/types.h> 21 #include <linux/string.h> 22 23 MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64"); 24 MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>"); 25 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_ALIAS_CRYPTO("sha224"); 28 MODULE_ALIAS_CRYPTO("sha256"); 29 30 asmlinkage void sha256_block_data_order(u32 *digest, const void *data, 31 unsigned int num_blks); 32 EXPORT_SYMBOL(sha256_block_data_order); 33 34 asmlinkage void sha256_block_neon(u32 *digest, const void *data, 35 unsigned int num_blks); 36 37 static int sha256_update(struct shash_desc *desc, const u8 *data, 38 unsigned int len) 39 { 40 return sha256_base_do_update(desc, data, len, 41 (sha256_block_fn *)sha256_block_data_order); 42 } 43 44 static int sha256_finup(struct shash_desc *desc, const u8 *data, 45 unsigned int len, u8 *out) 46 { 47 if (len) 48 sha256_base_do_update(desc, data, len, 49 (sha256_block_fn *)sha256_block_data_order); 50 sha256_base_do_finalize(desc, 51 (sha256_block_fn *)sha256_block_data_order); 52 53 return sha256_base_finish(desc, out); 54 } 55 56 static int sha256_final(struct shash_desc *desc, u8 *out) 57 { 58 return sha256_finup(desc, NULL, 0, out); 59 } 60 61 static struct shash_alg algs[] = { { 62 .digestsize = SHA256_DIGEST_SIZE, 63 .init = sha256_base_init, 64 .update = sha256_update, 65 .final = sha256_final, 66 .finup = sha256_finup, 67 .descsize = sizeof(struct sha256_state), 68 .base.cra_name = "sha256", 69 .base.cra_driver_name = "sha256-arm64", 70 .base.cra_priority = 100, 71 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 72 .base.cra_blocksize = SHA256_BLOCK_SIZE, 73 .base.cra_module = THIS_MODULE, 74 }, { 75 .digestsize = SHA224_DIGEST_SIZE, 76 .init = sha224_base_init, 77 .update = sha256_update, 78 .final = sha256_final, 79 .finup = sha256_finup, 80 .descsize = sizeof(struct sha256_state), 81 .base.cra_name = "sha224", 82 .base.cra_driver_name = "sha224-arm64", 83 .base.cra_priority = 100, 84 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 85 .base.cra_blocksize = SHA224_BLOCK_SIZE, 86 .base.cra_module = THIS_MODULE, 87 } }; 88 89 static int sha256_update_neon(struct shash_desc *desc, const u8 *data, 90 unsigned int len) 91 { 92 /* 93 * Stacking and unstacking a substantial slice of the NEON register 94 * file may significantly affect performance for small updates when 95 * executing in interrupt context, so fall back to the scalar code 96 * in that case. 97 */ 98 if (!may_use_simd()) 99 return sha256_base_do_update(desc, data, len, 100 (sha256_block_fn *)sha256_block_data_order); 101 102 kernel_neon_begin(); 103 sha256_base_do_update(desc, data, len, 104 (sha256_block_fn *)sha256_block_neon); 105 kernel_neon_end(); 106 107 return 0; 108 } 109 110 static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, 111 unsigned int len, u8 *out) 112 { 113 if (!may_use_simd()) { 114 if (len) 115 sha256_base_do_update(desc, data, len, 116 (sha256_block_fn *)sha256_block_data_order); 117 sha256_base_do_finalize(desc, 118 (sha256_block_fn *)sha256_block_data_order); 119 } else { 120 kernel_neon_begin(); 121 if (len) 122 sha256_base_do_update(desc, data, len, 123 (sha256_block_fn *)sha256_block_neon); 124 sha256_base_do_finalize(desc, 125 (sha256_block_fn *)sha256_block_neon); 126 kernel_neon_end(); 127 } 128 return sha256_base_finish(desc, out); 129 } 130 131 static int sha256_final_neon(struct shash_desc *desc, u8 *out) 132 { 133 return sha256_finup_neon(desc, NULL, 0, out); 134 } 135 136 static struct shash_alg neon_algs[] = { { 137 .digestsize = SHA256_DIGEST_SIZE, 138 .init = sha256_base_init, 139 .update = sha256_update_neon, 140 .final = sha256_final_neon, 141 .finup = sha256_finup_neon, 142 .descsize = sizeof(struct sha256_state), 143 .base.cra_name = "sha256", 144 .base.cra_driver_name = "sha256-arm64-neon", 145 .base.cra_priority = 150, 146 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 147 .base.cra_blocksize = SHA256_BLOCK_SIZE, 148 .base.cra_module = THIS_MODULE, 149 }, { 150 .digestsize = SHA224_DIGEST_SIZE, 151 .init = sha224_base_init, 152 .update = sha256_update_neon, 153 .final = sha256_final_neon, 154 .finup = sha256_finup_neon, 155 .descsize = sizeof(struct sha256_state), 156 .base.cra_name = "sha224", 157 .base.cra_driver_name = "sha224-arm64-neon", 158 .base.cra_priority = 150, 159 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 160 .base.cra_blocksize = SHA224_BLOCK_SIZE, 161 .base.cra_module = THIS_MODULE, 162 } }; 163 164 static int __init sha256_mod_init(void) 165 { 166 int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); 167 if (ret) 168 return ret; 169 170 if (elf_hwcap & HWCAP_ASIMD) { 171 ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs)); 172 if (ret) 173 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 174 } 175 return ret; 176 } 177 178 static void __exit sha256_mod_fini(void) 179 { 180 if (elf_hwcap & HWCAP_ASIMD) 181 crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs)); 182 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 183 } 184 185 module_init(sha256_mod_init); 186 module_exit(sha256_mod_fini); 187