1 /* 2 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64 3 * 4 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the Free 8 * Software Foundation; either version 2 of the License, or (at your option) 9 * any later version. 10 * 11 */ 12 13 #include <asm/hwcap.h> 14 #include <asm/neon.h> 15 #include <asm/simd.h> 16 #include <crypto/internal/hash.h> 17 #include <crypto/sha.h> 18 #include <crypto/sha256_base.h> 19 #include <linux/cryptohash.h> 20 #include <linux/types.h> 21 #include <linux/string.h> 22 23 MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64"); 24 MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>"); 25 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_ALIAS_CRYPTO("sha224"); 28 MODULE_ALIAS_CRYPTO("sha256"); 29 30 asmlinkage void sha256_block_data_order(u32 *digest, const void *data, 31 unsigned int num_blks); 32 33 asmlinkage void sha256_block_neon(u32 *digest, const void *data, 34 unsigned int num_blks); 35 36 static int sha256_update(struct shash_desc *desc, const u8 *data, 37 unsigned int len) 38 { 39 return sha256_base_do_update(desc, data, len, 40 (sha256_block_fn *)sha256_block_data_order); 41 } 42 43 static int sha256_finup(struct shash_desc *desc, const u8 *data, 44 unsigned int len, u8 *out) 45 { 46 if (len) 47 sha256_base_do_update(desc, data, len, 48 (sha256_block_fn *)sha256_block_data_order); 49 sha256_base_do_finalize(desc, 50 (sha256_block_fn *)sha256_block_data_order); 51 52 return sha256_base_finish(desc, out); 53 } 54 55 static int sha256_final(struct shash_desc *desc, u8 *out) 56 { 57 return sha256_finup(desc, NULL, 0, out); 58 } 59 60 static struct shash_alg algs[] = { { 61 .digestsize = SHA256_DIGEST_SIZE, 62 .init = sha256_base_init, 63 .update = sha256_update, 64 .final = sha256_final, 65 .finup = sha256_finup, 66 .descsize = sizeof(struct sha256_state), 67 .base.cra_name = "sha256", 68 .base.cra_driver_name = "sha256-arm64", 69 .base.cra_priority = 100, 70 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 71 .base.cra_blocksize = SHA256_BLOCK_SIZE, 72 .base.cra_module = THIS_MODULE, 73 }, { 74 .digestsize = SHA224_DIGEST_SIZE, 75 .init = sha224_base_init, 76 .update = sha256_update, 77 .final = sha256_final, 78 .finup = sha256_finup, 79 .descsize = sizeof(struct sha256_state), 80 .base.cra_name = "sha224", 81 .base.cra_driver_name = "sha224-arm64", 82 .base.cra_priority = 100, 83 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 84 .base.cra_blocksize = SHA224_BLOCK_SIZE, 85 .base.cra_module = THIS_MODULE, 86 } }; 87 88 static int sha256_update_neon(struct shash_desc *desc, const u8 *data, 89 unsigned int len) 90 { 91 /* 92 * Stacking and unstacking a substantial slice of the NEON register 93 * file may significantly affect performance for small updates when 94 * executing in interrupt context, so fall back to the scalar code 95 * in that case. 96 */ 97 if (!may_use_simd()) 98 return sha256_base_do_update(desc, data, len, 99 (sha256_block_fn *)sha256_block_data_order); 100 101 kernel_neon_begin(); 102 sha256_base_do_update(desc, data, len, 103 (sha256_block_fn *)sha256_block_neon); 104 kernel_neon_end(); 105 106 return 0; 107 } 108 109 static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, 110 unsigned int len, u8 *out) 111 { 112 if (!may_use_simd()) { 113 if (len) 114 sha256_base_do_update(desc, data, len, 115 (sha256_block_fn *)sha256_block_data_order); 116 sha256_base_do_finalize(desc, 117 (sha256_block_fn *)sha256_block_data_order); 118 } else { 119 kernel_neon_begin(); 120 if (len) 121 sha256_base_do_update(desc, data, len, 122 (sha256_block_fn *)sha256_block_neon); 123 sha256_base_do_finalize(desc, 124 (sha256_block_fn *)sha256_block_neon); 125 kernel_neon_end(); 126 } 127 return sha256_base_finish(desc, out); 128 } 129 130 static int sha256_final_neon(struct shash_desc *desc, u8 *out) 131 { 132 return sha256_finup_neon(desc, NULL, 0, out); 133 } 134 135 static struct shash_alg neon_algs[] = { { 136 .digestsize = SHA256_DIGEST_SIZE, 137 .init = sha256_base_init, 138 .update = sha256_update_neon, 139 .final = sha256_final_neon, 140 .finup = sha256_finup_neon, 141 .descsize = sizeof(struct sha256_state), 142 .base.cra_name = "sha256", 143 .base.cra_driver_name = "sha256-arm64-neon", 144 .base.cra_priority = 150, 145 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 146 .base.cra_blocksize = SHA256_BLOCK_SIZE, 147 .base.cra_module = THIS_MODULE, 148 }, { 149 .digestsize = SHA224_DIGEST_SIZE, 150 .init = sha224_base_init, 151 .update = sha256_update_neon, 152 .final = sha256_final_neon, 153 .finup = sha256_finup_neon, 154 .descsize = sizeof(struct sha256_state), 155 .base.cra_name = "sha224", 156 .base.cra_driver_name = "sha224-arm64-neon", 157 .base.cra_priority = 150, 158 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH, 159 .base.cra_blocksize = SHA224_BLOCK_SIZE, 160 .base.cra_module = THIS_MODULE, 161 } }; 162 163 static int __init sha256_mod_init(void) 164 { 165 int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); 166 if (ret) 167 return ret; 168 169 if (elf_hwcap & HWCAP_ASIMD) { 170 ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs)); 171 if (ret) 172 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 173 } 174 return ret; 175 } 176 177 static void __exit sha256_mod_fini(void) 178 { 179 if (elf_hwcap & HWCAP_ASIMD) 180 crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs)); 181 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 182 } 183 184 module_init(sha256_mod_init); 185 module_exit(sha256_mod_fini); 186