1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
4 *
5 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <crypto/internal/hash.h>
12 #include <crypto/internal/simd.h>
13 #include <crypto/sha2.h>
14 #include <crypto/sha256_base.h>
15 #include <linux/module.h>
16 #include <linux/string.h>
17 #include <linux/types.h>
18
19 MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
20 MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
21 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
22 MODULE_LICENSE("GPL v2");
23 MODULE_ALIAS_CRYPTO("sha224");
24 MODULE_ALIAS_CRYPTO("sha256");
25
26 asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
27 unsigned int num_blks);
28 EXPORT_SYMBOL(sha256_block_data_order);
29
__sha256_block_data_order(struct sha256_state * sst,u8 const * src,int blocks)30 static void __sha256_block_data_order(struct sha256_state *sst, u8 const *src,
31 int blocks)
32 {
33 sha256_block_data_order(sst->state, src, blocks);
34 }
35
36 asmlinkage void sha256_block_neon(u32 *digest, const void *data,
37 unsigned int num_blks);
38
__sha256_block_neon(struct sha256_state * sst,u8 const * src,int blocks)39 static void __sha256_block_neon(struct sha256_state *sst, u8 const *src,
40 int blocks)
41 {
42 sha256_block_neon(sst->state, src, blocks);
43 }
44
crypto_sha256_arm64_update(struct shash_desc * desc,const u8 * data,unsigned int len)45 static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data,
46 unsigned int len)
47 {
48 return sha256_base_do_update(desc, data, len,
49 __sha256_block_data_order);
50 }
51
crypto_sha256_arm64_finup(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out)52 static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data,
53 unsigned int len, u8 *out)
54 {
55 if (len)
56 sha256_base_do_update(desc, data, len,
57 __sha256_block_data_order);
58 sha256_base_do_finalize(desc, __sha256_block_data_order);
59
60 return sha256_base_finish(desc, out);
61 }
62
crypto_sha256_arm64_final(struct shash_desc * desc,u8 * out)63 static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out)
64 {
65 return crypto_sha256_arm64_finup(desc, NULL, 0, out);
66 }
67
68 static struct shash_alg algs[] = { {
69 .digestsize = SHA256_DIGEST_SIZE,
70 .init = sha256_base_init,
71 .update = crypto_sha256_arm64_update,
72 .final = crypto_sha256_arm64_final,
73 .finup = crypto_sha256_arm64_finup,
74 .descsize = sizeof(struct sha256_state),
75 .base.cra_name = "sha256",
76 .base.cra_driver_name = "sha256-arm64",
77 .base.cra_priority = 125,
78 .base.cra_blocksize = SHA256_BLOCK_SIZE,
79 .base.cra_module = THIS_MODULE,
80 }, {
81 .digestsize = SHA224_DIGEST_SIZE,
82 .init = sha224_base_init,
83 .update = crypto_sha256_arm64_update,
84 .final = crypto_sha256_arm64_final,
85 .finup = crypto_sha256_arm64_finup,
86 .descsize = sizeof(struct sha256_state),
87 .base.cra_name = "sha224",
88 .base.cra_driver_name = "sha224-arm64",
89 .base.cra_priority = 125,
90 .base.cra_blocksize = SHA224_BLOCK_SIZE,
91 .base.cra_module = THIS_MODULE,
92 } };
93
sha256_update_neon(struct shash_desc * desc,const u8 * data,unsigned int len)94 static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
95 unsigned int len)
96 {
97 struct sha256_state *sctx = shash_desc_ctx(desc);
98
99 if (!crypto_simd_usable())
100 return sha256_base_do_update(desc, data, len,
101 __sha256_block_data_order);
102
103 while (len > 0) {
104 unsigned int chunk = len;
105
106 /*
107 * Don't hog the CPU for the entire time it takes to process all
108 * input when running on a preemptible kernel, but process the
109 * data block by block instead.
110 */
111 if (IS_ENABLED(CONFIG_PREEMPTION) &&
112 chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE)
113 chunk = SHA256_BLOCK_SIZE -
114 sctx->count % SHA256_BLOCK_SIZE;
115
116 kernel_neon_begin();
117 sha256_base_do_update(desc, data, chunk, __sha256_block_neon);
118 kernel_neon_end();
119 data += chunk;
120 len -= chunk;
121 }
122 return 0;
123 }
124
sha256_finup_neon(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out)125 static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
126 unsigned int len, u8 *out)
127 {
128 if (!crypto_simd_usable()) {
129 if (len)
130 sha256_base_do_update(desc, data, len,
131 __sha256_block_data_order);
132 sha256_base_do_finalize(desc, __sha256_block_data_order);
133 } else {
134 if (len)
135 sha256_update_neon(desc, data, len);
136 kernel_neon_begin();
137 sha256_base_do_finalize(desc, __sha256_block_neon);
138 kernel_neon_end();
139 }
140 return sha256_base_finish(desc, out);
141 }
142
sha256_final_neon(struct shash_desc * desc,u8 * out)143 static int sha256_final_neon(struct shash_desc *desc, u8 *out)
144 {
145 return sha256_finup_neon(desc, NULL, 0, out);
146 }
147
148 static struct shash_alg neon_algs[] = { {
149 .digestsize = SHA256_DIGEST_SIZE,
150 .init = sha256_base_init,
151 .update = sha256_update_neon,
152 .final = sha256_final_neon,
153 .finup = sha256_finup_neon,
154 .descsize = sizeof(struct sha256_state),
155 .base.cra_name = "sha256",
156 .base.cra_driver_name = "sha256-arm64-neon",
157 .base.cra_priority = 150,
158 .base.cra_blocksize = SHA256_BLOCK_SIZE,
159 .base.cra_module = THIS_MODULE,
160 }, {
161 .digestsize = SHA224_DIGEST_SIZE,
162 .init = sha224_base_init,
163 .update = sha256_update_neon,
164 .final = sha256_final_neon,
165 .finup = sha256_finup_neon,
166 .descsize = sizeof(struct sha256_state),
167 .base.cra_name = "sha224",
168 .base.cra_driver_name = "sha224-arm64-neon",
169 .base.cra_priority = 150,
170 .base.cra_blocksize = SHA224_BLOCK_SIZE,
171 .base.cra_module = THIS_MODULE,
172 } };
173
sha256_mod_init(void)174 static int __init sha256_mod_init(void)
175 {
176 int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
177 if (ret)
178 return ret;
179
180 if (cpu_have_named_feature(ASIMD)) {
181 ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
182 if (ret)
183 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
184 }
185 return ret;
186 }
187
sha256_mod_fini(void)188 static void __exit sha256_mod_fini(void)
189 {
190 if (cpu_have_named_feature(ASIMD))
191 crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
192 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
193 }
194
195 module_init(sha256_mod_init);
196 module_exit(sha256_mod_fini);
197