1 /*
2  * Cryptographic API.
3  *
4  * Glue code for the SHA256 Secure Hash Algorithm assembler
5  * implementation using supplemental SSE3 / AVX / AVX2 instructions.
6  *
7  * This file is based on sha256_generic.c
8  *
9  * Copyright (C) 2013 Intel Corporation.
10  *
11  * Author:
12  *     Tim Chen <tim.c.chen@linux.intel.com>
13  *
14  * This program is free software; you can redistribute it and/or modify it
15  * under the terms of the GNU General Public License as published by the Free
16  * Software Foundation; either version 2 of the License, or (at your option)
17  * any later version.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
23  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
24  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26  * SOFTWARE.
27  */
28 
29 
30 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
31 
32 #include <crypto/internal/hash.h>
33 #include <crypto/internal/simd.h>
34 #include <linux/init.h>
35 #include <linux/module.h>
36 #include <linux/mm.h>
37 #include <linux/types.h>
38 #include <crypto/sha.h>
39 #include <crypto/sha256_base.h>
40 #include <linux/string.h>
41 #include <asm/simd.h>
42 
43 asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
44 				       const u8 *data, int blocks);
45 
46 static int _sha256_update(struct shash_desc *desc, const u8 *data,
47 			  unsigned int len, sha256_block_fn *sha256_xform)
48 {
49 	struct sha256_state *sctx = shash_desc_ctx(desc);
50 
51 	if (!crypto_simd_usable() ||
52 	    (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
53 		return crypto_sha256_update(desc, data, len);
54 
55 	/*
56 	 * Make sure struct sha256_state begins directly with the SHA256
57 	 * 256-bit internal state, as this is what the asm functions expect.
58 	 */
59 	BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
60 
61 	kernel_fpu_begin();
62 	sha256_base_do_update(desc, data, len, sha256_xform);
63 	kernel_fpu_end();
64 
65 	return 0;
66 }
67 
68 static int sha256_finup(struct shash_desc *desc, const u8 *data,
69 	      unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
70 {
71 	if (!crypto_simd_usable())
72 		return crypto_sha256_finup(desc, data, len, out);
73 
74 	kernel_fpu_begin();
75 	if (len)
76 		sha256_base_do_update(desc, data, len, sha256_xform);
77 	sha256_base_do_finalize(desc, sha256_xform);
78 	kernel_fpu_end();
79 
80 	return sha256_base_finish(desc, out);
81 }
82 
83 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
84 			 unsigned int len)
85 {
86 	return _sha256_update(desc, data, len, sha256_transform_ssse3);
87 }
88 
89 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
90 	      unsigned int len, u8 *out)
91 {
92 	return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
93 }
94 
95 /* Add padding and return the message digest. */
96 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
97 {
98 	return sha256_ssse3_finup(desc, NULL, 0, out);
99 }
100 
101 static struct shash_alg sha256_ssse3_algs[] = { {
102 	.digestsize	=	SHA256_DIGEST_SIZE,
103 	.init		=	sha256_base_init,
104 	.update		=	sha256_ssse3_update,
105 	.final		=	sha256_ssse3_final,
106 	.finup		=	sha256_ssse3_finup,
107 	.descsize	=	sizeof(struct sha256_state),
108 	.base		=	{
109 		.cra_name	=	"sha256",
110 		.cra_driver_name =	"sha256-ssse3",
111 		.cra_priority	=	150,
112 		.cra_blocksize	=	SHA256_BLOCK_SIZE,
113 		.cra_module	=	THIS_MODULE,
114 	}
115 }, {
116 	.digestsize	=	SHA224_DIGEST_SIZE,
117 	.init		=	sha224_base_init,
118 	.update		=	sha256_ssse3_update,
119 	.final		=	sha256_ssse3_final,
120 	.finup		=	sha256_ssse3_finup,
121 	.descsize	=	sizeof(struct sha256_state),
122 	.base		=	{
123 		.cra_name	=	"sha224",
124 		.cra_driver_name =	"sha224-ssse3",
125 		.cra_priority	=	150,
126 		.cra_blocksize	=	SHA224_BLOCK_SIZE,
127 		.cra_module	=	THIS_MODULE,
128 	}
129 } };
130 
131 static int register_sha256_ssse3(void)
132 {
133 	if (boot_cpu_has(X86_FEATURE_SSSE3))
134 		return crypto_register_shashes(sha256_ssse3_algs,
135 				ARRAY_SIZE(sha256_ssse3_algs));
136 	return 0;
137 }
138 
139 static void unregister_sha256_ssse3(void)
140 {
141 	if (boot_cpu_has(X86_FEATURE_SSSE3))
142 		crypto_unregister_shashes(sha256_ssse3_algs,
143 				ARRAY_SIZE(sha256_ssse3_algs));
144 }
145 
146 asmlinkage void sha256_transform_avx(struct sha256_state *state,
147 				     const u8 *data, int blocks);
148 
149 static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
150 			 unsigned int len)
151 {
152 	return _sha256_update(desc, data, len, sha256_transform_avx);
153 }
154 
155 static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
156 		      unsigned int len, u8 *out)
157 {
158 	return sha256_finup(desc, data, len, out, sha256_transform_avx);
159 }
160 
161 static int sha256_avx_final(struct shash_desc *desc, u8 *out)
162 {
163 	return sha256_avx_finup(desc, NULL, 0, out);
164 }
165 
166 static struct shash_alg sha256_avx_algs[] = { {
167 	.digestsize	=	SHA256_DIGEST_SIZE,
168 	.init		=	sha256_base_init,
169 	.update		=	sha256_avx_update,
170 	.final		=	sha256_avx_final,
171 	.finup		=	sha256_avx_finup,
172 	.descsize	=	sizeof(struct sha256_state),
173 	.base		=	{
174 		.cra_name	=	"sha256",
175 		.cra_driver_name =	"sha256-avx",
176 		.cra_priority	=	160,
177 		.cra_blocksize	=	SHA256_BLOCK_SIZE,
178 		.cra_module	=	THIS_MODULE,
179 	}
180 }, {
181 	.digestsize	=	SHA224_DIGEST_SIZE,
182 	.init		=	sha224_base_init,
183 	.update		=	sha256_avx_update,
184 	.final		=	sha256_avx_final,
185 	.finup		=	sha256_avx_finup,
186 	.descsize	=	sizeof(struct sha256_state),
187 	.base		=	{
188 		.cra_name	=	"sha224",
189 		.cra_driver_name =	"sha224-avx",
190 		.cra_priority	=	160,
191 		.cra_blocksize	=	SHA224_BLOCK_SIZE,
192 		.cra_module	=	THIS_MODULE,
193 	}
194 } };
195 
196 static bool avx_usable(void)
197 {
198 	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
199 		if (boot_cpu_has(X86_FEATURE_AVX))
200 			pr_info("AVX detected but unusable.\n");
201 		return false;
202 	}
203 
204 	return true;
205 }
206 
207 static int register_sha256_avx(void)
208 {
209 	if (avx_usable())
210 		return crypto_register_shashes(sha256_avx_algs,
211 				ARRAY_SIZE(sha256_avx_algs));
212 	return 0;
213 }
214 
215 static void unregister_sha256_avx(void)
216 {
217 	if (avx_usable())
218 		crypto_unregister_shashes(sha256_avx_algs,
219 				ARRAY_SIZE(sha256_avx_algs));
220 }
221 
222 asmlinkage void sha256_transform_rorx(struct sha256_state *state,
223 				      const u8 *data, int blocks);
224 
225 static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
226 			 unsigned int len)
227 {
228 	return _sha256_update(desc, data, len, sha256_transform_rorx);
229 }
230 
231 static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
232 		      unsigned int len, u8 *out)
233 {
234 	return sha256_finup(desc, data, len, out, sha256_transform_rorx);
235 }
236 
237 static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
238 {
239 	return sha256_avx2_finup(desc, NULL, 0, out);
240 }
241 
242 static struct shash_alg sha256_avx2_algs[] = { {
243 	.digestsize	=	SHA256_DIGEST_SIZE,
244 	.init		=	sha256_base_init,
245 	.update		=	sha256_avx2_update,
246 	.final		=	sha256_avx2_final,
247 	.finup		=	sha256_avx2_finup,
248 	.descsize	=	sizeof(struct sha256_state),
249 	.base		=	{
250 		.cra_name	=	"sha256",
251 		.cra_driver_name =	"sha256-avx2",
252 		.cra_priority	=	170,
253 		.cra_blocksize	=	SHA256_BLOCK_SIZE,
254 		.cra_module	=	THIS_MODULE,
255 	}
256 }, {
257 	.digestsize	=	SHA224_DIGEST_SIZE,
258 	.init		=	sha224_base_init,
259 	.update		=	sha256_avx2_update,
260 	.final		=	sha256_avx2_final,
261 	.finup		=	sha256_avx2_finup,
262 	.descsize	=	sizeof(struct sha256_state),
263 	.base		=	{
264 		.cra_name	=	"sha224",
265 		.cra_driver_name =	"sha224-avx2",
266 		.cra_priority	=	170,
267 		.cra_blocksize	=	SHA224_BLOCK_SIZE,
268 		.cra_module	=	THIS_MODULE,
269 	}
270 } };
271 
272 static bool avx2_usable(void)
273 {
274 	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
275 		    boot_cpu_has(X86_FEATURE_BMI2))
276 		return true;
277 
278 	return false;
279 }
280 
281 static int register_sha256_avx2(void)
282 {
283 	if (avx2_usable())
284 		return crypto_register_shashes(sha256_avx2_algs,
285 				ARRAY_SIZE(sha256_avx2_algs));
286 	return 0;
287 }
288 
289 static void unregister_sha256_avx2(void)
290 {
291 	if (avx2_usable())
292 		crypto_unregister_shashes(sha256_avx2_algs,
293 				ARRAY_SIZE(sha256_avx2_algs));
294 }
295 
296 #ifdef CONFIG_AS_SHA256_NI
297 asmlinkage void sha256_ni_transform(struct sha256_state *digest,
298 				    const u8 *data, int rounds);
299 
300 static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
301 			 unsigned int len)
302 {
303 	return _sha256_update(desc, data, len, sha256_ni_transform);
304 }
305 
306 static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
307 		      unsigned int len, u8 *out)
308 {
309 	return sha256_finup(desc, data, len, out, sha256_ni_transform);
310 }
311 
312 static int sha256_ni_final(struct shash_desc *desc, u8 *out)
313 {
314 	return sha256_ni_finup(desc, NULL, 0, out);
315 }
316 
317 static struct shash_alg sha256_ni_algs[] = { {
318 	.digestsize	=	SHA256_DIGEST_SIZE,
319 	.init		=	sha256_base_init,
320 	.update		=	sha256_ni_update,
321 	.final		=	sha256_ni_final,
322 	.finup		=	sha256_ni_finup,
323 	.descsize	=	sizeof(struct sha256_state),
324 	.base		=	{
325 		.cra_name	=	"sha256",
326 		.cra_driver_name =	"sha256-ni",
327 		.cra_priority	=	250,
328 		.cra_blocksize	=	SHA256_BLOCK_SIZE,
329 		.cra_module	=	THIS_MODULE,
330 	}
331 }, {
332 	.digestsize	=	SHA224_DIGEST_SIZE,
333 	.init		=	sha224_base_init,
334 	.update		=	sha256_ni_update,
335 	.final		=	sha256_ni_final,
336 	.finup		=	sha256_ni_finup,
337 	.descsize	=	sizeof(struct sha256_state),
338 	.base		=	{
339 		.cra_name	=	"sha224",
340 		.cra_driver_name =	"sha224-ni",
341 		.cra_priority	=	250,
342 		.cra_blocksize	=	SHA224_BLOCK_SIZE,
343 		.cra_module	=	THIS_MODULE,
344 	}
345 } };
346 
347 static int register_sha256_ni(void)
348 {
349 	if (boot_cpu_has(X86_FEATURE_SHA_NI))
350 		return crypto_register_shashes(sha256_ni_algs,
351 				ARRAY_SIZE(sha256_ni_algs));
352 	return 0;
353 }
354 
355 static void unregister_sha256_ni(void)
356 {
357 	if (boot_cpu_has(X86_FEATURE_SHA_NI))
358 		crypto_unregister_shashes(sha256_ni_algs,
359 				ARRAY_SIZE(sha256_ni_algs));
360 }
361 
362 #else
363 static inline int register_sha256_ni(void) { return 0; }
364 static inline void unregister_sha256_ni(void) { }
365 #endif
366 
367 static int __init sha256_ssse3_mod_init(void)
368 {
369 	if (register_sha256_ssse3())
370 		goto fail;
371 
372 	if (register_sha256_avx()) {
373 		unregister_sha256_ssse3();
374 		goto fail;
375 	}
376 
377 	if (register_sha256_avx2()) {
378 		unregister_sha256_avx();
379 		unregister_sha256_ssse3();
380 		goto fail;
381 	}
382 
383 	if (register_sha256_ni()) {
384 		unregister_sha256_avx2();
385 		unregister_sha256_avx();
386 		unregister_sha256_ssse3();
387 		goto fail;
388 	}
389 
390 	return 0;
391 fail:
392 	return -ENODEV;
393 }
394 
395 static void __exit sha256_ssse3_mod_fini(void)
396 {
397 	unregister_sha256_ni();
398 	unregister_sha256_avx2();
399 	unregister_sha256_avx();
400 	unregister_sha256_ssse3();
401 }
402 
403 module_init(sha256_ssse3_mod_init);
404 module_exit(sha256_ssse3_mod_fini);
405 
406 MODULE_LICENSE("GPL");
407 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
408 
409 MODULE_ALIAS_CRYPTO("sha256");
410 MODULE_ALIAS_CRYPTO("sha256-ssse3");
411 MODULE_ALIAS_CRYPTO("sha256-avx");
412 MODULE_ALIAS_CRYPTO("sha256-avx2");
413 MODULE_ALIAS_CRYPTO("sha224");
414 MODULE_ALIAS_CRYPTO("sha224-ssse3");
415 MODULE_ALIAS_CRYPTO("sha224-avx");
416 MODULE_ALIAS_CRYPTO("sha224-avx2");
417 #ifdef CONFIG_AS_SHA256_NI
418 MODULE_ALIAS_CRYPTO("sha256-ni");
419 MODULE_ALIAS_CRYPTO("sha224-ni");
420 #endif
421