1 /*
2  * Cryptographic API.
3  *
4  * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
5  * Supplemental SSE3 instructions.
6  *
7  * This file is based on sha1_generic.c
8  *
9  * Copyright (c) Alan Smithee.
10  * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
11  * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
12  * Copyright (c) Mathias Krause <minipli@googlemail.com>
13  * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
14  *
15  * This program is free software; you can redistribute it and/or modify it
16  * under the terms of the GNU General Public License as published by the Free
17  * Software Foundation; either version 2 of the License, or (at your option)
18  * any later version.
19  *
20  */
21 
22 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
23 
24 #include <crypto/internal/hash.h>
25 #include <linux/init.h>
26 #include <linux/module.h>
27 #include <linux/mm.h>
28 #include <linux/cryptohash.h>
29 #include <linux/types.h>
30 #include <crypto/sha.h>
31 #include <crypto/sha1_base.h>
32 #include <asm/i387.h>
33 #include <asm/xcr.h>
34 #include <asm/xsave.h>
35 
36 
37 asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
38 				     unsigned int rounds);
39 #ifdef CONFIG_AS_AVX
40 asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
41 				   unsigned int rounds);
42 #endif
43 #ifdef CONFIG_AS_AVX2
44 #define SHA1_AVX2_BLOCK_OPTSIZE	4	/* optimal 4*64 bytes of SHA1 blocks */
45 
46 asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
47 				    unsigned int rounds);
48 #endif
49 
50 static void (*sha1_transform_asm)(u32 *, const char *, unsigned int);
51 
52 static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
53 			     unsigned int len)
54 {
55 	struct sha1_state *sctx = shash_desc_ctx(desc);
56 
57 	if (!irq_fpu_usable() ||
58 	    (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
59 		return crypto_sha1_update(desc, data, len);
60 
61 	/* make sure casting to sha1_block_fn() is safe */
62 	BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
63 
64 	kernel_fpu_begin();
65 	sha1_base_do_update(desc, data, len,
66 			    (sha1_block_fn *)sha1_transform_asm);
67 	kernel_fpu_end();
68 
69 	return 0;
70 }
71 
72 static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
73 			      unsigned int len, u8 *out)
74 {
75 	if (!irq_fpu_usable())
76 		return crypto_sha1_finup(desc, data, len, out);
77 
78 	kernel_fpu_begin();
79 	if (len)
80 		sha1_base_do_update(desc, data, len,
81 				    (sha1_block_fn *)sha1_transform_asm);
82 	sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_asm);
83 	kernel_fpu_end();
84 
85 	return sha1_base_finish(desc, out);
86 }
87 
88 /* Add padding and return the message digest. */
89 static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
90 {
91 	return sha1_ssse3_finup(desc, NULL, 0, out);
92 }
93 
94 #ifdef CONFIG_AS_AVX2
95 static void sha1_apply_transform_avx2(u32 *digest, const char *data,
96 				unsigned int rounds)
97 {
98 	/* Select the optimal transform based on data block size */
99 	if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
100 		sha1_transform_avx2(digest, data, rounds);
101 	else
102 		sha1_transform_avx(digest, data, rounds);
103 }
104 #endif
105 
106 static struct shash_alg alg = {
107 	.digestsize	=	SHA1_DIGEST_SIZE,
108 	.init		=	sha1_base_init,
109 	.update		=	sha1_ssse3_update,
110 	.final		=	sha1_ssse3_final,
111 	.finup		=	sha1_ssse3_finup,
112 	.descsize	=	sizeof(struct sha1_state),
113 	.base		=	{
114 		.cra_name	=	"sha1",
115 		.cra_driver_name=	"sha1-ssse3",
116 		.cra_priority	=	150,
117 		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
118 		.cra_blocksize	=	SHA1_BLOCK_SIZE,
119 		.cra_module	=	THIS_MODULE,
120 	}
121 };
122 
123 #ifdef CONFIG_AS_AVX
124 static bool __init avx_usable(void)
125 {
126 	u64 xcr0;
127 
128 	if (!cpu_has_avx || !cpu_has_osxsave)
129 		return false;
130 
131 	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
132 	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
133 		pr_info("AVX detected but unusable.\n");
134 
135 		return false;
136 	}
137 
138 	return true;
139 }
140 
141 #ifdef CONFIG_AS_AVX2
142 static bool __init avx2_usable(void)
143 {
144 	if (avx_usable() && cpu_has_avx2 && boot_cpu_has(X86_FEATURE_BMI1) &&
145 	    boot_cpu_has(X86_FEATURE_BMI2))
146 		return true;
147 
148 	return false;
149 }
150 #endif
151 #endif
152 
153 static int __init sha1_ssse3_mod_init(void)
154 {
155 	char *algo_name;
156 
157 	/* test for SSSE3 first */
158 	if (cpu_has_ssse3) {
159 		sha1_transform_asm = sha1_transform_ssse3;
160 		algo_name = "SSSE3";
161 	}
162 
163 #ifdef CONFIG_AS_AVX
164 	/* allow AVX to override SSSE3, it's a little faster */
165 	if (avx_usable()) {
166 		sha1_transform_asm = sha1_transform_avx;
167 		algo_name = "AVX";
168 #ifdef CONFIG_AS_AVX2
169 		/* allow AVX2 to override AVX, it's a little faster */
170 		if (avx2_usable()) {
171 			sha1_transform_asm = sha1_apply_transform_avx2;
172 			algo_name = "AVX2";
173 		}
174 #endif
175 	}
176 #endif
177 
178 	if (sha1_transform_asm) {
179 		pr_info("Using %s optimized SHA-1 implementation\n", algo_name);
180 		return crypto_register_shash(&alg);
181 	}
182 	pr_info("Neither AVX nor AVX2 nor SSSE3 is available/usable.\n");
183 
184 	return -ENODEV;
185 }
186 
187 static void __exit sha1_ssse3_mod_fini(void)
188 {
189 	crypto_unregister_shash(&alg);
190 }
191 
192 module_init(sha1_ssse3_mod_init);
193 module_exit(sha1_ssse3_mod_fini);
194 
195 MODULE_LICENSE("GPL");
196 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
197 
198 MODULE_ALIAS_CRYPTO("sha1");
199