1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Cryptographic API. 4 * 5 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using 6 * Supplemental SSE3 instructions. 7 * 8 * This file is based on sha1_generic.c 9 * 10 * Copyright (c) Alan Smithee. 11 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> 12 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> 13 * Copyright (c) Mathias Krause <minipli@googlemail.com> 14 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com> 15 */ 16 17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 19 #include <crypto/internal/hash.h> 20 #include <crypto/internal/simd.h> 21 #include <linux/init.h> 22 #include <linux/module.h> 23 #include <linux/mm.h> 24 #include <linux/cryptohash.h> 25 #include <linux/types.h> 26 #include <crypto/sha.h> 27 #include <crypto/sha1_base.h> 28 #include <asm/simd.h> 29 30 static int sha1_update(struct shash_desc *desc, const u8 *data, 31 unsigned int len, sha1_block_fn *sha1_xform) 32 { 33 struct sha1_state *sctx = shash_desc_ctx(desc); 34 35 if (!crypto_simd_usable() || 36 (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) 37 return crypto_sha1_update(desc, data, len); 38 39 /* 40 * Make sure struct sha1_state begins directly with the SHA1 41 * 160-bit internal state, as this is what the asm functions expect. 42 */ 43 BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); 44 45 kernel_fpu_begin(); 46 sha1_base_do_update(desc, data, len, sha1_xform); 47 kernel_fpu_end(); 48 49 return 0; 50 } 51 52 static int sha1_finup(struct shash_desc *desc, const u8 *data, 53 unsigned int len, u8 *out, sha1_block_fn *sha1_xform) 54 { 55 if (!crypto_simd_usable()) 56 return crypto_sha1_finup(desc, data, len, out); 57 58 kernel_fpu_begin(); 59 if (len) 60 sha1_base_do_update(desc, data, len, sha1_xform); 61 sha1_base_do_finalize(desc, sha1_xform); 62 kernel_fpu_end(); 63 64 return sha1_base_finish(desc, out); 65 } 66 67 asmlinkage void sha1_transform_ssse3(struct sha1_state *state, 68 const u8 *data, int blocks); 69 70 static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, 71 unsigned int len) 72 { 73 return sha1_update(desc, data, len, sha1_transform_ssse3); 74 } 75 76 static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, 77 unsigned int len, u8 *out) 78 { 79 return sha1_finup(desc, data, len, out, sha1_transform_ssse3); 80 } 81 82 /* Add padding and return the message digest. */ 83 static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) 84 { 85 return sha1_ssse3_finup(desc, NULL, 0, out); 86 } 87 88 static struct shash_alg sha1_ssse3_alg = { 89 .digestsize = SHA1_DIGEST_SIZE, 90 .init = sha1_base_init, 91 .update = sha1_ssse3_update, 92 .final = sha1_ssse3_final, 93 .finup = sha1_ssse3_finup, 94 .descsize = sizeof(struct sha1_state), 95 .base = { 96 .cra_name = "sha1", 97 .cra_driver_name = "sha1-ssse3", 98 .cra_priority = 150, 99 .cra_blocksize = SHA1_BLOCK_SIZE, 100 .cra_module = THIS_MODULE, 101 } 102 }; 103 104 static int register_sha1_ssse3(void) 105 { 106 if (boot_cpu_has(X86_FEATURE_SSSE3)) 107 return crypto_register_shash(&sha1_ssse3_alg); 108 return 0; 109 } 110 111 static void unregister_sha1_ssse3(void) 112 { 113 if (boot_cpu_has(X86_FEATURE_SSSE3)) 114 crypto_unregister_shash(&sha1_ssse3_alg); 115 } 116 117 asmlinkage void sha1_transform_avx(struct sha1_state *state, 118 const u8 *data, int blocks); 119 120 static int sha1_avx_update(struct shash_desc *desc, const u8 *data, 121 unsigned int len) 122 { 123 return sha1_update(desc, data, len, sha1_transform_avx); 124 } 125 126 static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, 127 unsigned int len, u8 *out) 128 { 129 return sha1_finup(desc, data, len, out, sha1_transform_avx); 130 } 131 132 static int sha1_avx_final(struct shash_desc *desc, u8 *out) 133 { 134 return sha1_avx_finup(desc, NULL, 0, out); 135 } 136 137 static struct shash_alg sha1_avx_alg = { 138 .digestsize = SHA1_DIGEST_SIZE, 139 .init = sha1_base_init, 140 .update = sha1_avx_update, 141 .final = sha1_avx_final, 142 .finup = sha1_avx_finup, 143 .descsize = sizeof(struct sha1_state), 144 .base = { 145 .cra_name = "sha1", 146 .cra_driver_name = "sha1-avx", 147 .cra_priority = 160, 148 .cra_blocksize = SHA1_BLOCK_SIZE, 149 .cra_module = THIS_MODULE, 150 } 151 }; 152 153 static bool avx_usable(void) 154 { 155 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { 156 if (boot_cpu_has(X86_FEATURE_AVX)) 157 pr_info("AVX detected but unusable.\n"); 158 return false; 159 } 160 161 return true; 162 } 163 164 static int register_sha1_avx(void) 165 { 166 if (avx_usable()) 167 return crypto_register_shash(&sha1_avx_alg); 168 return 0; 169 } 170 171 static void unregister_sha1_avx(void) 172 { 173 if (avx_usable()) 174 crypto_unregister_shash(&sha1_avx_alg); 175 } 176 177 #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ 178 179 asmlinkage void sha1_transform_avx2(struct sha1_state *state, 180 const u8 *data, int blocks); 181 182 static bool avx2_usable(void) 183 { 184 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) 185 && boot_cpu_has(X86_FEATURE_BMI1) 186 && boot_cpu_has(X86_FEATURE_BMI2)) 187 return true; 188 189 return false; 190 } 191 192 static void sha1_apply_transform_avx2(struct sha1_state *state, 193 const u8 *data, int blocks) 194 { 195 /* Select the optimal transform based on data block size */ 196 if (blocks >= SHA1_AVX2_BLOCK_OPTSIZE) 197 sha1_transform_avx2(state, data, blocks); 198 else 199 sha1_transform_avx(state, data, blocks); 200 } 201 202 static int sha1_avx2_update(struct shash_desc *desc, const u8 *data, 203 unsigned int len) 204 { 205 return sha1_update(desc, data, len, sha1_apply_transform_avx2); 206 } 207 208 static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, 209 unsigned int len, u8 *out) 210 { 211 return sha1_finup(desc, data, len, out, sha1_apply_transform_avx2); 212 } 213 214 static int sha1_avx2_final(struct shash_desc *desc, u8 *out) 215 { 216 return sha1_avx2_finup(desc, NULL, 0, out); 217 } 218 219 static struct shash_alg sha1_avx2_alg = { 220 .digestsize = SHA1_DIGEST_SIZE, 221 .init = sha1_base_init, 222 .update = sha1_avx2_update, 223 .final = sha1_avx2_final, 224 .finup = sha1_avx2_finup, 225 .descsize = sizeof(struct sha1_state), 226 .base = { 227 .cra_name = "sha1", 228 .cra_driver_name = "sha1-avx2", 229 .cra_priority = 170, 230 .cra_blocksize = SHA1_BLOCK_SIZE, 231 .cra_module = THIS_MODULE, 232 } 233 }; 234 235 static int register_sha1_avx2(void) 236 { 237 if (avx2_usable()) 238 return crypto_register_shash(&sha1_avx2_alg); 239 return 0; 240 } 241 242 static void unregister_sha1_avx2(void) 243 { 244 if (avx2_usable()) 245 crypto_unregister_shash(&sha1_avx2_alg); 246 } 247 248 #ifdef CONFIG_AS_SHA1_NI 249 asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data, 250 int rounds); 251 252 static int sha1_ni_update(struct shash_desc *desc, const u8 *data, 253 unsigned int len) 254 { 255 return sha1_update(desc, data, len, sha1_ni_transform); 256 } 257 258 static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, 259 unsigned int len, u8 *out) 260 { 261 return sha1_finup(desc, data, len, out, sha1_ni_transform); 262 } 263 264 static int sha1_ni_final(struct shash_desc *desc, u8 *out) 265 { 266 return sha1_ni_finup(desc, NULL, 0, out); 267 } 268 269 static struct shash_alg sha1_ni_alg = { 270 .digestsize = SHA1_DIGEST_SIZE, 271 .init = sha1_base_init, 272 .update = sha1_ni_update, 273 .final = sha1_ni_final, 274 .finup = sha1_ni_finup, 275 .descsize = sizeof(struct sha1_state), 276 .base = { 277 .cra_name = "sha1", 278 .cra_driver_name = "sha1-ni", 279 .cra_priority = 250, 280 .cra_blocksize = SHA1_BLOCK_SIZE, 281 .cra_module = THIS_MODULE, 282 } 283 }; 284 285 static int register_sha1_ni(void) 286 { 287 if (boot_cpu_has(X86_FEATURE_SHA_NI)) 288 return crypto_register_shash(&sha1_ni_alg); 289 return 0; 290 } 291 292 static void unregister_sha1_ni(void) 293 { 294 if (boot_cpu_has(X86_FEATURE_SHA_NI)) 295 crypto_unregister_shash(&sha1_ni_alg); 296 } 297 298 #else 299 static inline int register_sha1_ni(void) { return 0; } 300 static inline void unregister_sha1_ni(void) { } 301 #endif 302 303 static int __init sha1_ssse3_mod_init(void) 304 { 305 if (register_sha1_ssse3()) 306 goto fail; 307 308 if (register_sha1_avx()) { 309 unregister_sha1_ssse3(); 310 goto fail; 311 } 312 313 if (register_sha1_avx2()) { 314 unregister_sha1_avx(); 315 unregister_sha1_ssse3(); 316 goto fail; 317 } 318 319 if (register_sha1_ni()) { 320 unregister_sha1_avx2(); 321 unregister_sha1_avx(); 322 unregister_sha1_ssse3(); 323 goto fail; 324 } 325 326 return 0; 327 fail: 328 return -ENODEV; 329 } 330 331 static void __exit sha1_ssse3_mod_fini(void) 332 { 333 unregister_sha1_ni(); 334 unregister_sha1_avx2(); 335 unregister_sha1_avx(); 336 unregister_sha1_ssse3(); 337 } 338 339 module_init(sha1_ssse3_mod_init); 340 module_exit(sha1_ssse3_mod_fini); 341 342 MODULE_LICENSE("GPL"); 343 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 344 345 MODULE_ALIAS_CRYPTO("sha1"); 346 MODULE_ALIAS_CRYPTO("sha1-ssse3"); 347 MODULE_ALIAS_CRYPTO("sha1-avx"); 348 MODULE_ALIAS_CRYPTO("sha1-avx2"); 349 #ifdef CONFIG_AS_SHA1_NI 350 MODULE_ALIAS_CRYPTO("sha1-ni"); 351 #endif 352