1 /* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using 5 * Supplemental SSE3 instructions. 6 * 7 * This file is based on sha1_generic.c 8 * 9 * Copyright (c) Alan Smithee. 10 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> 11 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> 12 * Copyright (c) Mathias Krause <minipli@googlemail.com> 13 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com> 14 * 15 * This program is free software; you can redistribute it and/or modify it 16 * under the terms of the GNU General Public License as published by the Free 17 * Software Foundation; either version 2 of the License, or (at your option) 18 * any later version. 19 * 20 */ 21 22 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 23 24 #include <crypto/internal/hash.h> 25 #include <linux/init.h> 26 #include <linux/module.h> 27 #include <linux/mm.h> 28 #include <linux/cryptohash.h> 29 #include <linux/types.h> 30 #include <crypto/sha.h> 31 #include <crypto/sha1_base.h> 32 #include <asm/fpu/api.h> 33 34 35 asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data, 36 unsigned int rounds); 37 #ifdef CONFIG_AS_AVX 38 asmlinkage void sha1_transform_avx(u32 *digest, const char *data, 39 unsigned int rounds); 40 #endif 41 #ifdef CONFIG_AS_AVX2 42 #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ 43 44 asmlinkage void sha1_transform_avx2(u32 *digest, const char *data, 45 unsigned int rounds); 46 #endif 47 #ifdef CONFIG_AS_SHA1_NI 48 asmlinkage void sha1_ni_transform(u32 *digest, const char *data, 49 unsigned int rounds); 50 #endif 51 52 static void (*sha1_transform_asm)(u32 *, const char *, unsigned int); 53 54 static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, 55 unsigned int len) 56 { 57 struct sha1_state *sctx = shash_desc_ctx(desc); 58 59 if (!irq_fpu_usable() || 60 (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) 61 return crypto_sha1_update(desc, data, len); 62 63 /* make sure casting to sha1_block_fn() is safe */ 64 BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); 65 66 kernel_fpu_begin(); 67 sha1_base_do_update(desc, data, len, 68 (sha1_block_fn *)sha1_transform_asm); 69 kernel_fpu_end(); 70 71 return 0; 72 } 73 74 static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, 75 unsigned int len, u8 *out) 76 { 77 if (!irq_fpu_usable()) 78 return crypto_sha1_finup(desc, data, len, out); 79 80 kernel_fpu_begin(); 81 if (len) 82 sha1_base_do_update(desc, data, len, 83 (sha1_block_fn *)sha1_transform_asm); 84 sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_asm); 85 kernel_fpu_end(); 86 87 return sha1_base_finish(desc, out); 88 } 89 90 /* Add padding and return the message digest. */ 91 static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) 92 { 93 return sha1_ssse3_finup(desc, NULL, 0, out); 94 } 95 96 #ifdef CONFIG_AS_AVX2 97 static void sha1_apply_transform_avx2(u32 *digest, const char *data, 98 unsigned int rounds) 99 { 100 /* Select the optimal transform based on data block size */ 101 if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE) 102 sha1_transform_avx2(digest, data, rounds); 103 else 104 sha1_transform_avx(digest, data, rounds); 105 } 106 #endif 107 108 static struct shash_alg alg = { 109 .digestsize = SHA1_DIGEST_SIZE, 110 .init = sha1_base_init, 111 .update = sha1_ssse3_update, 112 .final = sha1_ssse3_final, 113 .finup = sha1_ssse3_finup, 114 .descsize = sizeof(struct sha1_state), 115 .base = { 116 .cra_name = "sha1", 117 .cra_driver_name= "sha1-ssse3", 118 .cra_priority = 150, 119 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 120 .cra_blocksize = SHA1_BLOCK_SIZE, 121 .cra_module = THIS_MODULE, 122 } 123 }; 124 125 #ifdef CONFIG_AS_AVX 126 static bool __init avx_usable(void) 127 { 128 if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) { 129 if (cpu_has_avx) 130 pr_info("AVX detected but unusable.\n"); 131 return false; 132 } 133 134 return true; 135 } 136 137 #ifdef CONFIG_AS_AVX2 138 static bool __init avx2_usable(void) 139 { 140 if (avx_usable() && cpu_has_avx2 && boot_cpu_has(X86_FEATURE_BMI1) && 141 boot_cpu_has(X86_FEATURE_BMI2)) 142 return true; 143 144 return false; 145 } 146 #endif 147 #endif 148 149 static int __init sha1_ssse3_mod_init(void) 150 { 151 char *algo_name; 152 153 /* test for SSSE3 first */ 154 if (cpu_has_ssse3) { 155 sha1_transform_asm = sha1_transform_ssse3; 156 algo_name = "SSSE3"; 157 } 158 159 #ifdef CONFIG_AS_AVX 160 /* allow AVX to override SSSE3, it's a little faster */ 161 if (avx_usable()) { 162 sha1_transform_asm = sha1_transform_avx; 163 algo_name = "AVX"; 164 #ifdef CONFIG_AS_AVX2 165 /* allow AVX2 to override AVX, it's a little faster */ 166 if (avx2_usable()) { 167 sha1_transform_asm = sha1_apply_transform_avx2; 168 algo_name = "AVX2"; 169 } 170 #endif 171 } 172 #endif 173 #ifdef CONFIG_AS_SHA1_NI 174 if (boot_cpu_has(X86_FEATURE_SHA_NI)) { 175 sha1_transform_asm = sha1_ni_transform; 176 algo_name = "SHA-NI"; 177 } 178 #endif 179 180 if (sha1_transform_asm) { 181 pr_info("Using %s optimized SHA-1 implementation\n", algo_name); 182 return crypto_register_shash(&alg); 183 } 184 pr_info("Neither AVX nor AVX2 nor SSSE3/SHA-NI is available/usable.\n"); 185 186 return -ENODEV; 187 } 188 189 static void __exit sha1_ssse3_mod_fini(void) 190 { 191 crypto_unregister_shash(&alg); 192 } 193 194 module_init(sha1_ssse3_mod_init); 195 module_exit(sha1_ssse3_mod_fini); 196 197 MODULE_LICENSE("GPL"); 198 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 199 200 MODULE_ALIAS_CRYPTO("sha1"); 201