1 /* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA256 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha256_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation. 10 * 11 * Author: 12 * Tim Chen <tim.c.chen@linux.intel.com> 13 * 14 * This program is free software; you can redistribute it and/or modify it 15 * under the terms of the GNU General Public License as published by the Free 16 * Software Foundation; either version 2 of the License, or (at your option) 17 * any later version. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 * SOFTWARE. 27 */ 28 29 30 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 31 32 #include <crypto/internal/hash.h> 33 #include <linux/init.h> 34 #include <linux/module.h> 35 #include <linux/mm.h> 36 #include <linux/cryptohash.h> 37 #include <linux/types.h> 38 #include <crypto/sha.h> 39 #include <crypto/sha256_base.h> 40 #include <asm/i387.h> 41 #include <asm/xcr.h> 42 #include <asm/xsave.h> 43 #include <linux/string.h> 44 45 asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data, 46 u64 rounds); 47 #ifdef CONFIG_AS_AVX 48 asmlinkage void sha256_transform_avx(u32 *digest, const char *data, 49 u64 rounds); 50 #endif 51 #ifdef CONFIG_AS_AVX2 52 asmlinkage void sha256_transform_rorx(u32 *digest, const char *data, 53 u64 rounds); 54 #endif 55 56 static void (*sha256_transform_asm)(u32 *, const char *, u64); 57 58 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, 59 unsigned int len) 60 { 61 struct sha256_state *sctx = shash_desc_ctx(desc); 62 63 if (!irq_fpu_usable() || 64 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) 65 return crypto_sha256_update(desc, data, len); 66 67 /* make sure casting to sha256_block_fn() is safe */ 68 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); 69 70 kernel_fpu_begin(); 71 sha256_base_do_update(desc, data, len, 72 (sha256_block_fn *)sha256_transform_asm); 73 kernel_fpu_end(); 74 75 return 0; 76 } 77 78 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, 79 unsigned int len, u8 *out) 80 { 81 if (!irq_fpu_usable()) 82 return crypto_sha256_finup(desc, data, len, out); 83 84 kernel_fpu_begin(); 85 if (len) 86 sha256_base_do_update(desc, data, len, 87 (sha256_block_fn *)sha256_transform_asm); 88 sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_transform_asm); 89 kernel_fpu_end(); 90 91 return sha256_base_finish(desc, out); 92 } 93 94 /* Add padding and return the message digest. */ 95 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) 96 { 97 return sha256_ssse3_finup(desc, NULL, 0, out); 98 } 99 100 static struct shash_alg algs[] = { { 101 .digestsize = SHA256_DIGEST_SIZE, 102 .init = sha256_base_init, 103 .update = sha256_ssse3_update, 104 .final = sha256_ssse3_final, 105 .finup = sha256_ssse3_finup, 106 .descsize = sizeof(struct sha256_state), 107 .base = { 108 .cra_name = "sha256", 109 .cra_driver_name = "sha256-ssse3", 110 .cra_priority = 150, 111 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 112 .cra_blocksize = SHA256_BLOCK_SIZE, 113 .cra_module = THIS_MODULE, 114 } 115 }, { 116 .digestsize = SHA224_DIGEST_SIZE, 117 .init = sha224_base_init, 118 .update = sha256_ssse3_update, 119 .final = sha256_ssse3_final, 120 .finup = sha256_ssse3_finup, 121 .descsize = sizeof(struct sha256_state), 122 .base = { 123 .cra_name = "sha224", 124 .cra_driver_name = "sha224-ssse3", 125 .cra_priority = 150, 126 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 127 .cra_blocksize = SHA224_BLOCK_SIZE, 128 .cra_module = THIS_MODULE, 129 } 130 } }; 131 132 #ifdef CONFIG_AS_AVX 133 static bool __init avx_usable(void) 134 { 135 u64 xcr0; 136 137 if (!cpu_has_avx || !cpu_has_osxsave) 138 return false; 139 140 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 141 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { 142 pr_info("AVX detected but unusable.\n"); 143 144 return false; 145 } 146 147 return true; 148 } 149 #endif 150 151 static int __init sha256_ssse3_mod_init(void) 152 { 153 /* test for SSSE3 first */ 154 if (cpu_has_ssse3) 155 sha256_transform_asm = sha256_transform_ssse3; 156 157 #ifdef CONFIG_AS_AVX 158 /* allow AVX to override SSSE3, it's a little faster */ 159 if (avx_usable()) { 160 #ifdef CONFIG_AS_AVX2 161 if (boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2)) 162 sha256_transform_asm = sha256_transform_rorx; 163 else 164 #endif 165 sha256_transform_asm = sha256_transform_avx; 166 } 167 #endif 168 169 if (sha256_transform_asm) { 170 #ifdef CONFIG_AS_AVX 171 if (sha256_transform_asm == sha256_transform_avx) 172 pr_info("Using AVX optimized SHA-256 implementation\n"); 173 #ifdef CONFIG_AS_AVX2 174 else if (sha256_transform_asm == sha256_transform_rorx) 175 pr_info("Using AVX2 optimized SHA-256 implementation\n"); 176 #endif 177 else 178 #endif 179 pr_info("Using SSSE3 optimized SHA-256 implementation\n"); 180 return crypto_register_shashes(algs, ARRAY_SIZE(algs)); 181 } 182 pr_info("Neither AVX nor SSSE3 is available/usable.\n"); 183 184 return -ENODEV; 185 } 186 187 static void __exit sha256_ssse3_mod_fini(void) 188 { 189 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 190 } 191 192 module_init(sha256_ssse3_mod_init); 193 module_exit(sha256_ssse3_mod_fini); 194 195 MODULE_LICENSE("GPL"); 196 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 197 198 MODULE_ALIAS_CRYPTO("sha256"); 199 MODULE_ALIAS_CRYPTO("sha224"); 200