10f961f9fSEric Biggers // SPDX-License-Identifier: GPL-2.0 20f961f9fSEric Biggers /* 30f961f9fSEric Biggers * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum 40f961f9fSEric Biggers * (AVX2 accelerated version) 50f961f9fSEric Biggers * 60f961f9fSEric Biggers * Copyright 2018 Google LLC 70f961f9fSEric Biggers */ 80f961f9fSEric Biggers 90f961f9fSEric Biggers #include <crypto/internal/hash.h> 10f2abe0d7SEric Biggers #include <crypto/internal/simd.h> 110f961f9fSEric Biggers #include <crypto/nhpoly1305.h> 120f961f9fSEric Biggers #include <linux/module.h> 130c3dc787SHerbert Xu #include <linux/sizes.h> 14f2abe0d7SEric Biggers #include <asm/simd.h> 150f961f9fSEric Biggers 160f961f9fSEric Biggers asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, 17*0f8bc4bdSEric Biggers __le64 hash[NH_NUM_PASSES]); 180f961f9fSEric Biggers 190f961f9fSEric Biggers static int nhpoly1305_avx2_update(struct shash_desc *desc, 200f961f9fSEric Biggers const u8 *src, unsigned int srclen) 210f961f9fSEric Biggers { 22f2abe0d7SEric Biggers if (srclen < 64 || !crypto_simd_usable()) 230f961f9fSEric Biggers return crypto_nhpoly1305_update(desc, src, srclen); 240f961f9fSEric Biggers 250f961f9fSEric Biggers do { 26a9a8ba90SJason A. Donenfeld unsigned int n = min_t(unsigned int, srclen, SZ_4K); 270f961f9fSEric Biggers 280f961f9fSEric Biggers kernel_fpu_begin(); 29*0f8bc4bdSEric Biggers crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2); 300f961f9fSEric Biggers kernel_fpu_end(); 310f961f9fSEric Biggers src += n; 320f961f9fSEric Biggers srclen -= n; 330f961f9fSEric Biggers } while (srclen); 340f961f9fSEric Biggers return 0; 350f961f9fSEric Biggers } 360f961f9fSEric Biggers 370f961f9fSEric Biggers static struct shash_alg nhpoly1305_alg = { 380f961f9fSEric Biggers .base.cra_name = "nhpoly1305", 390f961f9fSEric Biggers .base.cra_driver_name = "nhpoly1305-avx2", 400f961f9fSEric Biggers .base.cra_priority = 300, 410f961f9fSEric Biggers .base.cra_ctxsize = sizeof(struct nhpoly1305_key), 420f961f9fSEric Biggers .base.cra_module = THIS_MODULE, 430f961f9fSEric Biggers .digestsize = POLY1305_DIGEST_SIZE, 440f961f9fSEric Biggers .init = crypto_nhpoly1305_init, 450f961f9fSEric Biggers .update = nhpoly1305_avx2_update, 460f961f9fSEric Biggers .final = crypto_nhpoly1305_final, 470f961f9fSEric Biggers .setkey = crypto_nhpoly1305_setkey, 480f961f9fSEric Biggers .descsize = sizeof(struct nhpoly1305_state), 490f961f9fSEric Biggers }; 500f961f9fSEric Biggers 510f961f9fSEric Biggers static int __init nhpoly1305_mod_init(void) 520f961f9fSEric Biggers { 530f961f9fSEric Biggers if (!boot_cpu_has(X86_FEATURE_AVX2) || 540f961f9fSEric Biggers !boot_cpu_has(X86_FEATURE_OSXSAVE)) 550f961f9fSEric Biggers return -ENODEV; 560f961f9fSEric Biggers 570f961f9fSEric Biggers return crypto_register_shash(&nhpoly1305_alg); 580f961f9fSEric Biggers } 590f961f9fSEric Biggers 600f961f9fSEric Biggers static void __exit nhpoly1305_mod_exit(void) 610f961f9fSEric Biggers { 620f961f9fSEric Biggers crypto_unregister_shash(&nhpoly1305_alg); 630f961f9fSEric Biggers } 640f961f9fSEric Biggers 650f961f9fSEric Biggers module_init(nhpoly1305_mod_init); 660f961f9fSEric Biggers module_exit(nhpoly1305_mod_exit); 670f961f9fSEric Biggers 680f961f9fSEric Biggers MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)"); 690f961f9fSEric Biggers MODULE_LICENSE("GPL v2"); 700f961f9fSEric Biggers MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>"); 710f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305"); 720f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305-avx2"); 73