10f961f9fSEric Biggers // SPDX-License-Identifier: GPL-2.0 20f961f9fSEric Biggers /* 30f961f9fSEric Biggers * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum 40f961f9fSEric Biggers * (AVX2 accelerated version) 50f961f9fSEric Biggers * 60f961f9fSEric Biggers * Copyright 2018 Google LLC 70f961f9fSEric Biggers */ 80f961f9fSEric Biggers 90f961f9fSEric Biggers #include <crypto/internal/hash.h> 10*f2abe0d7SEric Biggers #include <crypto/internal/simd.h> 110f961f9fSEric Biggers #include <crypto/nhpoly1305.h> 120f961f9fSEric Biggers #include <linux/module.h> 13*f2abe0d7SEric Biggers #include <asm/simd.h> 140f961f9fSEric Biggers 150f961f9fSEric Biggers asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, 160f961f9fSEric Biggers u8 hash[NH_HASH_BYTES]); 170f961f9fSEric Biggers 180f961f9fSEric Biggers /* wrapper to avoid indirect call to assembly, which doesn't work with CFI */ 190f961f9fSEric Biggers static void _nh_avx2(const u32 *key, const u8 *message, size_t message_len, 200f961f9fSEric Biggers __le64 hash[NH_NUM_PASSES]) 210f961f9fSEric Biggers { 220f961f9fSEric Biggers nh_avx2(key, message, message_len, (u8 *)hash); 230f961f9fSEric Biggers } 240f961f9fSEric Biggers 250f961f9fSEric Biggers static int nhpoly1305_avx2_update(struct shash_desc *desc, 260f961f9fSEric Biggers const u8 *src, unsigned int srclen) 270f961f9fSEric Biggers { 28*f2abe0d7SEric Biggers if (srclen < 64 || !crypto_simd_usable()) 290f961f9fSEric Biggers return crypto_nhpoly1305_update(desc, src, srclen); 300f961f9fSEric Biggers 310f961f9fSEric Biggers do { 320f961f9fSEric Biggers unsigned int n = min_t(unsigned int, srclen, PAGE_SIZE); 330f961f9fSEric Biggers 340f961f9fSEric Biggers kernel_fpu_begin(); 350f961f9fSEric Biggers crypto_nhpoly1305_update_helper(desc, src, n, _nh_avx2); 360f961f9fSEric Biggers kernel_fpu_end(); 370f961f9fSEric Biggers src += n; 380f961f9fSEric Biggers srclen -= n; 390f961f9fSEric Biggers } while (srclen); 400f961f9fSEric Biggers return 0; 410f961f9fSEric Biggers } 420f961f9fSEric Biggers 430f961f9fSEric Biggers static struct shash_alg nhpoly1305_alg = { 440f961f9fSEric Biggers .base.cra_name = "nhpoly1305", 450f961f9fSEric Biggers .base.cra_driver_name = "nhpoly1305-avx2", 460f961f9fSEric Biggers .base.cra_priority = 300, 470f961f9fSEric Biggers .base.cra_ctxsize = sizeof(struct nhpoly1305_key), 480f961f9fSEric Biggers .base.cra_module = THIS_MODULE, 490f961f9fSEric Biggers .digestsize = POLY1305_DIGEST_SIZE, 500f961f9fSEric Biggers .init = crypto_nhpoly1305_init, 510f961f9fSEric Biggers .update = nhpoly1305_avx2_update, 520f961f9fSEric Biggers .final = crypto_nhpoly1305_final, 530f961f9fSEric Biggers .setkey = crypto_nhpoly1305_setkey, 540f961f9fSEric Biggers .descsize = sizeof(struct nhpoly1305_state), 550f961f9fSEric Biggers }; 560f961f9fSEric Biggers 570f961f9fSEric Biggers static int __init nhpoly1305_mod_init(void) 580f961f9fSEric Biggers { 590f961f9fSEric Biggers if (!boot_cpu_has(X86_FEATURE_AVX2) || 600f961f9fSEric Biggers !boot_cpu_has(X86_FEATURE_OSXSAVE)) 610f961f9fSEric Biggers return -ENODEV; 620f961f9fSEric Biggers 630f961f9fSEric Biggers return crypto_register_shash(&nhpoly1305_alg); 640f961f9fSEric Biggers } 650f961f9fSEric Biggers 660f961f9fSEric Biggers static void __exit nhpoly1305_mod_exit(void) 670f961f9fSEric Biggers { 680f961f9fSEric Biggers crypto_unregister_shash(&nhpoly1305_alg); 690f961f9fSEric Biggers } 700f961f9fSEric Biggers 710f961f9fSEric Biggers module_init(nhpoly1305_mod_init); 720f961f9fSEric Biggers module_exit(nhpoly1305_mod_exit); 730f961f9fSEric Biggers 740f961f9fSEric Biggers MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)"); 750f961f9fSEric Biggers MODULE_LICENSE("GPL v2"); 760f961f9fSEric Biggers MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>"); 770f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305"); 780f961f9fSEric Biggers MODULE_ALIAS_CRYPTO("nhpoly1305-avx2"); 79