134f7f6c3SNathan Huckleberry // SPDX-License-Identifier: GPL-2.0-only
234f7f6c3SNathan Huckleberry /*
334f7f6c3SNathan Huckleberry * Glue code for POLYVAL using PCMULQDQ-NI
434f7f6c3SNathan Huckleberry *
534f7f6c3SNathan Huckleberry * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
634f7f6c3SNathan Huckleberry * Copyright (c) 2009 Intel Corp.
734f7f6c3SNathan Huckleberry * Author: Huang Ying <ying.huang@intel.com>
834f7f6c3SNathan Huckleberry * Copyright 2021 Google LLC
934f7f6c3SNathan Huckleberry */
1034f7f6c3SNathan Huckleberry
1134f7f6c3SNathan Huckleberry /*
1234f7f6c3SNathan Huckleberry * Glue code based on ghash-clmulni-intel_glue.c.
1334f7f6c3SNathan Huckleberry *
1434f7f6c3SNathan Huckleberry * This implementation of POLYVAL uses montgomery multiplication
1534f7f6c3SNathan Huckleberry * accelerated by PCLMULQDQ-NI to implement the finite field
1634f7f6c3SNathan Huckleberry * operations.
1734f7f6c3SNathan Huckleberry */
1834f7f6c3SNathan Huckleberry
1934f7f6c3SNathan Huckleberry #include <crypto/algapi.h>
2034f7f6c3SNathan Huckleberry #include <crypto/internal/hash.h>
2134f7f6c3SNathan Huckleberry #include <crypto/internal/simd.h>
2234f7f6c3SNathan Huckleberry #include <crypto/polyval.h>
2334f7f6c3SNathan Huckleberry #include <linux/crypto.h>
2434f7f6c3SNathan Huckleberry #include <linux/init.h>
2534f7f6c3SNathan Huckleberry #include <linux/kernel.h>
2634f7f6c3SNathan Huckleberry #include <linux/module.h>
2734f7f6c3SNathan Huckleberry #include <asm/cpu_device_id.h>
2834f7f6c3SNathan Huckleberry #include <asm/simd.h>
2934f7f6c3SNathan Huckleberry
30*9f6035afSNathan Huckleberry #define POLYVAL_ALIGN 16
31*9f6035afSNathan Huckleberry #define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN)
32*9f6035afSNathan Huckleberry #define POLYVAL_ALIGN_EXTRA ((POLYVAL_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
33*9f6035afSNathan Huckleberry #define POLYVAL_CTX_SIZE (sizeof(struct polyval_tfm_ctx) + POLYVAL_ALIGN_EXTRA)
3434f7f6c3SNathan Huckleberry #define NUM_KEY_POWERS 8
3534f7f6c3SNathan Huckleberry
3634f7f6c3SNathan Huckleberry struct polyval_tfm_ctx {
3734f7f6c3SNathan Huckleberry /*
3834f7f6c3SNathan Huckleberry * These powers must be in the order h^8, ..., h^1.
3934f7f6c3SNathan Huckleberry */
40*9f6035afSNathan Huckleberry u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE] POLYVAL_ALIGN_ATTR;
4134f7f6c3SNathan Huckleberry };
4234f7f6c3SNathan Huckleberry
4334f7f6c3SNathan Huckleberry struct polyval_desc_ctx {
4434f7f6c3SNathan Huckleberry u8 buffer[POLYVAL_BLOCK_SIZE];
4534f7f6c3SNathan Huckleberry u32 bytes;
4634f7f6c3SNathan Huckleberry };
4734f7f6c3SNathan Huckleberry
4834f7f6c3SNathan Huckleberry asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys,
4934f7f6c3SNathan Huckleberry const u8 *in, size_t nblocks, u8 *accumulator);
5034f7f6c3SNathan Huckleberry asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2);
5134f7f6c3SNathan Huckleberry
polyval_tfm_ctx(struct crypto_shash * tfm)52*9f6035afSNathan Huckleberry static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm)
53*9f6035afSNathan Huckleberry {
54*9f6035afSNathan Huckleberry return PTR_ALIGN(crypto_shash_ctx(tfm), POLYVAL_ALIGN);
55*9f6035afSNathan Huckleberry }
56*9f6035afSNathan Huckleberry
internal_polyval_update(const struct polyval_tfm_ctx * keys,const u8 * in,size_t nblocks,u8 * accumulator)5734f7f6c3SNathan Huckleberry static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
5834f7f6c3SNathan Huckleberry const u8 *in, size_t nblocks, u8 *accumulator)
5934f7f6c3SNathan Huckleberry {
6034f7f6c3SNathan Huckleberry if (likely(crypto_simd_usable())) {
6134f7f6c3SNathan Huckleberry kernel_fpu_begin();
6234f7f6c3SNathan Huckleberry clmul_polyval_update(keys, in, nblocks, accumulator);
6334f7f6c3SNathan Huckleberry kernel_fpu_end();
6434f7f6c3SNathan Huckleberry } else {
6534f7f6c3SNathan Huckleberry polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
6634f7f6c3SNathan Huckleberry nblocks, accumulator);
6734f7f6c3SNathan Huckleberry }
6834f7f6c3SNathan Huckleberry }
6934f7f6c3SNathan Huckleberry
internal_polyval_mul(u8 * op1,const u8 * op2)7034f7f6c3SNathan Huckleberry static void internal_polyval_mul(u8 *op1, const u8 *op2)
7134f7f6c3SNathan Huckleberry {
7234f7f6c3SNathan Huckleberry if (likely(crypto_simd_usable())) {
7334f7f6c3SNathan Huckleberry kernel_fpu_begin();
7434f7f6c3SNathan Huckleberry clmul_polyval_mul(op1, op2);
7534f7f6c3SNathan Huckleberry kernel_fpu_end();
7634f7f6c3SNathan Huckleberry } else {
7734f7f6c3SNathan Huckleberry polyval_mul_non4k(op1, op2);
7834f7f6c3SNathan Huckleberry }
7934f7f6c3SNathan Huckleberry }
8034f7f6c3SNathan Huckleberry
polyval_x86_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int keylen)8134f7f6c3SNathan Huckleberry static int polyval_x86_setkey(struct crypto_shash *tfm,
8234f7f6c3SNathan Huckleberry const u8 *key, unsigned int keylen)
8334f7f6c3SNathan Huckleberry {
84*9f6035afSNathan Huckleberry struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm);
8534f7f6c3SNathan Huckleberry int i;
8634f7f6c3SNathan Huckleberry
8734f7f6c3SNathan Huckleberry if (keylen != POLYVAL_BLOCK_SIZE)
8834f7f6c3SNathan Huckleberry return -EINVAL;
8934f7f6c3SNathan Huckleberry
9034f7f6c3SNathan Huckleberry memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
9134f7f6c3SNathan Huckleberry
9234f7f6c3SNathan Huckleberry for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
9334f7f6c3SNathan Huckleberry memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
9434f7f6c3SNathan Huckleberry internal_polyval_mul(tctx->key_powers[i],
9534f7f6c3SNathan Huckleberry tctx->key_powers[i+1]);
9634f7f6c3SNathan Huckleberry }
9734f7f6c3SNathan Huckleberry
9834f7f6c3SNathan Huckleberry return 0;
9934f7f6c3SNathan Huckleberry }
10034f7f6c3SNathan Huckleberry
polyval_x86_init(struct shash_desc * desc)10134f7f6c3SNathan Huckleberry static int polyval_x86_init(struct shash_desc *desc)
10234f7f6c3SNathan Huckleberry {
10334f7f6c3SNathan Huckleberry struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
10434f7f6c3SNathan Huckleberry
10534f7f6c3SNathan Huckleberry memset(dctx, 0, sizeof(*dctx));
10634f7f6c3SNathan Huckleberry
10734f7f6c3SNathan Huckleberry return 0;
10834f7f6c3SNathan Huckleberry }
10934f7f6c3SNathan Huckleberry
polyval_x86_update(struct shash_desc * desc,const u8 * src,unsigned int srclen)11034f7f6c3SNathan Huckleberry static int polyval_x86_update(struct shash_desc *desc,
11134f7f6c3SNathan Huckleberry const u8 *src, unsigned int srclen)
11234f7f6c3SNathan Huckleberry {
11334f7f6c3SNathan Huckleberry struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
114*9f6035afSNathan Huckleberry const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);
11534f7f6c3SNathan Huckleberry u8 *pos;
11634f7f6c3SNathan Huckleberry unsigned int nblocks;
11734f7f6c3SNathan Huckleberry unsigned int n;
11834f7f6c3SNathan Huckleberry
11934f7f6c3SNathan Huckleberry if (dctx->bytes) {
12034f7f6c3SNathan Huckleberry n = min(srclen, dctx->bytes);
12134f7f6c3SNathan Huckleberry pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
12234f7f6c3SNathan Huckleberry
12334f7f6c3SNathan Huckleberry dctx->bytes -= n;
12434f7f6c3SNathan Huckleberry srclen -= n;
12534f7f6c3SNathan Huckleberry
12634f7f6c3SNathan Huckleberry while (n--)
12734f7f6c3SNathan Huckleberry *pos++ ^= *src++;
12834f7f6c3SNathan Huckleberry
12934f7f6c3SNathan Huckleberry if (!dctx->bytes)
13034f7f6c3SNathan Huckleberry internal_polyval_mul(dctx->buffer,
13134f7f6c3SNathan Huckleberry tctx->key_powers[NUM_KEY_POWERS-1]);
13234f7f6c3SNathan Huckleberry }
13334f7f6c3SNathan Huckleberry
13434f7f6c3SNathan Huckleberry while (srclen >= POLYVAL_BLOCK_SIZE) {
13534f7f6c3SNathan Huckleberry /* Allow rescheduling every 4K bytes. */
13634f7f6c3SNathan Huckleberry nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
13734f7f6c3SNathan Huckleberry internal_polyval_update(tctx, src, nblocks, dctx->buffer);
13834f7f6c3SNathan Huckleberry srclen -= nblocks * POLYVAL_BLOCK_SIZE;
13934f7f6c3SNathan Huckleberry src += nblocks * POLYVAL_BLOCK_SIZE;
14034f7f6c3SNathan Huckleberry }
14134f7f6c3SNathan Huckleberry
14234f7f6c3SNathan Huckleberry if (srclen) {
14334f7f6c3SNathan Huckleberry dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
14434f7f6c3SNathan Huckleberry pos = dctx->buffer;
14534f7f6c3SNathan Huckleberry while (srclen--)
14634f7f6c3SNathan Huckleberry *pos++ ^= *src++;
14734f7f6c3SNathan Huckleberry }
14834f7f6c3SNathan Huckleberry
14934f7f6c3SNathan Huckleberry return 0;
15034f7f6c3SNathan Huckleberry }
15134f7f6c3SNathan Huckleberry
polyval_x86_final(struct shash_desc * desc,u8 * dst)15234f7f6c3SNathan Huckleberry static int polyval_x86_final(struct shash_desc *desc, u8 *dst)
15334f7f6c3SNathan Huckleberry {
15434f7f6c3SNathan Huckleberry struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
155*9f6035afSNathan Huckleberry const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);
15634f7f6c3SNathan Huckleberry
15734f7f6c3SNathan Huckleberry if (dctx->bytes) {
15834f7f6c3SNathan Huckleberry internal_polyval_mul(dctx->buffer,
15934f7f6c3SNathan Huckleberry tctx->key_powers[NUM_KEY_POWERS-1]);
16034f7f6c3SNathan Huckleberry }
16134f7f6c3SNathan Huckleberry
16234f7f6c3SNathan Huckleberry memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
16334f7f6c3SNathan Huckleberry
16434f7f6c3SNathan Huckleberry return 0;
16534f7f6c3SNathan Huckleberry }
16634f7f6c3SNathan Huckleberry
16734f7f6c3SNathan Huckleberry static struct shash_alg polyval_alg = {
16834f7f6c3SNathan Huckleberry .digestsize = POLYVAL_DIGEST_SIZE,
16934f7f6c3SNathan Huckleberry .init = polyval_x86_init,
17034f7f6c3SNathan Huckleberry .update = polyval_x86_update,
17134f7f6c3SNathan Huckleberry .final = polyval_x86_final,
17234f7f6c3SNathan Huckleberry .setkey = polyval_x86_setkey,
17334f7f6c3SNathan Huckleberry .descsize = sizeof(struct polyval_desc_ctx),
17434f7f6c3SNathan Huckleberry .base = {
17534f7f6c3SNathan Huckleberry .cra_name = "polyval",
17634f7f6c3SNathan Huckleberry .cra_driver_name = "polyval-clmulni",
17734f7f6c3SNathan Huckleberry .cra_priority = 200,
17834f7f6c3SNathan Huckleberry .cra_blocksize = POLYVAL_BLOCK_SIZE,
179*9f6035afSNathan Huckleberry .cra_ctxsize = POLYVAL_CTX_SIZE,
18034f7f6c3SNathan Huckleberry .cra_module = THIS_MODULE,
18134f7f6c3SNathan Huckleberry },
18234f7f6c3SNathan Huckleberry };
18334f7f6c3SNathan Huckleberry
18434f7f6c3SNathan Huckleberry __maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = {
18534f7f6c3SNathan Huckleberry X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
18634f7f6c3SNathan Huckleberry {}
18734f7f6c3SNathan Huckleberry };
18834f7f6c3SNathan Huckleberry MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
18934f7f6c3SNathan Huckleberry
polyval_clmulni_mod_init(void)19034f7f6c3SNathan Huckleberry static int __init polyval_clmulni_mod_init(void)
19134f7f6c3SNathan Huckleberry {
19234f7f6c3SNathan Huckleberry if (!x86_match_cpu(pcmul_cpu_id))
19334f7f6c3SNathan Huckleberry return -ENODEV;
19434f7f6c3SNathan Huckleberry
19534f7f6c3SNathan Huckleberry if (!boot_cpu_has(X86_FEATURE_AVX))
19634f7f6c3SNathan Huckleberry return -ENODEV;
19734f7f6c3SNathan Huckleberry
19834f7f6c3SNathan Huckleberry return crypto_register_shash(&polyval_alg);
19934f7f6c3SNathan Huckleberry }
20034f7f6c3SNathan Huckleberry
polyval_clmulni_mod_exit(void)20134f7f6c3SNathan Huckleberry static void __exit polyval_clmulni_mod_exit(void)
20234f7f6c3SNathan Huckleberry {
20334f7f6c3SNathan Huckleberry crypto_unregister_shash(&polyval_alg);
20434f7f6c3SNathan Huckleberry }
20534f7f6c3SNathan Huckleberry
20634f7f6c3SNathan Huckleberry module_init(polyval_clmulni_mod_init);
20734f7f6c3SNathan Huckleberry module_exit(polyval_clmulni_mod_exit);
20834f7f6c3SNathan Huckleberry
20934f7f6c3SNathan Huckleberry MODULE_LICENSE("GPL");
21034f7f6c3SNathan Huckleberry MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI");
21134f7f6c3SNathan Huckleberry MODULE_ALIAS_CRYPTO("polyval");
21234f7f6c3SNathan Huckleberry MODULE_ALIAS_CRYPTO("polyval-clmulni");
213