1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2019cd469SArd Biesheuvel /*
3019cd469SArd Biesheuvel * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
4019cd469SArd Biesheuvel *
5019cd469SArd Biesheuvel * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6019cd469SArd Biesheuvel */
7019cd469SArd Biesheuvel
8019cd469SArd Biesheuvel #include <asm/neon.h>
9019cd469SArd Biesheuvel #include <asm/simd.h>
10019cd469SArd Biesheuvel #include <asm/unaligned.h>
11019cd469SArd Biesheuvel #include <crypto/aes.h>
12*14386d47SHerbert Xu #include <crypto/algapi.h>
13e52b7023SEric Biggers #include <crypto/internal/simd.h>
14019cd469SArd Biesheuvel #include <linux/cpufeature.h>
15019cd469SArd Biesheuvel #include <linux/module.h>
16019cd469SArd Biesheuvel
17019cd469SArd Biesheuvel #include "aes-ce-setkey.h"
18019cd469SArd Biesheuvel
19019cd469SArd Biesheuvel MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
20019cd469SArd Biesheuvel MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
21019cd469SArd Biesheuvel MODULE_LICENSE("GPL v2");
22019cd469SArd Biesheuvel
23019cd469SArd Biesheuvel struct aes_block {
24019cd469SArd Biesheuvel u8 b[AES_BLOCK_SIZE];
25019cd469SArd Biesheuvel };
26019cd469SArd Biesheuvel
27019cd469SArd Biesheuvel asmlinkage void __aes_ce_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
28019cd469SArd Biesheuvel asmlinkage void __aes_ce_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
29019cd469SArd Biesheuvel
30019cd469SArd Biesheuvel asmlinkage u32 __aes_ce_sub(u32 l);
31019cd469SArd Biesheuvel asmlinkage void __aes_ce_invert(struct aes_block *out,
32019cd469SArd Biesheuvel const struct aes_block *in);
33019cd469SArd Biesheuvel
num_rounds(struct crypto_aes_ctx * ctx)34019cd469SArd Biesheuvel static int num_rounds(struct crypto_aes_ctx *ctx)
35019cd469SArd Biesheuvel {
36019cd469SArd Biesheuvel /*
37019cd469SArd Biesheuvel * # of rounds specified by AES:
38019cd469SArd Biesheuvel * 128 bit key 10 rounds
39019cd469SArd Biesheuvel * 192 bit key 12 rounds
40019cd469SArd Biesheuvel * 256 bit key 14 rounds
41019cd469SArd Biesheuvel * => n byte key => 6 + (n/4) rounds
42019cd469SArd Biesheuvel */
43019cd469SArd Biesheuvel return 6 + ctx->key_length / 4;
44019cd469SArd Biesheuvel }
45019cd469SArd Biesheuvel
aes_cipher_encrypt(struct crypto_tfm * tfm,u8 dst[],u8 const src[])46019cd469SArd Biesheuvel static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
47019cd469SArd Biesheuvel {
48019cd469SArd Biesheuvel struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
49019cd469SArd Biesheuvel
50e52b7023SEric Biggers if (!crypto_simd_usable()) {
514d3f9d89SArd Biesheuvel aes_encrypt(ctx, dst, src);
52019cd469SArd Biesheuvel return;
53019cd469SArd Biesheuvel }
54019cd469SArd Biesheuvel
55019cd469SArd Biesheuvel kernel_neon_begin();
56019cd469SArd Biesheuvel __aes_ce_encrypt(ctx->key_enc, dst, src, num_rounds(ctx));
57019cd469SArd Biesheuvel kernel_neon_end();
58019cd469SArd Biesheuvel }
59019cd469SArd Biesheuvel
aes_cipher_decrypt(struct crypto_tfm * tfm,u8 dst[],u8 const src[])60019cd469SArd Biesheuvel static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
61019cd469SArd Biesheuvel {
62019cd469SArd Biesheuvel struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
63019cd469SArd Biesheuvel
64e52b7023SEric Biggers if (!crypto_simd_usable()) {
654d3f9d89SArd Biesheuvel aes_decrypt(ctx, dst, src);
66019cd469SArd Biesheuvel return;
67019cd469SArd Biesheuvel }
68019cd469SArd Biesheuvel
69019cd469SArd Biesheuvel kernel_neon_begin();
70019cd469SArd Biesheuvel __aes_ce_decrypt(ctx->key_dec, dst, src, num_rounds(ctx));
71019cd469SArd Biesheuvel kernel_neon_end();
72019cd469SArd Biesheuvel }
73019cd469SArd Biesheuvel
ce_aes_expandkey(struct crypto_aes_ctx * ctx,const u8 * in_key,unsigned int key_len)74019cd469SArd Biesheuvel int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
75019cd469SArd Biesheuvel unsigned int key_len)
76019cd469SArd Biesheuvel {
77019cd469SArd Biesheuvel /*
78019cd469SArd Biesheuvel * The AES key schedule round constants
79019cd469SArd Biesheuvel */
80019cd469SArd Biesheuvel static u8 const rcon[] = {
81019cd469SArd Biesheuvel 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
82019cd469SArd Biesheuvel };
83019cd469SArd Biesheuvel
84019cd469SArd Biesheuvel u32 kwords = key_len / sizeof(u32);
85019cd469SArd Biesheuvel struct aes_block *key_enc, *key_dec;
86019cd469SArd Biesheuvel int i, j;
87019cd469SArd Biesheuvel
88019cd469SArd Biesheuvel if (key_len != AES_KEYSIZE_128 &&
89019cd469SArd Biesheuvel key_len != AES_KEYSIZE_192 &&
90019cd469SArd Biesheuvel key_len != AES_KEYSIZE_256)
91019cd469SArd Biesheuvel return -EINVAL;
92019cd469SArd Biesheuvel
93019cd469SArd Biesheuvel ctx->key_length = key_len;
94019cd469SArd Biesheuvel for (i = 0; i < kwords; i++)
95019cd469SArd Biesheuvel ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
96019cd469SArd Biesheuvel
97019cd469SArd Biesheuvel kernel_neon_begin();
98019cd469SArd Biesheuvel for (i = 0; i < sizeof(rcon); i++) {
99019cd469SArd Biesheuvel u32 *rki = ctx->key_enc + (i * kwords);
100019cd469SArd Biesheuvel u32 *rko = rki + kwords;
101019cd469SArd Biesheuvel
102019cd469SArd Biesheuvel rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
103019cd469SArd Biesheuvel rko[1] = rko[0] ^ rki[1];
104019cd469SArd Biesheuvel rko[2] = rko[1] ^ rki[2];
105019cd469SArd Biesheuvel rko[3] = rko[2] ^ rki[3];
106019cd469SArd Biesheuvel
107019cd469SArd Biesheuvel if (key_len == AES_KEYSIZE_192) {
108019cd469SArd Biesheuvel if (i >= 7)
109019cd469SArd Biesheuvel break;
110019cd469SArd Biesheuvel rko[4] = rko[3] ^ rki[4];
111019cd469SArd Biesheuvel rko[5] = rko[4] ^ rki[5];
112019cd469SArd Biesheuvel } else if (key_len == AES_KEYSIZE_256) {
113019cd469SArd Biesheuvel if (i >= 6)
114019cd469SArd Biesheuvel break;
115019cd469SArd Biesheuvel rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
116019cd469SArd Biesheuvel rko[5] = rko[4] ^ rki[5];
117019cd469SArd Biesheuvel rko[6] = rko[5] ^ rki[6];
118019cd469SArd Biesheuvel rko[7] = rko[6] ^ rki[7];
119019cd469SArd Biesheuvel }
120019cd469SArd Biesheuvel }
121019cd469SArd Biesheuvel
122019cd469SArd Biesheuvel /*
123019cd469SArd Biesheuvel * Generate the decryption keys for the Equivalent Inverse Cipher.
124019cd469SArd Biesheuvel * This involves reversing the order of the round keys, and applying
125019cd469SArd Biesheuvel * the Inverse Mix Columns transformation on all but the first and
126019cd469SArd Biesheuvel * the last one.
127019cd469SArd Biesheuvel */
128019cd469SArd Biesheuvel key_enc = (struct aes_block *)ctx->key_enc;
129019cd469SArd Biesheuvel key_dec = (struct aes_block *)ctx->key_dec;
130019cd469SArd Biesheuvel j = num_rounds(ctx);
131019cd469SArd Biesheuvel
132019cd469SArd Biesheuvel key_dec[0] = key_enc[j];
133019cd469SArd Biesheuvel for (i = 1, j--; j > 0; i++, j--)
134019cd469SArd Biesheuvel __aes_ce_invert(key_dec + i, key_enc + j);
135019cd469SArd Biesheuvel key_dec[i] = key_enc[0];
136019cd469SArd Biesheuvel
137019cd469SArd Biesheuvel kernel_neon_end();
138019cd469SArd Biesheuvel return 0;
139019cd469SArd Biesheuvel }
140019cd469SArd Biesheuvel EXPORT_SYMBOL(ce_aes_expandkey);
141019cd469SArd Biesheuvel
ce_aes_setkey(struct crypto_tfm * tfm,const u8 * in_key,unsigned int key_len)142019cd469SArd Biesheuvel int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
143019cd469SArd Biesheuvel unsigned int key_len)
144019cd469SArd Biesheuvel {
145019cd469SArd Biesheuvel struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
146019cd469SArd Biesheuvel
147674f368aSEric Biggers return ce_aes_expandkey(ctx, in_key, key_len);
148019cd469SArd Biesheuvel }
149019cd469SArd Biesheuvel EXPORT_SYMBOL(ce_aes_setkey);
150019cd469SArd Biesheuvel
151019cd469SArd Biesheuvel static struct crypto_alg aes_alg = {
152019cd469SArd Biesheuvel .cra_name = "aes",
153019cd469SArd Biesheuvel .cra_driver_name = "aes-ce",
154019cd469SArd Biesheuvel .cra_priority = 250,
155019cd469SArd Biesheuvel .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
156019cd469SArd Biesheuvel .cra_blocksize = AES_BLOCK_SIZE,
157019cd469SArd Biesheuvel .cra_ctxsize = sizeof(struct crypto_aes_ctx),
158019cd469SArd Biesheuvel .cra_module = THIS_MODULE,
159019cd469SArd Biesheuvel .cra_cipher = {
160019cd469SArd Biesheuvel .cia_min_keysize = AES_MIN_KEY_SIZE,
161019cd469SArd Biesheuvel .cia_max_keysize = AES_MAX_KEY_SIZE,
162019cd469SArd Biesheuvel .cia_setkey = ce_aes_setkey,
163019cd469SArd Biesheuvel .cia_encrypt = aes_cipher_encrypt,
164019cd469SArd Biesheuvel .cia_decrypt = aes_cipher_decrypt
165019cd469SArd Biesheuvel }
166019cd469SArd Biesheuvel };
167019cd469SArd Biesheuvel
aes_mod_init(void)168019cd469SArd Biesheuvel static int __init aes_mod_init(void)
169019cd469SArd Biesheuvel {
170019cd469SArd Biesheuvel return crypto_register_alg(&aes_alg);
171019cd469SArd Biesheuvel }
172019cd469SArd Biesheuvel
aes_mod_exit(void)173019cd469SArd Biesheuvel static void __exit aes_mod_exit(void)
174019cd469SArd Biesheuvel {
175019cd469SArd Biesheuvel crypto_unregister_alg(&aes_alg);
176019cd469SArd Biesheuvel }
177019cd469SArd Biesheuvel
178019cd469SArd Biesheuvel module_cpu_feature_match(AES, aes_mod_init);
179019cd469SArd Biesheuvel module_exit(aes_mod_exit);
180