1e22471c2SJia Jie Ho // SPDX-License-Identifier: GPL-2.0
2e22471c2SJia Jie Ho /*
3e22471c2SJia Jie Ho * StarFive AES acceleration driver
4e22471c2SJia Jie Ho *
5e22471c2SJia Jie Ho * Copyright (c) 2022 StarFive Technology
6e22471c2SJia Jie Ho */
7e22471c2SJia Jie Ho
8*982213e4SHerbert Xu #include <crypto/engine.h>
9e22471c2SJia Jie Ho #include <crypto/gcm.h>
10e22471c2SJia Jie Ho #include <crypto/internal/aead.h>
11e22471c2SJia Jie Ho #include <crypto/internal/skcipher.h>
12*982213e4SHerbert Xu #include <crypto/scatterwalk.h>
13e22471c2SJia Jie Ho #include "jh7110-cryp.h"
14*982213e4SHerbert Xu #include <linux/err.h>
15*982213e4SHerbert Xu #include <linux/iopoll.h>
16*982213e4SHerbert Xu #include <linux/kernel.h>
17*982213e4SHerbert Xu #include <linux/slab.h>
18*982213e4SHerbert Xu #include <linux/string.h>
19e22471c2SJia Jie Ho
20e22471c2SJia Jie Ho #define STARFIVE_AES_REGS_OFFSET 0x100
21e22471c2SJia Jie Ho #define STARFIVE_AES_AESDIO0R (STARFIVE_AES_REGS_OFFSET + 0x0)
22e22471c2SJia Jie Ho #define STARFIVE_AES_KEY0 (STARFIVE_AES_REGS_OFFSET + 0x4)
23e22471c2SJia Jie Ho #define STARFIVE_AES_KEY1 (STARFIVE_AES_REGS_OFFSET + 0x8)
24e22471c2SJia Jie Ho #define STARFIVE_AES_KEY2 (STARFIVE_AES_REGS_OFFSET + 0xC)
25e22471c2SJia Jie Ho #define STARFIVE_AES_KEY3 (STARFIVE_AES_REGS_OFFSET + 0x10)
26e22471c2SJia Jie Ho #define STARFIVE_AES_KEY4 (STARFIVE_AES_REGS_OFFSET + 0x14)
27e22471c2SJia Jie Ho #define STARFIVE_AES_KEY5 (STARFIVE_AES_REGS_OFFSET + 0x18)
28e22471c2SJia Jie Ho #define STARFIVE_AES_KEY6 (STARFIVE_AES_REGS_OFFSET + 0x1C)
29e22471c2SJia Jie Ho #define STARFIVE_AES_KEY7 (STARFIVE_AES_REGS_OFFSET + 0x20)
30e22471c2SJia Jie Ho #define STARFIVE_AES_CSR (STARFIVE_AES_REGS_OFFSET + 0x24)
31e22471c2SJia Jie Ho #define STARFIVE_AES_IV0 (STARFIVE_AES_REGS_OFFSET + 0x28)
32e22471c2SJia Jie Ho #define STARFIVE_AES_IV1 (STARFIVE_AES_REGS_OFFSET + 0x2C)
33e22471c2SJia Jie Ho #define STARFIVE_AES_IV2 (STARFIVE_AES_REGS_OFFSET + 0x30)
34e22471c2SJia Jie Ho #define STARFIVE_AES_IV3 (STARFIVE_AES_REGS_OFFSET + 0x34)
35e22471c2SJia Jie Ho #define STARFIVE_AES_NONCE0 (STARFIVE_AES_REGS_OFFSET + 0x3C)
36e22471c2SJia Jie Ho #define STARFIVE_AES_NONCE1 (STARFIVE_AES_REGS_OFFSET + 0x40)
37e22471c2SJia Jie Ho #define STARFIVE_AES_NONCE2 (STARFIVE_AES_REGS_OFFSET + 0x44)
38e22471c2SJia Jie Ho #define STARFIVE_AES_NONCE3 (STARFIVE_AES_REGS_OFFSET + 0x48)
39e22471c2SJia Jie Ho #define STARFIVE_AES_ALEN0 (STARFIVE_AES_REGS_OFFSET + 0x4C)
40e22471c2SJia Jie Ho #define STARFIVE_AES_ALEN1 (STARFIVE_AES_REGS_OFFSET + 0x50)
41e22471c2SJia Jie Ho #define STARFIVE_AES_MLEN0 (STARFIVE_AES_REGS_OFFSET + 0x54)
42e22471c2SJia Jie Ho #define STARFIVE_AES_MLEN1 (STARFIVE_AES_REGS_OFFSET + 0x58)
43e22471c2SJia Jie Ho #define STARFIVE_AES_IVLEN (STARFIVE_AES_REGS_OFFSET + 0x5C)
44e22471c2SJia Jie Ho
45e22471c2SJia Jie Ho #define FLG_MODE_MASK GENMASK(2, 0)
46e22471c2SJia Jie Ho #define FLG_ENCRYPT BIT(4)
47e22471c2SJia Jie Ho
48e22471c2SJia Jie Ho /* Misc */
49e22471c2SJia Jie Ho #define CCM_B0_ADATA 0x40
50e22471c2SJia Jie Ho #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))
51e22471c2SJia Jie Ho
starfive_aes_wait_busy(struct starfive_cryp_dev * cryp)52e22471c2SJia Jie Ho static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp)
53e22471c2SJia Jie Ho {
54e22471c2SJia Jie Ho u32 status;
55e22471c2SJia Jie Ho
56e22471c2SJia Jie Ho return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
57e22471c2SJia Jie Ho !(status & STARFIVE_AES_BUSY), 10, 100000);
58e22471c2SJia Jie Ho }
59e22471c2SJia Jie Ho
starfive_aes_wait_keydone(struct starfive_cryp_dev * cryp)60e22471c2SJia Jie Ho static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp)
61e22471c2SJia Jie Ho {
62e22471c2SJia Jie Ho u32 status;
63e22471c2SJia Jie Ho
64e22471c2SJia Jie Ho return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
65e22471c2SJia Jie Ho (status & STARFIVE_AES_KEY_DONE), 10, 100000);
66e22471c2SJia Jie Ho }
67e22471c2SJia Jie Ho
starfive_aes_wait_gcmdone(struct starfive_cryp_dev * cryp)68e22471c2SJia Jie Ho static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp)
69e22471c2SJia Jie Ho {
70e22471c2SJia Jie Ho u32 status;
71e22471c2SJia Jie Ho
72e22471c2SJia Jie Ho return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
73e22471c2SJia Jie Ho (status & STARFIVE_AES_GCM_DONE), 10, 100000);
74e22471c2SJia Jie Ho }
75e22471c2SJia Jie Ho
is_gcm(struct starfive_cryp_dev * cryp)76e22471c2SJia Jie Ho static inline int is_gcm(struct starfive_cryp_dev *cryp)
77e22471c2SJia Jie Ho {
78e22471c2SJia Jie Ho return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM;
79e22471c2SJia Jie Ho }
80e22471c2SJia Jie Ho
is_encrypt(struct starfive_cryp_dev * cryp)81e22471c2SJia Jie Ho static inline int is_encrypt(struct starfive_cryp_dev *cryp)
82e22471c2SJia Jie Ho {
83e22471c2SJia Jie Ho return cryp->flags & FLG_ENCRYPT;
84e22471c2SJia Jie Ho }
85e22471c2SJia Jie Ho
starfive_aes_aead_hw_start(struct starfive_cryp_ctx * ctx,u32 hw_mode)86e22471c2SJia Jie Ho static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode)
87e22471c2SJia Jie Ho {
88e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
89e22471c2SJia Jie Ho unsigned int value;
90e22471c2SJia Jie Ho
91e22471c2SJia Jie Ho switch (hw_mode) {
92e22471c2SJia Jie Ho case STARFIVE_AES_MODE_GCM:
93e22471c2SJia Jie Ho value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
94e22471c2SJia Jie Ho value |= STARFIVE_AES_GCM_START;
95e22471c2SJia Jie Ho writel(value, cryp->base + STARFIVE_AES_CSR);
96e22471c2SJia Jie Ho starfive_aes_wait_gcmdone(cryp);
97e22471c2SJia Jie Ho break;
98e22471c2SJia Jie Ho case STARFIVE_AES_MODE_CCM:
99e22471c2SJia Jie Ho value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
100e22471c2SJia Jie Ho value |= STARFIVE_AES_CCM_START;
101e22471c2SJia Jie Ho writel(value, cryp->base + STARFIVE_AES_CSR);
102e22471c2SJia Jie Ho break;
103e22471c2SJia Jie Ho }
104e22471c2SJia Jie Ho }
105e22471c2SJia Jie Ho
starfive_aes_set_ivlen(struct starfive_cryp_ctx * ctx)106e22471c2SJia Jie Ho static inline void starfive_aes_set_ivlen(struct starfive_cryp_ctx *ctx)
107e22471c2SJia Jie Ho {
108e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
109e22471c2SJia Jie Ho
110e22471c2SJia Jie Ho if (is_gcm(cryp))
111e22471c2SJia Jie Ho writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN);
112e22471c2SJia Jie Ho else
113e22471c2SJia Jie Ho writel(AES_BLOCK_SIZE, cryp->base + STARFIVE_AES_IVLEN);
114e22471c2SJia Jie Ho }
115e22471c2SJia Jie Ho
starfive_aes_set_alen(struct starfive_cryp_ctx * ctx)116e22471c2SJia Jie Ho static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx)
117e22471c2SJia Jie Ho {
118e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
119e22471c2SJia Jie Ho
120e22471c2SJia Jie Ho writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0);
121e22471c2SJia Jie Ho writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1);
122e22471c2SJia Jie Ho }
123e22471c2SJia Jie Ho
starfive_aes_set_mlen(struct starfive_cryp_ctx * ctx)124e22471c2SJia Jie Ho static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx)
125e22471c2SJia Jie Ho {
126e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
127e22471c2SJia Jie Ho
128e22471c2SJia Jie Ho writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0);
129e22471c2SJia Jie Ho writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1);
130e22471c2SJia Jie Ho }
131e22471c2SJia Jie Ho
starfive_aes_ccm_check_iv(const u8 * iv)132e22471c2SJia Jie Ho static inline int starfive_aes_ccm_check_iv(const u8 *iv)
133e22471c2SJia Jie Ho {
134e22471c2SJia Jie Ho /* 2 <= L <= 8, so 1 <= L' <= 7. */
135e22471c2SJia Jie Ho if (iv[0] < 1 || iv[0] > 7)
136e22471c2SJia Jie Ho return -EINVAL;
137e22471c2SJia Jie Ho
138e22471c2SJia Jie Ho return 0;
139e22471c2SJia Jie Ho }
140e22471c2SJia Jie Ho
starfive_aes_write_iv(struct starfive_cryp_ctx * ctx,u32 * iv)141e22471c2SJia Jie Ho static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv)
142e22471c2SJia Jie Ho {
143e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
144e22471c2SJia Jie Ho
145e22471c2SJia Jie Ho writel(iv[0], cryp->base + STARFIVE_AES_IV0);
146e22471c2SJia Jie Ho writel(iv[1], cryp->base + STARFIVE_AES_IV1);
147e22471c2SJia Jie Ho writel(iv[2], cryp->base + STARFIVE_AES_IV2);
148e22471c2SJia Jie Ho
149e22471c2SJia Jie Ho if (is_gcm(cryp)) {
150e22471c2SJia Jie Ho if (starfive_aes_wait_gcmdone(cryp))
151e22471c2SJia Jie Ho return -ETIMEDOUT;
152e22471c2SJia Jie Ho
153e22471c2SJia Jie Ho return 0;
154e22471c2SJia Jie Ho }
155e22471c2SJia Jie Ho
156e22471c2SJia Jie Ho writel(iv[3], cryp->base + STARFIVE_AES_IV3);
157e22471c2SJia Jie Ho
158e22471c2SJia Jie Ho return 0;
159e22471c2SJia Jie Ho }
160e22471c2SJia Jie Ho
starfive_aes_get_iv(struct starfive_cryp_dev * cryp,u32 * iv)161e22471c2SJia Jie Ho static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv)
162e22471c2SJia Jie Ho {
163e22471c2SJia Jie Ho iv[0] = readl(cryp->base + STARFIVE_AES_IV0);
164e22471c2SJia Jie Ho iv[1] = readl(cryp->base + STARFIVE_AES_IV1);
165e22471c2SJia Jie Ho iv[2] = readl(cryp->base + STARFIVE_AES_IV2);
166e22471c2SJia Jie Ho iv[3] = readl(cryp->base + STARFIVE_AES_IV3);
167e22471c2SJia Jie Ho }
168e22471c2SJia Jie Ho
starfive_aes_write_nonce(struct starfive_cryp_ctx * ctx,u32 * nonce)169e22471c2SJia Jie Ho static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce)
170e22471c2SJia Jie Ho {
171e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
172e22471c2SJia Jie Ho
173e22471c2SJia Jie Ho writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0);
174e22471c2SJia Jie Ho writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1);
175e22471c2SJia Jie Ho writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2);
176e22471c2SJia Jie Ho writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3);
177e22471c2SJia Jie Ho }
178e22471c2SJia Jie Ho
starfive_aes_write_key(struct starfive_cryp_ctx * ctx)179e22471c2SJia Jie Ho static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx)
180e22471c2SJia Jie Ho {
181e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
182e22471c2SJia Jie Ho u32 *key = (u32 *)ctx->key;
183e22471c2SJia Jie Ho
184e22471c2SJia Jie Ho if (ctx->keylen >= AES_KEYSIZE_128) {
185e22471c2SJia Jie Ho writel(key[0], cryp->base + STARFIVE_AES_KEY0);
186e22471c2SJia Jie Ho writel(key[1], cryp->base + STARFIVE_AES_KEY1);
187e22471c2SJia Jie Ho writel(key[2], cryp->base + STARFIVE_AES_KEY2);
188e22471c2SJia Jie Ho writel(key[3], cryp->base + STARFIVE_AES_KEY3);
189e22471c2SJia Jie Ho }
190e22471c2SJia Jie Ho
191e22471c2SJia Jie Ho if (ctx->keylen >= AES_KEYSIZE_192) {
192e22471c2SJia Jie Ho writel(key[4], cryp->base + STARFIVE_AES_KEY4);
193e22471c2SJia Jie Ho writel(key[5], cryp->base + STARFIVE_AES_KEY5);
194e22471c2SJia Jie Ho }
195e22471c2SJia Jie Ho
196e22471c2SJia Jie Ho if (ctx->keylen >= AES_KEYSIZE_256) {
197e22471c2SJia Jie Ho writel(key[6], cryp->base + STARFIVE_AES_KEY6);
198e22471c2SJia Jie Ho writel(key[7], cryp->base + STARFIVE_AES_KEY7);
199e22471c2SJia Jie Ho }
200e22471c2SJia Jie Ho
201e22471c2SJia Jie Ho if (starfive_aes_wait_keydone(cryp))
202e22471c2SJia Jie Ho return -ETIMEDOUT;
203e22471c2SJia Jie Ho
204e22471c2SJia Jie Ho return 0;
205e22471c2SJia Jie Ho }
206e22471c2SJia Jie Ho
starfive_aes_ccm_init(struct starfive_cryp_ctx * ctx)207e22471c2SJia Jie Ho static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx)
208e22471c2SJia Jie Ho {
209e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
210e22471c2SJia Jie Ho u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
211e22471c2SJia Jie Ho unsigned int textlen;
212e22471c2SJia Jie Ho
213e22471c2SJia Jie Ho memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE);
214e22471c2SJia Jie Ho memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
215e22471c2SJia Jie Ho
216e22471c2SJia Jie Ho /* Build B0 */
217e22471c2SJia Jie Ho memcpy(b0, iv, AES_BLOCK_SIZE);
218e22471c2SJia Jie Ho
219e22471c2SJia Jie Ho b0[0] |= (8 * ((cryp->authsize - 2) / 2));
220e22471c2SJia Jie Ho
221e22471c2SJia Jie Ho if (cryp->assoclen)
222e22471c2SJia Jie Ho b0[0] |= CCM_B0_ADATA;
223e22471c2SJia Jie Ho
224e22471c2SJia Jie Ho textlen = cryp->total_in;
225e22471c2SJia Jie Ho
226e22471c2SJia Jie Ho b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
227e22471c2SJia Jie Ho b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
228e22471c2SJia Jie Ho
229e22471c2SJia Jie Ho starfive_aes_write_nonce(ctx, (u32 *)b0);
230e22471c2SJia Jie Ho
231e22471c2SJia Jie Ho return 0;
232e22471c2SJia Jie Ho }
233e22471c2SJia Jie Ho
starfive_aes_hw_init(struct starfive_cryp_ctx * ctx)234e22471c2SJia Jie Ho static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx)
235e22471c2SJia Jie Ho {
236e22471c2SJia Jie Ho struct starfive_cryp_request_ctx *rctx = ctx->rctx;
237e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
238e22471c2SJia Jie Ho u32 hw_mode;
239e22471c2SJia Jie Ho
240e22471c2SJia Jie Ho /* reset */
241e22471c2SJia Jie Ho rctx->csr.aes.v = 0;
242e22471c2SJia Jie Ho rctx->csr.aes.aesrst = 1;
243e22471c2SJia Jie Ho writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
244e22471c2SJia Jie Ho
245e22471c2SJia Jie Ho /* csr setup */
246e22471c2SJia Jie Ho hw_mode = cryp->flags & FLG_MODE_MASK;
247e22471c2SJia Jie Ho
248e22471c2SJia Jie Ho rctx->csr.aes.v = 0;
249e22471c2SJia Jie Ho
250e22471c2SJia Jie Ho switch (ctx->keylen) {
251e22471c2SJia Jie Ho case AES_KEYSIZE_128:
252e22471c2SJia Jie Ho rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128;
253e22471c2SJia Jie Ho break;
254e22471c2SJia Jie Ho case AES_KEYSIZE_192:
255e22471c2SJia Jie Ho rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192;
256e22471c2SJia Jie Ho break;
257e22471c2SJia Jie Ho case AES_KEYSIZE_256:
258e22471c2SJia Jie Ho rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256;
259e22471c2SJia Jie Ho break;
260e22471c2SJia Jie Ho }
261e22471c2SJia Jie Ho
262e22471c2SJia Jie Ho rctx->csr.aes.mode = hw_mode;
263e22471c2SJia Jie Ho rctx->csr.aes.cmode = !is_encrypt(cryp);
264e22471c2SJia Jie Ho rctx->csr.aes.ie = 1;
265e22471c2SJia Jie Ho
266e22471c2SJia Jie Ho if (hw_mode == STARFIVE_AES_MODE_CFB ||
267e22471c2SJia Jie Ho hw_mode == STARFIVE_AES_MODE_OFB)
268e22471c2SJia Jie Ho rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_128;
269e22471c2SJia Jie Ho else
270e22471c2SJia Jie Ho rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1;
271e22471c2SJia Jie Ho
272e22471c2SJia Jie Ho if (cryp->side_chan) {
273e22471c2SJia Jie Ho rctx->csr.aes.delay_aes = 1;
274e22471c2SJia Jie Ho rctx->csr.aes.vaes_start = 1;
275e22471c2SJia Jie Ho }
276e22471c2SJia Jie Ho
277e22471c2SJia Jie Ho writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
278e22471c2SJia Jie Ho
279e22471c2SJia Jie Ho cryp->err = starfive_aes_write_key(ctx);
280e22471c2SJia Jie Ho if (cryp->err)
281e22471c2SJia Jie Ho return cryp->err;
282e22471c2SJia Jie Ho
283e22471c2SJia Jie Ho switch (hw_mode) {
284e22471c2SJia Jie Ho case STARFIVE_AES_MODE_GCM:
285e22471c2SJia Jie Ho starfive_aes_set_alen(ctx);
286e22471c2SJia Jie Ho starfive_aes_set_mlen(ctx);
287e22471c2SJia Jie Ho starfive_aes_set_ivlen(ctx);
288e22471c2SJia Jie Ho starfive_aes_aead_hw_start(ctx, hw_mode);
289e22471c2SJia Jie Ho starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv);
290e22471c2SJia Jie Ho break;
291e22471c2SJia Jie Ho case STARFIVE_AES_MODE_CCM:
292e22471c2SJia Jie Ho starfive_aes_set_alen(ctx);
293e22471c2SJia Jie Ho starfive_aes_set_mlen(ctx);
294e22471c2SJia Jie Ho starfive_aes_ccm_init(ctx);
295e22471c2SJia Jie Ho starfive_aes_aead_hw_start(ctx, hw_mode);
296e22471c2SJia Jie Ho break;
297e22471c2SJia Jie Ho case STARFIVE_AES_MODE_OFB:
298e22471c2SJia Jie Ho case STARFIVE_AES_MODE_CFB:
299e22471c2SJia Jie Ho case STARFIVE_AES_MODE_CBC:
300e22471c2SJia Jie Ho case STARFIVE_AES_MODE_CTR:
301e22471c2SJia Jie Ho starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv);
302e22471c2SJia Jie Ho break;
303e22471c2SJia Jie Ho default:
304e22471c2SJia Jie Ho break;
305e22471c2SJia Jie Ho }
306e22471c2SJia Jie Ho
307e22471c2SJia Jie Ho return cryp->err;
308e22471c2SJia Jie Ho }
309e22471c2SJia Jie Ho
starfive_aes_read_authtag(struct starfive_cryp_dev * cryp)310e22471c2SJia Jie Ho static int starfive_aes_read_authtag(struct starfive_cryp_dev *cryp)
311e22471c2SJia Jie Ho {
312e22471c2SJia Jie Ho int i, start_addr;
313e22471c2SJia Jie Ho
314e22471c2SJia Jie Ho if (starfive_aes_wait_busy(cryp))
315e22471c2SJia Jie Ho return dev_err_probe(cryp->dev, -ETIMEDOUT,
316e22471c2SJia Jie Ho "Timeout waiting for tag generation.");
317e22471c2SJia Jie Ho
318e22471c2SJia Jie Ho start_addr = STARFIVE_AES_NONCE0;
319e22471c2SJia Jie Ho
320e22471c2SJia Jie Ho if (is_gcm(cryp))
321e22471c2SJia Jie Ho for (i = 0; i < AES_BLOCK_32; i++, start_addr += 4)
322e22471c2SJia Jie Ho cryp->tag_out[i] = readl(cryp->base + start_addr);
323e22471c2SJia Jie Ho else
324e22471c2SJia Jie Ho for (i = 0; i < AES_BLOCK_32; i++)
325e22471c2SJia Jie Ho cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);
326e22471c2SJia Jie Ho
327e22471c2SJia Jie Ho if (is_encrypt(cryp)) {
328e22471c2SJia Jie Ho scatterwalk_copychunks(cryp->tag_out, &cryp->out_walk, cryp->authsize, 1);
329e22471c2SJia Jie Ho } else {
330e22471c2SJia Jie Ho scatterwalk_copychunks(cryp->tag_in, &cryp->in_walk, cryp->authsize, 0);
331e22471c2SJia Jie Ho
332e22471c2SJia Jie Ho if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize))
333e22471c2SJia Jie Ho return dev_err_probe(cryp->dev, -EBADMSG, "Failed tag verification\n");
334e22471c2SJia Jie Ho }
335e22471c2SJia Jie Ho
336e22471c2SJia Jie Ho return 0;
337e22471c2SJia Jie Ho }
338e22471c2SJia Jie Ho
starfive_aes_finish_req(struct starfive_cryp_dev * cryp)339e22471c2SJia Jie Ho static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp)
340e22471c2SJia Jie Ho {
341e22471c2SJia Jie Ho union starfive_aes_csr csr;
342e22471c2SJia Jie Ho int err = cryp->err;
343e22471c2SJia Jie Ho
344e22471c2SJia Jie Ho if (!err && cryp->authsize)
345e22471c2SJia Jie Ho err = starfive_aes_read_authtag(cryp);
346e22471c2SJia Jie Ho
347e22471c2SJia Jie Ho if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC ||
348e22471c2SJia Jie Ho (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR))
349e22471c2SJia Jie Ho starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv);
350e22471c2SJia Jie Ho
351e22471c2SJia Jie Ho /* reset irq flags*/
352e22471c2SJia Jie Ho csr.v = 0;
353e22471c2SJia Jie Ho csr.aesrst = 1;
354e22471c2SJia Jie Ho writel(csr.v, cryp->base + STARFIVE_AES_CSR);
355e22471c2SJia Jie Ho
356e22471c2SJia Jie Ho if (cryp->authsize)
357e22471c2SJia Jie Ho crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err);
358e22471c2SJia Jie Ho else
359e22471c2SJia Jie Ho crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq,
360e22471c2SJia Jie Ho err);
361e22471c2SJia Jie Ho }
362e22471c2SJia Jie Ho
starfive_aes_done_task(unsigned long param)363e22471c2SJia Jie Ho void starfive_aes_done_task(unsigned long param)
364e22471c2SJia Jie Ho {
365e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param;
366e22471c2SJia Jie Ho u32 block[AES_BLOCK_32];
367e22471c2SJia Jie Ho u32 stat;
368e22471c2SJia Jie Ho int i;
369e22471c2SJia Jie Ho
370e22471c2SJia Jie Ho for (i = 0; i < AES_BLOCK_32; i++)
371e22471c2SJia Jie Ho block[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);
372e22471c2SJia Jie Ho
373e22471c2SJia Jie Ho scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, AES_BLOCK_SIZE,
374e22471c2SJia Jie Ho cryp->total_out), 1);
375e22471c2SJia Jie Ho
376e22471c2SJia Jie Ho cryp->total_out -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_out);
377e22471c2SJia Jie Ho
378e22471c2SJia Jie Ho if (!cryp->total_out) {
379e22471c2SJia Jie Ho starfive_aes_finish_req(cryp);
380e22471c2SJia Jie Ho return;
381e22471c2SJia Jie Ho }
382e22471c2SJia Jie Ho
383e22471c2SJia Jie Ho memset(block, 0, AES_BLOCK_SIZE);
384e22471c2SJia Jie Ho scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
385e22471c2SJia Jie Ho cryp->total_in), 0);
386e22471c2SJia Jie Ho cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
387e22471c2SJia Jie Ho
388e22471c2SJia Jie Ho for (i = 0; i < AES_BLOCK_32; i++)
389e22471c2SJia Jie Ho writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);
390e22471c2SJia Jie Ho
391e22471c2SJia Jie Ho stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
392e22471c2SJia Jie Ho stat &= ~STARFIVE_IE_MASK_AES_DONE;
393e22471c2SJia Jie Ho writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
394e22471c2SJia Jie Ho }
395e22471c2SJia Jie Ho
starfive_aes_gcm_write_adata(struct starfive_cryp_ctx * ctx)396e22471c2SJia Jie Ho static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx)
397e22471c2SJia Jie Ho {
398e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
399e22471c2SJia Jie Ho struct starfive_cryp_request_ctx *rctx = ctx->rctx;
400e22471c2SJia Jie Ho u32 *buffer;
401e22471c2SJia Jie Ho int total_len, loop;
402e22471c2SJia Jie Ho
403e22471c2SJia Jie Ho total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int);
404e22471c2SJia Jie Ho buffer = (u32 *)rctx->adata;
405e22471c2SJia Jie Ho
406e22471c2SJia Jie Ho for (loop = 0; loop < total_len; loop += 4) {
407e22471c2SJia Jie Ho writel(*buffer, cryp->base + STARFIVE_AES_NONCE0);
408e22471c2SJia Jie Ho buffer++;
409e22471c2SJia Jie Ho writel(*buffer, cryp->base + STARFIVE_AES_NONCE1);
410e22471c2SJia Jie Ho buffer++;
411e22471c2SJia Jie Ho writel(*buffer, cryp->base + STARFIVE_AES_NONCE2);
412e22471c2SJia Jie Ho buffer++;
413e22471c2SJia Jie Ho writel(*buffer, cryp->base + STARFIVE_AES_NONCE3);
414e22471c2SJia Jie Ho buffer++;
415e22471c2SJia Jie Ho }
416e22471c2SJia Jie Ho
417e22471c2SJia Jie Ho if (starfive_aes_wait_gcmdone(cryp))
418e22471c2SJia Jie Ho return dev_err_probe(cryp->dev, -ETIMEDOUT,
419e22471c2SJia Jie Ho "Timeout processing gcm aad block");
420e22471c2SJia Jie Ho
421e22471c2SJia Jie Ho return 0;
422e22471c2SJia Jie Ho }
423e22471c2SJia Jie Ho
starfive_aes_ccm_write_adata(struct starfive_cryp_ctx * ctx)424e22471c2SJia Jie Ho static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx)
425e22471c2SJia Jie Ho {
426e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
427e22471c2SJia Jie Ho struct starfive_cryp_request_ctx *rctx = ctx->rctx;
428e22471c2SJia Jie Ho u32 *buffer;
429e22471c2SJia Jie Ho u8 *ci;
430e22471c2SJia Jie Ho int total_len, loop;
431e22471c2SJia Jie Ho
432e22471c2SJia Jie Ho total_len = cryp->assoclen;
433e22471c2SJia Jie Ho
434e22471c2SJia Jie Ho ci = rctx->adata;
435e22471c2SJia Jie Ho writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
436e22471c2SJia Jie Ho ci++;
437e22471c2SJia Jie Ho writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
438e22471c2SJia Jie Ho ci++;
439e22471c2SJia Jie Ho total_len -= 2;
440e22471c2SJia Jie Ho buffer = (u32 *)ci;
441e22471c2SJia Jie Ho
442e22471c2SJia Jie Ho for (loop = 0; loop < 3; loop++, buffer++)
443e22471c2SJia Jie Ho writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
444e22471c2SJia Jie Ho
445e22471c2SJia Jie Ho total_len -= 12;
446e22471c2SJia Jie Ho
447e22471c2SJia Jie Ho while (total_len > 0) {
448e22471c2SJia Jie Ho for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++)
449e22471c2SJia Jie Ho writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
450e22471c2SJia Jie Ho
451e22471c2SJia Jie Ho total_len -= AES_BLOCK_SIZE;
452e22471c2SJia Jie Ho }
453e22471c2SJia Jie Ho
454e22471c2SJia Jie Ho if (starfive_aes_wait_busy(cryp))
455e22471c2SJia Jie Ho return dev_err_probe(cryp->dev, -ETIMEDOUT,
456e22471c2SJia Jie Ho "Timeout processing ccm aad block");
457e22471c2SJia Jie Ho
458e22471c2SJia Jie Ho return 0;
459e22471c2SJia Jie Ho }
460e22471c2SJia Jie Ho
starfive_aes_prepare_req(struct skcipher_request * req,struct aead_request * areq)461e22471c2SJia Jie Ho static int starfive_aes_prepare_req(struct skcipher_request *req,
462e22471c2SJia Jie Ho struct aead_request *areq)
463e22471c2SJia Jie Ho {
464e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx;
465e22471c2SJia Jie Ho struct starfive_cryp_request_ctx *rctx;
466e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp;
467e22471c2SJia Jie Ho
468e22471c2SJia Jie Ho if (!req && !areq)
469e22471c2SJia Jie Ho return -EINVAL;
470e22471c2SJia Jie Ho
471e22471c2SJia Jie Ho ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) :
472e22471c2SJia Jie Ho crypto_aead_ctx(crypto_aead_reqtfm(areq));
473e22471c2SJia Jie Ho
474e22471c2SJia Jie Ho cryp = ctx->cryp;
475e22471c2SJia Jie Ho rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq);
476e22471c2SJia Jie Ho
477e22471c2SJia Jie Ho if (req) {
478e22471c2SJia Jie Ho cryp->req.sreq = req;
479e22471c2SJia Jie Ho cryp->total_in = req->cryptlen;
480e22471c2SJia Jie Ho cryp->total_out = req->cryptlen;
481e22471c2SJia Jie Ho cryp->assoclen = 0;
482e22471c2SJia Jie Ho cryp->authsize = 0;
483e22471c2SJia Jie Ho } else {
484e22471c2SJia Jie Ho cryp->req.areq = areq;
485e22471c2SJia Jie Ho cryp->assoclen = areq->assoclen;
486e22471c2SJia Jie Ho cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
487e22471c2SJia Jie Ho if (is_encrypt(cryp)) {
488e22471c2SJia Jie Ho cryp->total_in = areq->cryptlen;
489e22471c2SJia Jie Ho cryp->total_out = areq->cryptlen;
490e22471c2SJia Jie Ho } else {
491e22471c2SJia Jie Ho cryp->total_in = areq->cryptlen - cryp->authsize;
492e22471c2SJia Jie Ho cryp->total_out = cryp->total_in;
493e22471c2SJia Jie Ho }
494e22471c2SJia Jie Ho }
495e22471c2SJia Jie Ho
496e22471c2SJia Jie Ho rctx->in_sg = req ? req->src : areq->src;
497e22471c2SJia Jie Ho scatterwalk_start(&cryp->in_walk, rctx->in_sg);
498e22471c2SJia Jie Ho
499e22471c2SJia Jie Ho rctx->out_sg = req ? req->dst : areq->dst;
500e22471c2SJia Jie Ho scatterwalk_start(&cryp->out_walk, rctx->out_sg);
501e22471c2SJia Jie Ho
502e22471c2SJia Jie Ho if (cryp->assoclen) {
503e22471c2SJia Jie Ho rctx->adata = kzalloc(ALIGN(cryp->assoclen, AES_BLOCK_SIZE), GFP_KERNEL);
50433b53749SYang Yingliang if (!rctx->adata)
50533b53749SYang Yingliang return dev_err_probe(cryp->dev, -ENOMEM,
506e22471c2SJia Jie Ho "Failed to alloc memory for adata");
507e22471c2SJia Jie Ho
508e22471c2SJia Jie Ho scatterwalk_copychunks(rctx->adata, &cryp->in_walk, cryp->assoclen, 0);
509e22471c2SJia Jie Ho scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->assoclen, 2);
510e22471c2SJia Jie Ho }
511e22471c2SJia Jie Ho
512e22471c2SJia Jie Ho ctx->rctx = rctx;
513e22471c2SJia Jie Ho
514e22471c2SJia Jie Ho return starfive_aes_hw_init(ctx);
515e22471c2SJia Jie Ho }
516e22471c2SJia Jie Ho
starfive_aes_do_one_req(struct crypto_engine * engine,void * areq)517e22471c2SJia Jie Ho static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq)
518e22471c2SJia Jie Ho {
519e22471c2SJia Jie Ho struct skcipher_request *req =
520e22471c2SJia Jie Ho container_of(areq, struct skcipher_request, base);
521e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx =
522e22471c2SJia Jie Ho crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
523e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
524e22471c2SJia Jie Ho u32 block[AES_BLOCK_32];
525e22471c2SJia Jie Ho u32 stat;
52650c546d7SHerbert Xu int err;
527e22471c2SJia Jie Ho int i;
528e22471c2SJia Jie Ho
52950c546d7SHerbert Xu err = starfive_aes_prepare_req(req, NULL);
53050c546d7SHerbert Xu if (err)
53150c546d7SHerbert Xu return err;
53250c546d7SHerbert Xu
533e22471c2SJia Jie Ho /*
534e22471c2SJia Jie Ho * Write first plain/ciphertext block to start the module
535e22471c2SJia Jie Ho * then let irq tasklet handle the rest of the data blocks.
536e22471c2SJia Jie Ho */
537e22471c2SJia Jie Ho scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
538e22471c2SJia Jie Ho cryp->total_in), 0);
539e22471c2SJia Jie Ho cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
540e22471c2SJia Jie Ho
541e22471c2SJia Jie Ho for (i = 0; i < AES_BLOCK_32; i++)
542e22471c2SJia Jie Ho writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);
543e22471c2SJia Jie Ho
544e22471c2SJia Jie Ho stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
545e22471c2SJia Jie Ho stat &= ~STARFIVE_IE_MASK_AES_DONE;
546e22471c2SJia Jie Ho writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
547e22471c2SJia Jie Ho
548e22471c2SJia Jie Ho return 0;
549e22471c2SJia Jie Ho }
550e22471c2SJia Jie Ho
starfive_aes_init_tfm(struct crypto_skcipher * tfm)551e22471c2SJia Jie Ho static int starfive_aes_init_tfm(struct crypto_skcipher *tfm)
552e22471c2SJia Jie Ho {
553e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
554e22471c2SJia Jie Ho
555e22471c2SJia Jie Ho ctx->cryp = starfive_cryp_find_dev(ctx);
556e22471c2SJia Jie Ho if (!ctx->cryp)
557e22471c2SJia Jie Ho return -ENODEV;
558e22471c2SJia Jie Ho
559e22471c2SJia Jie Ho crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) +
560e22471c2SJia Jie Ho sizeof(struct skcipher_request));
561e22471c2SJia Jie Ho
562e22471c2SJia Jie Ho return 0;
563e22471c2SJia Jie Ho }
564e22471c2SJia Jie Ho
starfive_aes_aead_do_one_req(struct crypto_engine * engine,void * areq)565e22471c2SJia Jie Ho static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq)
566e22471c2SJia Jie Ho {
567e22471c2SJia Jie Ho struct aead_request *req =
568e22471c2SJia Jie Ho container_of(areq, struct aead_request, base);
569e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx =
570e22471c2SJia Jie Ho crypto_aead_ctx(crypto_aead_reqtfm(req));
571e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
572e22471c2SJia Jie Ho struct starfive_cryp_request_ctx *rctx = ctx->rctx;
573e22471c2SJia Jie Ho u32 block[AES_BLOCK_32];
574e22471c2SJia Jie Ho u32 stat;
57550c546d7SHerbert Xu int err;
576e22471c2SJia Jie Ho int i;
577e22471c2SJia Jie Ho
57850c546d7SHerbert Xu err = starfive_aes_prepare_req(NULL, req);
57950c546d7SHerbert Xu if (err)
58050c546d7SHerbert Xu return err;
58150c546d7SHerbert Xu
582e22471c2SJia Jie Ho if (!cryp->assoclen)
583e22471c2SJia Jie Ho goto write_text;
584e22471c2SJia Jie Ho
585e22471c2SJia Jie Ho if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM)
586e22471c2SJia Jie Ho cryp->err = starfive_aes_ccm_write_adata(ctx);
587e22471c2SJia Jie Ho else
588e22471c2SJia Jie Ho cryp->err = starfive_aes_gcm_write_adata(ctx);
589e22471c2SJia Jie Ho
590e22471c2SJia Jie Ho kfree(rctx->adata);
591e22471c2SJia Jie Ho
592e22471c2SJia Jie Ho if (cryp->err)
593e22471c2SJia Jie Ho return cryp->err;
594e22471c2SJia Jie Ho
595e22471c2SJia Jie Ho write_text:
596e22471c2SJia Jie Ho if (!cryp->total_in)
597e22471c2SJia Jie Ho goto finish_req;
598e22471c2SJia Jie Ho
599e22471c2SJia Jie Ho /*
600e22471c2SJia Jie Ho * Write first plain/ciphertext block to start the module
601e22471c2SJia Jie Ho * then let irq tasklet handle the rest of the data blocks.
602e22471c2SJia Jie Ho */
603e22471c2SJia Jie Ho scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
604e22471c2SJia Jie Ho cryp->total_in), 0);
605e22471c2SJia Jie Ho cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
606e22471c2SJia Jie Ho
607e22471c2SJia Jie Ho for (i = 0; i < AES_BLOCK_32; i++)
608e22471c2SJia Jie Ho writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);
609e22471c2SJia Jie Ho
610e22471c2SJia Jie Ho stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
611e22471c2SJia Jie Ho stat &= ~STARFIVE_IE_MASK_AES_DONE;
612e22471c2SJia Jie Ho writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
613e22471c2SJia Jie Ho
614e22471c2SJia Jie Ho return 0;
615e22471c2SJia Jie Ho
616e22471c2SJia Jie Ho finish_req:
617e22471c2SJia Jie Ho starfive_aes_finish_req(cryp);
618e22471c2SJia Jie Ho return 0;
619e22471c2SJia Jie Ho }
620e22471c2SJia Jie Ho
starfive_aes_aead_init_tfm(struct crypto_aead * tfm)621e22471c2SJia Jie Ho static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm)
622e22471c2SJia Jie Ho {
623e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
624e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
625e22471c2SJia Jie Ho struct crypto_tfm *aead = crypto_aead_tfm(tfm);
626e22471c2SJia Jie Ho struct crypto_alg *alg = aead->__crt_alg;
627e22471c2SJia Jie Ho
628e22471c2SJia Jie Ho ctx->cryp = starfive_cryp_find_dev(ctx);
629e22471c2SJia Jie Ho if (!ctx->cryp)
630e22471c2SJia Jie Ho return -ENODEV;
631e22471c2SJia Jie Ho
632e22471c2SJia Jie Ho if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
633e22471c2SJia Jie Ho ctx->aead_fbk = crypto_alloc_aead(alg->cra_name, 0,
634e22471c2SJia Jie Ho CRYPTO_ALG_NEED_FALLBACK);
635e22471c2SJia Jie Ho if (IS_ERR(ctx->aead_fbk))
636e22471c2SJia Jie Ho return dev_err_probe(cryp->dev, PTR_ERR(ctx->aead_fbk),
637e22471c2SJia Jie Ho "%s() failed to allocate fallback for %s\n",
638e22471c2SJia Jie Ho __func__, alg->cra_name);
639e22471c2SJia Jie Ho }
640e22471c2SJia Jie Ho
641e22471c2SJia Jie Ho crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_ctx) +
642e22471c2SJia Jie Ho sizeof(struct aead_request));
643e22471c2SJia Jie Ho
644e22471c2SJia Jie Ho return 0;
645e22471c2SJia Jie Ho }
646e22471c2SJia Jie Ho
starfive_aes_aead_exit_tfm(struct crypto_aead * tfm)647e22471c2SJia Jie Ho static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm)
648e22471c2SJia Jie Ho {
649e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
650e22471c2SJia Jie Ho
651e22471c2SJia Jie Ho crypto_free_aead(ctx->aead_fbk);
652e22471c2SJia Jie Ho }
653e22471c2SJia Jie Ho
starfive_aes_crypt(struct skcipher_request * req,unsigned long flags)654e22471c2SJia Jie Ho static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags)
655e22471c2SJia Jie Ho {
656e22471c2SJia Jie Ho struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
657e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
658e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
659e22471c2SJia Jie Ho unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1;
660e22471c2SJia Jie Ho
661e22471c2SJia Jie Ho cryp->flags = flags;
662e22471c2SJia Jie Ho
663e22471c2SJia Jie Ho if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB ||
664e22471c2SJia Jie Ho (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC)
665e22471c2SJia Jie Ho if (req->cryptlen & blocksize_align)
666e22471c2SJia Jie Ho return -EINVAL;
667e22471c2SJia Jie Ho
668e22471c2SJia Jie Ho return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);
669e22471c2SJia Jie Ho }
670e22471c2SJia Jie Ho
starfive_aes_aead_crypt(struct aead_request * req,unsigned long flags)671e22471c2SJia Jie Ho static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags)
672e22471c2SJia Jie Ho {
673e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
674e22471c2SJia Jie Ho struct starfive_cryp_dev *cryp = ctx->cryp;
675e22471c2SJia Jie Ho
676e22471c2SJia Jie Ho cryp->flags = flags;
677e22471c2SJia Jie Ho
678e22471c2SJia Jie Ho /*
679e22471c2SJia Jie Ho * HW engine could not perform CCM tag verification on
680e22471c2SJia Jie Ho * non-blocksize aligned text, use fallback algo instead
681e22471c2SJia Jie Ho */
682e22471c2SJia Jie Ho if (ctx->aead_fbk && !is_encrypt(cryp)) {
683e22471c2SJia Jie Ho struct aead_request *subreq = aead_request_ctx(req);
684e22471c2SJia Jie Ho
685e22471c2SJia Jie Ho aead_request_set_tfm(subreq, ctx->aead_fbk);
686e22471c2SJia Jie Ho aead_request_set_callback(subreq, req->base.flags,
687e22471c2SJia Jie Ho req->base.complete, req->base.data);
688e22471c2SJia Jie Ho aead_request_set_crypt(subreq, req->src,
689e22471c2SJia Jie Ho req->dst, req->cryptlen, req->iv);
690e22471c2SJia Jie Ho aead_request_set_ad(subreq, req->assoclen);
691e22471c2SJia Jie Ho
692e22471c2SJia Jie Ho return crypto_aead_decrypt(subreq);
693e22471c2SJia Jie Ho }
694e22471c2SJia Jie Ho
695e22471c2SJia Jie Ho return crypto_transfer_aead_request_to_engine(cryp->engine, req);
696e22471c2SJia Jie Ho }
697e22471c2SJia Jie Ho
starfive_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)698e22471c2SJia Jie Ho static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
699e22471c2SJia Jie Ho unsigned int keylen)
700e22471c2SJia Jie Ho {
701e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
702e22471c2SJia Jie Ho
703e22471c2SJia Jie Ho if (!key || !keylen)
704e22471c2SJia Jie Ho return -EINVAL;
705e22471c2SJia Jie Ho
706e22471c2SJia Jie Ho if (keylen != AES_KEYSIZE_128 &&
707e22471c2SJia Jie Ho keylen != AES_KEYSIZE_192 &&
708e22471c2SJia Jie Ho keylen != AES_KEYSIZE_256)
709e22471c2SJia Jie Ho return -EINVAL;
710e22471c2SJia Jie Ho
711e22471c2SJia Jie Ho memcpy(ctx->key, key, keylen);
712e22471c2SJia Jie Ho ctx->keylen = keylen;
713e22471c2SJia Jie Ho
714e22471c2SJia Jie Ho return 0;
715e22471c2SJia Jie Ho }
716e22471c2SJia Jie Ho
starfive_aes_aead_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)717e22471c2SJia Jie Ho static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
718e22471c2SJia Jie Ho unsigned int keylen)
719e22471c2SJia Jie Ho {
720e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
721e22471c2SJia Jie Ho
722e22471c2SJia Jie Ho if (!key || !keylen)
723e22471c2SJia Jie Ho return -EINVAL;
724e22471c2SJia Jie Ho
725e22471c2SJia Jie Ho if (keylen != AES_KEYSIZE_128 &&
726e22471c2SJia Jie Ho keylen != AES_KEYSIZE_192 &&
727e22471c2SJia Jie Ho keylen != AES_KEYSIZE_256)
728e22471c2SJia Jie Ho return -EINVAL;
729e22471c2SJia Jie Ho
730e22471c2SJia Jie Ho memcpy(ctx->key, key, keylen);
731e22471c2SJia Jie Ho ctx->keylen = keylen;
732e22471c2SJia Jie Ho
733e22471c2SJia Jie Ho if (ctx->aead_fbk)
734e22471c2SJia Jie Ho return crypto_aead_setkey(ctx->aead_fbk, key, keylen);
735e22471c2SJia Jie Ho
736e22471c2SJia Jie Ho return 0;
737e22471c2SJia Jie Ho }
738e22471c2SJia Jie Ho
starfive_aes_gcm_setauthsize(struct crypto_aead * tfm,unsigned int authsize)739e22471c2SJia Jie Ho static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm,
740e22471c2SJia Jie Ho unsigned int authsize)
741e22471c2SJia Jie Ho {
742e22471c2SJia Jie Ho return crypto_gcm_check_authsize(authsize);
743e22471c2SJia Jie Ho }
744e22471c2SJia Jie Ho
starfive_aes_ccm_setauthsize(struct crypto_aead * tfm,unsigned int authsize)745e22471c2SJia Jie Ho static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm,
746e22471c2SJia Jie Ho unsigned int authsize)
747e22471c2SJia Jie Ho {
748e22471c2SJia Jie Ho struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
749e22471c2SJia Jie Ho
750e22471c2SJia Jie Ho switch (authsize) {
751e22471c2SJia Jie Ho case 4:
752e22471c2SJia Jie Ho case 6:
753e22471c2SJia Jie Ho case 8:
754e22471c2SJia Jie Ho case 10:
755e22471c2SJia Jie Ho case 12:
756e22471c2SJia Jie Ho case 14:
757e22471c2SJia Jie Ho case 16:
758e22471c2SJia Jie Ho break;
759e22471c2SJia Jie Ho default:
760e22471c2SJia Jie Ho return -EINVAL;
761e22471c2SJia Jie Ho }
762e22471c2SJia Jie Ho
763e22471c2SJia Jie Ho return crypto_aead_setauthsize(ctx->aead_fbk, authsize);
764e22471c2SJia Jie Ho }
765e22471c2SJia Jie Ho
starfive_aes_ecb_encrypt(struct skcipher_request * req)766e22471c2SJia Jie Ho static int starfive_aes_ecb_encrypt(struct skcipher_request *req)
767e22471c2SJia Jie Ho {
768e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT);
769e22471c2SJia Jie Ho }
770e22471c2SJia Jie Ho
starfive_aes_ecb_decrypt(struct skcipher_request * req)771e22471c2SJia Jie Ho static int starfive_aes_ecb_decrypt(struct skcipher_request *req)
772e22471c2SJia Jie Ho {
773e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB);
774e22471c2SJia Jie Ho }
775e22471c2SJia Jie Ho
starfive_aes_cbc_encrypt(struct skcipher_request * req)776e22471c2SJia Jie Ho static int starfive_aes_cbc_encrypt(struct skcipher_request *req)
777e22471c2SJia Jie Ho {
778e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT);
779e22471c2SJia Jie Ho }
780e22471c2SJia Jie Ho
starfive_aes_cbc_decrypt(struct skcipher_request * req)781e22471c2SJia Jie Ho static int starfive_aes_cbc_decrypt(struct skcipher_request *req)
782e22471c2SJia Jie Ho {
783e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC);
784e22471c2SJia Jie Ho }
785e22471c2SJia Jie Ho
starfive_aes_cfb_encrypt(struct skcipher_request * req)786e22471c2SJia Jie Ho static int starfive_aes_cfb_encrypt(struct skcipher_request *req)
787e22471c2SJia Jie Ho {
788e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB | FLG_ENCRYPT);
789e22471c2SJia Jie Ho }
790e22471c2SJia Jie Ho
starfive_aes_cfb_decrypt(struct skcipher_request * req)791e22471c2SJia Jie Ho static int starfive_aes_cfb_decrypt(struct skcipher_request *req)
792e22471c2SJia Jie Ho {
793e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB);
794e22471c2SJia Jie Ho }
795e22471c2SJia Jie Ho
starfive_aes_ofb_encrypt(struct skcipher_request * req)796e22471c2SJia Jie Ho static int starfive_aes_ofb_encrypt(struct skcipher_request *req)
797e22471c2SJia Jie Ho {
798e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB | FLG_ENCRYPT);
799e22471c2SJia Jie Ho }
800e22471c2SJia Jie Ho
starfive_aes_ofb_decrypt(struct skcipher_request * req)801e22471c2SJia Jie Ho static int starfive_aes_ofb_decrypt(struct skcipher_request *req)
802e22471c2SJia Jie Ho {
803e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB);
804e22471c2SJia Jie Ho }
805e22471c2SJia Jie Ho
starfive_aes_ctr_encrypt(struct skcipher_request * req)806e22471c2SJia Jie Ho static int starfive_aes_ctr_encrypt(struct skcipher_request *req)
807e22471c2SJia Jie Ho {
808e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT);
809e22471c2SJia Jie Ho }
810e22471c2SJia Jie Ho
starfive_aes_ctr_decrypt(struct skcipher_request * req)811e22471c2SJia Jie Ho static int starfive_aes_ctr_decrypt(struct skcipher_request *req)
812e22471c2SJia Jie Ho {
813e22471c2SJia Jie Ho return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR);
814e22471c2SJia Jie Ho }
815e22471c2SJia Jie Ho
starfive_aes_gcm_encrypt(struct aead_request * req)816e22471c2SJia Jie Ho static int starfive_aes_gcm_encrypt(struct aead_request *req)
817e22471c2SJia Jie Ho {
818e22471c2SJia Jie Ho return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT);
819e22471c2SJia Jie Ho }
820e22471c2SJia Jie Ho
starfive_aes_gcm_decrypt(struct aead_request * req)821e22471c2SJia Jie Ho static int starfive_aes_gcm_decrypt(struct aead_request *req)
822e22471c2SJia Jie Ho {
823e22471c2SJia Jie Ho return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM);
824e22471c2SJia Jie Ho }
825e22471c2SJia Jie Ho
starfive_aes_ccm_encrypt(struct aead_request * req)826e22471c2SJia Jie Ho static int starfive_aes_ccm_encrypt(struct aead_request *req)
827e22471c2SJia Jie Ho {
828e22471c2SJia Jie Ho int ret;
829e22471c2SJia Jie Ho
830e22471c2SJia Jie Ho ret = starfive_aes_ccm_check_iv(req->iv);
831e22471c2SJia Jie Ho if (ret)
832e22471c2SJia Jie Ho return ret;
833e22471c2SJia Jie Ho
834e22471c2SJia Jie Ho return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT);
835e22471c2SJia Jie Ho }
836e22471c2SJia Jie Ho
starfive_aes_ccm_decrypt(struct aead_request * req)837e22471c2SJia Jie Ho static int starfive_aes_ccm_decrypt(struct aead_request *req)
838e22471c2SJia Jie Ho {
839e22471c2SJia Jie Ho int ret;
840e22471c2SJia Jie Ho
841e22471c2SJia Jie Ho ret = starfive_aes_ccm_check_iv(req->iv);
842e22471c2SJia Jie Ho if (ret)
843e22471c2SJia Jie Ho return ret;
844e22471c2SJia Jie Ho
845e22471c2SJia Jie Ho return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM);
846e22471c2SJia Jie Ho }
847e22471c2SJia Jie Ho
848*982213e4SHerbert Xu static struct skcipher_engine_alg skcipher_algs[] = {
849e22471c2SJia Jie Ho {
850*982213e4SHerbert Xu .base.init = starfive_aes_init_tfm,
851*982213e4SHerbert Xu .base.setkey = starfive_aes_setkey,
852*982213e4SHerbert Xu .base.encrypt = starfive_aes_ecb_encrypt,
853*982213e4SHerbert Xu .base.decrypt = starfive_aes_ecb_decrypt,
854*982213e4SHerbert Xu .base.min_keysize = AES_MIN_KEY_SIZE,
855*982213e4SHerbert Xu .base.max_keysize = AES_MAX_KEY_SIZE,
856*982213e4SHerbert Xu .base.base = {
857e22471c2SJia Jie Ho .cra_name = "ecb(aes)",
858e22471c2SJia Jie Ho .cra_driver_name = "starfive-ecb-aes",
859e22471c2SJia Jie Ho .cra_priority = 200,
860e22471c2SJia Jie Ho .cra_flags = CRYPTO_ALG_ASYNC,
861e22471c2SJia Jie Ho .cra_blocksize = AES_BLOCK_SIZE,
862e22471c2SJia Jie Ho .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
863e22471c2SJia Jie Ho .cra_alignmask = 0xf,
864e22471c2SJia Jie Ho .cra_module = THIS_MODULE,
865e22471c2SJia Jie Ho },
866*982213e4SHerbert Xu .op = {
867*982213e4SHerbert Xu .do_one_request = starfive_aes_do_one_req,
868*982213e4SHerbert Xu },
869e22471c2SJia Jie Ho }, {
870*982213e4SHerbert Xu .base.init = starfive_aes_init_tfm,
871*982213e4SHerbert Xu .base.setkey = starfive_aes_setkey,
872*982213e4SHerbert Xu .base.encrypt = starfive_aes_cbc_encrypt,
873*982213e4SHerbert Xu .base.decrypt = starfive_aes_cbc_decrypt,
874*982213e4SHerbert Xu .base.min_keysize = AES_MIN_KEY_SIZE,
875*982213e4SHerbert Xu .base.max_keysize = AES_MAX_KEY_SIZE,
876*982213e4SHerbert Xu .base.ivsize = AES_BLOCK_SIZE,
877*982213e4SHerbert Xu .base.base = {
878e22471c2SJia Jie Ho .cra_name = "cbc(aes)",
879e22471c2SJia Jie Ho .cra_driver_name = "starfive-cbc-aes",
880e22471c2SJia Jie Ho .cra_priority = 200,
881e22471c2SJia Jie Ho .cra_flags = CRYPTO_ALG_ASYNC,
882e22471c2SJia Jie Ho .cra_blocksize = AES_BLOCK_SIZE,
883e22471c2SJia Jie Ho .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
884e22471c2SJia Jie Ho .cra_alignmask = 0xf,
885e22471c2SJia Jie Ho .cra_module = THIS_MODULE,
886e22471c2SJia Jie Ho },
887*982213e4SHerbert Xu .op = {
888*982213e4SHerbert Xu .do_one_request = starfive_aes_do_one_req,
889*982213e4SHerbert Xu },
890e22471c2SJia Jie Ho }, {
891*982213e4SHerbert Xu .base.init = starfive_aes_init_tfm,
892*982213e4SHerbert Xu .base.setkey = starfive_aes_setkey,
893*982213e4SHerbert Xu .base.encrypt = starfive_aes_ctr_encrypt,
894*982213e4SHerbert Xu .base.decrypt = starfive_aes_ctr_decrypt,
895*982213e4SHerbert Xu .base.min_keysize = AES_MIN_KEY_SIZE,
896*982213e4SHerbert Xu .base.max_keysize = AES_MAX_KEY_SIZE,
897*982213e4SHerbert Xu .base.ivsize = AES_BLOCK_SIZE,
898*982213e4SHerbert Xu .base.base = {
899e22471c2SJia Jie Ho .cra_name = "ctr(aes)",
900e22471c2SJia Jie Ho .cra_driver_name = "starfive-ctr-aes",
901e22471c2SJia Jie Ho .cra_priority = 200,
902e22471c2SJia Jie Ho .cra_flags = CRYPTO_ALG_ASYNC,
903e22471c2SJia Jie Ho .cra_blocksize = 1,
904e22471c2SJia Jie Ho .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
905e22471c2SJia Jie Ho .cra_alignmask = 0xf,
906e22471c2SJia Jie Ho .cra_module = THIS_MODULE,
907e22471c2SJia Jie Ho },
908*982213e4SHerbert Xu .op = {
909*982213e4SHerbert Xu .do_one_request = starfive_aes_do_one_req,
910*982213e4SHerbert Xu },
911e22471c2SJia Jie Ho }, {
912*982213e4SHerbert Xu .base.init = starfive_aes_init_tfm,
913*982213e4SHerbert Xu .base.setkey = starfive_aes_setkey,
914*982213e4SHerbert Xu .base.encrypt = starfive_aes_cfb_encrypt,
915*982213e4SHerbert Xu .base.decrypt = starfive_aes_cfb_decrypt,
916*982213e4SHerbert Xu .base.min_keysize = AES_MIN_KEY_SIZE,
917*982213e4SHerbert Xu .base.max_keysize = AES_MAX_KEY_SIZE,
918*982213e4SHerbert Xu .base.ivsize = AES_BLOCK_SIZE,
919*982213e4SHerbert Xu .base.base = {
920e22471c2SJia Jie Ho .cra_name = "cfb(aes)",
921e22471c2SJia Jie Ho .cra_driver_name = "starfive-cfb-aes",
922e22471c2SJia Jie Ho .cra_priority = 200,
923e22471c2SJia Jie Ho .cra_flags = CRYPTO_ALG_ASYNC,
924e22471c2SJia Jie Ho .cra_blocksize = 1,
925e22471c2SJia Jie Ho .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
926e22471c2SJia Jie Ho .cra_alignmask = 0xf,
927e22471c2SJia Jie Ho .cra_module = THIS_MODULE,
928e22471c2SJia Jie Ho },
929*982213e4SHerbert Xu .op = {
930*982213e4SHerbert Xu .do_one_request = starfive_aes_do_one_req,
931*982213e4SHerbert Xu },
932e22471c2SJia Jie Ho }, {
933*982213e4SHerbert Xu .base.init = starfive_aes_init_tfm,
934*982213e4SHerbert Xu .base.setkey = starfive_aes_setkey,
935*982213e4SHerbert Xu .base.encrypt = starfive_aes_ofb_encrypt,
936*982213e4SHerbert Xu .base.decrypt = starfive_aes_ofb_decrypt,
937*982213e4SHerbert Xu .base.min_keysize = AES_MIN_KEY_SIZE,
938*982213e4SHerbert Xu .base.max_keysize = AES_MAX_KEY_SIZE,
939*982213e4SHerbert Xu .base.ivsize = AES_BLOCK_SIZE,
940*982213e4SHerbert Xu .base.base = {
941e22471c2SJia Jie Ho .cra_name = "ofb(aes)",
942e22471c2SJia Jie Ho .cra_driver_name = "starfive-ofb-aes",
943e22471c2SJia Jie Ho .cra_priority = 200,
944e22471c2SJia Jie Ho .cra_flags = CRYPTO_ALG_ASYNC,
945e22471c2SJia Jie Ho .cra_blocksize = 1,
946e22471c2SJia Jie Ho .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
947e22471c2SJia Jie Ho .cra_alignmask = 0xf,
948e22471c2SJia Jie Ho .cra_module = THIS_MODULE,
949e22471c2SJia Jie Ho },
950*982213e4SHerbert Xu .op = {
951*982213e4SHerbert Xu .do_one_request = starfive_aes_do_one_req,
952*982213e4SHerbert Xu },
953e22471c2SJia Jie Ho },
954e22471c2SJia Jie Ho };
955e22471c2SJia Jie Ho
956*982213e4SHerbert Xu static struct aead_engine_alg aead_algs[] = {
957e22471c2SJia Jie Ho {
958*982213e4SHerbert Xu .base.setkey = starfive_aes_aead_setkey,
959*982213e4SHerbert Xu .base.setauthsize = starfive_aes_gcm_setauthsize,
960*982213e4SHerbert Xu .base.encrypt = starfive_aes_gcm_encrypt,
961*982213e4SHerbert Xu .base.decrypt = starfive_aes_gcm_decrypt,
962*982213e4SHerbert Xu .base.init = starfive_aes_aead_init_tfm,
963*982213e4SHerbert Xu .base.exit = starfive_aes_aead_exit_tfm,
964*982213e4SHerbert Xu .base.ivsize = GCM_AES_IV_SIZE,
965*982213e4SHerbert Xu .base.maxauthsize = AES_BLOCK_SIZE,
966*982213e4SHerbert Xu .base.base = {
967e22471c2SJia Jie Ho .cra_name = "gcm(aes)",
968e22471c2SJia Jie Ho .cra_driver_name = "starfive-gcm-aes",
969e22471c2SJia Jie Ho .cra_priority = 200,
970e22471c2SJia Jie Ho .cra_flags = CRYPTO_ALG_ASYNC,
971e22471c2SJia Jie Ho .cra_blocksize = 1,
972e22471c2SJia Jie Ho .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
973e22471c2SJia Jie Ho .cra_alignmask = 0xf,
974e22471c2SJia Jie Ho .cra_module = THIS_MODULE,
975e22471c2SJia Jie Ho },
976*982213e4SHerbert Xu .op = {
977*982213e4SHerbert Xu .do_one_request = starfive_aes_aead_do_one_req,
978*982213e4SHerbert Xu },
979e22471c2SJia Jie Ho }, {
980*982213e4SHerbert Xu .base.setkey = starfive_aes_aead_setkey,
981*982213e4SHerbert Xu .base.setauthsize = starfive_aes_ccm_setauthsize,
982*982213e4SHerbert Xu .base.encrypt = starfive_aes_ccm_encrypt,
983*982213e4SHerbert Xu .base.decrypt = starfive_aes_ccm_decrypt,
984*982213e4SHerbert Xu .base.init = starfive_aes_aead_init_tfm,
985*982213e4SHerbert Xu .base.exit = starfive_aes_aead_exit_tfm,
986*982213e4SHerbert Xu .base.ivsize = AES_BLOCK_SIZE,
987*982213e4SHerbert Xu .base.maxauthsize = AES_BLOCK_SIZE,
988*982213e4SHerbert Xu .base.base = {
989e22471c2SJia Jie Ho .cra_name = "ccm(aes)",
990e22471c2SJia Jie Ho .cra_driver_name = "starfive-ccm-aes",
991e22471c2SJia Jie Ho .cra_priority = 200,
992e22471c2SJia Jie Ho .cra_flags = CRYPTO_ALG_ASYNC |
993e22471c2SJia Jie Ho CRYPTO_ALG_NEED_FALLBACK,
994e22471c2SJia Jie Ho .cra_blocksize = 1,
995e22471c2SJia Jie Ho .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
996e22471c2SJia Jie Ho .cra_alignmask = 0xf,
997e22471c2SJia Jie Ho .cra_module = THIS_MODULE,
998e22471c2SJia Jie Ho },
999*982213e4SHerbert Xu .op = {
1000*982213e4SHerbert Xu .do_one_request = starfive_aes_aead_do_one_req,
1001*982213e4SHerbert Xu },
1002e22471c2SJia Jie Ho },
1003e22471c2SJia Jie Ho };
1004e22471c2SJia Jie Ho
starfive_aes_register_algs(void)1005e22471c2SJia Jie Ho int starfive_aes_register_algs(void)
1006e22471c2SJia Jie Ho {
1007e22471c2SJia Jie Ho int ret;
1008e22471c2SJia Jie Ho
1009*982213e4SHerbert Xu ret = crypto_engine_register_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
1010e22471c2SJia Jie Ho if (ret)
1011e22471c2SJia Jie Ho return ret;
1012e22471c2SJia Jie Ho
1013*982213e4SHerbert Xu ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
1014e22471c2SJia Jie Ho if (ret)
1015*982213e4SHerbert Xu crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
1016e22471c2SJia Jie Ho
1017e22471c2SJia Jie Ho return ret;
1018e22471c2SJia Jie Ho }
1019e22471c2SJia Jie Ho
starfive_aes_unregister_algs(void)1020e22471c2SJia Jie Ho void starfive_aes_unregister_algs(void)
1021e22471c2SJia Jie Ho {
1022*982213e4SHerbert Xu crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
1023*982213e4SHerbert Xu crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
1024e22471c2SJia Jie Ho }
1025